diff --git a/.bazelrc b/.bazelrc index 475706072..1246d336b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -10,6 +10,8 @@ build:linux --copt=-Wno-deprecated-declarations # you will typically need to spell out the compiler for local dev # BAZEL_VC= # BAZEL_VC_FULL_VERSION=14.44.3520 +# Some dependencies rely on bash so you will likely need msys2 +# BAZEL_SH=C:\msys64\usr\bin\bash.exe build:msvc --cxxopt="-std:c++20" --cxxopt="-utf-8" --host_cxxopt="-std:c++20" build:msvc --define=protobuf_allow_msvc=true build:msvc --test_tag_filters=-benchmark,-notap,-no_test_msvc diff --git a/.bazelversion b/.bazelversion index eab246c06..df5119ec6 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -7.3.2 +8.7.0 diff --git a/.github/workflows/bazel_test_post_merge.yml b/.github/workflows/bazel_test_post_merge.yml new file mode 100644 index 000000000..d9cd1b294 --- /dev/null +++ b/.github/workflows/bazel_test_post_merge.yml @@ -0,0 +1,16 @@ +name: Bazel Test (Post-Merge) + +on: + push: + branches: + - master + +jobs: + trigger-win-latest-test: + # This prevents the workflow from running automatically when someone + # pushes to their fork. + if: github.repository == 'cel-expr/cel-cpp' + uses: ./.github/workflows/windows_bazel_test.yml + trigger-mac-latest-test: + if: github.repository == 'cel-expr/cel-cpp' + uses: ./.github/workflows/macos_bazel_test.yml diff --git a/.github/workflows/macos_bazel_test.yml b/.github/workflows/macos_bazel_test.yml new file mode 100644 index 000000000..c0d448503 --- /dev/null +++ b/.github/workflows/macos_bazel_test.yml @@ -0,0 +1,27 @@ +name: MacOS Bazel Test + +on: + workflow_call: + workflow_dispatch: + +jobs: + test: + name: Run Bazel Tests + runs-on: macos-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup Bazel and Bazelisk + uses: bazel-contrib/setup-bazel@0.19.0 + with: + bazelisk-cache: true + disk-cache: ${{ github.workflow }} + repository-cache: true + + - name: Run Tests + shell: bash + # TODO(b/506179116): is not supported on all target versions in xcode, so not + # running policy conformance. + run: | + bazelisk test --copt=-Wno-deprecated-declarations --copt=-Wno-nullability-completeness conformance:all conformance/policy:all diff --git a/.github/workflows/windows_bazel_test.yml b/.github/workflows/windows_bazel_test.yml new file mode 100644 index 000000000..4fb93c04a --- /dev/null +++ b/.github/workflows/windows_bazel_test.yml @@ -0,0 +1,28 @@ +name: Windows Bazel Test + +on: + workflow_call: + workflow_dispatch: + +jobs: + test: + name: Run Bazel Tests + runs-on: windows-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup Bazel and Bazelisk + uses: bazel-contrib/setup-bazel@0.19.0 + with: + bazelisk-cache: true + disk-cache: ${{ github.workflow }} + repository-cache: true + + - name: Run Tests + # msys2 'bash' on Windows will try to 'fix' the label prefix to + # work as a directory. + # //... won't work. + shell: bash + run: | + bazelisk test --config=msvc conformance:all conformance/policy:all diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c2c2915be..000000000 --- a/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -# This Dockerfile is used to create a container around gcc9 and bazel for -# building the CEL C++ library on GitHub. -# -# To update a new version of this container, use gcloud. You may need to run -# `gcloud auth login` and `gcloud auth configure-docker` first. -# -# Note, if you need to run docker using `sudo` use the following commands -# instead: -# -# sudo gcloud auth login --no-launch-browser -# sudo gcloud auth configure-docker -# -# Run the following command from the root of the CEL repository: -# -# gcloud builds submit --region=us -t gcr.io/cel-analysis/gcc9 . -# -# Once complete get the sha256 digest from the output using the following -# command: -# -# gcloud artifacts versions list --package=gcc9 --repository=gcr.io \ -# --location=us -# -# The cloudbuild.yaml file must be updated to use the new digest like so: -# -# - name: 'gcr.io/cel-analysis/gcc9@' -FROM gcc:9 - -# Install Bazel prerequesites and required tools. -# See https://docs.bazel.build/versions/master/install-ubuntu.html -RUN apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - git \ - libssl-dev \ - make \ - pkg-config \ - python3 \ - unzip \ - wget \ - zip \ - zlib1g-dev \ - default-jdk-headless \ - clang-11 && \ - apt-get clean - -# Install Bazel. -# https://github.com/bazelbuild/bazel/releases -ARG BAZEL_VERSION="7.3.2" -ADD https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh /tmp/install_bazel.sh -RUN /bin/bash /tmp/install_bazel.sh && rm /tmp/install_bazel.sh - -RUN mkdir -p /workspace -RUN mkdir -p /bazel - -ENTRYPOINT ["/usr/local/bin/bazel"] diff --git a/MODULE.bazel b/MODULE.bazel index 02404b645..d6ac0cb81 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -17,11 +17,11 @@ bazel_dep( ) bazel_dep( name = "rules_cc", - version = "0.2.14", + version = "0.2.18", ) bazel_dep( name = "rules_java", - version = "8.6.1", + version = "8.14.0", ) bazel_dep( name = "rules_proto", @@ -31,59 +31,50 @@ bazel_dep( name = "rules_python", version = "1.6.3", ) +bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep( name = "protobuf", - version = "33.4", + version = "35.1", repo_name = "com_google_protobuf", ) bazel_dep( name = "abseil-cpp", - version = "20260107.0", + version = "20260526.0", repo_name = "com_google_absl", ) - bazel_dep( name = "googletest", version = "1.17.0.bcr.2", - dev_dependency = True, repo_name = "com_google_googletest", ) bazel_dep( name = "google_benchmark", version = "1.9.2", - dev_dependency = True, repo_name = "com_github_google_benchmark", ) - bazel_dep( name = "re2", version = "2025-11-05.bcr.1", repo_name = "com_googlesource_code_re2", ) -bazel_dep( - name = "flatbuffers", - version = "25.9.23", - repo_name = "com_github_google_flatbuffers", -) bazel_dep( name = "cel-spec", - version = "0.25.1", + version = "0.25.3", repo_name = "com_google_cel_spec", ) +git_override( + module_name = "cel-spec", + commit = "6dccde40b346f89761ddb890abc6ca503a453134", + remote = "https://github.com/cel-expr/cel-spec", +) + bazel_dep( name = "platforms", - version = "1.0.0", + version = "1.1.0", ) - -ANTLR4_VERSION = "4.13.2" - bazel_dep( name = "antlr4-cpp-runtime", - version = ANTLR4_VERSION, -) -single_version_override( - module_name = "antlr4-cpp-runtime", - patches = ["//bazel:antlr.patch"], + version = "4.13.2.bcr.2", ) python = use_extension("@rules_python//python/extensions:python.bzl", "python") @@ -95,6 +86,8 @@ python.toolchain( http_jar = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar") +ANTLR4_VERSION = "4.13.2" + http_jar( name = "antlr4_jar", sha256 = "eae2dfa119a64327444672aff63e9ec35a20180dc5b8090b7a6ab85125df4d76", @@ -105,3 +98,16 @@ bazel_dep( name = "yaml-cpp", version = "0.9.0", ) + +_CEL_POLICY_TAG = "8e4cf0378e3f0d3406b537afbcc6f59562652ad6" + +_CEL_POLICY_SHA = "c3aaed647f04efb7f77d67de21f21720d298938a066cec7f94dd9e24f9a8d88f" + +http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "cel_policy", + sha256 = _CEL_POLICY_SHA, + strip_prefix = "cel-policy-%s" % _CEL_POLICY_TAG, + url = "https://github.com/cel-expr/cel-policy/archive/%s.tar.gz" % _CEL_POLICY_TAG, +) diff --git a/README.md b/README.md index 23afe2b00..7c3c26be0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ # C++ Implementations of the Common Expression Language +> [!WARNING] +> **On June 16, 2026, this repository will move to +> github.com/cel-expr/cel-cpp!** +> +> Please update your links and dependencies. See the [pinned +> issue](https://github.com/google/cel-cpp/issues/2029) for details. + For background on the Common Expression Language see the [cel-spec][1] repo. This is a C++ implementation of a [Common Expression Language][1] runtime, @@ -8,4 +15,4 @@ parser, and type checker. Released under the [Apache License](LICENSE). -[1]: https://github.com/google/cel-spec +[1]: https://github.com/cel-expr/cel-spec diff --git a/base/BUILD b/base/BUILD index a239d4751..9becb06cb 100644 --- a/base/BUILD +++ b/base/BUILD @@ -33,16 +33,64 @@ cc_library( ], deps = [ ":kind", + "//internal:lexis", "//internal:status_macros", + "//internal:strings", + "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:btree", + "@com_google_absl//absl/functional:overload", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/meta:type_traits", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:compare", + "@com_google_absl//absl/types:optional_ref", "@com_google_absl//absl/types:span", - "@com_google_absl//absl/types:variant", + ], +) + +cc_test( + name = "attribute_test", + srcs = ["attribute_test.cc"], + deps = [ + ":attributes", + "//internal:testing", + "@com_google_absl//absl/strings:string_view", + ], +) + +cc_library( + name = "attribute_matcher", + srcs = ["attribute_matcher.cc"], + hdrs = ["attribute_matcher.h"], + deps = [ + ":attributes", + "//internal:status_macros", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/functional:function_ref", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:source_location", + "@com_google_absl//absl/types:span", + ], +) + +cc_test( + name = "attribute_matcher_test", + srcs = ["attribute_matcher_test.cc"], + deps = [ + ":attribute_matcher", + ":attributes", + "//internal:testing", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", ], ) @@ -119,7 +167,11 @@ cc_library( hdrs = [ "function_result.h", ], - deps = [":function_descriptor"], + deps = [ + ":function_descriptor", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/strings:string_view", + ], ) cc_library( diff --git a/base/attribute.cc b/base/attribute.cc index f750a1850..2d8d8cada 100644 --- a/base/attribute.cc +++ b/base/attribute.cc @@ -14,18 +14,27 @@ #include "base/attribute.h" +#include #include #include #include +#include +#include -#include "absl/base/macros.h" +#include "absl/algorithm/container.h" +#include "absl/base/attributes.h" #include "absl/base/nullability.h" +#include "absl/log/absl_check.h" +#include "absl/meta/type_traits.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" -#include "absl/types/variant.h" +#include "absl/strings/string_view.h" +#include "absl/types/compare.h" #include "base/kind.h" +#include "internal/lexis.h" #include "internal/status_macros.h" +#include "internal/strings.h" namespace cel { @@ -36,14 +45,10 @@ class AttributeStringPrinter { public: // String representation for the given qualifier is appended to output. // output must be non-null. - explicit AttributeStringPrinter(std::string* output, Kind type) - : output_(*output), type_(type) {} + explicit AttributeStringPrinter(std::string* output) : output_(*output) {} - absl::Status operator()(const Kind& ignored) const { - // Attributes are represented as a variant, with illegal attribute - // qualifiers represented with their type as the first alternative. - return absl::InvalidArgumentError( - absl::StrCat("Unsupported attribute qualifier ", KindToString(type_))); + absl::Status operator()(std::monostate) const { + return absl::InvalidArgumentError("bad attribute qualifier"); } absl::Status operator()(int64_t index) { @@ -68,22 +73,19 @@ class AttributeStringPrinter { private: std::string& output_; - Kind type_; }; // Visitor for appending string representation for different qualifier kinds. class AttributeQualifierStringPrinter { public: // String representation for the given qualifier is appended to output. - explicit AttributeQualifierStringPrinter(std::string* absl_nonnull output, - Kind type) - : output_(*output), type_(type) {} + explicit AttributeQualifierStringPrinter(std::string* absl_nonnull output) + : output_(*output) {} - absl::Status operator()(const Kind& ignored) const { + absl::Status operator()(std::monostate) const { // Attributes are represented as a variant, with illegal attribute // qualifiers represented with their type as the first alternative. - return absl::InvalidArgumentError( - absl::StrCat("Unsupported attribute qualifier ", KindToString(type_))); + return absl::InvalidArgumentError("bad attribute qualifier"); } absl::Status operator()(int64_t index) { @@ -108,223 +110,635 @@ class AttributeQualifierStringPrinter { private: std::string& output_; - Kind type_; }; struct AttributeQualifierTypeVisitor final { - Kind operator()(const Kind& type) const { return type; } + Kind operator()(std::monostate) const { return Kind::kNull; } - Kind operator()(int64_t ignored) const { - static_cast(ignored); - return Kind::kInt64; - } + Kind operator()(int64_t) const { return Kind::kInt64; } + + Kind operator()(uint64_t) const { return Kind::kUint64; } + + Kind operator()(const std::string&) const { return Kind::kString; } - Kind operator()(uint64_t ignored) const { - static_cast(ignored); - return Kind::kUint64; + Kind operator()(bool) const { return Kind::kBool; } +}; + +struct AttributeQualifierToString { + void operator()(std::monostate) const {} + + void operator()(bool value) const { output.append(value ? "true" : "false"); } + + void operator()(int64_t value) const { absl::StrAppend(&output, value); } + + void operator()(uint64_t value) const { + absl::StrAppend(&output, value, "u"); } - Kind operator()(const std::string& ignored) const { - static_cast(ignored); - return Kind::kString; + void operator()(absl::string_view value) const { + if (internal::LexisIsIdentifier(value)) { + absl::StrAppend(&output, value); + } else { + internal::FormatStringLiteralTo(value, &output); + } } - Kind operator()(bool ignored) const { - static_cast(ignored); - return Kind::kBool; + void operator()(common_internal::WildcardType) const { + output.push_back('*'); } + + std::string& output; }; -struct AttributeQualifierTypeComparator final { - const Kind lhs; +struct AttributeQualifierJoinString { + void operator()(std::monostate) const {} - bool operator()(const Kind& rhs) const { - return static_cast(lhs) < static_cast(rhs); + void operator()(bool value) const { + output.push_back('['); + output.append(value ? "true" : "false"); + output.push_back(']'); } - bool operator()(int64_t) const { return false; } + void operator()(int64_t value) const { + absl::StrAppend(&output, "[", value, "]"); + } - bool operator()(uint64_t other) const { return false; } + void operator()(uint64_t value) const { + absl::StrAppend(&output, "[", value, "u", "]"); + } - bool operator()(const std::string&) const { return false; } + void operator()(absl::string_view value) const { + if (internal::LexisIsIdentifier(value)) { + absl::StrAppend(&output, ".", value); + } else { + output.push_back('['); + internal::FormatStringLiteralTo(value, &output); + output.push_back(']'); + } + } + + void operator()(common_internal::WildcardType) const { output.append(".*"); } - bool operator()(bool other) const { return false; } + std::string& output; }; -struct AttributeQualifierIntComparator final { - const int64_t lhs; +template +void AttributeQualifierAppendJoinString(std::string* out, + absl::Span qualifiers) { + for (const auto& qualifier : qualifiers) { + std::visit(AttributeQualifierJoinString{*out}, + common_internal::AsVariant(qualifier)); + } +} + +} // namespace - bool operator()(const Kind&) const { return true; } +Kind AttributeQualifier::kind() const { + return std::visit(AttributeQualifierTypeVisitor{}, value_); +} - bool operator()(int64_t rhs) const { return lhs < rhs; } +absl::StatusOr Attribute::AsString() const { + if (variable_name().empty()) { + return absl::InvalidArgumentError( + "Only ident rooted attributes are supported."); + } - bool operator()(uint64_t) const { return true; } + std::string result = std::string(variable_name()); - bool operator()(const std::string&) const { return true; } + for (const auto& qualifier : qualifier_path()) { + CEL_RETURN_IF_ERROR(std::visit(AttributeStringPrinter(&result), + common_internal::AsVariant(qualifier))); + } - bool operator()(bool) const { return false; } -}; + return result; +} -struct AttributeQualifierUintComparator final { - const uint64_t lhs; +std::string Attribute::ToString() const { + std::string result; + result.append(variable_name()); + AttributeQualifier::AppendJoinString(&result, qualifier_path()); + return result; +} - bool operator()(const Kind&) const { return true; } +std::string AttributePattern::ToString() const { + std::string result; + result.append(variable()); + AttributeQualifierPattern::AppendJoinString(&result, qualifier_path()); + return result; +} - bool operator()(int64_t) const { return false; } +void AttributeQualifier::AppendToString(std::string* absl_nonnull out) const { + std::visit(AttributeQualifierToString{*out}, value_); +} - bool operator()(uint64_t rhs) const { return lhs < rhs; } +void AttributeQualifierView::AppendToString( + std::string* absl_nonnull out) const { + std::visit(AttributeQualifierToString{*out}, value_); +} - bool operator()(const std::string&) const { return true; } +void AttributeQualifierPattern::AppendToString( + std::string* absl_nonnull out) const { + std::visit(AttributeQualifierToString{*out}, value_); +} - bool operator()(bool) const { return false; } -}; +void AttributeQualifier::AppendJoinString(std::string* absl_nonnull out) const { + std::visit(AttributeQualifierJoinString{*out}, value_); +} -struct AttributeQualifierStringComparator final { - const std::string& lhs; +void AttributeQualifierView::AppendJoinString( + std::string* absl_nonnull out) const { + std::visit(AttributeQualifierJoinString{*out}, value_); +} - bool operator()(const Kind&) const { return true; } +void AttributeQualifierPattern::AppendJoinString( + std::string* absl_nonnull out) const { + std::visit(AttributeQualifierJoinString{*out}, value_); +} - bool operator()(int64_t) const { return false; } +void AttributeQualifier::AppendJoinString( + std::string* absl_nonnull out, + absl::Span qualifiers) { + AttributeQualifierAppendJoinString(out, qualifiers); +} - bool operator()(uint64_t) const { return false; } +void AttributeQualifierView::AppendJoinString( + std::string* absl_nonnull out, + absl::Span qualifiers) { + AttributeQualifierAppendJoinString(out, qualifiers); +} - bool operator()(const std::string& rhs) const { return lhs < rhs; } +void AttributeQualifierPattern::AppendJoinString( + std::string* absl_nonnull out, + absl::Span qualifiers) { + AttributeQualifierAppendJoinString(out, qualifiers); +} - bool operator()(bool) const { return false; } -}; +bool AttributeQualifier::IsMatch(const AttributeQualifier& other) const { + ABSL_DCHECK(*this) << "bad attribute qualifier"; + ABSL_DCHECK(other) << "bad attribute qualifier"; + if (std::holds_alternative(value_) || + std::holds_alternative(other.value_)) { + return false; + } + return *this == other; +} -struct AttributeQualifierBoolComparator final { - const bool lhs; +bool AttributeQualifier::IsMatch(absl::string_view other_key) const { + ABSL_DCHECK(*this) << "bad attribute qualifier"; + if (auto string = AsString(); string.has_value()) { + return *string == other_key; + } + return false; +} - bool operator()(const Kind&) const { return true; } +bool AttributeQualifierPattern::IsMatch( + const AttributeQualifier& qualifier) const { + ABSL_DCHECK(*this) << "bad attribute qualifier pattern"; + ABSL_DCHECK(qualifier) << "bad attribute qualifier"; + if (!*this || !qualifier) { + return false; + } + if (IsWildcard()) { + return true; + } + return *this == qualifier; +} - bool operator()(int64_t) const { return true; } +bool AttributeQualifierPattern::IsMatch(absl::string_view other_key) const { + ABSL_DCHECK(*this) << "bad attribute qualifier pattern"; + if (IsWildcard()) { + return true; + } + if (auto string = AsString(); string.has_value()) { + return *string == other_key; + } + return false; +} - bool operator()(uint64_t) const { return true; } +namespace { - bool operator()(const std::string&) const { return true; } +struct AttributeQualifierEqualTo { + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(const std::monostate&, + const std::monostate&) const { + return false; + } - bool operator()(bool rhs) const { return lhs < rhs; } -}; + template + ABSL_ATTRIBUTE_ALWAYS_INLINE std::enable_if_t< + !std::is_same_v, std::monostate>, bool> + operator()(const std::monostate&, const T&) const { + return false; + } -} // namespace + template + ABSL_ATTRIBUTE_ALWAYS_INLINE std::enable_if_t< + !std::is_same_v, std::monostate>, bool> + operator()(const T&, const std::monostate&) const { + return false; + } -struct AttributeQualifier::ComparatorVisitor final { - const AttributeQualifier::Variant& rhs; + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(bool lhs, bool rhs) const { + return lhs == rhs; + } - bool operator()(const Kind& lhs) const { - return absl::visit(AttributeQualifierTypeComparator{lhs}, rhs); + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(bool, int64_t) const { + return false; } - bool operator()(int64_t lhs) const { - return absl::visit(AttributeQualifierIntComparator{lhs}, rhs); + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(bool, uint64_t) const { + return false; } - bool operator()(uint64_t lhs) const { - return absl::visit(AttributeQualifierUintComparator{lhs}, rhs); + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(bool, absl::string_view) const { + return false; } - bool operator()(const std::string& lhs) const { - return absl::visit(AttributeQualifierStringComparator{lhs}, rhs); + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(int64_t, bool) const { + return false; } - bool operator()(bool lhs) const { - return absl::visit(AttributeQualifierBoolComparator{lhs}, rhs); + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(int64_t lhs, int64_t rhs) const { + return lhs == rhs; } -}; -Kind AttributeQualifier::kind() const { - return absl::visit(AttributeQualifierTypeVisitor{}, value_); -} + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(int64_t, uint64_t) const { + return false; + } -bool AttributeQualifier::operator<(const AttributeQualifier& other) const { - // The order is not publicly documented because it is subject to change. - // Currently we sort in the following order, with each type being sorted - // against itself: bool, int, uint, string, type. - return absl::visit(ComparatorVisitor{other.value_}, value_); -} + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(int64_t, + absl::string_view) const { + return false; + } -bool Attribute::operator==(const Attribute& other) const { - // We cannot check pointer equality as a short circuit because we have to - // treat all invalid AttributeQualifier as not equal to each other. - // TODO(issues/41) we only support Ident-rooted attributes at the moment. - if (variable_name() != other.variable_name()) { + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(uint64_t, bool) const { return false; } - if (qualifier_path().size() != other.qualifier_path().size()) { + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(uint64_t, int64_t) const { return false; } - for (size_t i = 0; i < qualifier_path().size(); i++) { - if (!(qualifier_path()[i] == other.qualifier_path()[i])) { - return false; - } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(uint64_t lhs, + uint64_t rhs) const { + return lhs == rhs; } - return true; -} + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(uint64_t, + absl::string_view) const { + return false; + } -bool Attribute::operator<(const Attribute& other) const { - if (impl_.get() == other.impl_.get()) { + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(absl::string_view, bool) const { return false; } - auto lhs_begin = qualifier_path().begin(); - auto lhs_end = qualifier_path().end(); - auto rhs_begin = other.qualifier_path().begin(); - auto rhs_end = other.qualifier_path().end(); - while (lhs_begin != lhs_end && rhs_begin != rhs_end) { - if (*lhs_begin < *rhs_begin) { - return true; - } - if (!(*lhs_begin == *rhs_begin)) { - return false; - } - lhs_begin++; - rhs_begin++; + + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(absl::string_view, + int64_t) const { + return false; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(absl::string_view, + uint64_t) const { + return false; } - if (lhs_begin == lhs_end && rhs_begin == rhs_end) { - // Neither has any elements left, they are equal. Compare variable names. - return variable_name() < other.variable_name(); + + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(absl::string_view lhs, + absl::string_view rhs) const { + return lhs == rhs; } - if (lhs_begin == lhs_end) { - // Left has no more elements. Right is greater. + + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()( + const common_internal::WildcardType&, + const common_internal::WildcardType&) const { return true; } - // Right has no more elements. Left is greater. - ABSL_ASSERT(rhs_begin == rhs_end); - return false; + + template + ABSL_ATTRIBUTE_ALWAYS_INLINE std::enable_if_t< + !std::is_same_v, common_internal::WildcardType> && + !std::is_same_v, std::monostate>, + bool> + operator()(const common_internal::WildcardType&, const T&) const { + return false; + } + + template + ABSL_ATTRIBUTE_ALWAYS_INLINE std::enable_if_t< + !std::is_same_v, common_internal::WildcardType> && + !std::is_same_v, std::monostate>, + bool> + operator()(const T&, const common_internal::WildcardType&) const { + return false; + } +}; + +template +[[nodiscard]] +ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering SynthesizeCompare(T lhs, + T rhs) { +#if defined(__cpp_impl_three_way_comparison) && \ + defined(__cpp_lib_three_way_comparison) + return lhs <=> rhs; +#else + if (lhs < rhs) { + return absl::strong_ordering::less; + } + if (rhs < lhs) { + return absl::strong_ordering::greater; + } + return absl::strong_ordering::equal; +#endif } -const absl::StatusOr Attribute::AsString() const { - if (variable_name().empty()) { - return absl::InvalidArgumentError( - "Only ident rooted attributes are supported."); +struct AttributeQualifierCompare { + // monostate < int < uint < string < bool < wildcard + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + const std::monostate&, const std::monostate&) const { + return absl::strong_ordering::equal; } - std::string result = std::string(variable_name()); + template + ABSL_ATTRIBUTE_ALWAYS_INLINE + std::enable_if_t, std::monostate>, + absl::strong_ordering> + operator()(const std::monostate&, const T&) const { + return absl::strong_ordering::less; + } - for (const auto& qualifier : qualifier_path()) { - CEL_RETURN_IF_ERROR(absl::visit( - AttributeStringPrinter(&result, qualifier.kind()), qualifier.value_)); + template + ABSL_ATTRIBUTE_ALWAYS_INLINE + std::enable_if_t, std::monostate>, + absl::strong_ordering> + operator()(const T&, const std::monostate&) const { + return absl::strong_ordering::greater; } - return result; + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + bool lhs, bool rhs) const { + return (SynthesizeCompare)(lhs, rhs); + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()(bool, + int64_t) const { + return absl::strong_ordering::greater; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + bool, uint64_t) const { + return absl::strong_ordering::greater; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + bool, absl::string_view) const { + return absl::strong_ordering::greater; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()(int64_t, + bool) const { + return absl::strong_ordering::less; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + int64_t lhs, int64_t rhs) const { + return (SynthesizeCompare)(lhs, rhs); + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + int64_t, uint64_t) const { + return absl::strong_ordering::less; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + int64_t, absl::string_view) const { + return absl::strong_ordering::less; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()(uint64_t, + bool) const { + return absl::strong_ordering::less; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()(uint64_t, + int64_t) const { + return absl::strong_ordering::greater; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + uint64_t lhs, uint64_t rhs) const { + return (SynthesizeCompare)(lhs, rhs); + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + uint64_t, absl::string_view) const { + return absl::strong_ordering::less; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + absl::string_view, bool) const { + return absl::strong_ordering::less; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + absl::string_view, int64_t) const { + return absl::strong_ordering::greater; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + absl::string_view, uint64_t) const { + return absl::strong_ordering::greater; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + absl::string_view lhs, absl::string_view rhs) const { + return (SynthesizeCompare)(lhs, rhs); + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + const common_internal::WildcardType&, + const common_internal::WildcardType&) const { + return absl::strong_ordering::equal; + } + + template + ABSL_ATTRIBUTE_ALWAYS_INLINE std::enable_if_t< + !std::is_same_v, common_internal::WildcardType> && + !std::is_same_v, std::monostate>, + absl::strong_ordering> + operator()(const common_internal::WildcardType&, const T&) const { + return absl::strong_ordering::greater; + } + + template + ABSL_ATTRIBUTE_ALWAYS_INLINE std::enable_if_t< + !std::is_same_v, common_internal::WildcardType> && + !std::is_same_v, std::monostate>, + absl::strong_ordering> + operator()(const T&, const common_internal::WildcardType&) const { + return absl::strong_ordering::less; + } +}; + +} // namespace + +absl::strong_ordering AttributeQualifier::Compare( + const AttributeQualifier& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); } -bool AttributeQualifier::IsMatch(const AttributeQualifier& other) const { - if (absl::holds_alternative(value_) || - absl::holds_alternative(other.value_)) { - return false; +absl::strong_ordering AttributeQualifier::Compare( + const AttributeQualifierView& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifier::Compare( + const AttributeQualifierPattern& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifierView::Compare( + const AttributeQualifier& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifierView::Compare( + const AttributeQualifierView& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifierView::Compare( + const AttributeQualifierPattern& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifierPattern::Compare( + const AttributeQualifier& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifierPattern::Compare( + const AttributeQualifierView& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifierPattern::Compare( + const AttributeQualifierPattern& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering Attribute::Compare(const Attribute& other) const { + if (absl::strong_ordering order = + (SynthesizeCompare)(variable_name(), other.variable_name()); + order != absl::strong_ordering::equal) { + return order; } - return value_ == other.value_; + const auto& qualifiers = qualifier_path(); + const auto& other_qualifiers = other.qualifier_path(); + const size_t min_size = std::min(qualifiers.size(), other_qualifiers.size()); + for (size_t i = 0; i < min_size; ++i) { + if (absl::strong_ordering order = + qualifiers[i].Compare(other_qualifiers[i]); + order != absl::strong_ordering::equal) { + return order; + } + } + if (qualifiers.size() < other_qualifiers.size()) { + return absl::strong_ordering::less; + } + if (qualifiers.size() > other_qualifiers.size()) { + return absl::strong_ordering::greater; + } + return absl::strong_ordering::equal; } -absl::StatusOr AttributeQualifier::AsString() const { - std::string result; - CEL_RETURN_IF_ERROR( - absl::visit(AttributeQualifierStringPrinter(&result, kind()), value_)); - return result; +bool operator==(const AttributeQualifier& lhs, const AttributeQualifier& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifierView& lhs, + const AttributeQualifierView& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifierPattern& lhs, + const AttributeQualifierPattern& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifier& lhs, + const AttributeQualifierView& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifier& lhs, + const AttributeQualifierPattern& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifierView& lhs, + const AttributeQualifier& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifierView& lhs, + const AttributeQualifierPattern& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifierPattern& lhs, + const AttributeQualifier& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifierPattern& lhs, + const AttributeQualifierView& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const Attribute& lhs, const Attribute& rhs) { + return lhs.variable_name() == rhs.variable_name() && + absl::c_equal(lhs.qualifier_path(), rhs.qualifier_path()); +} + +bool operator==(const AttributePattern& lhs, const AttributePattern& rhs) { + return lhs.variable() == rhs.variable() && + absl::c_equal(lhs.qualifier_path(), rhs.qualifier_path()); } } // namespace cel diff --git a/base/attribute.h b/base/attribute.h index 69dcaf161..7e9e98c67 100644 --- a/base/attribute.h +++ b/base/attribute.h @@ -15,108 +15,259 @@ #ifndef THIRD_PARTY_CEL_CPP_BASE_ATTRIBUTE_H_ #define THIRD_PARTY_CEL_CPP_BASE_ATTRIBUTE_H_ +#include #include +#include #include #include #include #include +#include #include +#include "absl/base/attributes.h" +#include "absl/base/macros.h" +#include "absl/base/nullability.h" +#include "absl/functional/overload.h" +#include "absl/log/absl_check.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" +#include "absl/types/compare.h" +#include "absl/types/optional_ref.h" #include "absl/types/span.h" -#include "absl/types/variant.h" #include "base/kind.h" namespace cel { -// AttributeQualifier represents a segment in -// attribute resolutuion path. A segment can be qualified by values of -// following types: string/int64_t/uint64_t/bool. -class AttributeQualifier final { - private: - struct ComparatorVisitor; +namespace common_internal { +struct WildcardType {}; +// Important. The types in the variants must align with each other and the order +// matters. It must be int < uint < string < bool. If you change it here in the +// header you must change the logic in the source. +using AttributeQualifierVariant = + std::variant; +using AttributeQualifierPatternVariant = + std::variant; +using AttributeQualifierViewVariant = + std::variant; +} // namespace common_internal - using Variant = absl::variant; +class AttributeQualifier; +class AttributeQualifierPattern; +class Attribute; +class AttributePattern; +class AttributeQualifierView; +namespace common_internal { +[[nodiscard]] +const AttributeQualifierVariant& AsVariant( + const AttributeQualifier& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND); +[[nodiscard]] +AttributeQualifierVariant&& AsVariant( + AttributeQualifier&& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND); +[[nodiscard]] +const AttributeQualifierPatternVariant& AsVariant( + const AttributeQualifierPattern& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND); +[[nodiscard]] +AttributeQualifierPatternVariant&& AsVariant( + AttributeQualifierPattern&& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND); +[[nodiscard]] +const AttributeQualifierViewVariant& AsVariant( + const AttributeQualifierView& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND); +[[nodiscard]] +AttributeQualifierViewVariant&& AsVariant( + AttributeQualifierView&& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND); +} // namespace common_internal + +// AttributeQualifier represents a segment in the +// attribute resolution path. A segment can be qualified by values of +// following types: string/int64_t/uint64_t/bool. +class AttributeQualifier { public: static AttributeQualifier OfInt(int64_t value) { - return AttributeQualifier(absl::in_place_type, std::move(value)); + return AttributeQualifier(std::in_place_type, std::move(value)); } static AttributeQualifier OfUint(uint64_t value) { - return AttributeQualifier(absl::in_place_type, std::move(value)); + return AttributeQualifier(std::in_place_type, std::move(value)); + } + + static AttributeQualifier OfString(const char* value) { + return OfString(absl::string_view(value)); } static AttributeQualifier OfString(std::string value) { - return AttributeQualifier(absl::in_place_type, + return AttributeQualifier(std::in_place_type, std::move(value)); } + static AttributeQualifier OfString(absl::string_view value) { + return AttributeQualifier(std::in_place_type, + std::string(value)); + } + static AttributeQualifier OfBool(bool value) { - return AttributeQualifier(absl::in_place_type, std::move(value)); + return AttributeQualifier(std::in_place_type, std::move(value)); } AttributeQualifier() = default; - AttributeQualifier(const AttributeQualifier&) = default; AttributeQualifier(AttributeQualifier&&) = default; - AttributeQualifier& operator=(const AttributeQualifier&) = default; AttributeQualifier& operator=(AttributeQualifier&&) = default; + explicit AttributeQualifier(const AttributeQualifierView& other); + Kind kind() const; - // Family of Get... methods. Return values if requested type matches the - // stored one. - absl::optional GetInt64Key() const { - return absl::holds_alternative(value_) - ? absl::optional(absl::get<1>(value_)) - : absl::nullopt; + ABSL_DEPRECATE_AND_INLINE() + std::optional GetInt64Key() const { return AsInt(); } + + ABSL_DEPRECATE_AND_INLINE() + std::optional GetUint64Key() const { return AsUint(); } + + ABSL_DEPRECATED("Use AsString") + std::optional GetStringKey() const { + if (auto string = AsString(); string.has_value()) { + return *string; + } + return std::nullopt; } - absl::optional GetUint64Key() const { - return absl::holds_alternative(value_) - ? absl::optional(absl::get<2>(value_)) - : absl::nullopt; + ABSL_DEPRECATE_AND_INLINE() + std::optional GetBoolKey() const { return AsBool(); } + + explicit operator bool() const { + return !std::holds_alternative(value_); } - absl::optional GetStringKey() const { - return absl::holds_alternative(value_) - ? absl::optional(absl::get<3>(value_)) - : absl::nullopt; + [[nodiscard]] + bool IsBool() const { + return std::holds_alternative(value_); } - absl::optional GetBoolKey() const { - return absl::holds_alternative(value_) - ? absl::optional(absl::get<4>(value_)) - : absl::nullopt; + [[nodiscard]] + bool IsInt() const { + return std::holds_alternative(value_); } - bool operator==(const AttributeQualifier& other) const { - return IsMatch(other); + [[nodiscard]] + bool IsUint() const { + return std::holds_alternative(value_); } - bool operator<(const AttributeQualifier& other) const; + [[nodiscard]] + bool IsString() const { + return std::holds_alternative(value_); + } - bool IsMatch(absl::string_view other_key) const { - absl::optional key = GetStringKey(); - return (key.has_value() && key.value() == other_key); + [[nodiscard]] + bool GetBool() const { + ABSL_DCHECK(IsBool()); + return std::get(value_); } - absl::StatusOr AsString() const; + [[nodiscard]] + int64_t GetInt() const { + ABSL_DCHECK(IsInt()); + return std::get(value_); + } - private: - friend class Attribute; - friend struct ComparatorVisitor; + [[nodiscard]] + uint64_t GetUint() const { + ABSL_DCHECK(IsUint()); + return std::get(value_); + } - template - AttributeQualifier(absl::in_place_type_t in_place_type, T&& value) - : value_(in_place_type, std::forward(value)) {} + [[nodiscard]] + const std::string& GetString() const { + ABSL_DCHECK(IsString()); + return std::get(value_); + } + + [[nodiscard]] + std::optional AsBool() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] + std::optional AsInt() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; + } + [[nodiscard]] + std::optional AsUint() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] + absl::optional_ref AsString() const { + if (const auto* value = std::get_if(&value_); + value != nullptr) { + return *value; + } + return std::nullopt; + } + + void AppendToString(std::string* absl_nonnull out) const; + + [[nodiscard]] + std::string ToString() const { + std::string out; + AppendToString(&out); + return out; + } + + void AppendJoinString(std::string* absl_nonnull out) const; + + [[nodiscard]] + std::string JoinString() const { + std::string out; + AppendJoinString(&out); + return out; + } + + static void AppendJoinString(std::string* absl_nonnull out, + absl::Span qualifiers); + + [[nodiscard]] bool IsMatch(const AttributeQualifier& other) const; + [[nodiscard]] + bool IsMatch(absl::string_view other_key) const; + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifier& other) const; + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifierView& other) const; + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifierPattern& other) const; + + private: + friend const common_internal::AttributeQualifierVariant& + common_internal::AsVariant(const AttributeQualifier& qualifier); + friend common_internal::AttributeQualifierVariant&& + common_internal::AsVariant(AttributeQualifier&& qualifier); + + template + explicit AttributeQualifier(std::in_place_type_t in_place_type, + Args&&... args) + : value_(in_place_type, std::forward(args)...) {} + + using Variant = common_internal::AttributeQualifierVariant; + // The previous implementation of Attribute preserved all value // instances, regardless of whether they are supported in this context or not. // We represented unsupported types by using the first alternative and thus @@ -124,58 +275,748 @@ class AttributeQualifier final { Variant value_; }; -// AttributeQualifierPattern matches a segment in -// attribute resolutuion path. AttributeQualifierPattern is capable of -// matching path elements of types string/int64/uint64/bool. -class AttributeQualifierPattern final { +class AttributeQualifierView { + public: + static AttributeQualifierView OfInt(int64_t value) { + return AttributeQualifierView(std::in_place_type, + std::move(value)); + } + + static AttributeQualifierView OfUint(uint64_t value) { + return AttributeQualifierView(std::in_place_type, + std::move(value)); + } + + static AttributeQualifierView OfString(const char* value) { + return OfString(absl::string_view(value)); + } + + static AttributeQualifierView OfString(absl::string_view value) { + return AttributeQualifierView(std::in_place_type, + std::move(value)); + } + + static AttributeQualifierView OfString(std::string&&) = delete; + + static AttributeQualifierView OfBool(bool value) { + return AttributeQualifierView(std::in_place_type, std::move(value)); + } + + AttributeQualifierView() = default; + AttributeQualifierView(const AttributeQualifierView&) = default; + AttributeQualifierView& operator=(const AttributeQualifierView&) = default; + + // NOLINTNEXTLINE(google-explicit-constructor) + AttributeQualifierView( + const AttributeQualifier& other ABSL_ATTRIBUTE_LIFETIME_BOUND) + : value_(std::visit( + absl::Overload( + [](std::monostate value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](bool value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](int64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](uint64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](const std::string& value) -> Variant { + return Variant(std::in_place_type, value); + }), + common_internal::AsVariant(other))) {} + + // NOLINTNEXTLINE(google-explicit-constructor) + AttributeQualifierView& operator=( + const AttributeQualifier& other ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return *this = AttributeQualifierView(other); + } + + AttributeQualifierView& operator=(AttributeQualifier&&) = delete; + + explicit operator bool() const { + return !std::holds_alternative(value_); + } + + ABSL_DEPRECATE_AND_INLINE() + std::optional GetInt64Key() const { return AsInt(); } + + ABSL_DEPRECATE_AND_INLINE() + std::optional GetUint64Key() const { return AsUint(); } + + ABSL_DEPRECATE_AND_INLINE() + std::optional GetStringKey() const { return AsString(); } + + ABSL_DEPRECATE_AND_INLINE() + std::optional GetBoolKey() const { return AsBool(); } + + [[nodiscard]] + bool IsBool() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsInt() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsUint() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsString() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool GetBool() const { + ABSL_DCHECK(IsBool()); + return std::get(value_); + } + + [[nodiscard]] + int64_t GetInt() const { + ABSL_DCHECK(IsInt()); + return std::get(value_); + } + + [[nodiscard]] + uint64_t GetUint() const { + ABSL_DCHECK(IsUint()); + return std::get(value_); + } + + [[nodiscard]] + absl::string_view GetString() const { + ABSL_DCHECK(IsString()); + return std::get(value_); + } + + [[nodiscard]] + std::optional AsBool() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] + std::optional AsInt() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] + std::optional AsUint() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] + std::optional AsString() const { + if (const auto* value = std::get_if(&value_); + value != nullptr) { + return *value; + } + return std::nullopt; + } + + void AppendToString(std::string* absl_nonnull out) const; + + [[nodiscard]] + std::string ToString() const { + std::string out; + AppendToString(&out); + return out; + } + + void AppendJoinString(std::string* absl_nonnull out) const; + + [[nodiscard]] + std::string JoinString() const { + std::string out; + AppendJoinString(&out); + return out; + } + + static void AppendJoinString( + std::string* absl_nonnull out, + absl::Span qualifiers); + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifier& other) const; + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifierView& other) const; + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifierPattern& other) const; + private: - // Qualifier value. If not set, treated as wildcard. - std::optional value_; + friend const common_internal::AttributeQualifierViewVariant& + common_internal::AsVariant(const AttributeQualifierView& qualifier); + friend common_internal::AttributeQualifierViewVariant&& + common_internal::AsVariant(AttributeQualifierView&& qualifier); - explicit AttributeQualifierPattern(std::optional value) - : value_(std::move(value)) {} + using Variant = common_internal::AttributeQualifierViewVariant; + template + AttributeQualifierView(std::in_place_type_t in_place_type, T&& value) + : value_(in_place_type, std::forward(value)) {} + + Variant value_; +}; + +inline AttributeQualifier::AttributeQualifier( + const AttributeQualifierView& other) + : value_(std::visit( + absl::Overload([](std::monostate) -> Variant { return Variant(); }, + [](int64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](uint64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](absl::string_view value) -> Variant { + return Variant(std::in_place_type, + value); + }, + [](bool value) -> Variant { + return Variant(std::in_place_type, value); + }), + common_internal::AsVariant(other))) {} + +// AttributeQualifierPattern matches a segment in +// attribute resolution path. AttributeQualifierPattern is capable of +// matching path elements of types string/int64/uint64/bool. +class AttributeQualifierPattern { public: static AttributeQualifierPattern OfInt(int64_t value) { - return AttributeQualifierPattern(AttributeQualifier::OfInt(value)); + return AttributeQualifierPattern(std::in_place_type, value); } static AttributeQualifierPattern OfUint(uint64_t value) { - return AttributeQualifierPattern(AttributeQualifier::OfUint(value)); + return AttributeQualifierPattern(std::in_place_type, value); + } + + static AttributeQualifierPattern OfString(const char* value) { + return OfString(absl::string_view(value)); } static AttributeQualifierPattern OfString(std::string value) { - return AttributeQualifierPattern( - AttributeQualifier::OfString(std::move(value))); + return AttributeQualifierPattern(std::in_place_type, + std::move(value)); + } + + static AttributeQualifierPattern OfString(absl::string_view value) { + return AttributeQualifierPattern(std::in_place_type, + std::string(value)); } static AttributeQualifierPattern OfBool(bool value) { - return AttributeQualifierPattern(AttributeQualifier::OfBool(value)); + return AttributeQualifierPattern(std::in_place_type, value); + } + + ABSL_DEPRECATE_AND_INLINE() + static AttributeQualifierPattern CreateWildcard() { return Wildcard(); } + + static AttributeQualifierPattern Wildcard() { + return AttributeQualifierPattern(std::in_place_type); + } + + AttributeQualifierPattern() = default; + AttributeQualifierPattern(const AttributeQualifierPattern&) = default; + AttributeQualifierPattern(AttributeQualifierPattern&&) = default; + AttributeQualifierPattern& operator=(const AttributeQualifierPattern&) = + default; + AttributeQualifierPattern& operator=(AttributeQualifierPattern&&) = default; + + // NOLINTNEXTLINE(google-explicit-constructor) + AttributeQualifierPattern(const AttributeQualifier& value) + : value_(std::visit( + absl::Overload( + [](std::monostate value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](bool value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](int64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](uint64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](const std::string& value) -> Variant { + return Variant(std::in_place_type, value); + }), + common_internal::AsVariant(value))) {} + + // NOLINTNEXTLINE(google-explicit-constructor) + AttributeQualifierPattern(AttributeQualifier&& value) + : value_(std::visit( + absl::Overload( + [](std::monostate value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](bool value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](int64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](uint64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](std::string&& value) -> Variant { + return Variant(std::in_place_type, + std::move(value)); + }), + common_internal::AsVariant(std::move(value)))) {} + + explicit operator bool() const { + return !std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsBool() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsInt() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsUint() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsString() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsWildcard() const { + return std::holds_alternative(value_); } - static AttributeQualifierPattern CreateWildcard() { - return AttributeQualifierPattern(std::nullopt); + [[nodiscard]] + bool GetBool() const { + ABSL_DCHECK(IsBool()); + return std::get(value_); } - explicit AttributeQualifierPattern(AttributeQualifier qualifier) - : AttributeQualifierPattern( - std::optional(std::move(qualifier))) {} + [[nodiscard]] + int64_t GetInt() const { + ABSL_DCHECK(IsInt()); + return std::get(value_); + } + + [[nodiscard]] + uint64_t GetUint() const { + ABSL_DCHECK(IsUint()); + return std::get(value_); + } - bool IsWildcard() const { return !value_.has_value(); } + [[nodiscard]] + const std::string& GetString() const { + ABSL_DCHECK(IsString()); + return std::get(value_); + } - bool IsMatch(const AttributeQualifier& qualifier) const { - if (IsWildcard()) return true; - return value_.value() == qualifier; + [[nodiscard]] + std::optional AsBool() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; } - bool IsMatch(absl::string_view other_key) const { - if (!value_.has_value()) return true; - return value_->IsMatch(other_key); + [[nodiscard]] + std::optional AsInt() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; } + + [[nodiscard]] + std::optional AsUint() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] + absl::optional_ref AsString() const { + if (const auto* value = std::get_if(&value_); + value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] + std::optional ToQualifier() const { + return std::visit( + absl::Overload( + [](std::monostate) -> std::optional { + return AttributeQualifier(); + }, + [](bool value) -> std::optional { + return AttributeQualifier::OfBool(value); + }, + [](int64_t value) -> std::optional { + return AttributeQualifier::OfInt(value); + }, + [](uint64_t value) -> std::optional { + return AttributeQualifier::OfUint(value); + }, + [](const std::string& value) -> std::optional { + return AttributeQualifier::OfString(value); + }, + [](common_internal::WildcardType value) + -> std::optional { return std::nullopt; }), + value_); + } + + [[nodiscard]] + std::optional ToQualifierView() const + ABSL_ATTRIBUTE_LIFETIME_BOUND { + return std::visit( + absl::Overload( + [](std::monostate) -> std::optional { + return AttributeQualifierView(); + }, + [](bool value) -> std::optional { + return AttributeQualifierView::OfBool(value); + }, + [](int64_t value) -> std::optional { + return AttributeQualifierView::OfInt(value); + }, + [](uint64_t value) -> std::optional { + return AttributeQualifierView::OfUint(value); + }, + [](const std::string& value) + -> std::optional { + return AttributeQualifierView::OfString(value); + }, + [](common_internal::WildcardType value) + -> std::optional { + return std::nullopt; + }), + value_); + } + + void AppendToString(std::string* absl_nonnull out) const; + + [[nodiscard]] + std::string ToString() const { + std::string out; + AppendToString(&out); + return out; + } + + void AppendJoinString(std::string* absl_nonnull out) const; + + [[nodiscard]] + std::string JoinString() const { + std::string out; + AppendJoinString(&out); + return out; + } + + static void AppendJoinString( + std::string* absl_nonnull out, + absl::Span qualifiers); + + [[nodiscard]] + bool IsMatch(const AttributeQualifier& qualifier) const; + + [[nodiscard]] + bool IsMatch(absl::string_view other_key) const; + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifier& other) const; + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifierView& other) const; + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifierPattern& other) const; + + private: + friend const common_internal::AttributeQualifierPatternVariant& + common_internal::AsVariant(const AttributeQualifierPattern& qualifier); + friend common_internal::AttributeQualifierPatternVariant&& + common_internal::AsVariant(AttributeQualifierPattern&& qualifier); + + using Variant = common_internal::AttributeQualifierPatternVariant; + using WildcardType = common_internal::WildcardType; + + template + explicit AttributeQualifierPattern(std::in_place_type_t in_place_type, + Args&&... args) + : value_(in_place_type, std::forward(args)...) {} + + // Qualifier value. If not set, treated as wildcard. + common_internal::AttributeQualifierPatternVariant value_; }; +[[nodiscard]] +bool operator==(const AttributeQualifier& lhs, const AttributeQualifier& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifierView& lhs, + const AttributeQualifierView& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifierPattern& lhs, + const AttributeQualifierPattern& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifier& lhs, + const AttributeQualifierView& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifier& lhs, + const AttributeQualifierPattern& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifierView& lhs, + const AttributeQualifier& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifierView& lhs, + const AttributeQualifierPattern& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifierPattern& lhs, + const AttributeQualifier& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifierPattern& lhs, + const AttributeQualifierView& rhs); + +#if defined(__cpp_impl_three_way_comparison) && \ + defined(__cpp_lib_three_way_comparison) +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifier& lhs, + const AttributeQualifier& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifier& lhs, + const AttributeQualifierView& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifier& lhs, + const AttributeQualifierPattern& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifierView& lhs, + const AttributeQualifier& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifierView& lhs, + const AttributeQualifierView& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifierView& lhs, + const AttributeQualifierPattern& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifierPattern& lhs, + const AttributeQualifier& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifierPattern& lhs, + const AttributeQualifierView& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifierPattern& lhs, + const AttributeQualifierPattern& rhs) { + return lhs.Compare(rhs); +} +#else +[[nodiscard]] +inline bool operator!=(const AttributeQualifier& lhs, + const AttributeQualifier& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifierView& lhs, + const AttributeQualifierView& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifierPattern& lhs, + const AttributeQualifierPattern& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifier& lhs, + const AttributeQualifierView& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifier& lhs, + const AttributeQualifierPattern& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifierView& lhs, + const AttributeQualifier& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifierView& lhs, + const AttributeQualifierPattern& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifierPattern& lhs, + const AttributeQualifier& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifierPattern& lhs, + const AttributeQualifierView& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator<(const AttributeQualifier& lhs, + const AttributeQualifier& rhs) { + return lhs.Compare(rhs) < 0; +} + +[[nodiscard]] +inline bool operator<(const AttributeQualifierView& lhs, + const AttributeQualifierView& rhs) { + return lhs.Compare(rhs) < 0; +} + +[[nodiscard]] +inline bool operator<(const AttributeQualifierPattern& lhs, + const AttributeQualifierPattern& rhs) { + return lhs.Compare(rhs) < 0; +} + +[[nodiscard]] +inline bool operator<(const AttributeQualifier& lhs, + const AttributeQualifierView& rhs) { + return lhs.Compare(rhs) < 0; +} + +[[nodiscard]] +inline bool operator<(const AttributeQualifier& lhs, + const AttributeQualifierPattern& rhs) { + return lhs.Compare(rhs) < 0; +} + +[[nodiscard]] +inline bool operator<(const AttributeQualifierView& lhs, + const AttributeQualifier& rhs) { + return lhs.Compare(rhs) < 0; +} + +[[nodiscard]] +inline bool operator<(const AttributeQualifierView& lhs, + const AttributeQualifierPattern& rhs) { + return lhs.Compare(rhs) < 0; +} + +[[nodiscard]] +inline bool operator<(const AttributeQualifierPattern& lhs, + const AttributeQualifier& rhs) { + return lhs.Compare(rhs) < 0; +} + +[[nodiscard]] +inline bool operator<(const AttributeQualifierPattern& lhs, + const AttributeQualifierView& rhs) { + return lhs.Compare(rhs) < 0; +} +#endif + +namespace common_internal { + +[[nodiscard]] +inline const AttributeQualifierVariant& AsVariant( + const AttributeQualifier& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return qualifier.value_; +} + +[[nodiscard]] +inline AttributeQualifierVariant&& AsVariant( + AttributeQualifier&& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return std::move(qualifier.value_); +} + +[[nodiscard]] +inline const AttributeQualifierPatternVariant& AsVariant( + const AttributeQualifierPattern& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return qualifier.value_; +} + +[[nodiscard]] +inline AttributeQualifierPatternVariant&& AsVariant( + AttributeQualifierPattern&& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return std::move(qualifier.value_); +} + +[[nodiscard]] +inline const AttributeQualifierViewVariant& AsVariant( + const AttributeQualifierView& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return qualifier.value_; +} + +[[nodiscard]] +inline AttributeQualifierViewVariant&& AsVariant( + AttributeQualifierView&& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return std::move(qualifier.value_); +} + +} // namespace common_internal + // Attribute represents resolved attribute path. -class Attribute final { +class Attribute { public: explicit Attribute(std::string variable_name) : Attribute(std::move(variable_name), {}) {} @@ -193,11 +1034,13 @@ class Attribute final { return impl_->qualifier_path; } - bool operator==(const Attribute& other) const; + absl::StatusOr AsString() const; - bool operator<(const Attribute& other) const; + [[nodiscard]] + std::string ToString() const; - const absl::StatusOr AsString() const; + [[nodiscard]] + absl::strong_ordering Compare(const Attribute& other) const; private: struct Impl final { @@ -213,12 +1056,39 @@ class Attribute final { std::shared_ptr impl_; }; +[[nodiscard]] +bool operator==(const Attribute& lhs, const Attribute& rhs); + +#if defined(__cpp_impl_three_way_comparison) && \ + defined(__cpp_lib_three_way_comparison) +[[nodiscard]] +inline absl::strong_ordering operator<=>(const Attribute& lhs, + const Attribute& rhs) { + return lhs.Compare(rhs); +} +#else +[[nodiscard]] +inline bool operator!=(const Attribute& lhs, const Attribute& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator<(const Attribute& lhs, const Attribute& rhs) { + return lhs.Compare(rhs) < 0; +} +#endif + +template +void AbslStringify(S& sink, const Attribute& attribute) { + sink.Append(attribute.ToString()); +} + // AttributePattern is a fully-qualified absolute attribute path pattern. // Supported segments steps in the path are: // - field selection; // - map lookup by key; // - list access by index. -class AttributePattern final { +class AttributePattern { public: // MatchType enum specifies how closely pattern is matching the attribute: enum class MatchType { @@ -227,8 +1097,9 @@ class AttributePattern final { FULL // Pattern matches an attribute itself. }; - AttributePattern(std::string variable, - std::vector qualifier_path) + explicit AttributePattern( + std::string variable, + std::vector qualifier_path = {}) : variable_(std::move(variable)), qualifier_path_(std::move(qualifier_path)) {} @@ -261,17 +1132,34 @@ class AttributePattern final { return result; } + [[nodiscard]] + std::string ToString() const; + private: std::string variable_; std::vector qualifier_path_; }; +template +void AbslStringify(S& sink, const AttributePattern& pattern) { + sink.Append(pattern.ToString()); +} + +[[nodiscard]] +bool operator==(const AttributePattern& lhs, const AttributePattern& rhs); + +[[nodiscard]] +inline bool operator!=(const AttributePattern& lhs, + const AttributePattern& rhs) { + return !operator==(lhs, rhs); +} + struct FieldSpecifier { int64_t number; std::string name; }; -using SelectQualifier = absl::variant; +using SelectQualifier = std::variant; } // namespace cel diff --git a/base/attribute_matcher.cc b/base/attribute_matcher.cc new file mode 100644 index 000000000..328045aa1 --- /dev/null +++ b/base/attribute_matcher.cc @@ -0,0 +1,366 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "base/attribute_matcher.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/no_destructor.h" +#include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/source_location.h" +#include "absl/types/span.h" +#include "base/attribute.h" +#include "internal/status_macros.h" + +namespace cel { + +namespace common_internal { + +std::pair AttributeMatcherNode::Add( + const AttributeQualifier& pattern, bool prev_emplaced) { + // If this node was not just created and it has no children, it is a full + // match and additional children cannot be added. + if (wildcard_ == nullptr && children_ == nullptr && !prev_emplaced) { + return {nullptr, false}; + } + // If this node has a wildcard already, no more children can be added. + if (wildcard_ != nullptr) { + return {nullptr, false}; + } + if (children_ == nullptr) { + children_ = std::make_unique>(); + } + auto [child, emplaced] = children_->emplace(this, pattern); + return {const_cast(&*child), emplaced}; +} + +std::pair AttributeMatcherNode::Add( + const AttributeQualifierPattern& pattern, bool prev_emplaced) { + if (auto qualifier = pattern.ToQualifier(); qualifier.has_value()) { + return Add(*qualifier, prev_emplaced); + } + // If this node was not just created and it has no children, it is a full + // match and additional children cannot be added. + if (wildcard_ == nullptr && children_ == nullptr && !prev_emplaced) { + return {nullptr, false}; + } + // If this node has children already, no wildcard can be added. + if (children_ != nullptr) { + return {nullptr, false}; + } + if (wildcard_ == nullptr) { + wildcard_ = std::make_unique( + this, AttributeQualifierPattern::Wildcard()); + return {wildcard_.get(), true}; + } + return {wildcard_.get(), false}; +} + +size_t AttributeMatcherNode::RemoveChildren() { + if (children_ == nullptr && wildcard_ == nullptr) { + return 0; + } + return RemoveChildrenInternal(); +} + +size_t AttributeMatcherNode::RemoveChildrenInternal() { + if (children_ == nullptr && wildcard_ == nullptr) { + return 1; + } + size_t count = 0; + if (children_ != nullptr) { + auto children_begin = children_->begin(); + const auto children_end = children_->end(); + while (children_begin != children_end) { + count += const_cast(&*children_begin) + ->RemoveChildrenInternal(); + children_begin = children_->erase(children_begin); + } + children_.reset(); + } + if (wildcard_ != nullptr) { + count += wildcard_->RemoveChildrenInternal(); + wildcard_.reset(); + } + return count; +} + +bool AttributeMatcherNode::Remove() { + AttributeMatcherNode* parent = this->parent(); + if (parent == nullptr) { + return children_ == nullptr && wildcard_ == nullptr; + } + if (children_ == nullptr && wildcard_ == nullptr) { + if (key_.IsWildcard()) { + ABSL_DCHECK_EQ(parent->wildcard_.get(), this); + parent->wildcard_.reset(); + } else { + parent->children_->erase(*this); + if (parent->children_->empty()) { + parent->children_.reset(); + } + } + return parent->ChildRemoved(); + } + return false; +} + +bool AttributeMatcherNode::ChildRemoved() { return Remove(); } + +} // namespace common_internal + +namespace { + +absl::Status AttributePatternNotFound( + absl::string_view pattern, + absl::SourceLocation source_location = absl::SourceLocation::current()) { + return absl::NotFoundError( + absl::StrCat("attribute pattern ", pattern, " not found"), + source_location); +} + +bool IsWildcard(const AttributeQualifierPattern& qualifier) { + return qualifier.IsWildcard(); +} + +bool IsWildcard(const AttributeQualifier&) { return false; } + +template +absl::Status ValidateAttribute(absl::string_view variable, + absl::Span qualifiers, + absl::FunctionRef to_string) { + if (variable.empty()) { + return absl::InvalidArgumentError( + "attribute pattern variable name is required"); + } + for (size_t i = 0; i < qualifiers.size(); ++i) { + if (!qualifiers[i]) { + return absl::InvalidArgumentError("bad attribute qualifier pattern"); + } + if (IsWildcard(qualifiers[i]) && i != qualifiers.size() - 1) { + return absl::InvalidArgumentError( + "wildcard qualifier in attribute pattern must be last"); + } + } + return absl::OkStatus(); +} + +} // namespace + +template +absl::Status AttributeMatcher::InsertAttribute( + absl::string_view variable, absl::Span qualifiers, + absl::FunctionRef to_string) { + CEL_RETURN_IF_ERROR(ValidateAttribute(variable, qualifiers, to_string)); + auto [node, emplaced] = AddVariable(variable); + for (const auto& qualifier : qualifiers) { + std::tie(node, emplaced) = node->Add(qualifier, emplaced); + if (node == nullptr) { + break; + } + } + if (node == nullptr) { + return absl::AlreadyExistsError( + absl::StrCat("attribute pattern which is a superset of ", to_string(), + " already exists")); + } + if (!emplaced) { + return absl::AlreadyExistsError(absl::StrCat( + "attribute pattern covering ", to_string(), " already exists")); + } + ++attributes_; + return absl::OkStatus(); +} + +absl::Status AttributeMatcher::InsertAttribute( + const AttributePattern& pattern) { + return InsertAttribute( + pattern.variable(), pattern.qualifier_path(), + [&pattern]() -> std::string { return pattern.ToString(); }); +} + +absl::Status AttributeMatcher::InsertAttribute(const Attribute& attribute) { + return InsertAttribute( + attribute.variable_name(), attribute.qualifier_path(), + [&attribute]() -> std::string { return attribute.ToString(); }); +} + +template +absl::Status AttributeMatcher::UpsertAttribute( + absl::string_view variable, absl::Span qualifiers, + absl::FunctionRef to_string) { + CEL_RETURN_IF_ERROR(ValidateAttribute(variable, qualifiers, to_string)); + auto [node, emplaced] = AddVariable(variable); + for (const auto& qualifier : qualifiers) { + std::tie(node, emplaced) = node->Add(qualifier, emplaced); + if (node == nullptr) { + // subset + return absl::OkStatus(); + } + } + if (node->children_ == nullptr && node->wildcard_ == nullptr) { + // duplicate. + return absl::OkStatus(); + } + if (!emplaced) { + // superset + attributes_ -= node->RemoveChildren(); + ++attributes_; + } + return absl::OkStatus(); +} + +absl::Status AttributeMatcher::UpsertAttribute( + const AttributePattern& pattern) { + return UpsertAttribute( + pattern.variable(), pattern.qualifier_path(), + [&pattern]() -> std::string { return pattern.ToString(); }); +} + +absl::Status AttributeMatcher::UpsertAttribute(const Attribute& attribute) { + return UpsertAttribute( + attribute.variable_name(), attribute.qualifier_path(), + [&attribute]() -> std::string { return attribute.ToString(); }); +} + +template +absl::Status AttributeMatcher::RemoveAttribute( + absl::string_view variable, absl::Span qualifiers, + absl::FunctionRef to_string) { + CEL_RETURN_IF_ERROR(ValidateAttribute(variable, qualifiers, to_string)); + auto children_it = children_.find(AttributeQualifierView::OfString(variable)); + if (children_it == children_.end()) { + return AttributePatternNotFound(to_string()); + } + Node* variable_node = const_cast(&*children_it); + Node* node = variable_node; + for (const auto& qualifier : qualifiers) { + if (IsWildcard(qualifier)) { + if (node->wildcard_ == nullptr) { + return AttributePatternNotFound(to_string()); + } + node = node->wildcard_.get(); + } else { + if (node->children_ == nullptr) { + return AttributePatternNotFound(to_string()); + } + children_it = node->children_->find(qualifier); + if (children_it == node->children_->end()) { + return AttributePatternNotFound(to_string()); + } + node = const_cast(&*children_it); + } + } + if (node->children_ != nullptr || node->wildcard_ != nullptr) { + return AttributePatternNotFound(to_string()); + } + --attributes_; + if (node->Remove()) { + children_.erase(*variable_node); + } + return absl::OkStatus(); +} + +absl::Status AttributeMatcher::RemoveAttribute( + const AttributePattern& pattern) { + return RemoveAttribute( + pattern.variable(), pattern.qualifier_path(), + [&pattern]() -> std::string { return pattern.ToString(); }); +} + +absl::Status AttributeMatcher::RemoveAttribute(const Attribute& attribute) { + return RemoveAttribute( + attribute.variable_name(), attribute.qualifier_path(), + [&attribute]() -> std::string { return attribute.ToString(); }); +} + +struct AttributeMatcher::AttributeCollector { + std::vector result; + absl::string_view variable; + std::vector qualifiers; + + explicit AttributeCollector(size_t capacity) { result.reserve(capacity); } + + void CollectVariable(absl::string_view variable, + const common_internal::AttributeMatcherNode* node) { + if (node->children_ == nullptr && node->wildcard_ == nullptr) { + result.push_back(AttributePattern(std::string(variable), {})); + return; + } + this->variable = variable; + qualifiers.clear(); + if (node->children_ != nullptr) { + for (const auto& child : *node->children_) { + CollectQualifier(child.key(), &child); + } + } + if (node->wildcard_ != nullptr) { + CollectQualifier(AttributeQualifierPattern::Wildcard(), + node->wildcard_.get()); + } + } + + void CollectQualifier(const AttributeQualifierPattern& qualifier, + const common_internal::AttributeMatcherNode* node) { + qualifiers.emplace_back(qualifier); + if (node->children_ == nullptr && node->wildcard_ == nullptr) { + result.push_back(AttributePattern(std::string(variable), qualifiers)); + } else { + if (node->children_ != nullptr) { + for (const auto& child : *node->children_) { + CollectQualifier(child.key(), &child); + } + } + if (node->wildcard_ != nullptr) { + CollectQualifier(AttributeQualifierPattern::Wildcard(), + node->wildcard_.get()); + } + } + qualifiers.pop_back(); + } +}; + +[[nodiscard]] +std::vector AttributeMatcher::GetAttributes() const { + AttributeCollector collector(attributes_); + for (const auto& child : children_) { + // Guaranteed to be string. + collector.CollectVariable(child.key().GetString(), &child); + } + return std::move(collector.result); +} + +std::pair AttributeMatcher::AddVariable( + absl::string_view variable) { + auto [child, emplaced] = + children_.emplace(nullptr, AttributeQualifierPattern::OfString(variable)); + return {const_cast(&*child), emplaced}; +} + +const AttributeMatcher& EmptyAttributeMatcher() { + static const absl::NoDestructor empty; + return *empty; +} + +} // namespace cel diff --git a/base/attribute_matcher.h b/base/attribute_matcher.h new file mode 100644 index 000000000..a243e56b8 --- /dev/null +++ b/base/attribute_matcher.h @@ -0,0 +1,435 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_BASE_ATTRIBUTE_MATCHER_H_ +#define THIRD_PARTY_CEL_CPP_BASE_ATTRIBUTE_MATCHER_H_ + +#include +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/base/optimization.h" +#include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "base/attribute.h" + +namespace cel { + +class AttributeMatcher; +class AttributeMatch; + +namespace common_internal { + +// Node in the tree used by AttributeMatcher, represents either a variable or a +// qualifier. Nodes with no parent are variables and all others are qualifiers. +class AttributeMatcherNode { + public: + struct Less { + using is_transparent = void; + + bool operator()(const AttributeMatcherNode& lhs, + const AttributeMatcherNode& rhs) const { + return lhs.key() < rhs.key(); + } + + bool operator()(const AttributeMatcherNode& lhs, + const AttributeQualifierPattern& rhs) const { + return lhs.key() < rhs; + } + + bool operator()(const AttributeQualifierPattern& lhs, + const AttributeMatcherNode& rhs) const { + return lhs < rhs.key(); + } + + bool operator()(const AttributeMatcherNode& lhs, + const AttributeQualifierView& rhs) const { + return lhs.key() < rhs; + } + + bool operator()(const AttributeQualifierView& lhs, + const AttributeMatcherNode& rhs) const { + return lhs < rhs.key(); + } + + bool operator()(const AttributeMatcherNode& lhs, + const AttributeQualifier& rhs) const { + return lhs.key() < rhs; + } + + bool operator()(const AttributeQualifier& lhs, + const AttributeMatcherNode& rhs) const { + return lhs < rhs.key(); + } + + bool operator()(absl::string_view lhs, + const AttributeQualifierPattern& rhs) const { + // Fast path for top-level variable matching, should only be called in + // that context. + ABSL_DCHECK(rhs.IsString()); + return lhs < rhs.GetString(); + } + + bool operator()(const AttributeQualifierPattern& lhs, + absl::string_view rhs) const { + // Fast path for top-level variable matching, should only be called in + // that context. + ABSL_DCHECK(lhs.IsString()); + return lhs.GetString() < rhs; + } + + bool operator()(absl::string_view lhs, + const AttributeMatcherNode& rhs) const { + return (*this)(lhs, rhs.key()); + } + + bool operator()(const AttributeMatcherNode& lhs, + absl::string_view rhs) const { + return (*this)(lhs.key(), rhs); + } + }; + + AttributeMatcherNode(AttributeMatcherNode* parent, + const AttributeQualifierPattern& key) + : parent_(parent), + key_(key), + depth_(parent != nullptr ? parent->depth_ + 1 : 0) {} + + [[nodiscard]] + bool IsPartial() const { + return children_ != nullptr || wildcard_ != nullptr; + } + + // True if this is a partial match and the only child of this node is a + // wildcard and the wildcard has no children. + [[nodiscard]] + bool IsPartialWildcard() const { + return children_ == nullptr && wildcard_ != nullptr && wildcard_->IsFull(); + } + + [[nodiscard]] + bool IsFull() const { + return !IsPartial(); + } + + [[nodiscard]] + const AttributeQualifierPattern& key() const { + return key_; + } + + [[nodiscard]] + AttributeMatcherNode* parent() const { + return parent_; + } + + [[nodiscard]] + size_t depth() const { + return depth_; + } + + [[nodiscard]] + const AttributeMatcherNode* MatchQualifier( + const AttributeQualifierView& qualifier) const { + if (children_ != nullptr) { + auto it = children_->find(qualifier); + if (it != children_->end()) { + return &*it; + } + } else if (wildcard_ != nullptr) { + return wildcard_.get(); + } + return nullptr; + } + + [[nodiscard]] + const AttributeMatcherNode* MatchQualifier( + const AttributeQualifier& qualifier) const { + if (children_ != nullptr) { + auto it = children_->find(qualifier); + if (it != children_->end()) { + return &*it; + } + } else if (wildcard_ != nullptr) { + return wildcard_.get(); + } + return nullptr; + } + + [[nodiscard]] + std::pair Add(const AttributeQualifier& pattern, + bool prev_emplaced); + [[nodiscard]] + std::pair Add( + const AttributeQualifierPattern& pattern, bool prev_emplaced); + + [[nodiscard]] + bool Remove(); + + [[nodiscard]] + size_t RemoveChildren(); + + private: + friend class cel::AttributeMatcher; + + [[nodiscard]] + bool ChildRemoved(); + + [[nodiscard]] + size_t RemoveChildrenInternal(); + + AttributeMatcherNode* const parent_; + const AttributeQualifierPattern key_; + const size_t depth_; + // Pointer stability is required, so use std::set. + std::unique_ptr> children_; + std::unique_ptr wildcard_; +}; + +AttributeMatch MakeAttributeMatch( + const common_internal::AttributeMatcherNode* node); + +[[nodiscard]] +const AttributeMatcherNode* GetAttributeMatcherNode(AttributeMatch match); + +} // namespace common_internal + +class [[nodiscard]] AttributeMatch { + public: + using Type = AttributePattern::MatchType; + + AttributeMatch() = default; + AttributeMatch(const AttributeMatch&) = default; + AttributeMatch& operator=(const AttributeMatch&) = default; + + [[nodiscard]] + Type GetType() const { + return node_ == nullptr ? Type::NONE + : node_->IsPartial() ? Type::PARTIAL + : Type::FULL; + } + + [[nodiscard]] + bool IsNone() const { + return node_ == nullptr; + } + + [[nodiscard]] + bool IsPartial() const { + return node_ != nullptr && node_->IsPartial(); + } + + // True if this is a partial match and the only possible next match is a + // wildcard and there is no next match after the wildcard. + [[nodiscard]] + bool IsPartialWildcard() const { + return node_ != nullptr && node_->IsPartialWildcard(); + } + + [[nodiscard]] + bool IsFull() const { + return node_ != nullptr && node_->IsFull(); + } + + // Attempts to match the next qualifier. This should not be called on full + // matches, as those do not have children by definition. Instead the full + // match should be handled. + AttributeMatch MatchQualifier(const AttributeQualifierView& qualifier) const { + ABSL_DCHECK(!IsFull()); + ABSL_DCHECK(qualifier); + if (IsNone()) { + return AttributeMatch(); + } + return AttributeMatch(node_->MatchQualifier(qualifier)); + } + + private: + friend class AttributeMatcher; + friend AttributeMatch common_internal::MakeAttributeMatch( + const common_internal::AttributeMatcherNode* node); + friend const common_internal::AttributeMatcherNode* + common_internal::GetAttributeMatcherNode(AttributeMatch match); + + AttributeMatch NotNoneMatchQualifier( + const AttributeQualifierView& qualifier) const { + ABSL_DCHECK(!IsNone()); + ABSL_DCHECK(qualifier); + return AttributeMatch(node_->MatchQualifier(qualifier)); + } + + explicit AttributeMatch(const common_internal::AttributeMatcherNode* node) + : node_(node) {} + + const common_internal::AttributeMatcherNode* node_ = nullptr; +}; + +namespace common_internal { + +inline AttributeMatch MakeAttributeMatch( + const common_internal::AttributeMatcherNode* node) { + return AttributeMatch(node); +} + +[[nodiscard]] +inline const AttributeMatcherNode* GetAttributeMatcherNode( + AttributeMatch match) { + return match.node_; +} + +} // namespace common_internal + +// AttributeMatcher allows for efficiently checking for unknown or missing +// attributes at runtime in a scalable manner. AttributeMatcher is +// thread-compatible however it must not be modified during any evaluation where +// it is being used. +class AttributeMatcher { + private: + using Node = common_internal::AttributeMatcherNode; + + public: + AttributeMatcher() = default; + AttributeMatcher(const AttributeMatcher&) = delete; + AttributeMatcher(AttributeMatcher&&) = default; + AttributeMatcher& operator=(const AttributeMatcher&) = delete; + AttributeMatcher& operator=(AttributeMatcher&&) = default; + + // Inserts an attribute to the matcher. Returns an error if the attribute + // or attribute pattern being added overlaps with a previously added + // attribute. + absl::Status InsertAttribute(const AttributePattern& pattern); + absl::Status InsertAttribute(const Attribute& attribute); + + // Inserts or updates the attribute in the matcher. If the attribute being + // added is a superset of others, the subsets will be removed and replaced + // with this attribute. If the attribute being added is a subset of others, + // no action is taken. Otherwise the attribute is inserted. + absl::Status UpsertAttribute(const AttributePattern& pattern); + absl::Status UpsertAttribute(const Attribute& attribute); + + // Removes a previously added unknown attribute from the matcher. + absl::Status RemoveAttribute(const AttributePattern& pattern); + absl::Status RemoveAttribute(const Attribute& attribute); + + // Resets the matcher such that all observable behavior is equivalent to a + // default constructed matcher. + void ClearAttributes() { + children_.clear(); + attributes_ = 0; + } + + AttributeMatch MatchVariable(absl::string_view variable) const + ABSL_ATTRIBUTE_LIFETIME_BOUND { + if (!variable.empty()) { + auto it = children_.find(variable); + if (it != children_.end()) { + return common_internal::MakeAttributeMatch(&*it); + } + } + return AttributeMatch(); + } + + AttributeMatch MatchAttribute( + absl::string_view variable, + absl::Span qualifiers) const { + AttributeMatch match = MatchVariable(variable); + for (const auto& qualifier : qualifiers) { + if (ABSL_PREDICT_FALSE(!qualifier)) { + return AttributeMatch(); + } + if (match.IsNone()) { + break; + } + match = match.NotNoneMatchQualifier(qualifier); + } + return match; + } + + AttributeMatch MatchAttribute( + absl::string_view variable, + absl::Span qualifiers) const { + AttributeMatch match = MatchVariable(variable); + for (const auto& qualifier : qualifiers) { + if (ABSL_PREDICT_FALSE(!qualifier)) { + return AttributeMatch(); + } + if (match.IsNone()) { + break; + } + match = match.NotNoneMatchQualifier(qualifier); + } + return match; + } + + AttributeMatch MatchAttribute(const Attribute& attribute) const { + return MatchAttribute(attribute.variable_name(), + attribute.qualifier_path()); + } + + // Returns an array of attribute patterns corresponding to the + // previously added attribute patterns. The order of the attribute + // patterns in the array is not guaranteed. + [[nodiscard]] + std::vector GetAttributes() const; + + // Tests whether any attribute patterns have been added to this + // matcher. + [[nodiscard]] + bool HasAttributes() const { + return !children_.empty(); + } + + friend void swap(AttributeMatcher& lhs, AttributeMatcher& rhs) { + using std::swap; + swap(lhs.children_, rhs.children_); + swap(lhs.attributes_, rhs.attributes_); + } + + private: + struct AttributeCollector; + + [[nodiscard]] + std::pair AddVariable(absl::string_view variable); + + template + absl::Status InsertAttribute(absl::string_view variable, + absl::Span qualifiers, + absl::FunctionRef to_string); + + template + absl::Status UpsertAttribute(absl::string_view variable, + absl::Span qualifiers, + absl::FunctionRef to_string); + + template + absl::Status RemoveAttribute(absl::string_view variable, + absl::Span qualifiers, + absl::FunctionRef to_string); + + // Pointer stability is required. + std::set children_; + size_t attributes_ = 0; +}; + +[[nodiscard]] +const AttributeMatcher& EmptyAttributeMatcher(); + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_BASE_ATTRIBUTE_MATCHER_H_ diff --git a/base/attribute_matcher_test.cc b/base/attribute_matcher_test.cc new file mode 100644 index 000000000..32f002e7e --- /dev/null +++ b/base/attribute_matcher_test.cc @@ -0,0 +1,455 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "base/attribute_matcher.h" + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "base/attribute.h" +#include "internal/testing.h" + +namespace cel { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::StatusIs; +using ::testing::_; +using ::testing::UnorderedElementsAre; + +using MatchType = AttributePattern::MatchType; + +MATCHER_P(AttributeMatchIs, type, "") { + return ::testing::ExplainMatchResult(::testing::Eq(type), arg.GetType(), + result_listener); +} + +using ::testing::IsEmpty; + +TEST(AttributeMatcher, Empty) { + AttributeMatcher matcher; + EXPECT_FALSE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), IsEmpty()); + EXPECT_THAT(matcher.MatchVariable("foo"), AttributeMatchIs(MatchType::NONE)); +} + +TEST(AttributeMatcher, BadAttribute) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(Attribute("")), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern("")), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.InsertAttribute(Attribute("foo", {AttributeQualifier()})), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.InsertAttribute( + AttributePattern("foo", {AttributeQualifierPattern()})), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.UpsertAttribute(Attribute("")), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.UpsertAttribute(AttributePattern("")), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.UpsertAttribute(Attribute("foo", {AttributeQualifier()})), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.UpsertAttribute( + AttributePattern("foo", {AttributeQualifierPattern()})), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.RemoveAttribute(Attribute("")), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.RemoveAttribute(AttributePattern("")), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.RemoveAttribute(Attribute("foo", {AttributeQualifier()})), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.RemoveAttribute( + AttributePattern("foo", {AttributeQualifierPattern()})), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +TEST(AttributeMatcher, Attribute_OneVariable_ZeroQualifier_Subset) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(Attribute("foo")), IsOk()); + EXPECT_THAT(matcher.InsertAttribute(Attribute("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::Wildcard()})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern("foo"))); +} + +TEST(AttributeMatcher, + Unknown_AttributePattern_OneVariable_ZeroQualifier_Subset) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(AttributePattern("foo")), IsOk()); + EXPECT_THAT(matcher.InsertAttribute(Attribute("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::Wildcard()})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern("foo"))); +} + +TEST(AttributeMatcher, Unknown_Attribute_OneVariable_TwoQualifier_Superset) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + IsOk()); + EXPECT_THAT(matcher.InsertAttribute(Attribute("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz")}))); +} + +TEST(AttributeMatcher, + Unknown_AttributePattern_OneVariable_TwoQualifier_Superset) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz")})), + IsOk()); + EXPECT_THAT(matcher.InsertAttribute(Attribute("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz")}))); +} + +TEST(AttributeMatcher, Unknown_Attribute_OneVariable_ZeroQualifier) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(Attribute("foo")), IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern("foo"))); + EXPECT_THAT(matcher.MatchVariable("foo"), AttributeMatchIs(MatchType::FULL)); +} + +TEST(AttributeMatcher, Unknown_Attribute_OneVariable_OneQualifier) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")}))); + EXPECT_THAT(matcher.MatchVariable("foo"), + AttributeMatchIs(MatchType::PARTIAL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("bar")), + AttributeMatchIs(MatchType::FULL)); +} + +TEST(AttributeMatcher, Unknown_Attribute_OneVariable_TwoQualifier) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + IsOk()); + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("baz")})), + IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT( + matcher.GetAttributes(), + UnorderedElementsAre( + AttributePattern("foo", {AttributeQualifierPattern::OfString("bar")}), + AttributePattern("foo", + {AttributeQualifierPattern::OfString("baz")}))); + EXPECT_THAT(matcher.MatchVariable("foo"), + AttributeMatchIs(MatchType::PARTIAL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("bar")), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("baz")), + AttributeMatchIs(MatchType::FULL)); +} + +TEST(AttributeMatcher, Unknown_AttributePattern_OneVariable_ZeroQualifier) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(AttributePattern("foo")), IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern("foo"))); + EXPECT_THAT(matcher.MatchVariable("foo"), AttributeMatchIs(MatchType::FULL)); +} + +TEST(AttributeMatcher, Unknown_AttributePattern_OneVariable_OneQualifier) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")})), + IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")}))); + EXPECT_THAT(matcher.MatchVariable("foo"), + AttributeMatchIs(MatchType::PARTIAL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("bar")), + AttributeMatchIs(MatchType::FULL)); +} + +TEST(AttributeMatcher, Unknown_AttributePattern_OneVariable_TwoQualifier) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")})), + IsOk()); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("baz")})), + IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT( + matcher.GetAttributes(), + UnorderedElementsAre( + AttributePattern("foo", {AttributeQualifierPattern::OfString("bar")}), + AttributePattern("foo", + {AttributeQualifierPattern::OfString("baz")}))); + EXPECT_THAT(matcher.MatchVariable("foo"), + AttributeMatchIs(MatchType::PARTIAL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("bar")), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("baz")), + AttributeMatchIs(MatchType::FULL)); +} + +TEST(AttributeMatcher, RemoveAttribute) { + const std::vector unknown_attributes( + {AttributePattern("foo", {AttributeQualifierPattern::OfString("bar")}), + AttributePattern("bar", {AttributeQualifierPattern::OfString("foo")}), + AttributePattern("baz", {AttributeQualifierPattern::OfString("foo")})}); + AttributeMatcher matcher; + std::vector removal_order({0, 1, 2}); + do { + for (const auto& attribute : unknown_attributes) { + EXPECT_THAT(matcher.InsertAttribute(attribute), IsOk()); + } + // 0 + EXPECT_THAT(matcher.RemoveAttribute(unknown_attributes[removal_order[0]]), + IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(unknown_attributes[removal_order[1]], + unknown_attributes[removal_order[2]])); + // 1 + EXPECT_THAT(matcher.RemoveAttribute(unknown_attributes[removal_order[1]]), + IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(unknown_attributes[removal_order[2]])); + // 2 + EXPECT_THAT(matcher.RemoveAttribute(unknown_attributes[removal_order[2]]), + IsOk()); + EXPECT_FALSE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), IsEmpty()); + } while (std::next_permutation(removal_order.begin(), removal_order.end())); +} + +TEST(AttributeMatcher, Wildcards) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::Wildcard(), + AttributeQualifierPattern::Wildcard()})), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::Wildcard()})), + IsOk()); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "bar", {AttributeQualifierPattern::OfString("baz"), + AttributeQualifierPattern::Wildcard()})), + IsOk()); + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute( + Attribute("bar", {AttributeQualifier::OfString("baz")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("bar")), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfInt(1)), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfUint(1)), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfBool(true)), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT( + matcher.GetAttributes(), + UnorderedElementsAre( + AttributePattern("foo", {AttributeQualifierPattern::Wildcard()}), + AttributePattern("bar", {AttributeQualifierPattern::OfString("baz"), + AttributeQualifierPattern::Wildcard()}))); + EXPECT_THAT(matcher.RemoveAttribute(Attribute("foo")), + StatusIs(absl::StatusCode::kNotFound)); + EXPECT_THAT(matcher.RemoveAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + StatusIs(absl::StatusCode::kNotFound)); + EXPECT_THAT(matcher.RemoveAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + StatusIs(absl::StatusCode::kNotFound)); + EXPECT_THAT(matcher.RemoveAttribute(Attribute("bar")), + StatusIs(absl::StatusCode::kNotFound)); + EXPECT_THAT(matcher.RemoveAttribute( + Attribute("bar", {AttributeQualifier::OfString("baz")})), + StatusIs(absl::StatusCode::kNotFound)); + EXPECT_THAT(matcher.RemoveAttribute( + Attribute("bar", {AttributeQualifier::OfString("baz"), + AttributeQualifier::OfString("foo")})), + StatusIs(absl::StatusCode::kNotFound)); +} + +TEST(AttributeMatcher, Upsert) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + IsOk()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz")}))); + + // Upserting the same attribute does nothing. + EXPECT_THAT(matcher.UpsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + IsOk()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz")}))); + + // Upserting a more specific attribute does nothing. + EXPECT_THAT(matcher.UpsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz"), + AttributeQualifier::OfString("qux")})), + IsOk()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz")}))); + + // Upserting a less specific attribute replaces others. + EXPECT_THAT(matcher.UpsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("qux")})), + IsOk()); + EXPECT_THAT( + matcher.GetAttributes(), + UnorderedElementsAre( + AttributePattern("foo", {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz")}), + AttributePattern("foo", + {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("qux")}))); + EXPECT_THAT(matcher.UpsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + IsOk()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")}))); +} + +TEST(AttributeMatcher, PastFullMatch) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + IsOk()); + EXPECT_DEBUG_DEATH( + static_cast( + matcher.MatchVariable("foo") + .MatchQualifier(AttributeQualifierView::OfString("bar")) + .MatchQualifier(AttributeQualifierView::OfString("baz"))), + _); +} + +TEST(AttributeMatcher, MatchAttribute) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + IsOk()); + EXPECT_THAT(matcher.MatchAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT( + matcher.MatchAttribute("foo", {AttributeQualifierView::OfString("bar"), + AttributeQualifierView::OfString("baz")}), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT(matcher.MatchAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + AttributeMatchIs(MatchType::PARTIAL)); + EXPECT_THAT( + matcher.MatchAttribute("foo", {AttributeQualifierView::OfString("bar")}), + AttributeMatchIs(MatchType::PARTIAL)); + EXPECT_THAT(matcher.MatchAttribute( + Attribute("bar", {AttributeQualifier::OfString("baz")})), + AttributeMatchIs(MatchType::NONE)); + EXPECT_THAT( + matcher.MatchAttribute("bar", {AttributeQualifierView::OfString("baz")}), + AttributeMatchIs(MatchType::NONE)); +} + +} // namespace +} // namespace cel diff --git a/base/attribute_set.h b/base/attribute_set.h index 078f37881..625893fd9 100644 --- a/base/attribute_set.h +++ b/base/attribute_set.h @@ -15,6 +15,9 @@ #ifndef THIRD_PARTY_CEL_CPP_BASE_ATTRIBUTE_SET_H_ #define THIRD_PARTY_CEL_CPP_BASE_ATTRIBUTE_SET_H_ +#include + +#include "absl/base/no_destructor.h" #include "absl/container/btree_set.h" #include "absl/types/span.h" #include "base/attribute.h" @@ -54,27 +57,43 @@ class AttributeSet final { } } - AttributeSet(const AttributeSet& set1, const AttributeSet& set2) - : attributes_(set1.attributes_) { - for (const auto& attr : set2.attributes_) { - Add(attr); - } + AttributeSet(const AttributeSet& set1, const AttributeSet& set2) { + Add(set1); + Add(set2); } - iterator begin() const { return attributes_.begin(); } + iterator begin() const { + return attributes_ != nullptr ? attributes_->begin() + : EmptyContainer().begin(); + } - const_iterator cbegin() const { return attributes_.cbegin(); } + const_iterator cbegin() const { + return attributes_ != nullptr ? attributes_->cbegin() + : EmptyContainer().cbegin(); + } - iterator end() const { return attributes_.end(); } + iterator end() const { + return attributes_ != nullptr ? attributes_->end() : EmptyContainer().end(); + } - const_iterator cend() const { return attributes_.cend(); } + const_iterator cend() const { + return attributes_ != nullptr ? attributes_->cend() + : EmptyContainer().cend(); + } - size_type size() const { return attributes_.size(); } + size_type size() const { + return attributes_ != nullptr ? attributes_->size() : 0; + } - bool empty() const { return attributes_.empty(); } + bool empty() const { return attributes_ == nullptr || attributes_->empty(); } bool operator==(const AttributeSet& other) const { - return this == &other || attributes_ == other.attributes_; + if (attributes_ == other.attributes_) { + return true; + } + return (attributes_ != nullptr ? *attributes_ : EmptyContainer()) == + (other.attributes_ != nullptr ? *other.attributes_ + : EmptyContainer()); } bool operator!=(const AttributeSet& other) const { @@ -91,16 +110,31 @@ class AttributeSet final { friend class UnknownValue; friend class base_internal::UnknownSet; - void Add(const Attribute& attribute) { attributes_.insert(attribute); } + static const Container& EmptyContainer() { + static const absl::NoDestructor container; + return *container; + } + + void Add(const Attribute& attribute) { + if (attributes_ == nullptr) { + attributes_ = std::make_shared(); + } + attributes_->insert(attribute); + } void Add(const AttributeSet& other) { - for (const auto& attribute : other) { - Add(attribute); + if (!other.empty()) { + if (attributes_ == nullptr) { + attributes_ = std::make_shared(); + } + for (const auto& attribute : other) { + attributes_->insert(attribute); + } } } // Attribute container. - Container attributes_; + std::shared_ptr attributes_; }; } // namespace cel diff --git a/base/attribute_test.cc b/base/attribute_test.cc new file mode 100644 index 000000000..d9aa1ef98 --- /dev/null +++ b/base/attribute_test.cc @@ -0,0 +1,169 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "base/attribute.h" + +#include +#include + +#include "absl/strings/string_view.h" +#include "internal/testing.h" + +namespace cel { +namespace { + +using ::testing::Optional; + +TEST(AttributeQualifierView, Bool) { + AttributeQualifierView qualifier = AttributeQualifierView::OfBool(true); + EXPECT_TRUE(qualifier.IsBool()); + EXPECT_FALSE(qualifier.IsInt()); + EXPECT_FALSE(qualifier.IsUint()); + EXPECT_FALSE(qualifier.IsString()); + EXPECT_TRUE(qualifier.GetBool()); + EXPECT_THAT(qualifier.AsBool(), Optional(true)); + EXPECT_THAT(qualifier.AsInt(), std::nullopt); + EXPECT_THAT(qualifier.AsUint(), std::nullopt); + EXPECT_THAT(qualifier.AsString(), std::nullopt); +} + +TEST(AttributeQualifierView, Int) { + AttributeQualifierView qualifier = AttributeQualifierView::OfInt(1); + EXPECT_FALSE(qualifier.IsBool()); + EXPECT_TRUE(qualifier.IsInt()); + EXPECT_FALSE(qualifier.IsUint()); + EXPECT_FALSE(qualifier.IsString()); + EXPECT_EQ(qualifier.GetInt(), 1); + EXPECT_THAT(qualifier.AsBool(), std::nullopt); + EXPECT_THAT(qualifier.AsInt(), Optional(1)); + EXPECT_THAT(qualifier.AsUint(), std::nullopt); + EXPECT_THAT(qualifier.AsString(), std::nullopt); +} + +TEST(AttributeQualifierView, Uint) { + AttributeQualifierView qualifier = AttributeQualifierView::OfUint(1); + EXPECT_FALSE(qualifier.IsBool()); + EXPECT_FALSE(qualifier.IsInt()); + EXPECT_TRUE(qualifier.IsUint()); + EXPECT_FALSE(qualifier.IsString()); + EXPECT_EQ(qualifier.GetUint(), 1); + EXPECT_THAT(qualifier.AsBool(), std::nullopt); + EXPECT_THAT(qualifier.AsInt(), std::nullopt); + EXPECT_THAT(qualifier.AsUint(), Optional(1)); + EXPECT_THAT(qualifier.AsString(), std::nullopt); +} + +TEST(AttributeQualifierView, String) { + AttributeQualifierView qualifier = AttributeQualifierView::OfString("foo"); + EXPECT_FALSE(qualifier.IsBool()); + EXPECT_FALSE(qualifier.IsInt()); + EXPECT_FALSE(qualifier.IsUint()); + EXPECT_TRUE(qualifier.IsString()); + EXPECT_EQ(qualifier.GetString(), "foo"); + EXPECT_THAT(qualifier.AsBool(), std::nullopt); + EXPECT_THAT(qualifier.AsInt(), std::nullopt); + EXPECT_THAT(qualifier.AsUint(), std::nullopt); + EXPECT_THAT(qualifier.AsString(), Optional(absl::string_view("foo"))); +} + +TEST(Attribute, ToString) { + EXPECT_EQ(Attribute("foo", {AttributeQualifier::OfBool(true), + AttributeQualifier::OfInt(2), + AttributeQualifier::OfUint(3), + AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz qux")}) + .ToString(), + "foo[true][2][3u].bar[\"baz qux\"]"); +} + +TEST(AttributePattern, ToString) { + EXPECT_EQ( + AttributePattern("foo", {AttributeQualifierPattern::OfBool(true), + AttributeQualifierPattern::OfInt(2), + AttributeQualifierPattern::OfUint(3), + AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz qux"), + AttributeQualifierPattern::Wildcard()}) + .ToString(), + "foo[true][2][3u].bar[\"baz qux\"].*"); +} + +TEST(AttributeQualifierView, Order) { + // int < uint < string < bool + EXPECT_LT(AttributeQualifierView::OfInt(int64_t{0}), + AttributeQualifierView::OfUint(uint64_t{0})); + EXPECT_LT(AttributeQualifierView::OfInt(int64_t{0}), + AttributeQualifierView::OfString("")); + EXPECT_LT(AttributeQualifierView::OfInt(int64_t{0}), + AttributeQualifierView::OfBool(false)); + EXPECT_LT(AttributeQualifierView::OfUint(uint64_t{0}), + AttributeQualifierView::OfString("")); + EXPECT_LT(AttributeQualifierView::OfUint(uint64_t{0}), + AttributeQualifierView::OfBool(false)); +} + +TEST(AttributeQualifierView, TransparentOrder) { + EXPECT_EQ(AttributeQualifierView::OfInt(int64_t{0}), + AttributeQualifier::OfInt(0)); + EXPECT_EQ(AttributeQualifierView::OfUint(uint64_t{0}), + AttributeQualifier::OfUint(0)); + EXPECT_EQ(AttributeQualifierView::OfString("0"), + AttributeQualifier::OfString("0")); + EXPECT_EQ(AttributeQualifierView::OfBool(false), + AttributeQualifier::OfBool(false)); + EXPECT_EQ(AttributeQualifier::OfInt(0), + AttributeQualifierView::OfInt(int64_t{0})); + EXPECT_EQ(AttributeQualifier::OfUint(0), + AttributeQualifierView::OfUint(uint64_t{0})); + EXPECT_EQ(AttributeQualifier::OfString("0"), + AttributeQualifierView::OfString("0")); + EXPECT_EQ(AttributeQualifier::OfBool(false), + AttributeQualifierView::OfBool(false)); + EXPECT_LT(AttributeQualifierView::OfInt(int64_t{0}), + AttributeQualifier::OfInt(1)); + EXPECT_LT(AttributeQualifierView::OfUint(uint64_t{0}), + AttributeQualifier::OfUint(1)); + EXPECT_LT(AttributeQualifierView::OfString("0"), + AttributeQualifier::OfString("1")); + EXPECT_LT(AttributeQualifierView::OfBool(false), + AttributeQualifier::OfBool(true)); + EXPECT_LT(AttributeQualifier::OfInt(0), + AttributeQualifierView::OfInt(int64_t{1})); + EXPECT_LT(AttributeQualifier::OfUint(0), + AttributeQualifierView::OfUint(uint64_t{1})); + EXPECT_LT(AttributeQualifier::OfString("0"), + AttributeQualifierView::OfString("1")); + EXPECT_LT(AttributeQualifier::OfBool(false), + AttributeQualifierView::OfBool(true)); +} + +template +void TestAttributeQualifierConversion() { + EXPECT_EQ(To(From::OfInt(1)), To::OfInt(1)); + EXPECT_EQ(To(From::OfUint(1)), To::OfUint(1)); + EXPECT_EQ(To(From::OfString("foo")), To::OfString("foo")); + EXPECT_EQ(To(From::OfBool(true)), To::OfBool(true)); +} + +TEST(AttributeQualifier, Conversion) { + TestAttributeQualifierConversion(); + TestAttributeQualifierConversion(); + TestAttributeQualifierConversion(); +} + +} // namespace +} // namespace cel diff --git a/base/function_result.h b/base/function_result.h index 977ceeb90..b511dd744 100644 --- a/base/function_result.h +++ b/base/function_result.h @@ -16,8 +16,11 @@ #define THIRD_PARTY_CEL_CPP_BASE_FUNCTION_RESULT_H_ #include +#include #include +#include "absl/base/attributes.h" +#include "absl/strings/string_view.h" #include "base/function_descriptor.h" namespace cel { @@ -32,29 +35,21 @@ class FunctionResult final { FunctionResult& operator=(const FunctionResult&) = default; FunctionResult& operator=(FunctionResult&&) = default; - FunctionResult(FunctionDescriptor descriptor, int64_t expr_id) - : descriptor_(std::move(descriptor)), expr_id_(expr_id) {} + explicit FunctionResult(std::string_view name) : name_(name) {} - // The descriptor of the called function that return Unknown. - const FunctionDescriptor& descriptor() const { return descriptor_; } - - // The id of the |Expr| that triggered the function call step. Provided - // informationally -- if two different |Expr|s generate the same unknown call, - // they will be treated as the same unknown function result. - int64_t call_expr_id() const { return expr_id_; } + absl::string_view name() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return name_; } // Equality operator provided for testing. Compatible with set less-than // comparator. // Compares descriptor then arguments elementwise. bool IsEqualTo(const FunctionResult& other) const { - return descriptor() == other.descriptor(); + return name() == other.name(); } // TODO(uncreated-issue/5): re-implement argument capture private: - FunctionDescriptor descriptor_; - int64_t expr_id_; + std::string name_; }; inline bool operator==(const FunctionResult& lhs, const FunctionResult& rhs) { @@ -62,7 +57,7 @@ inline bool operator==(const FunctionResult& lhs, const FunctionResult& rhs) { } inline bool operator<(const FunctionResult& lhs, const FunctionResult& rhs) { - return lhs.descriptor() < rhs.descriptor(); + return lhs.name() < rhs.name(); } } // namespace cel diff --git a/base/operators.cc b/base/operators.cc index 805acc5a1..b7df40b27 100644 --- a/base/operators.cc +++ b/base/operators.cc @@ -179,13 +179,13 @@ CEL_INTERNAL_TERNARY_OPERATORS_ENUM(CEL_TERNARY_OPERATOR) absl::optional Operator::FindByName(absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(operators_by_name.cbegin(), operators_by_name.cend(), input, OperatorDataNameComparer{}); if (it == operators_by_name.cend() || (*it)->name != input) { - return absl::nullopt; + return std::nullopt; } return Operator(*it); } @@ -193,13 +193,13 @@ absl::optional Operator::FindByName(absl::string_view input) { absl::optional Operator::FindByDisplayName(absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(operators_by_display_name.cbegin(), operators_by_display_name.cend(), input, OperatorDataDisplayNameComparer{}); if (it == operators_by_name.cend() || (*it)->display_name != input) { - return absl::nullopt; + return std::nullopt; } return Operator(*it); } @@ -208,13 +208,13 @@ absl::optional UnaryOperator::FindByName( absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(unary_operators_by_name.cbegin(), unary_operators_by_name.cend(), input, OperatorDataNameComparer{}); if (it == unary_operators_by_name.cend() || (*it)->name != input) { - return absl::nullopt; + return std::nullopt; } return UnaryOperator(*it); } @@ -223,14 +223,14 @@ absl::optional UnaryOperator::FindByDisplayName( absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(unary_operators_by_display_name.cbegin(), unary_operators_by_display_name.cend(), input, OperatorDataDisplayNameComparer{}); if (it == unary_operators_by_display_name.cend() || (*it)->display_name != input) { - return absl::nullopt; + return std::nullopt; } return UnaryOperator(*it); } @@ -239,13 +239,13 @@ absl::optional BinaryOperator::FindByName( absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(binary_operators_by_name.cbegin(), binary_operators_by_name.cend(), input, OperatorDataNameComparer{}); if (it == binary_operators_by_name.cend() || (*it)->name != input) { - return absl::nullopt; + return std::nullopt; } return BinaryOperator(*it); } @@ -254,14 +254,14 @@ absl::optional BinaryOperator::FindByDisplayName( absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(binary_operators_by_display_name.cbegin(), binary_operators_by_display_name.cend(), input, OperatorDataDisplayNameComparer{}); if (it == binary_operators_by_display_name.cend() || (*it)->display_name != input) { - return absl::nullopt; + return std::nullopt; } return BinaryOperator(*it); } @@ -270,13 +270,13 @@ absl::optional TernaryOperator::FindByName( absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(ternary_operators_by_name.cbegin(), ternary_operators_by_name.cend(), input, OperatorDataNameComparer{}); if (it == ternary_operators_by_name.cend() || (*it)->name != input) { - return absl::nullopt; + return std::nullopt; } return TernaryOperator(*it); } @@ -285,14 +285,14 @@ absl::optional TernaryOperator::FindByDisplayName( absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(ternary_operators_by_display_name.cbegin(), ternary_operators_by_display_name.cend(), input, OperatorDataDisplayNameComparer{}); if (it == ternary_operators_by_display_name.cend() || (*it)->display_name != input) { - return absl::nullopt; + return std::nullopt; } return TernaryOperator(*it); } diff --git a/base/operators_test.cc b/base/operators_test.cc index fdf95e7ae..6049f76c8 100644 --- a/base/operators_test.cc +++ b/base/operators_test.cc @@ -130,55 +130,55 @@ CEL_INTERNAL_TERNARY_OPERATORS_ENUM(CEL_TERNARY_OPERATOR) TEST(Operator, FindByName) { EXPECT_THAT(Operator::FindByName("@in"), Optional(Eq(Operator::In()))); EXPECT_THAT(Operator::FindByName("_in_"), Optional(Eq(Operator::OldIn()))); - EXPECT_THAT(Operator::FindByName("in"), Eq(absl::nullopt)); - EXPECT_THAT(Operator::FindByName(""), Eq(absl::nullopt)); + EXPECT_THAT(Operator::FindByName("in"), Eq(std::nullopt)); + EXPECT_THAT(Operator::FindByName(""), Eq(std::nullopt)); } TEST(Operator, FindByDisplayName) { EXPECT_THAT(Operator::FindByDisplayName("-"), Optional(Eq(Operator::Subtract()))); - EXPECT_THAT(Operator::FindByDisplayName("@in"), Eq(absl::nullopt)); - EXPECT_THAT(Operator::FindByDisplayName(""), Eq(absl::nullopt)); + EXPECT_THAT(Operator::FindByDisplayName("@in"), Eq(std::nullopt)); + EXPECT_THAT(Operator::FindByDisplayName(""), Eq(std::nullopt)); } TEST(UnaryOperator, FindByName) { EXPECT_THAT(UnaryOperator::FindByName("-_"), Optional(Eq(Operator::Negate()))); - EXPECT_THAT(UnaryOperator::FindByName("_-_"), Eq(absl::nullopt)); - EXPECT_THAT(UnaryOperator::FindByName(""), Eq(absl::nullopt)); + EXPECT_THAT(UnaryOperator::FindByName("_-_"), Eq(std::nullopt)); + EXPECT_THAT(UnaryOperator::FindByName(""), Eq(std::nullopt)); } TEST(UnaryOperator, FindByDisplayName) { EXPECT_THAT(UnaryOperator::FindByDisplayName("-"), Optional(Eq(Operator::Negate()))); - EXPECT_THAT(UnaryOperator::FindByDisplayName("&&"), Eq(absl::nullopt)); - EXPECT_THAT(UnaryOperator::FindByDisplayName(""), Eq(absl::nullopt)); + EXPECT_THAT(UnaryOperator::FindByDisplayName("&&"), Eq(std::nullopt)); + EXPECT_THAT(UnaryOperator::FindByDisplayName(""), Eq(std::nullopt)); } TEST(BinaryOperator, FindByName) { EXPECT_THAT(BinaryOperator::FindByName("_-_"), Optional(Eq(Operator::Subtract()))); - EXPECT_THAT(BinaryOperator::FindByName("-_"), Eq(absl::nullopt)); - EXPECT_THAT(BinaryOperator::FindByName(""), Eq(absl::nullopt)); + EXPECT_THAT(BinaryOperator::FindByName("-_"), Eq(std::nullopt)); + EXPECT_THAT(BinaryOperator::FindByName(""), Eq(std::nullopt)); } TEST(BinaryOperator, FindByDisplayName) { EXPECT_THAT(BinaryOperator::FindByDisplayName("-"), Optional(Eq(Operator::Subtract()))); - EXPECT_THAT(BinaryOperator::FindByDisplayName("!"), Eq(absl::nullopt)); - EXPECT_THAT(BinaryOperator::FindByDisplayName(""), Eq(absl::nullopt)); + EXPECT_THAT(BinaryOperator::FindByDisplayName("!"), Eq(std::nullopt)); + EXPECT_THAT(BinaryOperator::FindByDisplayName(""), Eq(std::nullopt)); } TEST(TernaryOperator, FindByName) { EXPECT_THAT(TernaryOperator::FindByName("_?_:_"), Optional(Eq(TernaryOperator::Conditional()))); - EXPECT_THAT(TernaryOperator::FindByName("-_"), Eq(absl::nullopt)); - EXPECT_THAT(TernaryOperator::FindByName(""), Eq(absl::nullopt)); + EXPECT_THAT(TernaryOperator::FindByName("-_"), Eq(std::nullopt)); + EXPECT_THAT(TernaryOperator::FindByName(""), Eq(std::nullopt)); } TEST(TernaryOperator, FindByDisplayName) { - EXPECT_THAT(TernaryOperator::FindByDisplayName(""), Eq(absl::nullopt)); - EXPECT_THAT(TernaryOperator::FindByDisplayName("!"), Eq(absl::nullopt)); + EXPECT_THAT(TernaryOperator::FindByDisplayName(""), Eq(std::nullopt)); + EXPECT_THAT(TernaryOperator::FindByDisplayName("!"), Eq(std::nullopt)); } TEST(Operator, SupportsAbslHash) { diff --git a/bazel/BUILD b/bazel/BUILD index 5b3cb2d2c..5edddadda 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -19,7 +19,9 @@ exports_files( cc_binary( name = "cel_cc_embed", srcs = ["cel_cc_embed.cc"], - visibility = ["//:__subpackages__"], + visibility = [ + "//:__subpackages__", + ], deps = [ "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:parse", @@ -33,7 +35,9 @@ cc_binary( cc_binary( name = "cat_param_file", srcs = ["cat_param_file.cc"], - visibility = ["//:__subpackages__"], + visibility = [ + "//:__subpackages__", + ], deps = [ "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", diff --git a/bazel/antlr.bzl b/bazel/antlr.bzl index 2abbb6dbd..a4d28cdf8 100644 --- a/bazel/antlr.bzl +++ b/bazel/antlr.bzl @@ -55,6 +55,7 @@ def antlr_cc_library(name, src, package): generated, "@antlr4-cpp-runtime//:antlr4-cpp-runtime", ], + copts = ["-fexceptions"], linkstatic = 1, ) diff --git a/bazel/antlr.patch b/bazel/antlr.patch deleted file mode 100644 index c1aa9080c..000000000 --- a/bazel/antlr.patch +++ /dev/null @@ -1,30 +0,0 @@ ---- BUILD.bazel -+++ BUILD.bazel -@@ -17,21 +17,21 @@ - cc_library( - name = "antlr4-cpp-runtime", - srcs = glob(["runtime/src/**/*.cpp"]), - hdrs = ["runtime/src/antlr4-runtime.h"], - copts = ["-fexceptions"], -- defines = ["ANTLR4CPP_USING_ABSEIL"], -+ defines = ["ANTLR4CPP_USING_ABSEIL", "ANTLR4CPP_STATIC"], - features = ["-use_header_modules"], - includes = ["runtime/src"], - textual_hdrs = glob( - ["runtime/src/**/*.h"], - exclude = ["runtime/src/antlr4-runtime.h"], - ), - visibility = ["//visibility:public"], - deps = [ - "@com_google_absl//absl/base", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/synchronization", - ], - ) - ---- VERSION -+++ /dev/null -@@ -1,1 +1,0 @@ --4.13.2 \ No newline at end of file diff --git a/checker/BUILD b/checker/BUILD index f1e0cef3c..7f3ccfef7 100644 --- a/checker/BUILD +++ b/checker/BUILD @@ -49,8 +49,11 @@ cc_library( deps = [ ":type_check_issue", "//common:ast", + "//common:decl", "//common:source", + "//common:type", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -74,11 +77,14 @@ cc_test( cc_library( name = "type_checker", + srcs = ["type_checker.cc"], hdrs = ["type_checker.h"], deps = [ ":validation_result", "//common:ast", + "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status:statusor", + "@com_google_protobuf//:protobuf", ], ) @@ -224,6 +230,8 @@ cc_library( hdrs = ["type_checker_subset_factory.h"], deps = [ ":type_checker_builder", + "//common:decl", + "//common:signature", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/types:span", diff --git a/checker/checker_options.h b/checker/checker_options.h index cb85337fa..da6004d74 100644 --- a/checker/checker_options.h +++ b/checker/checker_options.h @@ -94,7 +94,7 @@ struct CheckerOptions { // // Temporary flag to allow rolling out the change. No functional changes to // evaluation behavior in either mode. - bool enable_function_name_in_reference = true; + bool enable_function_name_in_reference = false; // If true, the checker will use the proto json field names for protobuf // messages. Unlike protojson parsers, it will not accept the standard proto diff --git a/checker/internal/BUILD b/checker/internal/BUILD index 1c560cdb9..1957a3a8b 100644 --- a/checker/internal/BUILD +++ b/checker/internal/BUILD @@ -65,10 +65,12 @@ cc_library( srcs = ["type_check_env.cc"], hdrs = ["type_check_env.h"], deps = [ - ":descriptor_pool_type_introspector", + ":proto_type_mask", + ":proto_type_mask_registry", "//common:constant", "//common:container", "//common:decl", + "//common:descriptor_pool_type_introspector", "//common:type", "//internal:status_macros", "@com_google_absl//absl/base:core_headers", @@ -76,6 +78,7 @@ cc_library( "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", @@ -127,8 +130,8 @@ cc_library( "type_checker_impl.h", ], deps = [ - ":format_type_name", ":namespace_generator", + ":proto_type_mask", ":type_check_env", ":type_inference_context", "//checker:checker_options", @@ -145,6 +148,8 @@ cc_library( "//common:container", "//common:decl", "//common:expr", + "//common:format_type_name", + "//common:standard_definitions", "//common:type", "//common:type_kind", "//internal:lexis", @@ -153,8 +158,10 @@ cc_library( "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/container:btree", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -179,21 +186,27 @@ cc_test( "//checker:type_checker_builder", "//checker:validation_result", "//common:ast", + "//common:ast_proto", "//common:container", "//common:decl", "//common:expr", + "//common:format_type_name", "//common:source", "//common:type", "//internal:status_macros", "//internal:testing", "//internal:testing_descriptor_pool", + "//parser", + "//parser:macro_registry", "//testutil:baseline_tests", + "//testutil:test_macros", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", @@ -219,6 +232,7 @@ cc_test( "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/types:optional", "@com_google_protobuf//:protobuf", @@ -230,10 +244,13 @@ cc_library( srcs = ["type_inference_context.cc"], hdrs = ["type_inference_context.h"], deps = [ - ":format_type_name", + ":builtins_arena", "//common:decl", + "//common:format_type_name", + "//common:standard_definitions", "//common:type", "//common:type_kind", + "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:node_hash_map", "@com_google_absl//absl/log:absl_check", @@ -252,39 +269,86 @@ cc_test( deps = [ ":type_inference_context", "//common:decl", + "//common:format_type_name", "//common:type", "//common:type_kind", "//internal:testing", "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/types:optional", "@com_google_protobuf//:protobuf", ], ) cc_library( - name = "format_type_name", - srcs = ["format_type_name.cc"], - hdrs = ["format_type_name.h"], + name = "field_path", + srcs = ["field_path.cc"], + hdrs = ["field_path.h"], deps = [ - "//common:type", - "//common:type_kind", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + ], +) + +cc_test( + name = "field_path_test", + srcs = ["field_path_test.cc"], + deps = [ + ":field_path", + "//internal:testing", "@com_google_absl//absl/strings", ], ) cc_library( - name = "descriptor_pool_type_introspector", - srcs = ["descriptor_pool_type_introspector.cc"], - hdrs = ["descriptor_pool_type_introspector.h"], + name = "proto_type_mask", + srcs = ["proto_type_mask.cc"], + hdrs = ["proto_type_mask.h"], deps = [ + ":field_path", + "//internal:status_macros", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:btree", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "proto_type_mask_test", + srcs = ["proto_type_mask_test.cc"], + deps = [ + ":field_path", + ":proto_type_mask", + "//internal:testing", + "//internal:testing_descriptor_pool", + "@com_google_absl//absl/container:btree", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + ], +) + +cc_library( + name = "proto_type_mask_registry", + srcs = ["proto_type_mask_registry.cc"], + hdrs = ["proto_type_mask_registry.h"], + deps = [ + ":field_path", + ":proto_type_mask", "//common:type", - "@com_google_absl//absl/base:core_headers", + "//internal:status_macros", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:btree", "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", - "@com_google_absl//absl/synchronization", "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_protobuf//:protobuf", @@ -292,15 +356,18 @@ cc_library( ) cc_test( - name = "descriptor_pool_type_introspector_test", - srcs = ["descriptor_pool_type_introspector_test.cc"], + name = "proto_type_mask_registry_test", + srcs = ["proto_type_mask_registry_test.cc"], deps = [ - ":descriptor_pool_type_introspector", - "//common:type", + ":proto_type_mask", + ":proto_type_mask_registry", "//internal:testing", "//internal:testing_descriptor_pool", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", ], ) diff --git a/common/reference_count.h b/checker/internal/field_path.cc similarity index 57% rename from common/reference_count.h rename to checker/internal/field_path.cc index 0a07670bd..5ecc4219b 100644 --- a/common/reference_count.h +++ b/checker/internal/field_path.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2026 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,15 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_COMMON_REFERENCE_COUNT_H_ -#define THIRD_PARTY_CEL_CPP_COMMON_REFERENCE_COUNT_H_ +#include "checker/internal/field_path.h" -#include "common/internal/reference_count.h" +#include -namespace cel { +#include "absl/strings/str_join.h" +#include "absl/strings/substitute.h" -using ReferenceCount = common_internal::ReferenceCount; +namespace cel::checker_internal { -} // namespace cel +std::string FieldPath::DebugString() const { + return absl::Substitute( + "FieldPath { field path: '$0', field selection: {'$1'} }", path_, + absl::StrJoin(field_selection_, "', '")); +} -#endif // THIRD_PARTY_CEL_CPP_COMMON_REFERENCE_COUNT_H_ +} // namespace cel::checker_internal diff --git a/checker/internal/field_path.h b/checker/internal/field_path.h new file mode 100644 index 000000000..d67d9b935 --- /dev/null +++ b/checker/internal/field_path.h @@ -0,0 +1,77 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_CHECKER_FIELD_PATH_H_ +#define THIRD_PARTY_CEL_CPP_CHECKER_FIELD_PATH_H_ + +#include +#include +#include + +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" + +namespace cel::checker_internal { + +// Represents a single path within a FieldMask. +class FieldPath { + public: + explicit FieldPath(std::string path) + : path_(std::move(path)), + field_selection_(absl::StrSplit(path_, kPathDelimiter)) {} + + // Returns the input path. + // For example: "f.b.d". + absl::string_view GetPath() const { return path_; } + + // Returns the list of nested field names in the path. + // For example: {"f", "b", "d"}. + absl::Span GetFieldSelection() const { + return field_selection_; + } + + // Returns the first field name in the path. + // For example: "f". + std::string GetFieldName() const { return field_selection_.front(); } + + template + friend void AbslStringify(Sink& sink, const FieldPath& field_path) { + sink.Append(field_path.DebugString()); + } + + private: + static constexpr char kPathDelimiter = '.'; + + std::string DebugString() const; + + // The input path. For example: "f.b.d". + std::string path_; + // The list of nested field names in the path. For example: {"f", "b", "d"}. + std::vector field_selection_; +}; + +inline bool operator==(const FieldPath& lhs, const FieldPath& rhs) { + return lhs.GetFieldSelection() == rhs.GetFieldSelection(); +} + +// Compares the field selections in the field paths. +// This is only intended as an arbitrary ordering for a set. +inline bool operator<(const FieldPath& lhs, const FieldPath& rhs) { + return lhs.GetFieldSelection() < rhs.GetFieldSelection(); +} + +} // namespace cel::checker_internal + +#endif // THIRD_PARTY_CEL_CPP_CHECKER_FIELD_PATH_H_ diff --git a/checker/internal/field_path_test.cc b/checker/internal/field_path_test.cc new file mode 100644 index 000000000..9a1434954 --- /dev/null +++ b/checker/internal/field_path_test.cc @@ -0,0 +1,85 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "checker/internal/field_path.h" + +#include "absl/strings/str_cat.h" +#include "internal/testing.h" + +namespace cel::checker_internal { +namespace { + +using ::testing::ElementsAre; + +TEST(FieldPathTest, EmptyPathReturnsEmptyString) { + FieldPath field_path(""); + EXPECT_EQ(field_path.GetPath(), ""); + EXPECT_THAT(field_path.GetFieldSelection(), ElementsAre("")); + EXPECT_EQ(field_path.GetFieldName(), ""); +} + +TEST(FieldPathTest, DelimiterPathReturnsEmptyStrings) { + FieldPath field_path("."); + EXPECT_EQ(field_path.GetPath(), "."); + EXPECT_THAT(field_path.GetFieldSelection(), ElementsAre("", "")); + EXPECT_EQ(field_path.GetFieldName(), ""); +} + +TEST(FieldPathTest, FieldPathReturnsFields) { + FieldPath field_path("resource.name.other_field"); + EXPECT_EQ(field_path.GetPath(), "resource.name.other_field"); + EXPECT_THAT(field_path.GetFieldSelection(), + ElementsAre("resource", "name", "other_field")); + EXPECT_EQ(field_path.GetFieldName(), "resource"); +} + +TEST(FieldPathTest, AbslStringifyPrintsFieldSelection) { + FieldPath field_path("resource.name"); + EXPECT_EQ(absl::StrCat(field_path), + "FieldPath { field path: 'resource.name', field selection: " + "{'resource', 'name'} }"); +} + +TEST(FieldPathTest, EqualsComparesFieldSelectionAndReturnsTrue) { + FieldPath field_path_1("resource.name"); + FieldPath field_path_2("resource.name"); + EXPECT_TRUE(field_path_1 == field_path_2); +} + +TEST(FieldPathTest, EqualsComparesFieldSelectionAndReturnsFalse) { + FieldPath field_path_1("resource.name"); + FieldPath field_path_2("resource.type"); + EXPECT_FALSE(field_path_1 == field_path_2); +} + +TEST(FieldPathTest, LessThanComparesFieldSelectionAndReturnsTrue) { + FieldPath field_path_1("resource.name"); + FieldPath field_path_2("resource.type"); + EXPECT_TRUE(field_path_1 < field_path_2); +} + +TEST(FieldPathTest, LessThanComparesIdenticalFieldSelectionAndReturnsFalse) { + FieldPath field_path_1("resource.name"); + FieldPath field_path_2("resource.name"); + EXPECT_FALSE(field_path_1 < field_path_2); +} + +TEST(FieldPathTest, LessThanComparesFieldSelectionAndReturnsFalse) { + FieldPath field_path_1("resource.type"); + FieldPath field_path_2("resource.name"); + EXPECT_FALSE(field_path_1 < field_path_2); +} + +} // namespace +} // namespace cel::checker_internal diff --git a/checker/internal/proto_type_mask.cc b/checker/internal/proto_type_mask.cc new file mode 100644 index 000000000..85e39cb69 --- /dev/null +++ b/checker/internal/proto_type_mask.cc @@ -0,0 +1,87 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "checker/internal/proto_type_mask.h" + +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/btree_set.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/strings/substitute.h" +#include "checker/internal/field_path.h" +#include "internal/status_macros.h" +#include "google/protobuf/descriptor.h" + +namespace cel::checker_internal { + +using ::google::protobuf::Descriptor; +using ::google::protobuf::DescriptorPool; +using ::google::protobuf::FieldDescriptor; + +absl::StatusOr FindMessage( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + absl::string_view type_name) { + const Descriptor* descriptor = + descriptor_pool->FindMessageTypeByName(type_name); + if (descriptor == nullptr) { + return absl::InvalidArgumentError( + absl::Substitute("type '$0' not found", type_name)); + } + return descriptor; +} + +absl::StatusOr FindField(const Descriptor* descriptor, + absl::string_view field_name) { + const FieldDescriptor* field_descriptor = + descriptor->FindFieldByName(field_name); + if (field_descriptor == nullptr) { + return absl::InvalidArgumentError( + absl::Substitute("could not select field '$0' from type '$1'", + field_name, descriptor->full_name())); + } + return field_descriptor; +} + +absl::StatusOr> ProtoTypeMask::GetFieldNames( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool) const { + CEL_ASSIGN_OR_RETURN(const Descriptor* descriptor, + FindMessage(descriptor_pool, this->GetTypeName())); + absl::btree_set field_names; + for (const FieldPath& field_path : this->GetFieldPaths()) { + std::string field_name = field_path.GetFieldName(); + CEL_ASSIGN_OR_RETURN(const FieldDescriptor* field_descriptor, + FindField(descriptor, field_name)); + field_names.insert(field_descriptor->name()); + } + return field_names; +} + +std::string ProtoTypeMask::DebugString() const { + // Represent each FieldPath by its path because it is easiest to read. + std::vector paths; + paths.reserve(field_paths_.size()); + for (const FieldPath& field_path : field_paths_) { + paths.emplace_back(field_path.GetPath()); + } + return absl::Substitute( + "ProtoTypeMask { type name: '$0', field paths: { '$1' } }", type_name_, + absl::StrJoin(paths, "', '")); +} + +} // namespace cel::checker_internal diff --git a/checker/internal/proto_type_mask.h b/checker/internal/proto_type_mask.h new file mode 100644 index 000000000..f7d522cba --- /dev/null +++ b/checker/internal/proto_type_mask.h @@ -0,0 +1,111 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_CHECKER_PROTO_TYPE_MASK_H_ +#define THIRD_PARTY_CEL_CPP_CHECKER_PROTO_TYPE_MASK_H_ + +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/btree_set.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "checker/internal/field_path.h" +#include "google/protobuf/descriptor.h" + +namespace cel::checker_internal { + +// Returns a descriptor for the input type name. +// Returns an error if the type name is not found. +absl::StatusOr FindMessage( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + absl::string_view type_name); + +// Returns a field descriptor for the input field name. +// Returns an error if the field name is not found. +absl::StatusOr FindField( + const google::protobuf::Descriptor* descriptor, absl::string_view field_name); + +// Represents the fraction of a protobuf type's object graph that should be +// visible within CEL expressions. +class ProtoTypeMask { + public: + explicit ProtoTypeMask(std::string type_name, + const std::vector& field_paths) + : type_name_(std::move(type_name)) { + for (const std::string& field_path : field_paths) { + field_paths_.insert(FieldPath(field_path)); + } + } + + // Returns a set of field names. The set includes the first field name from + // each field path. We are able to return a set of absl::string_view because + // the result is backed by the descriptor pool. + absl::StatusOr> GetFieldNames( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool) const; + + // Returns the type's full name. + // For example: "google.rpc.context.AttributeContext". + absl::string_view GetTypeName() const { return type_name_; } + + // Returns a representation of the FieldMask, which is a set of field paths. + // For example: + // { + // FieldPath { + // field path: 'resource.name', + // field selection: {'resource', 'name'} + // }, + // FieldPath { + // field path: 'request.auth.claims', + // field selection: {'request', 'auth', 'claims'} + // } + // } + const absl::btree_set& GetFieldPaths() const { + return field_paths_; + } + + template + friend void AbslStringify(Sink& sink, const ProtoTypeMask& proto_type_mask) { + sink.Append(proto_type_mask.DebugString()); + } + + private: + std::string DebugString() const; + + // A type's full name. For example: "google.rpc.context.AttributeContext". + std::string type_name_; + // A representation of a FieldMask, which is a set of field paths. + // For example: + // { + // FieldPath { + // field path: 'resource.name', + // field selection: {'resource', 'name'} + // }, + // FieldPath { + // field path: 'request.auth.claims', + // field selection: {'request', 'auth', 'claims'} + // } + // } + // A FieldMask contains one or more paths which contain identifier characters + // that have been dot delimited, e.g. resource.name, request.auth.claims. + // For each path, all descendent fields after the last element in the path are + // visible. An empty set means all fields are hidden. + absl::btree_set field_paths_; +}; + +} // namespace cel::checker_internal + +#endif // THIRD_PARTY_CEL_CPP_CHECKER_PROTO_TYPE_MASK_H_ diff --git a/checker/internal/proto_type_mask_registry.cc b/checker/internal/proto_type_mask_registry.cc new file mode 100644 index 000000000..9c50c9784 --- /dev/null +++ b/checker/internal/proto_type_mask_registry.cc @@ -0,0 +1,180 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "checker/internal/proto_type_mask_registry.h" + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/btree_set.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/strings/substitute.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" +#include "checker/internal/field_path.h" +#include "checker/internal/proto_type_mask.h" +#include "common/type.h" +#include "internal/status_macros.h" +#include "google/protobuf/descriptor.h" + +namespace cel::checker_internal { +namespace { + +using ::google::protobuf::Descriptor; +using ::google::protobuf::DescriptorPool; +using ::google::protobuf::FieldDescriptor; +using TypeMap = + absl::flat_hash_map>; + +// Returns a message type descriptor for the input field descriptor. +// Returns an error if the field is not a message type. +absl::StatusOr GetMessage( + const FieldDescriptor* field_descriptor) { + cel::MessageTypeField field(field_descriptor); + cel::Type type = field.GetType(); + absl::optional message_type = type.AsMessage(); + if (!message_type.has_value()) { + return absl::InvalidArgumentError(absl::Substitute( + "field '$0' is not a message type", field_descriptor->name())); + } + return &(*message_type.value()); +} + +// Inserts the type name with an empty set into types_and_visible_fields. +// Returns an error if the type name is already present with a non-empty set. +absl::Status AddAllHiddenFields(TypeMap& types_and_visible_fields, + absl::string_view type_name) { + auto result = types_and_visible_fields.find(type_name); + if (result != types_and_visible_fields.end()) { + if (!result->second.empty()) { + return absl::InvalidArgumentError( + absl::Substitute("cannot insert a proto type mask with all hidden " + "fields when type '$0' has already been inserted " + "with a proto type mask with a visible field", + type_name)); + } + return absl::OkStatus(); + } + types_and_visible_fields.insert({std::string(type_name), {}}); + return absl::OkStatus(); +} + +// Inserts the type name and field name into types_and_visible_fields. +// Returns an error if the type name is already present with an empty set. +absl::Status AddVisibleField(TypeMap& types_and_visible_fields, + absl::string_view type_name, + absl::string_view field_name) { + auto result = types_and_visible_fields.find(type_name); + if (result != types_and_visible_fields.end()) { + if (result->second.empty()) { + return absl::InvalidArgumentError(absl::Substitute( + "cannot insert a proto type mask with visible " + "field '$0' when type '$1' has already been inserted " + "with a proto type mask with all hidden fields", + field_name, type_name)); + } + result->second.insert(std::string(field_name)); + return absl::OkStatus(); + } + types_and_visible_fields.insert( + {std::string(type_name), {std::string(field_name)}}); + return absl::OkStatus(); +} + +// Processes the input proto type masks to create and return the +// types_and_visible_fields map. +// Returns an error if one of the proto type masks is not valid. For example, +// if a type is not found in the descriptor pool, if a field name is not +// found, or if a field is not a message type when we are expecting it to be. +// Returns an error if there is a conflict in field visibility when +// updating the map. +absl::StatusOr ComputeVisibleFieldsMap( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + const std::vector& proto_type_masks) { + TypeMap types_and_visible_fields; + for (const ProtoTypeMask& proto_type_mask : proto_type_masks) { + absl::string_view type_name = proto_type_mask.GetTypeName(); + CEL_ASSIGN_OR_RETURN(const Descriptor* descriptor, + FindMessage(descriptor_pool, type_name)); + const absl::btree_set& field_paths = + proto_type_mask.GetFieldPaths(); + if (field_paths.empty()) { + CEL_RETURN_IF_ERROR( + AddAllHiddenFields(types_and_visible_fields, type_name)); + } + for (const FieldPath& field_path : field_paths) { + const Descriptor* target_descriptor = descriptor; + absl::Span field_selection = + field_path.GetFieldSelection(); + for (auto iterator = field_selection.begin(); + iterator != field_selection.end(); ++iterator) { + CEL_ASSIGN_OR_RETURN(const FieldDescriptor* field_descriptor, + FindField(target_descriptor, *iterator)); + CEL_RETURN_IF_ERROR(AddVisibleField(types_and_visible_fields, + target_descriptor->full_name(), + *iterator)); + if (std::next(iterator) != field_selection.end()) { + CEL_ASSIGN_OR_RETURN(target_descriptor, GetMessage(field_descriptor)); + } + } + } + } + return types_and_visible_fields; +} + +} // namespace + +absl::StatusOr> +ProtoTypeMaskRegistry::Create( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + const std::vector& proto_type_masks) { + CEL_ASSIGN_OR_RETURN( + auto types_and_visible_fields, + ComputeVisibleFieldsMap(descriptor_pool, proto_type_masks)); + std::shared_ptr proto_type_mask_registry = + absl::WrapUnique(new ProtoTypeMaskRegistry(types_and_visible_fields)); + return proto_type_mask_registry; +} + +bool ProtoTypeMaskRegistry::FieldIsVisible(absl::string_view type_name, + absl::string_view field_name) const { + auto iterator = types_and_visible_fields_.find(type_name); + if (iterator != types_and_visible_fields_.end() && + !iterator->second.contains(field_name)) { + return false; + } + return true; +} + +std::string ProtoTypeMaskRegistry::DebugString() const { + std::string output = "ProtoTypeMaskRegistry { "; + for (auto& element : types_and_visible_fields_) { + absl::StrAppend(&output, "{type: '", element.first, "', visible_fields: '", + absl::StrJoin(element.second, "', '"), "'} "); + } + absl::StrAppend(&output, "}"); + return output; +} + +} // namespace cel::checker_internal diff --git a/checker/internal/proto_type_mask_registry.h b/checker/internal/proto_type_mask_registry.h new file mode 100644 index 000000000..338353e7d --- /dev/null +++ b/checker/internal/proto_type_mask_registry.h @@ -0,0 +1,83 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_CHECKER_PROTO_TYPE_MASK_REGISTRY_H_ +#define THIRD_PARTY_CEL_CPP_CHECKER_PROTO_TYPE_MASK_REGISTRY_H_ + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "checker/internal/proto_type_mask.h" +#include "google/protobuf/descriptor.h" + +namespace cel::checker_internal { + +// Stores information related to ProtoTypeMasks. Visibility is defined per type, +// meaning that all messages of a type have the same visible fields. +class ProtoTypeMaskRegistry { + public: + // Processes the input proto type masks to create a ProtoTypeMaskRegistry. + // Returns an error if one of the proto type masks is not valid. For example, + // if a type is not found in the descriptor pool, if a field name is not + // found, or if a field is not a message type when we are expecting it to be. + // Returns an error if there is a conflict in field visibility when + // updating the map. + static absl::StatusOr> Create( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + const std::vector& proto_type_masks); + + const absl::flat_hash_map>& + GetTypesAndVisibleFields() const { + return types_and_visible_fields_; + } + + // Returns true when the field name is visible. A field is visible if: + // 1. The type name is not a key in the map. + // 2. The type name is a key in the map and the field name is in the set of + // field names that are visible for the type. + bool FieldIsVisible(absl::string_view type_name, + absl::string_view field_name) const; + + template + friend void AbslStringify( + Sink& sink, + const std::shared_ptr& proto_type_mask_registry) { + sink.Append(proto_type_mask_registry->DebugString()); + } + + private: + explicit ProtoTypeMaskRegistry( + absl::flat_hash_map> + types_and_visible_fields) + : types_and_visible_fields_(std::move(types_and_visible_fields)) {} + + std::string DebugString() const; + + // Map of types that have a field mask where the keys are + // fully qualified type names and the values are the set of field names that + // are visible for the type. + absl::flat_hash_map> + types_and_visible_fields_; +}; + +} // namespace cel::checker_internal + +#endif // THIRD_PARTY_CEL_CPP_CHECKER_PROTO_TYPE_MASK_REGISTRY_H_ diff --git a/checker/internal/proto_type_mask_registry_test.cc b/checker/internal/proto_type_mask_registry_test.cc new file mode 100644 index 000000000..3a73c8823 --- /dev/null +++ b/checker/internal/proto_type_mask_registry_test.cc @@ -0,0 +1,402 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "checker/internal/proto_type_mask_registry.h" + +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "checker/internal/proto_type_mask.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" + +namespace cel::checker_internal { +namespace { + +using ::absl_testing::StatusIs; +using ::cel::internal::GetSharedTestingDescriptorPool; +using ::testing::AllOf; +using ::testing::HasSubstr; +using ::testing::IsEmpty; +using ::testing::Pair; +using ::testing::UnorderedElementsAre; + +using TypeMap = + absl::flat_hash_map>; + +TEST(ProtoTypeMaskRegistryTest, + CreateWithEmptyInputSucceedsAndAllFieldsAreVisible) { + std::vector proto_type_masks = {}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT(proto_type_mask_registry->GetTypesAndVisibleFields(), IsEmpty()); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, CreateWithEmptyTypeReturnsError) { + std::vector proto_type_masks = {ProtoTypeMask("", {})}; + EXPECT_THAT(ProtoTypeMaskRegistry::Create( + GetSharedTestingDescriptorPool().get(), proto_type_masks), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("type '' not found"))); +} + +TEST(ProtoTypeMaskRegistryTest, CreateWithUnknownTypeReturnsError) { + std::vector proto_type_masks = { + ProtoTypeMask("com.example.UnknownType", {})}; + EXPECT_THAT(ProtoTypeMaskRegistry::Create( + GetSharedTestingDescriptorPool().get(), proto_type_masks), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("type 'com.example.UnknownType' not found"))); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithEmptySetFieldPathSucceedsAndFieldsAreHidden) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", {})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT(proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre( + Pair("cel.expr.conformance.proto3.TestAllTypes", IsEmpty()))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDuplicateEmptySetFieldPathSucceedsAndFieldsAreHidden) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", {}), + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", {})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT(proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre( + Pair("cel.expr.conformance.proto3.TestAllTypes", IsEmpty()))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, CreateWithEmptyFieldPathReturnsError) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", {""})}; + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("could not select field '' from type " + "'cel.expr.conformance.proto3.TestAllTypes'"))); +} + +TEST(ProtoTypeMaskRegistryTest, CreateWithDelimiterFieldPathReturnsError) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", {"."})}; + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("could not select field '' from type " + "'cel.expr.conformance.proto3.TestAllTypes'"))); +} + +TEST(ProtoTypeMaskRegistryTest, CreateWithUnknownFieldReturnsError) { + std::vector proto_type_masks = {ProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"unknown_field"})}; + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("could not select field 'unknown_field' from type " + "'cel.expr.conformance.proto3.TestAllTypes'"))); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDepthOneNonMessageFieldsSucceedsAndFieldsAreVisible) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", + {"single_int32", "single_any", "single_timestamp"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT(proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre( + Pair("cel.expr.conformance.proto3.TestAllTypes", + UnorderedElementsAre("single_int32", "single_any", + "single_timestamp")))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "single_int32")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "single_any")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "single_timestamp")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, CreateWithDepthTwoNonMessageFieldReturnsError) { + std::vector proto_type_masks; + proto_type_masks.push_back( + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", + {"single_int32.any_field_name"})); + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("field 'single_int32' is not a message type"))); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDepthOneMessageFieldSucceedsAndFieldsAreVisible) { + std::vector proto_type_masks = {ProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"standalone_message"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT( + proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre(Pair("cel.expr.conformance.proto3.TestAllTypes", + UnorderedElementsAre("standalone_message")))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "standalone_message")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDepthTwoMessageFieldSucceedsAndFieldsAreVisible) { + std::vector proto_type_masks = {ProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"standalone_message.bb"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT(proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre( + Pair("cel.expr.conformance.proto3.TestAllTypes", + UnorderedElementsAre("standalone_message")), + Pair("cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + UnorderedElementsAre("bb")))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "standalone_message")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage", "bb")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDepthTwoMessageUnknownFieldReturnsError) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", + {"standalone_message.unknown_field"})}; + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs( + absl::StatusCode::kInvalidArgument, + HasSubstr( + "could not select field 'unknown_field' from type " + "'cel.expr.conformance.proto3.TestAllTypes.NestedMessage'"))); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDepthThreeMessageFieldSucceedsAndFieldsAreVisible) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.NestedTestAllTypes", + {"payload.standalone_message.bb"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT(proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre( + Pair("cel.expr.conformance.proto3.NestedTestAllTypes", + UnorderedElementsAre("payload")), + Pair("cel.expr.conformance.proto3.TestAllTypes", + UnorderedElementsAre("standalone_message")), + Pair("cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + UnorderedElementsAre("bb")))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.NestedTestAllTypes", "payload")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.NestedTestAllTypes", "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "standalone_message")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage", "bb")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDepthOneRepeatedMessageFieldSucceedsAndFieldsAreVisible) { + std::vector proto_type_masks = {ProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"repeated_nested_message"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT(proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre( + Pair("cel.expr.conformance.proto3.TestAllTypes", + UnorderedElementsAre("repeated_nested_message")))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "repeated_nested_message")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDepthTwoRepeatedMessageFieldReturnsError) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", + {"repeated_nested_message.bb"})}; + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs( + absl::StatusCode::kInvalidArgument, + HasSubstr("field 'repeated_nested_message' is not a message type"))); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithListOfFieldPathsSucceedsAndFieldsAreVisible) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.NestedTestAllTypes", + {"payload.standalone_message.bb", "payload.single_int32"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT( + proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre( + Pair("cel.expr.conformance.proto3.NestedTestAllTypes", + UnorderedElementsAre("payload")), + Pair("cel.expr.conformance.proto3.TestAllTypes", + UnorderedElementsAre("standalone_message", "single_int32")), + Pair("cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + UnorderedElementsAre("bb")))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.NestedTestAllTypes", "payload")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.NestedTestAllTypes", "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "standalone_message")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "single_int32")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage", "bb")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateAddVisibleFieldThenAllHiddenFieldsReturnsError) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", + {"standalone_message.bb"}), + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + {})}; + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs( + absl::StatusCode::kInvalidArgument, + HasSubstr( + "cannot insert a proto type mask with all hidden fields when " + "type 'cel.expr.conformance.proto3.TestAllTypes.NestedMessage' " + "has already been inserted with a proto type mask with a visible " + "field"))); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateAddAllHiddenThenVisibleFieldReturnsError) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + {}), + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", + {"standalone_message.bb"})}; + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs( + absl::StatusCode::kInvalidArgument, + HasSubstr( + "cannot insert a proto type mask with visible field 'bb' when " + "type 'cel.expr.conformance.proto3.TestAllTypes.NestedMessage' " + "has already been inserted with a proto type mask with all " + "hidden fields"))); +} + +TEST(ProtoTypeMaskRegistryTest, AbslStringifyPrintsTypesAndVisibleFieldsMap) { + std::vector proto_type_masks = {ProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"standalone_message.bb"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT( + absl::StrCat(proto_type_mask_registry), + AllOf(HasSubstr("ProtoTypeMaskRegistry {"), + HasSubstr("{type: 'cel.expr.conformance.proto3.TestAllTypes', " + "visible_fields: 'standalone_message'}"), + HasSubstr("{type: " + "'cel.expr.conformance.proto3.TestAllTypes.NestedMessage'" + ", visible_fields: 'bb'}"))); +} + +} // namespace +} // namespace cel::checker_internal diff --git a/checker/internal/proto_type_mask_test.cc b/checker/internal/proto_type_mask_test.cc new file mode 100644 index 000000000..0c534f8cf --- /dev/null +++ b/checker/internal/proto_type_mask_test.cc @@ -0,0 +1,143 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "checker/internal/proto_type_mask.h" + +#include +#include + +#include "absl/container/btree_set.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "checker/internal/field_path.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" + +namespace cel::checker_internal { +namespace { + +using ::absl_testing::StatusIs; +using ::cel::internal::GetSharedTestingDescriptorPool; +using ::testing::HasSubstr; +using ::testing::IsEmpty; +using ::testing::UnorderedElementsAre; + +TEST(ProtoTypeMaskTest, EmptyTypeNameAndEmptyFieldPathsSucceeds) { + std::string type_name = ""; + std::vector field_paths; + ProtoTypeMask proto_type_mask(type_name, field_paths); + EXPECT_EQ(proto_type_mask.GetTypeName(), ""); + EXPECT_THAT(proto_type_mask.GetFieldPaths(), IsEmpty()); +} + +TEST(ProtoTypeMaskTest, NotEmptyTypeNameAndNotEmptyFieldPathsSucceeds) { + std::string type_name = "google.type.Expr"; + std::vector field_paths = {"resource.name", "resource.type"}; + ProtoTypeMask proto_type_mask(type_name, field_paths); + EXPECT_EQ(proto_type_mask.GetTypeName(), "google.type.Expr"); + EXPECT_THAT(proto_type_mask.GetFieldPaths(), + UnorderedElementsAre(FieldPath("resource.name"), + FieldPath("resource.type"))); +} + +TEST(ProtoTypeMaskTest, GetFieldNamesWithEmptyTypeReturnsError) { + ProtoTypeMask proto_type_mask("", {}); + EXPECT_THAT( + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("type '' not found"))); +} + +TEST(ProtoTypeMaskTest, GetFieldNamesWithUnknownTypeReturnsError) { + ProtoTypeMask proto_type_mask("com.example.UnknownType", {}); + EXPECT_THAT( + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("type 'com.example.UnknownType' not found"))); +} + +TEST(ProtoTypeMaskTest, + GetFieldNamesWithEmptySetFieldPathSucceedsAndReturnsEmptySet) { + ProtoTypeMask proto_type_mask("cel.expr.conformance.proto3.TestAllTypes", {}); + ASSERT_OK_AND_ASSIGN( + absl::btree_set field_names, + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get())); + EXPECT_THAT(field_names, IsEmpty()); +} + +TEST(ProtoTypeMaskTest, GetFieldNamesWithEmptyFieldPathReturnsError) { + ProtoTypeMask proto_type_mask("cel.expr.conformance.proto3.TestAllTypes", + {""}); + EXPECT_THAT( + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("could not select field '' from type " + "'cel.expr.conformance.proto3.TestAllTypes'"))); +} + +TEST(ProtoTypeMaskTest, GetFieldNamesWithDelimiterFieldPathReturnsError) { + ProtoTypeMask proto_type_mask("cel.expr.conformance.proto3.TestAllTypes", + {"single_int32", "."}); + EXPECT_THAT( + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("could not select field '' from type " + "'cel.expr.conformance.proto3.TestAllTypes'"))); +} + +TEST(ProtoTypeMaskTest, GetFieldNamesWithUnknownFieldReturnsError) { + ProtoTypeMask proto_type_mask("cel.expr.conformance.proto3.TestAllTypes", + {"unknown_field"}); + EXPECT_THAT( + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("could not select field 'unknown_field' from type " + "'cel.expr.conformance.proto3.TestAllTypes'"))); +} + +TEST(ProtoTypeMaskTest, + GetFieldNamesWithValidFieldsSucceedsAndReturnsFieldNames) { + ProtoTypeMask proto_type_mask("cel.expr.conformance.proto3.TestAllTypes", + {"single_int32", "single_string"}); + ASSERT_OK_AND_ASSIGN( + absl::btree_set field_names, + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get())); + EXPECT_THAT(field_names, + UnorderedElementsAre("single_int32", "single_string")); +} + +TEST(ProtoTypeMaskTest, + GetFieldNamesWithValidFieldPathsSucceedsAndReturnsFieldNames) { + ProtoTypeMask proto_type_mask( + "cel.expr.conformance.proto3.NestedTestAllTypes", + {"payload.standalone_message.bb", "payload.single_int32", + "child.any_field_name"}); + ASSERT_OK_AND_ASSIGN( + absl::btree_set field_names, + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get())); + EXPECT_THAT(field_names, UnorderedElementsAre("payload", "child")); +} + +TEST(ProtoTypeMaskTest, AbslStringifyPrintsTypeNameAndFieldPaths) { + std::string type_name = "google.type.Expr"; + std::vector field_paths = {"resource.name", "resource.type"}; + ProtoTypeMask proto_type_mask(type_name, field_paths); + EXPECT_THAT(absl::StrCat(proto_type_mask), + HasSubstr("ProtoTypeMask { type name: 'google.type.Expr', field " + "paths: { 'resource.name', 'resource.type' } }")); +} + +} // namespace +} // namespace cel::checker_internal diff --git a/checker/internal/type_check_env.cc b/checker/internal/type_check_env.cc index c080326cb..8dc83518d 100644 --- a/checker/internal/type_check_env.cc +++ b/checker/internal/type_check_env.cc @@ -16,6 +16,7 @@ #include #include +#include #include "absl/base/nullability.h" #include "absl/status/statusor.h" @@ -48,7 +49,7 @@ const FunctionDecl* absl_nullable TypeCheckEnv::LookupFunction( return nullptr; } -absl::StatusOr> TypeCheckEnv::LookupTypeName( +absl::StatusOr> TypeCheckEnv::LookupTypeName( absl::string_view name) const { for (auto iter = type_providers_.begin(); iter != type_providers_.end(); ++iter) { @@ -57,10 +58,10 @@ absl::StatusOr> TypeCheckEnv::LookupTypeName( return type; } } - return absl::nullopt; + return std::nullopt; } -absl::StatusOr> TypeCheckEnv::LookupEnumConstant( +absl::StatusOr> TypeCheckEnv::LookupEnumConstant( absl::string_view type, absl::string_view value) const { for (auto iter = type_providers_.begin(); iter != type_providers_.end(); ++iter) { @@ -74,12 +75,12 @@ absl::StatusOr> TypeCheckEnv::LookupEnumConstant( return decl; } } - return absl::nullopt; + return std::nullopt; } -absl::StatusOr> TypeCheckEnv::LookupTypeConstant( +absl::StatusOr> TypeCheckEnv::LookupTypeConstant( google::protobuf::Arena* absl_nonnull arena, absl::string_view name) const { - CEL_ASSIGN_OR_RETURN(absl::optional type, LookupTypeName(name)); + CEL_ASSIGN_OR_RETURN(std::optional type, LookupTypeName(name)); if (type.has_value()) { return MakeVariableDecl(type->name(), TypeType(arena, *type)); } @@ -91,11 +92,15 @@ absl::StatusOr> TypeCheckEnv::LookupTypeConstant( return LookupEnumConstant(enum_name_candidate, value_name_candidate); } - return absl::nullopt; + return std::nullopt; } -absl::StatusOr> TypeCheckEnv::LookupStructField( +absl::StatusOr> TypeCheckEnv::LookupStructField( absl::string_view type_name, absl::string_view field_name) const { + if (proto_type_mask_registry_ != nullptr && + !proto_type_mask_registry_->FieldIsVisible(type_name, field_name)) { + return std::nullopt; + } // Check the type providers in registration order. // Note: this doesn't allow for shadowing a type with a subset type of the // same name -- the later type provider will still be considered when @@ -108,7 +113,7 @@ absl::StatusOr> TypeCheckEnv::LookupStructField( return field; } } - return absl::nullopt; + return std::nullopt; } const VariableDecl* absl_nullable VariableScope::LookupLocalVariable( diff --git a/checker/internal/type_check_env.h b/checker/internal/type_check_env.h index 15f8ecc4d..0fda608ab 100644 --- a/checker/internal/type_check_env.h +++ b/checker/internal/type_check_env.h @@ -25,16 +25,20 @@ #include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" #include "absl/memory/memory.h" +#include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "absl/types/span.h" -#include "checker/internal/descriptor_pool_type_introspector.h" +#include "checker/internal/proto_type_mask.h" +#include "checker/internal/proto_type_mask_registry.h" #include "common/constant.h" #include "common/container.h" #include "common/decl.h" +#include "common/descriptor_pool_type_introspector.h" #include "common/type.h" #include "common/type_introspector.h" +#include "internal/status_macros.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -154,6 +158,14 @@ class TypeCheckEnv { variables_[decl.name()] = std::move(decl); } + absl::Status CreateProtoTypeMaskRegistry( + const std::vector& proto_type_masks) { + CEL_ASSIGN_OR_RETURN(proto_type_mask_registry_, + ProtoTypeMaskRegistry::Create(descriptor_pool_.get(), + proto_type_masks)); + return absl::OkStatus(); + } + const absl::flat_hash_map& functions() const { return functions_; } @@ -224,6 +236,8 @@ class TypeCheckEnv { absl::flat_hash_map variables_; absl::flat_hash_map functions_; + std::shared_ptr proto_type_mask_registry_; + // Type providers for custom types. std::vector> type_providers_; diff --git a/checker/internal/type_checker_builder_impl.cc b/checker/internal/type_checker_builder_impl.cc index 94a05602e..ab0caa713 100644 --- a/checker/internal/type_checker_builder_impl.cc +++ b/checker/internal/type_checker_builder_impl.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -23,13 +24,16 @@ #include "absl/base/no_destructor.h" #include "absl/base/nullability.h" #include "absl/cleanup/cleanup.h" +#include "absl/container/btree_set.h" #include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" #include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "checker/internal/proto_type_mask.h" #include "checker/internal/type_check_env.h" #include "checker/internal/type_checker_impl.h" #include "checker/type_checker.h" @@ -86,10 +90,19 @@ absl::Status CheckStdMacroOverlap(const FunctionDecl& decl) { } absl::Status AddWellKnownContextDeclarationVariables( - const google::protobuf::Descriptor* absl_nonnull descriptor, TypeCheckEnv& env, - bool use_json_name) { + const google::protobuf::Descriptor* absl_nonnull descriptor, + const absl::flat_hash_map>& + context_type_fields, + TypeCheckEnv& env, bool use_json_name) { for (int i = 0; i < descriptor->field_count(); ++i) { const google::protobuf::FieldDescriptor* field = descriptor->field(i); + // Skip fields that are hidden because of a proto type mask. + auto map_iterator = context_type_fields.find(descriptor->full_name()); + if (map_iterator != context_type_fields.end() && + !map_iterator->second.contains(field->name())) { + continue; + } Type type = MessageTypeField(field).GetType(); if (type.IsEnum()) { type = IntType(); @@ -109,11 +122,15 @@ absl::Status AddWellKnownContextDeclarationVariables( } absl::Status AddContextDeclarationVariables( - const google::protobuf::Descriptor* absl_nonnull descriptor, TypeCheckEnv& env) { + const google::protobuf::Descriptor* absl_nonnull descriptor, + const absl::flat_hash_map>& + context_type_fields, + TypeCheckEnv& env) { const bool use_json_name = env.proto_type_introspector().use_json_name(); if (IsWellKnownMessageType(descriptor)) { - return AddWellKnownContextDeclarationVariables(descriptor, env, - use_json_name); + return AddWellKnownContextDeclarationVariables( + descriptor, context_type_fields, env, use_json_name); } CEL_ASSIGN_OR_RETURN(auto fields, env.proto_type_introspector().ListFieldsForStructType( @@ -131,6 +148,13 @@ absl::Status AddContextDeclarationVariables( absl::string_view name = field_entry.name; + // Skip fields that are hidden because of a proto type mask. + auto map_iterator = context_type_fields.find(descriptor->full_name()); + if (map_iterator != context_type_fields.end() && + !map_iterator->second.contains(name)) { + continue; + } + if (!env.InsertVariableIfAbsent(MakeVariableDecl(name, type))) { return absl::AlreadyExistsError( absl::StrCat("variable '", name, @@ -158,13 +182,13 @@ absl::StatusOr MergeFunctionDecls( return merged_decl; } -absl::optional FilterDecl(FunctionDecl decl, - const TypeCheckerSubset& subset) { +std::optional FilterDecl(FunctionDecl decl, + const TypeCheckerSubset& subset) { FunctionDecl filtered; std::string name = decl.release_name(); std::vector overloads = decl.release_overloads(); - for (const auto& ovl : overloads) { - if (subset.should_include_overload(name, ovl.id())) { + for (auto& ovl : overloads) { + if (subset.should_include_overload(name, ovl)) { absl::Status s = filtered.AddOverload(std::move(ovl)); if (!s.ok()) { // Should not be possible to construct the original decl in a way that @@ -174,7 +198,7 @@ absl::optional FilterDecl(FunctionDecl decl, } } if (filtered.overloads().empty()) { - return absl::nullopt; + return std::nullopt; } filtered.set_name(std::move(name)); return filtered; @@ -221,14 +245,16 @@ absl::Status ValidateType(const Type& t, bool check_type_param_name, return absl::OkStatus(); } case TypeKind::kOpaque: { - for (Type type_param : t.AsOpaque()->GetParameters()) { + OpaqueType opaque_type = t.GetOpaque(); + for (Type type_param : opaque_type.GetParameters()) { CEL_RETURN_IF_ERROR(ValidateType(type_param, check_type_param_name, depth_limit, remaining_depth)); } return absl::OkStatus(); } case TypeKind::kType: { - for (Type type_param : t.AsType()->GetParameters()) { + TypeType type_type = t.GetType(); + for (Type type_param : type_type.GetParameters()) { CEL_RETURN_IF_ERROR(ValidateType(type_param, check_type_param_name, depth_limit, remaining_depth)); } @@ -283,7 +309,7 @@ absl::Status TypeCheckerBuilderImpl::ApplyConfig( for (FunctionDeclRecord& fn : config.functions) { FunctionDecl decl = std::move(fn.decl); if (subset != nullptr) { - absl::optional filtered = + std::optional filtered = FilterDecl(std::move(decl), *subset); if (!filtered.has_value()) { continue; @@ -317,7 +343,8 @@ absl::Status TypeCheckerBuilderImpl::ApplyConfig( } for (const google::protobuf::Descriptor* context_type : config.context_types) { - CEL_RETURN_IF_ERROR(AddContextDeclarationVariables(context_type, env)); + CEL_RETURN_IF_ERROR(AddContextDeclarationVariables( + context_type, config.context_type_fields, env)); } for (VariableDeclRecord& var : config.variables) { @@ -339,6 +366,8 @@ absl::Status TypeCheckerBuilderImpl::ApplyConfig( } } + CEL_RETURN_IF_ERROR(env.CreateProtoTypeMaskRegistry(config.proto_type_masks)); + return absl::OkStatus(); } @@ -462,6 +491,23 @@ absl::Status TypeCheckerBuilderImpl::AddContextDeclaration( return absl::OkStatus(); } +absl::Status TypeCheckerBuilderImpl::AddContextDeclarationWithProtoTypeMask( + absl::string_view type, std::vector field_paths) { + if (field_paths.empty()) { + return absl::InvalidArgumentError("field paths cannot be the empty set"); + } + + ProtoTypeMask proto_type_mask(std::string(type), field_paths); + target_config_->proto_type_masks.push_back(proto_type_mask); + + CEL_RETURN_IF_ERROR(AddContextDeclaration(type)); + CEL_ASSIGN_OR_RETURN( + absl::btree_set field_names, + proto_type_mask.GetFieldNames(template_env_.descriptor_pool())); + target_config_->context_type_fields.insert({type, std::move(field_names)}); + return absl::OkStatus(); +} + absl::Status TypeCheckerBuilderImpl::AddFunction(const FunctionDecl& decl) { CEL_RETURN_IF_ERROR( ValidateFunctionDecl(decl, options_.enable_type_parameter_name_validation, diff --git a/checker/internal/type_checker_builder_impl.h b/checker/internal/type_checker_builder_impl.h index 646a5d16f..9895a8aee 100644 --- a/checker/internal/type_checker_builder_impl.h +++ b/checker/internal/type_checker_builder_impl.h @@ -21,6 +21,7 @@ #include #include "absl/base/nullability.h" +#include "absl/container/btree_set.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/status/status.h" @@ -28,6 +29,7 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "checker/checker_options.h" +#include "checker/internal/proto_type_mask.h" #include "checker/internal/type_check_env.h" #include "checker/type_checker.h" #include "checker/type_checker_builder.h" @@ -76,6 +78,8 @@ class TypeCheckerBuilderImpl : public TypeCheckerBuilder { absl::Status AddVariable(const VariableDecl& decl) override; absl::Status AddOrReplaceVariable(const VariableDecl& decl) override; absl::Status AddContextDeclaration(absl::string_view type) override; + absl::Status AddContextDeclarationWithProtoTypeMask( + absl::string_view type, std::vector field_paths) override; absl::Status AddFunction(const FunctionDecl& decl) override; absl::Status MergeFunction(const FunctionDecl& decl) override; @@ -130,6 +134,11 @@ class TypeCheckerBuilderImpl : public TypeCheckerBuilder { std::vector functions; std::vector> type_providers; std::vector context_types; + // Maps context type names to fields names to add as variables. + // Only includes context types that are defined with proto type masks. + absl::flat_hash_map> + context_type_fields; + std::vector proto_type_masks; }; absl::Status BuildLibraryConfig(const CheckerLibrary& library, diff --git a/checker/internal/type_checker_builder_impl_test.cc b/checker/internal/type_checker_builder_impl_test.cc index f7a3dff97..8b142b432 100644 --- a/checker/internal/type_checker_builder_impl_test.cc +++ b/checker/internal/type_checker_builder_impl_test.cc @@ -15,12 +15,15 @@ #include "checker/internal/type_checker_builder_impl.h" #include +#include #include #include +#include #include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "checker/checker_options.h" @@ -92,6 +95,123 @@ INSTANTIATE_TEST_SUITE_P( ContextDeclsTestCase{"repeated_bytes", TypeSpec(ListTypeSpec(std::make_unique( PrimitiveType::kBytes)))}, + ContextDeclsTestCase{"repeated_nested_enum", + TypeSpec(ListTypeSpec(std::make_unique( + PrimitiveType::kInt64)))}, + ContextDeclsTestCase{ + "repeated_nested_message", + TypeSpec(ListTypeSpec(std::make_unique(MessageTypeSpec( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage"))))}, + ContextDeclsTestCase{ + "map_int32_timestamp", + TypeSpec(MapTypeSpec( + std::make_unique(PrimitiveType::kInt64), + std::make_unique(WellKnownTypeSpec::kTimestamp)))}, + ContextDeclsTestCase{ + "single_struct", + TypeSpec( + MapTypeSpec(std::make_unique(PrimitiveType::kString), + std::make_unique(DynTypeSpec())))})); + +TEST(ContextDeclsWithProtoTypeMaskTest, ErrorOnEmptyFieldPaths) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {}), + StatusIs(absl::StatusCode::kInvalidArgument, + "field paths cannot be the empty set")); +} + +TEST(ContextDeclsWithProtoTypeMaskTest, ErrorOnUnknownFieldPath) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT( + builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"unknown_field"}), + StatusIs(absl::StatusCode::kInvalidArgument, + "could not select field 'unknown_field' from type " + "'cel.expr.conformance.proto3.TestAllTypes'")); +} + +class ContextDeclsWithProtoTypeMaskFieldsDefinedTest + : public testing::TestWithParam {}; + +std::string LogFieldName(absl::string_view field_name, absl::string_view expr) { + return absl::StrCat("field_name: ", field_name, ", expr: ", expr); +} + +TEST_P(ContextDeclsWithProtoTypeMaskFieldsDefinedTest, + ContextDeclsWithProtoTypeMaskFieldsDefined) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT( + builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {GetParam().expr}), + IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr type_checker, + builder.Build()); + std::vector field_names = {"single_int64", + "single_uint32", + "single_double", + "single_string", + "single_any", + "single_duration", + "single_bool_wrapper", + "list_value", + "standalone_message", + "standalone_enum", + "repeated_bytes", + "repeated_nested_enum", + "repeated_nested_message", + "map_int32_timestamp", + "single_struct"}; + for (auto& field_name : field_names) { + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst(field_name)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + type_checker->Check(std::move(ast))); + if (field_name == GetParam().expr) { + // The field name that is part of the proto type mask is visible. + ASSERT_TRUE(result.IsValid()) + << LogFieldName(field_name, GetParam().expr); + EXPECT_EQ(result.GetAst()->GetReturnType(), GetParam().expected_type) + << LogFieldName(field_name, GetParam().expr); + } else { + // The field names that are not part of the proto type mask are not + // visible. + EXPECT_FALSE(result.IsValid()) + << LogFieldName(field_name, GetParam().expr); + } + } +} + +INSTANTIATE_TEST_SUITE_P( + TestAllTypes, ContextDeclsWithProtoTypeMaskFieldsDefinedTest, + testing::Values( + ContextDeclsTestCase{"single_int64", TypeSpec(PrimitiveType::kInt64)}, + ContextDeclsTestCase{"single_uint32", TypeSpec(PrimitiveType::kUint64)}, + ContextDeclsTestCase{"single_double", TypeSpec(PrimitiveType::kDouble)}, + ContextDeclsTestCase{"single_string", TypeSpec(PrimitiveType::kString)}, + ContextDeclsTestCase{"single_any", TypeSpec(WellKnownTypeSpec::kAny)}, + ContextDeclsTestCase{"single_duration", + TypeSpec(WellKnownTypeSpec::kDuration)}, + ContextDeclsTestCase{ + "single_bool_wrapper", + TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBool))}, + ContextDeclsTestCase{ + "list_value", + TypeSpec(ListTypeSpec(std::make_unique(DynTypeSpec())))}, + ContextDeclsTestCase{ + "standalone_message", + TypeSpec(MessageTypeSpec( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage"))}, + ContextDeclsTestCase{"standalone_enum", + TypeSpec(PrimitiveType::kInt64)}, + ContextDeclsTestCase{"repeated_bytes", + TypeSpec(ListTypeSpec(std::make_unique( + PrimitiveType::kBytes)))}, + ContextDeclsTestCase{"repeated_nested_enum", + TypeSpec(ListTypeSpec(std::make_unique( + PrimitiveType::kInt64)))}, ContextDeclsTestCase{ "repeated_nested_message", TypeSpec(ListTypeSpec(std::make_unique(MessageTypeSpec( @@ -107,6 +227,66 @@ INSTANTIATE_TEST_SUITE_P( MapTypeSpec(std::make_unique(PrimitiveType::kString), std::make_unique(DynTypeSpec())))})); +TEST(ContextDeclsWithProtoTypeMaskTest, FieldsInMaskAreVisibleFieldAccess) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.NestedTestAllTypes", + {"payload.standalone_message.bb", "payload.single_int32"}), + IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr type_checker, + builder.Build()); + // Visible field: standalone_message.bb + ASSERT_OK_AND_ASSIGN(auto ast, + MakeTestParsedAst("payload.standalone_message.bb")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + type_checker->Check(std::move(ast))); + ASSERT_TRUE(result.IsValid()); + EXPECT_EQ(result.GetAst()->GetReturnType(), TypeSpec(PrimitiveType::kInt64)); + // Visible field: single_int32 + ASSERT_OK_AND_ASSIGN(ast, MakeTestParsedAst("payload.single_int32")); + ASSERT_OK_AND_ASSIGN(result, type_checker->Check(std::move(ast))); + ASSERT_TRUE(result.IsValid()); + EXPECT_EQ(result.GetAst()->GetReturnType(), TypeSpec(PrimitiveType::kInt64)); + // Not Visible field: single_int64 + ASSERT_OK_AND_ASSIGN(ast, MakeTestParsedAst("payload.single_int64")); + ASSERT_OK_AND_ASSIGN(result, type_checker->Check(std::move(ast))); + ASSERT_FALSE(result.IsValid()); +} + +TEST(ContextDeclsWithProtoTypeMaskTest, FieldsInMaskAreVisibleFieldAssignment) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.NestedTestAllTypes", + {"payload.standalone_message.bb", "payload.single_int32"}), + IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr type_checker, + builder.Build()); + // Visible field: standalone_message.bb + ASSERT_OK_AND_ASSIGN( + auto ast, + MakeTestParsedAst( + R"(cel.expr.conformance.proto3.TestAllTypes.NestedMessage{bb: 12345})")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + type_checker->Check(std::move(ast))); + ASSERT_TRUE(result.IsValid()); + // Visible field: single_int32 + ASSERT_OK_AND_ASSIGN( + ast, + MakeTestParsedAst( + R"(cel.expr.conformance.proto3.TestAllTypes{single_int32: 12345})")); + ASSERT_OK_AND_ASSIGN(result, type_checker->Check(std::move(ast))); + ASSERT_TRUE(result.IsValid()); + // Not Visible field: single_int64 + ASSERT_OK_AND_ASSIGN( + ast, + MakeTestParsedAst( + R"(cel.expr.conformance.proto3.TestAllTypes{single_int64: 12345})")); + ASSERT_OK_AND_ASSIGN(result, type_checker->Check(std::move(ast))); + ASSERT_FALSE(result.IsValid()); +} + TEST(ContextDeclsTest, ErrorOnDuplicateContextDeclaration) { TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), {}); @@ -120,6 +300,20 @@ TEST(ContextDeclsTest, ErrorOnDuplicateContextDeclaration) { "already exists")); } +TEST(ContextDeclsWithProtoTypeMaskTest, ErrorOnDuplicateContextDeclaration) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT( + builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"standalone_message"}), + IsOk()); + EXPECT_THAT( + builder.AddContextDeclaration("cel.expr.conformance.proto3.TestAllTypes"), + StatusIs(absl::StatusCode::kAlreadyExists, + "context declaration 'cel.expr.conformance.proto3.TestAllTypes' " + "already exists")); +} + TEST(ContextDeclsTest, ErrorOnContextDeclarationNotFound) { TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), {}); @@ -129,6 +323,16 @@ TEST(ContextDeclsTest, ErrorOnContextDeclarationNotFound) { "context declaration 'com.example.UnknownType' not found")); } +TEST(ContextDeclsWithProtoTypeMaskTest, ErrorOnContextDeclarationNotFound) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + EXPECT_THAT( + builder.AddContextDeclarationWithProtoTypeMask("com.example.UnknownType", + {"any_field_name"}), + StatusIs(absl::StatusCode::kNotFound, + "context declaration 'com.example.UnknownType' not found")); +} + TEST(ContextDeclsTest, ErrorOnNonStructMessageType) { TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), {}); @@ -139,17 +343,28 @@ TEST(ContextDeclsTest, ErrorOnNonStructMessageType) { "context declaration 'google.protobuf.Timestamp' is not a struct")); } +TEST(ContextDeclsWithProtoTypeMaskTest, ErrorOnNonStructMessageType) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + EXPECT_THAT( + builder.AddContextDeclarationWithProtoTypeMask( + "google.protobuf.Timestamp", {"any_field_name"}), + StatusIs( + absl::StatusCode::kInvalidArgument, + "context declaration 'google.protobuf.Timestamp' is not a struct")); +} + TEST(ContextDeclsTest, CustomStructNotSupported) { TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), {}); class MyTypeProvider : public cel::TypeIntrospector { public: - absl::StatusOr> FindTypeImpl( + absl::StatusOr> FindTypeImpl( absl::string_view name) const override { if (name == "com.example.MyStruct") { return common_internal::MakeBasicStructType("com.example.MyStruct"); } - return absl::nullopt; + return std::nullopt; } }; @@ -160,6 +375,28 @@ TEST(ContextDeclsTest, CustomStructNotSupported) { "context declaration 'com.example.MyStruct' not found")); } +TEST(ContextDeclsWithProtoTypeMaskTest, CustomStructNotSupported) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + class MyTypeProvider : public cel::TypeIntrospector { + public: + absl::StatusOr> FindTypeImpl( + absl::string_view name) const override { + if (name == "com.example.MyStruct") { + return common_internal::MakeBasicStructType("com.example.MyStruct"); + } + return std::nullopt; + } + }; + + builder.AddTypeProvider(std::make_unique()); + + EXPECT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "com.example.MyStruct", {"any_field_name"}), + StatusIs(absl::StatusCode::kNotFound, + "context declaration 'com.example.MyStruct' not found")); +} + TEST(ContextDeclsTest, ErrorOnOverlappingContextDeclaration) { TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), {}); @@ -179,6 +416,69 @@ TEST(ContextDeclsTest, ErrorOnOverlappingContextDeclaration) { "declaration: 'cel.expr.conformance.proto2.TestAllTypes')")); } +TEST(ContextDeclsWithProtoTypeMaskTest, ErrorOnOverlappingContextDeclaration) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT( + builder.AddContextDeclaration("cel.expr.conformance.proto3.TestAllTypes"), + IsOk()); + // We resolve the context declaration variables at the Build() call, so the + // error surfaces then. + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto2.TestAllTypes", {"single_int32"}), + IsOk()); + + EXPECT_THAT( + builder.Build(), + StatusIs(absl::StatusCode::kAlreadyExists, + "variable 'single_int32' declared multiple times (from context " + "declaration: 'cel.expr.conformance.proto2.TestAllTypes')")); +} + +TEST(ContextDeclsWithProtoTypeMaskTest, + ErrorOnOverlappingContextDeclarationBothProtoTypeMasks) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"single_int32"}), + IsOk()); + // We resolve the context declaration variables at the Build() call, so the + // error surfaces then. + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto2.TestAllTypes", {"single_int32"}), + IsOk()); + + EXPECT_THAT( + builder.Build(), + StatusIs(absl::StatusCode::kAlreadyExists, + "variable 'single_int32' declared multiple times (from context " + "declaration: 'cel.expr.conformance.proto2.TestAllTypes')")); +} + +TEST(ContextDeclsWithProtoTypeMaskTest, + NonOverlappingContextDeclarationBothProtoTypeMasks) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"single_int32"}), + IsOk()); + // We resolve the context declaration variables at the Build() call, so the + // error surfaces then. + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto2.NestedTestAllTypes", + {"payload.single_int64"}), + IsOk()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr type_checker, + builder.Build()); + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst("single_int32")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + type_checker->Check(std::move(ast))); + ASSERT_TRUE(result.IsValid()); + ASSERT_OK_AND_ASSIGN(ast, MakeTestParsedAst("payload.single_int64")); + ASSERT_OK_AND_ASSIGN(result, type_checker->Check(std::move(ast))); +} + TEST(ContextDeclsTest, ErrorOnOverlappingVariableDeclaration) { TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), {}); @@ -193,6 +493,32 @@ TEST(ContextDeclsTest, ErrorOnOverlappingVariableDeclaration) { "variable 'single_int64' declared multiple times")); } +TEST(ContextDeclsWithProtoTypeMaskTest, ErrorOnOverlappingVariableDeclaration) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"single_int64"}), + IsOk()); + ASSERT_THAT(builder.AddVariable(MakeVariableDecl("single_int64", IntType())), + IsOk()); + + EXPECT_THAT(builder.Build(), + StatusIs(absl::StatusCode::kAlreadyExists, + "variable 'single_int64' declared multiple times")); +} + +TEST(ContextDeclsWithProtoTypeMaskTest, NonOverlappingVariableDeclaration) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"single_int32"}), + IsOk()); + ASSERT_THAT(builder.AddVariable(MakeVariableDecl("single_int64", IntType())), + IsOk()); + + EXPECT_THAT(builder.Build(), IsOk()); +} + TEST(TypeCheckerBuilderImplTest, InvalidTypeParamNameVariableValidationDisabled) { CheckerOptions options; diff --git a/checker/internal/type_checker_impl.cc b/checker/internal/type_checker_impl.cc index 8f67efbde..74bafdcb7 100644 --- a/checker/internal/type_checker_impl.cc +++ b/checker/internal/type_checker_impl.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,7 @@ #include "absl/base/nullability.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" @@ -34,7 +36,6 @@ #include "absl/types/optional.h" #include "absl/types/span.h" #include "checker/checker_options.h" -#include "checker/internal/format_type_name.h" #include "checker/internal/namespace_generator.h" #include "checker/internal/type_check_env.h" #include "checker/internal/type_checker_builder_impl.h" @@ -50,6 +51,8 @@ #include "common/constant.h" #include "common/decl.h" #include "common/expr.h" +#include "common/format_type_name.h" +#include "common/standard_definitions.h" #include "common/type.h" #include "common/type_kind.h" #include "internal/status_macros.h" @@ -58,6 +61,15 @@ namespace cel::checker_internal { namespace { +bool MatchesBlock(const Expr& expr) { + if (!expr.has_call_expr()) { + return false; + } + const auto& call = expr.call_expr(); + return call.function() == "cel.@block" && call.args().size() == 2 && + call.args()[0].has_list_expr(); +} + using AstType = cel::TypeSpec; using Severity = TypeCheckIssue::Severity; @@ -187,6 +199,7 @@ class ResolveVisitor : public AstVisitorBase { struct AttributeResolution { const VariableDecl* decl; bool requires_disambiguation; + bool local; }; ResolveVisitor(NamespaceGenerator namespace_generator, @@ -203,13 +216,23 @@ class ResolveVisitor : public AstVisitorBase { arena_(arena), current_scope_(&root_scope_) {} - void PreVisitExpr(const Expr& expr) override { expr_stack_.push_back(&expr); } + void PreVisitExpr(const Expr& expr) override { + expr_stack_.push_back(&expr); + if (expr_stack_.size() == 1 && MatchesBlock(expr)) { + ABSL_DCHECK_EQ(expr.call_expr().args().size(), 2); + ABSL_DCHECK(block_init_list_ == nullptr); + block_init_list_ = &expr.call_expr().args()[0]; + } + } void PostVisitExpr(const Expr& expr) override { if (expr_stack_.empty()) { return; } expr_stack_.pop_back(); + if (expr_stack_.size() == 2 && expr_stack_.back() == block_init_list_) { + HandleBlockIndex(&expr); + } } void PostVisitConst(const Expr& expr, const Constant& constant) override; @@ -358,7 +381,7 @@ class ResolveVisitor : public AstVisitorBase { // Lookup message type by name to support WellKnownType creation. CEL_ASSIGN_OR_RETURN( - absl::optional field_info, + std::optional field_info, env_->LookupStructField(resolved_name, field.name())); if (!field_info.has_value()) { ReportUndefinedField(field.id(), field.name(), resolved_name); @@ -384,10 +407,11 @@ class ResolveVisitor : public AstVisitorBase { return absl::OkStatus(); } - absl::optional CheckFieldType(int64_t expr_id, const Type& operand_type, - absl::string_view field_name); + std::optional CheckFieldType(int64_t expr_id, const Type& operand_type, + absl::string_view field_name); void HandleOptSelect(const Expr& expr); + void HandleBlockIndex(const Expr* expr); // Get the assigned type of the given subexpression. Should only be called if // the given subexpression is expected to have already been checked. @@ -420,6 +444,7 @@ class ResolveVisitor : public AstVisitorBase { std::vector expr_stack_; absl::flat_hash_map> maybe_namespaced_functions_; + const Expr* block_init_list_ = nullptr; // Select operations that need to be resolved outside of the traversal. // These are handled separately to disambiguate between namespaces and field // accesses @@ -585,9 +610,11 @@ void ResolveVisitor::PostVisitMap(const Expr& expr, const MapExpr& map) { for (const auto& entry : map.entries()) { const Expr* value = &entry.value(); Type value_type = GetDeducedType(value); + OptionalType opt_value_type; if (entry.optional()) { if (value_type.IsOptional()) { - value_type = value_type.GetOptional().GetParameter(); + opt_value_type = value_type.GetOptional(); + value_type = opt_value_type.GetParameter(); } else { ReportTypeMismatch(entry.value().id(), OptionalType(arena_, value_type), value_type); @@ -608,17 +635,26 @@ void ResolveVisitor::PostVisitMap(const Expr& expr, const MapExpr& map) { } void ResolveVisitor::PostVisitList(const Expr& expr, const ListExpr& list) { - // Follows list type inferencing behavior in Go (see map comments above). + if (&expr == block_init_list_) { + // Don't try to coalesce list type here because it can influence the + // resolved type of the list elements. cel.@block is always list and + // the elements are treated independently at runtime. + types_[&expr] = ListType(); + return; + } + // Follows list type inferencing behavior in Go (see map comments above). Type overall_elem_type = inference_context_->InstantiateTypeParams(TypeParamType("E")); auto assignability_context = inference_context_->CreateAssignabilityContext(); for (const auto& element : list.elements()) { const Expr* value = &element.expr(); Type value_type = GetDeducedType(value); + OptionalType opt_value_type; if (element.optional()) { if (value_type.IsOptional()) { - value_type = value_type.GetOptional().GetParameter(); + opt_value_type = value_type.GetOptional(); + value_type = opt_value_type.GetParameter(); } else { ReportTypeMismatch(element.expr().id(), OptionalType(arena_, value_type), value_type); @@ -805,8 +841,17 @@ void ResolveVisitor::PostVisitComprehensionSubexpression( MakeVariableDecl(comprehension.accu_var(), GetDeducedType(&comprehension.accu_init()))); break; + case ComprehensionArg::LOOP_CONDITION: { + Type condition_type = GetDeducedType(&comprehension.loop_condition()); + if (!inference_context_->IsAssignable(condition_type, BoolType())) { + ReportTypeMismatch(comprehension.loop_condition().id(), BoolType(), + condition_type); + } + break; + } case ComprehensionArg::ITER_RANGE: { - Type range_type = GetDeducedType(&comprehension.iter_range()); + Type range_type = inference_context_->FullySubstitute( + GetDeducedType(&comprehension.iter_range()), /*free_to_dyn=*/false); Type iter_type = DynType(); // iter_var for non comprehensions v2. Type iter_type1 = DynType(); // iter_var for comprehensions v2. Type iter_type2 = DynType(); // iter_var2 for comprehensions v2. @@ -819,7 +864,14 @@ void ResolveVisitor::PostVisitComprehensionSubexpression( iter_type = iter_type1 = range_type.GetMap().key(); iter_type2 = range_type.GetMap().value(); break; + case TypeKind::kTypeParam: + // Set the range type to DYN to prevent assignment to a potentially + // incorrect type at a later point in type-checking. The IsAssignable + // call will update the type substitutions for the type param. + inference_context_->IsAssignable(DynType(), range_type); + break; case TypeKind::kDyn: + case TypeKind::kError: break; default: ReportIssue(TypeCheckIssue::CreateError( @@ -864,8 +916,12 @@ const FunctionDecl* ResolveVisitor::ResolveFunctionCallShape( if (decl == nullptr) { return true; } + bool is_logical_op = (candidate == cel::StandardFunctions::kAnd || + candidate == cel::StandardFunctions::kOr) && + arg_count >= 2; for (const auto& ovl : decl->overloads()) { - if (ovl.member() == is_receiver && ovl.args().size() == arg_count) { + if (ovl.member() == is_receiver && + (ovl.args().size() == arg_count || is_logical_op)) { return false; } } @@ -889,7 +945,7 @@ void ResolveVisitor::ResolveFunctionOverloads(const Expr& expr, arg_types.push_back(GetDeducedType(&expr.call_expr().args()[i])); } - absl::optional resolution = + std::optional resolution = inference_context_->ResolveOverload(decl, arg_types, is_receiver); if (!resolution.has_value()) { @@ -924,11 +980,10 @@ void ResolveVisitor::ResolveFunctionOverloads(const Expr& expr, const VariableDecl* absl_nullable ResolveVisitor::LookupLocalIdentifier( absl::string_view name) { - // Note: if we see a leading dot, this shouldn't resolve to a local variable, - // but we need to check whether we need to disambiguate against a global in - // the reference map. if (absl::StartsWith(name, ".")) { - name = name.substr(1); + // Should not happen for normally parsed CEL, but prevent lookup in case + // of hand-crafted ASTs. + return nullptr; } return current_scope_->LookupLocalVariable(name); } @@ -938,7 +993,7 @@ const VariableDecl* absl_nullable ResolveVisitor::LookupGlobalIdentifier( if (const VariableDecl* decl = env_->LookupVariable(name); decl != nullptr) { return decl; } - absl::StatusOr> constant = + absl::StatusOr> constant = env_->LookupTypeConstant(arena_, name); if (!constant.ok()) { @@ -963,13 +1018,15 @@ void ResolveVisitor::ResolveSimpleIdentifier(const Expr& expr, absl::string_view name) { // Local variables (comprehension, bind) are simple identifiers so we can // skip generating the different namespace-qualified candidates. - const VariableDecl* local_decl = LookupLocalIdentifier(name); - - if (local_decl != nullptr && !absl::StartsWith(name, ".")) { - attributes_[&expr] = {local_decl, false}; - types_[&expr] = - inference_context_->InstantiateTypeParams(local_decl->type()); - return; + if (!absl::StartsWith(name, ".")) { + const VariableDecl* local_decl = LookupLocalIdentifier(name); + if (local_decl != nullptr) { + attributes_[&expr] = {local_decl, /*requires_disambiguation=*/false, + /*local=*/true}; + types_[&expr] = + inference_context_->InstantiateTypeParams(local_decl->type()); + return; + } } const VariableDecl* decl = nullptr; @@ -980,9 +1037,13 @@ void ResolveVisitor::ResolveSimpleIdentifier(const Expr& expr, return decl == nullptr; }); + bool requires_disambiguation = false; + if (absl::StartsWith(name, ".")) { + requires_disambiguation = LookupLocalIdentifier(name.substr(1)) != nullptr; + } + if (decl != nullptr) { - attributes_[&expr] = {decl, - /* requires_disambiguation= */ local_decl != nullptr}; + attributes_[&expr] = {decl, requires_disambiguation, /*local=*/false}; types_[&expr] = inference_context_->InstantiateTypeParams(decl->type()); return; } @@ -998,35 +1059,49 @@ void ResolveVisitor::ResolveQualifiedIdentifier( return; } + int matched_segment_index = -1; + const VariableDecl* decl = nullptr; + bool requires_disambiguation = false; + bool is_local = false; // Local variables (comprehension, bind) are simple identifiers so we can // skip generating the different namespace-qualified candidates. - const VariableDecl* local_decl = LookupLocalIdentifier(qualifiers[0]); - const VariableDecl* decl = nullptr; - - int matched_segment_index = -1; - - if (local_decl != nullptr && !absl::StartsWith(qualifiers[0], ".")) { - decl = local_decl; - matched_segment_index = 0; - } else { - namespace_generator_.GenerateCandidates( - qualifiers, [&decl, &matched_segment_index, this]( - absl::string_view candidate, int segment_index) { - decl = LookupGlobalIdentifier(candidate); - if (decl != nullptr) { - matched_segment_index = segment_index; - return false; - } - return true; - }); + if (!absl::StartsWith(qualifiers[0], ".")) { + const VariableDecl* local_decl = LookupLocalIdentifier(qualifiers[0]); + if (local_decl != nullptr) { + decl = local_decl; + matched_segment_index = 0; + is_local = true; + goto resolve_select_trail; + } } + namespace_generator_.GenerateCandidates( + qualifiers, [&decl, &matched_segment_index, this]( + absl::string_view candidate, int segment_index) { + decl = LookupGlobalIdentifier(candidate); + if (decl != nullptr) { + matched_segment_index = segment_index; + return false; + } + return true; + }); + if (decl == nullptr) { ReportMissingReference(expr, FormatCandidate(qualifiers)); types_[&expr] = ErrorType(); return; } + if (absl::StartsWith(qualifiers[0], ".")) { + const VariableDecl* local_decl = + LookupLocalIdentifier(qualifiers[0].substr(1)); + if (local_decl != nullptr) { + requires_disambiguation = true; + } + } + +resolve_select_trail: + const int num_select_opts = qualifiers.size() - matched_segment_index - 1; const Expr* root = &expr; @@ -1037,9 +1112,7 @@ void ResolveVisitor::ResolveQualifiedIdentifier( root = &root->select_expr().operand(); } - attributes_[root] = {decl, - /* requires_disambiguation= */ decl != local_decl && - local_decl != nullptr}; + attributes_[root] = {decl, requires_disambiguation, is_local}; types_[root] = inference_context_->InstantiateTypeParams(decl->type()); // fix-up select operations that were deferred. @@ -1049,9 +1122,9 @@ void ResolveVisitor::ResolveQualifiedIdentifier( } } -absl::optional ResolveVisitor::CheckFieldType(int64_t id, - const Type& operand_type, - absl::string_view field) { +std::optional ResolveVisitor::CheckFieldType(int64_t id, + const Type& operand_type, + absl::string_view field) { if (operand_type.kind() == TypeKind::kDyn || operand_type.kind() == TypeKind::kAny) { return DynType(); @@ -1063,11 +1136,11 @@ absl::optional ResolveVisitor::CheckFieldType(int64_t id, auto field_info = env_->LookupStructField(struct_type.name(), field); if (!field_info.ok()) { status_.Update(field_info.status()); - return absl::nullopt; + return std::nullopt; } if (!field_info->has_value()) { ReportUndefinedField(id, field, struct_type.name()); - return absl::nullopt; + return std::nullopt; } auto type = field_info->value().GetType(); if (type.kind() == TypeKind::kEnum) { @@ -1099,7 +1172,7 @@ absl::optional ResolveVisitor::CheckFieldType(int64_t id, "expression of type '", FormatTypeName(inference_context_->FinalizeType(operand_type)), "' cannot be the operand of a select operation"))); - return absl::nullopt; + return std::nullopt; } void ResolveVisitor::ResolveSelectOperation(const Expr& expr, @@ -1107,7 +1180,7 @@ void ResolveVisitor::ResolveSelectOperation(const Expr& expr, const Expr& operand) { const Type& operand_type = GetDeducedType(&operand); - absl::optional result_type; + std::optional result_type; int64_t id = expr.id(); // Support short-hand optional chaining. if (operand_type.IsOptional()) { @@ -1150,11 +1223,13 @@ void ResolveVisitor::HandleOptSelect(const Expr& expr) { } Type operand_type = GetDeducedType(operand); + OptionalType opt_operand_type; if (operand_type.IsOptional()) { - operand_type = operand_type.GetOptional().GetParameter(); + opt_operand_type = operand_type.GetOptional(); + operand_type = opt_operand_type.GetParameter(); } - absl::optional field_type = CheckFieldType( + std::optional field_type = CheckFieldType( expr.id(), operand_type, field->const_expr().string_value()); if (!field_type.has_value()) { types_[&expr] = ErrorType(); @@ -1171,16 +1246,56 @@ void ResolveVisitor::HandleOptSelect(const Expr& expr) { } } +void ResolveVisitor::HandleBlockIndex(const Expr* expr) { + ABSL_DCHECK(block_init_list_ != nullptr); + ABSL_DCHECK(block_init_list_->has_list_expr()); + const auto& elements = block_init_list_->list_expr().elements(); + int index = -1; + for (size_t i = 0; i < elements.size(); ++i) { + if (&elements[i].expr() == expr) { + index = i; + break; + } + } + if (index < 0) { + status_.Update(absl::InternalError( + "could not resolve expression as a cel.@block subexpression")); + return; + } + std::string var_name = absl::StrCat("@index", index); + + // Block is typically manually assembled from logically separate + // expressions so fix the type instead of inferring any remaining free type + // params as for normal subexpressions. + auto type = inference_context_->FinalizeType(GetDeducedType(expr)); + + VariableDecl decl = MakeVariableDecl(var_name, std::move(type)); + + // The C++ runtime requires that the indexes are topologically ordered. + // They just come into scope in order as we walk the AST so we don't need + // to do any additional work to check references to other initializers in + // an init expr. + // + // TODO(uncreated-issue/90): This is slightly inconsistent with the java + // runtime implementation which just requires the references to be acyclic. + auto* scope = + comprehension_vars_.emplace_back(current_scope_->MakeNestedScope()).get(); + scope->InsertVariableIfAbsent(std::move(decl)); + current_scope_ = scope; +} + class ResolveRewriter : public AstRewriterBase { public: explicit ResolveRewriter(const ResolveVisitor& visitor, const TypeInferenceContext& inference_context, const CheckerOptions& options, - Ast::ReferenceMap& references, Ast::TypeMap& types) + Ast::ReferenceMap& references, Ast::TypeMap& types, + ValidationResult::TypeMap& resolved_types) : visitor_(visitor), inference_context_(inference_context), reference_map_(references), type_map_(types), + resolved_types_(resolved_types), options_(options) {} bool PostVisitRewrite(Expr& expr) override { bool rewritten = false; @@ -1227,14 +1342,15 @@ class ResolveRewriter : public AstRewriterBase { if (auto iter = visitor_.types().find(&expr); iter != visitor_.types().end()) { - auto flattened_type = - FlattenType(inference_context_.FinalizeType(iter->second)); + cel::Type finalized_type = inference_context_.FinalizeType(iter->second); + auto flattened_type = FlattenType(finalized_type); if (!flattened_type.ok()) { status_.Update(flattened_type.status()); return rewritten; } type_map_[expr.id()] = *std::move(flattened_type); + resolved_types_[expr.id()] = finalized_type; rewritten = true; } @@ -1249,23 +1365,31 @@ class ResolveRewriter : public AstRewriterBase { const TypeInferenceContext& inference_context_; Ast::ReferenceMap& reference_map_; Ast::TypeMap& type_map_; + ValidationResult::TypeMap& resolved_types_; const CheckerOptions& options_; }; } // namespace -absl::StatusOr TypeCheckerImpl::Check( - std::unique_ptr ast) const { - google::protobuf::Arena type_arena; +absl::StatusOr TypeCheckerImpl::CheckImpl( + std::unique_ptr ast, google::protobuf::Arena* arena) const { + ast->mutable_reference_map().clear(); + ast->mutable_type_map().clear(); + + std::optional type_arena; + if (arena == nullptr) { + type_arena.emplace(); + arena = &(*type_arena); + } std::vector issues; CEL_ASSIGN_OR_RETURN(auto generator, NamespaceGenerator::Create(env_.container())); TypeInferenceContext type_inference_context( - &type_arena, options_.enable_legacy_null_assignment); + arena, options_.enable_legacy_null_assignment); ResolveVisitor visitor(std::move(generator), env_, *ast, - type_inference_context, issues, &type_arena); + type_inference_context, issues, arena); TraversalOptions opts; opts.use_comprehension_callbacks = true; @@ -1310,9 +1434,10 @@ absl::StatusOr TypeCheckerImpl::Check( // Apply updates as needed. // Happens in a second pass to simplify validating that pointers haven't // been invalidated by other updates. + ValidationResult::TypeMap resolved_types; ResolveRewriter rewriter(visitor, type_inference_context, options_, ast->mutable_reference_map(), - ast->mutable_type_map()); + ast->mutable_type_map(), resolved_types); AstRewrite(ast->mutable_root_expr(), rewriter); CEL_RETURN_IF_ERROR(rewriter.status()); @@ -1325,7 +1450,15 @@ absl::StatusOr TypeCheckerImpl::Check( {cel::ExtensionSpec::Component::kRuntime})); } - return ValidationResult(std::move(ast), std::move(issues)); + auto result = ValidationResult(std::move(ast), std::move(issues)); + if (!type_arena.has_value()) { + // cel::Type values will expire after this function returns when the local + // arena is destructed. Only set the resolved type map if we're using the + // caller's arena. + result.SetResolvedTypeMap(std::move(resolved_types)); + } + + return result; } std::unique_ptr TypeCheckerImpl::ToBuilder() const { diff --git a/checker/internal/type_checker_impl.h b/checker/internal/type_checker_impl.h index 71683276d..9ee9a50d0 100644 --- a/checker/internal/type_checker_impl.h +++ b/checker/internal/type_checker_impl.h @@ -42,8 +42,8 @@ class TypeCheckerImpl : public TypeChecker { TypeCheckerImpl(TypeCheckerImpl&&) = delete; TypeCheckerImpl& operator=(TypeCheckerImpl&&) = delete; - absl::StatusOr Check( - std::unique_ptr ast) const override; + absl::StatusOr CheckImpl( + std::unique_ptr ast, google::protobuf::Arena* arena) const override; std::unique_ptr ToBuilder() const override; diff --git a/checker/internal/type_checker_impl_test.cc b/checker/internal/type_checker_impl_test.cc index e6cd641d6..bc08900be 100644 --- a/checker/internal/type_checker_impl_test.cc +++ b/checker/internal/type_checker_impl_test.cc @@ -26,6 +26,7 @@ #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" @@ -36,20 +37,26 @@ #include "checker/type_checker_builder.h" #include "checker/validation_result.h" #include "common/ast.h" +#include "common/ast_proto.h" #include "common/container.h" #include "common/decl.h" #include "common/expr.h" +#include "common/format_type_name.h" #include "common/source.h" #include "common/type.h" #include "common/type_introspector.h" #include "internal/status_macros.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" +#include "parser/macro_registry.h" +#include "parser/parser.h" #include "testutil/baseline_tests.h" +#include "testutil/test_macros.h" #include "cel/expr/conformance/proto2/test_all_types.pb.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" #include "google/protobuf/arena.h" #include "google/protobuf/message.h" +#include "google/protobuf/text_format.h" namespace cel { namespace checker_internal { @@ -108,6 +115,17 @@ google::protobuf::Arena* absl_nonnull TestTypeArena() { return &(*kArena); } +absl::StatusOr> MakeTestParsedAstWithMacros( + absl::string_view expression, const cel::MacroRegistry& registry) { + CEL_ASSIGN_OR_RETURN( + auto source, + cel::NewSource(expression, /*description=*/std::string(expression))); + CEL_ASSIGN_OR_RETURN(auto parsed_expr, google::api::expr::parser::Parse( + *source, registry, + {.enable_optional_syntax = true})); + return cel::CreateAstFromParsedExpr(parsed_expr); +} + FunctionDecl MakeIdentFunction() { auto decl = MakeFunctionDecl( "identity", @@ -272,6 +290,12 @@ absl::Status RegisterMinimalBuiltins(google::protobuf::Arena* absl_nonnull arena /*return_type=*/TypeType(arena, TypeParamType("A")), TypeParamType("A")))); + Type kParam(TypeParamType("T")); + CEL_ASSIGN_OR_RETURN( + auto block_decl, + MakeFunctionDecl("cel.@block", MakeOverloadDecl("cel_block_list", kParam, + ListType(), kParam))); + env.InsertFunctionIfAbsent(std::move(not_op)); env.InsertFunctionIfAbsent(std::move(not_strictly_false)); env.InsertFunctionIfAbsent(std::move(add_op)); @@ -289,6 +313,7 @@ absl::Status RegisterMinimalBuiltins(google::protobuf::Arena* absl_nonnull arena env.InsertFunctionIfAbsent(std::move(to_type)); env.InsertFunctionIfAbsent(std::move(to_duration)); env.InsertFunctionIfAbsent(std::move(to_timestamp)); + env.InsertFunctionIfAbsent(std::move(block_decl)); return absl::OkStatus(); } @@ -308,6 +333,78 @@ TEST(TypeCheckerImplTest, SmokeTest) { EXPECT_THAT(result.GetIssues(), IsEmpty()); } +TEST(TypeCheckerImplTest, BlockMacroSupport) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + MacroRegistry registry; + ASSERT_THAT(cel::test::RegisterTestMacros(registry), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN( + auto ast, + MakeTestParsedAstWithMacros( + "cel.block([1, 2], cel.index(0) + cel.index(1))", registry)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + + EXPECT_TRUE(result.IsValid()); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + + // Overall type should be int. + ASSERT_OK_AND_ASSIGN(auto checked_ast, result.ReleaseAst()); + auto root_id = checked_ast->root_expr().id(); + EXPECT_EQ(checked_ast->type_map().at(root_id).primitive(), + PrimitiveType::kInt64); +} + +TEST(TypeCheckerImplTest, BlockMacroSupportMixedTypes) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + MacroRegistry registry; + ASSERT_THAT(cel::test::RegisterTestMacros(registry), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN( + auto ast, MakeTestParsedAstWithMacros("cel.block([1, 'a'], cel.index(1))", + registry)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + + EXPECT_TRUE(result.IsValid()); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + + // cel.index(1) refers to 'a' which is string. + // So overall type should be string. + ASSERT_OK_AND_ASSIGN(auto checked_ast, result.ReleaseAst()); + auto root_id = checked_ast->root_expr().id(); + EXPECT_EQ(checked_ast->type_map().at(root_id).primitive(), + PrimitiveType::kString); +} + +TEST(TypeCheckerImplTest, BadIndex) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + MacroRegistry registry; + ASSERT_THAT(cel::test::RegisterTestMacros(registry), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN( + auto ast, MakeTestParsedAstWithMacros("cel.block([1, 'a'], cel.index(2))", + registry)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatError(), + HasSubstr("undeclared reference to '@index2' (in container")); +} + TEST(TypeCheckerImplTest, SimpleIdentsResolved) { TypeCheckEnv env(GetSharedTestingDescriptorPool()); @@ -682,6 +779,169 @@ TEST(TypeCheckerImplTest, FreeMapTypeToDyn) { EXPECT_TRUE(checked_ast->type_map().at(1).map_type().value_type().has_dyn()); } +struct TypeDecayTestCase { + std::string expr; + std::string expected_type; + // If set, the expression is expected to fail type checking with an error + // containing this substring. + std::string error_substring = ""; +}; + +class TypeDecayTest : public testing::TestWithParam {}; + +TEST_P(TypeDecayTest, Check) { + const TypeDecayTestCase& test_case = GetParam(); + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst(test_case.expr)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + impl.Check(std::move(ast), &arena)); + + if (!test_case.error_substring.empty()) { + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatError(), HasSubstr(test_case.error_substring)); + return; + } + ASSERT_TRUE(result.IsValid()) << result.FormatError(); + auto it = result.GetResolvedTypeMap().find(result.GetAst()->root_expr().id()); + ASSERT_TRUE(it != result.GetResolvedTypeMap().end()); + EXPECT_EQ(FormatTypeName(it->second), test_case.expected_type); +} + +INSTANTIATE_TEST_SUITE_P( + TypeDecayTest, TypeDecayTest, + testing::Values( + TypeDecayTestCase{"[type(1), type(2)]", "list(type(int))"}, + TypeDecayTestCase{"[type(1), type(true)]", "list(type(dyn))"}, + TypeDecayTestCase{"[type(1), type(true), type(1u)]", "list(type(dyn))"}, + TypeDecayTestCase{"true ? type(1) : type(1u)", "type(dyn)"}, + TypeDecayTestCase{"[[type(1)], [type(1u)]]", "list(list(type(dyn)))"}, + TypeDecayTestCase{"[type([]), type(1)]", "list(type(dyn))"}, + TypeDecayTestCase{"[type(1), type([])]", "list(type(dyn))"}, + TypeDecayTestCase{"type(1) == type('a')", "bool"}, + TypeDecayTestCase{"[{1: type(1)}, {1: type(1u)}]", + "list(map(int, type(dyn)))"}, + TypeDecayTestCase{"true ? {1: type(1)} : {1: type(1u)}", + "map(int, type(dyn))"}, + // Mismatched non-type(...) parameters do not unify, even if sibling + // type(...) parameters decay. + TypeDecayTestCase{"[{1: type(1)}, {'a': type(1u)}]", "list(dyn)"}, + TypeDecayTestCase{"true ? {1: type(1)} : {'a': type(1u)}", "", + "no matching overload for '_?_:_'"}, + TypeDecayTestCase{"{1: type(1)} == {'a': type(1u)}", "", + "no matching overload for '_==_'"}, + TypeDecayTestCase{"[{1: type(1)}] == [{'a': type(1u)}]", "", + "no matching overload for '_==_'"}, + TypeDecayTestCase{"true ? [{1: type(1)}] : [{'a': type(1u)}]", "", + "no matching overload for '_?_:_'"})); + +// Builds expressions with unique ids for hand-constructed test ASTs. +class TestExprBuilder { + public: + Expr IntConst(int64_t value) { + Expr expr = NewExpr(); + expr.mutable_const_expr().set_int_value(value); + return expr; + } + + Expr BoolConst(bool value) { + Expr expr = NewExpr(); + expr.mutable_const_expr().set_bool_value(value); + return expr; + } + + Expr StringConst(absl::string_view value) { + Expr expr = NewExpr(); + expr.mutable_const_expr().set_string_value(value); + return expr; + } + + Expr Ident(absl::string_view name) { + Expr expr = NewExpr(); + expr.mutable_ident_expr().set_name(name); + return expr; + } + + template + Expr Call(absl::string_view function, Args&&... args) { + Expr expr = NewExpr(); + CallExpr& call = expr.mutable_call_expr(); + call.set_function(function); + ((call.add_args() = std::forward(args)), ...); + return expr; + } + + template + Expr List(Args&&... elements) { + Expr expr = NewExpr(); + ListExpr& list = expr.mutable_list_expr(); + (list.add_elements().set_expr(std::forward(elements)), ...); + return expr; + } + + // Builds a comprehension over the range `[1, 2]` (iteration variable `i`) + // with the given accu_init, loop_condition and loop_step, and result + // `__result__`. + std::unique_ptr ComprehensionAst(Expr accu_init, Expr loop_condition, + Expr loop_step) { + Expr expr = NewExpr(); + ComprehensionExpr& comprehension = expr.mutable_comprehension_expr(); + comprehension.set_iter_var("i"); + comprehension.set_iter_range(List(IntConst(1), IntConst(2))); + comprehension.set_accu_var("__result__"); + comprehension.set_accu_init(std::move(accu_init)); + comprehension.set_loop_condition(std::move(loop_condition)); + comprehension.set_loop_step(std::move(loop_step)); + comprehension.set_result(Ident("__result__")); + return std::make_unique(std::move(expr), SourceInfo()); + } + + private: + Expr NewExpr() { + Expr expr; + expr.set_id(next_id_++); + return expr; + } + + int64_t next_id_ = 1; +}; + +TEST(TypeCheckerImplTest, ComprehensionLoopConditionNotBool) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + TypeCheckerImpl impl(std::move(env)); + + TestExprBuilder b; + std::unique_ptr ast = + b.ComprehensionAst(/*accu_init=*/b.IntConst(0), + /*loop_condition=*/b.IntConst(1), + /*loop_step=*/b.IntConst(1)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.GetIssues(), + Contains(IsIssueWithSubstring( + Severity::kError, "expected type 'bool' but found 'int'"))); +} + +TEST(TypeCheckerImplTest, ComprehensionLoopConditionDynIsAccepted) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + TypeCheckerImpl impl(std::move(env)); + + TestExprBuilder b; + std::unique_ptr ast = b.ComprehensionAst( + /*accu_init=*/b.IntConst(0), + /*loop_condition=*/b.Call("dyn", b.BoolConst(true)), + /*loop_step=*/b.IntConst(1)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + EXPECT_TRUE(result.IsValid()) << result.FormatError(); +} + TEST(TypeCheckerImplTest, MapTypeWithMixedKeys) { TypeCheckEnv env(GetSharedTestingDescriptorPool()); @@ -1191,6 +1451,76 @@ TEST(TypeCheckerImplTest, ComprehensionDynRange) { EXPECT_THAT(result.GetIssues(), IsEmpty()); } +TEST(TypeCheckerImplTest, EmptyListRangeNestedListComprehension) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst("[].map(x, x.map(y, y))")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + + EXPECT_TRUE(result.IsValid()) << result.FormatError(); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + + ASSERT_OK_AND_ASSIGN(auto checked_ast, result.ReleaseAst()); + EXPECT_EQ(checked_ast->GetReturnType(), + AstType(ListTypeSpec(std::make_unique( + ListTypeSpec(std::make_unique(DynTypeSpec())))))); +} + +TEST(TypeCheckerImplTest, EmptyMapRangeNestedComprehension) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst("{}.map(k, k.map(y, y))")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + + EXPECT_TRUE(result.IsValid()) << result.FormatError(); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + + ASSERT_OK_AND_ASSIGN(auto checked_ast, result.ReleaseAst()); + EXPECT_EQ(checked_ast->GetReturnType(), + AstType(ListTypeSpec(std::make_unique( + ListTypeSpec(std::make_unique(DynTypeSpec())))))); +} + +TEST(TypeCheckerImplTest, EmptyRangeNestedComprehensionPredicate) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN(auto ast, + MakeTestParsedAst("[].all(x, x.all(y, y == 1))")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + + EXPECT_TRUE(result.IsValid()) << result.FormatError(); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + + ASSERT_OK_AND_ASSIGN(auto checked_ast, result.ReleaseAst()); + EXPECT_EQ(checked_ast->GetReturnType(), AstType(PrimitiveType::kBool)); +} + +TEST(TypeCheckerImplTest, EmptyRangeNestedComprehensionBeforeConstraint) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN( + auto ast, MakeTestParsedAst("[].all(x, x.all(y, y == 1) && x == 1)")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + + EXPECT_TRUE(result.IsValid()) << result.FormatError(); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + + ASSERT_OK_AND_ASSIGN(auto checked_ast, result.ReleaseAst()); + EXPECT_EQ(checked_ast->GetReturnType(), AstType(PrimitiveType::kBool)); +} + TEST(TypeCheckerImplTest, BasicOvlResolution) { TypeCheckEnv env(GetSharedTestingDescriptorPool()); google::protobuf::Arena arena; @@ -1376,6 +1706,93 @@ TEST(TypeCheckerImplTest, TypeInferredFromStructCreation) { std::make_unique(DynTypeSpec()))))))); } +struct VariadicLogicalCheckerTestCase { + std::string expr; +}; + +class VariadicLogicalCheckerTest + : public testing::TestWithParam {}; + +TEST_P(VariadicLogicalCheckerTest, Check) { + const auto& test_case = GetParam(); + + auto builder = cel::NewParserBuilder(); + builder->GetOptions().enable_variadic_logical_operators = true; + ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(test_case.expr)); + ASSERT_OK_AND_ASSIGN(auto parsed_ast, parser->Parse(*source)); + + google::protobuf::Arena arena; + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + TypeCheckerImpl impl(std::move(env)); + auto checker_builder = impl.ToBuilder(); + ASSERT_THAT(checker_builder->AddVariable(MakeVariableDecl("a", BoolType())), + IsOk()); + ASSERT_THAT(checker_builder->AddVariable(MakeVariableDecl("b", BoolType())), + IsOk()); + ASSERT_THAT(checker_builder->AddVariable(MakeVariableDecl("c", BoolType())), + IsOk()); + ASSERT_THAT(checker_builder->AddVariable(MakeVariableDecl("d", BoolType())), + IsOk()); + ASSERT_THAT(checker_builder->AddVariable(MakeVariableDecl("e", BoolType())), + IsOk()); + + ASSERT_OK_AND_ASSIGN(auto checker, checker_builder->Build()); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + checker->Check(std::move(parsed_ast))); + + ASSERT_TRUE(result.IsValid()) + << absl::StrJoin(result.GetIssues(), "\n", + [](std::string* out, const TypeCheckIssue& issue) { + absl::StrAppend(out, issue.message()); + }); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr checked_ast, result.ReleaseAst()); + EXPECT_THAT(checked_ast->type_map(), + Contains(Pair(checked_ast->root_expr().id(), + Eq(AstType(PrimitiveType::kBool))))); +} + +INSTANTIATE_TEST_SUITE_P( + VariadicLogicalChecker, VariadicLogicalCheckerTest, + testing::Values(VariadicLogicalCheckerTestCase{"true && false && true"}, + VariadicLogicalCheckerTestCase{"a && b && c && d"}, + VariadicLogicalCheckerTestCase{"a || b || c || d"}, + VariadicLogicalCheckerTestCase{"a && b && (c || d || e)"}, + VariadicLogicalCheckerTestCase{"a && b && c"}, + VariadicLogicalCheckerTestCase{"a || b || c"}, + VariadicLogicalCheckerTestCase{"[a, b, c].exists(x, x)"}, + VariadicLogicalCheckerTestCase{"[a, b, c].all(x, x)"})); + +TEST(TypeCheckerImplTest, VariadicLogicalOperatorsError) { + cel::expr::ParsedExpr parsed_expr; + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( + R"pb( + expr { + call_expr { + function: "_&&_" + args { const_expr { bool_value: true } } + } + } + )pb", + &parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto parsed_ast, + cel::CreateAstFromParsedExpr(parsed_expr)); + + google::protobuf::Arena arena; + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + impl.Check(std::move(parsed_ast))); + + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT( + result.GetIssues(), + Contains(IsIssueWithSubstring(Severity::kError, "undeclared reference"))); +} + TEST(TypeCheckerImplTest, ExpectedTypeMatches) { google::protobuf::Arena arena; TypeCheckEnv env(GetSharedTestingDescriptorPool()); @@ -2633,6 +3050,32 @@ INSTANTIATE_TEST_SUITE_P( .error_substring = "no matching overload for '_==_'", })); +TEST(TypeCheckerImplTest, RecheckClearsStaleMetadata) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), absl_testing::IsOk()); + + env.InsertVariableIfAbsent(MakeVariableDecl("x", IntType())); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, MakeTestParsedAst("x")); + + // Artificially inject stale metadata for IDs that do not exist. + constexpr int64_t kStaleId = 9999; + ast->mutable_reference_map()[kStaleId] = + Reference("stale_ref", {}, Constant()); + ast->mutable_type_map()[kStaleId] = TypeSpec(PrimitiveType::kInt64); + + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + EXPECT_TRUE(result.IsValid()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr checked_ast, result.ReleaseAst()); + + EXPECT_EQ(checked_ast->reference_map().find(kStaleId), + checked_ast->reference_map().end()); + EXPECT_EQ(checked_ast->type_map().find(kStaleId), + checked_ast->type_map().end()); +} + } // namespace } // namespace checker_internal } // namespace cel diff --git a/checker/internal/type_inference_context.cc b/checker/internal/type_inference_context.cc index 96d985071..9cf111e4f 100644 --- a/checker/internal/type_inference_context.cc +++ b/checker/internal/type_inference_context.cc @@ -15,10 +15,12 @@ #include "checker/internal/type_inference_context.h" #include +#include #include #include #include +#include "absl/base/no_destructor.h" #include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" @@ -28,14 +30,23 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "absl/types/span.h" -#include "checker/internal/format_type_name.h" +#include "checker/internal/builtins_arena.h" #include "common/decl.h" +#include "common/format_type_name.h" +#include "common/standard_definitions.h" #include "common/type.h" #include "common/type_kind.h" namespace cel::checker_internal { namespace { +// Returns `type(dyn)`. A parameterless `type` is equivalent to `type(dyn)`. +const Type& TypeOfDyn() { + static const absl::NoDestructor kInstance( + TypeType(BuiltinsArena(), DynType())); + return *kInstance; +} + bool IsWildCardType(Type type) { switch (type.kind()) { case TypeKind::kAny: @@ -133,7 +144,7 @@ FunctionOverloadInstance InstantiateFunctionOverload( // Converts a wrapper type to its corresponding primitive type. // Returns nullopt if the type is not a wrapper type. -absl::optional WrapperToPrimitive(const Type& t) { +std::optional WrapperToPrimitive(const Type& t) { switch (t.kind()) { case TypeKind::kBoolWrapper: return BoolType(); @@ -148,8 +159,21 @@ absl::optional WrapperToPrimitive(const Type& t) { case TypeKind::kUintWrapper: return UintType(); default: - return absl::nullopt; + return std::nullopt; + } +} + +// Tests whether the type contains any type params directly or transitively. +bool HasTypeParam(const Type& type) { + if (type.kind() == TypeKind::kTypeParam) { + return true; + } + for (const auto& param : type.GetParameters()) { + if (HasTypeParam(param)) { + return true; + } } + return false; } } // namespace @@ -183,9 +207,11 @@ Type TypeInferenceContext::InstantiateTypeParams( case TypeKind::kUint: case TypeKind::kIntWrapper: case TypeKind::kUintWrapper: + case TypeKind::kEnum: return type; case TypeKind::kTypeParam: { - absl::string_view name = type.AsTypeParam()->name(); + TypeParamType type_param = type.GetTypeParam(); + absl::string_view name = type_param.name(); if (IsTypeVar(name)) { // Already instantiated (e.g. list comprehension variable). return type; @@ -271,6 +297,19 @@ bool TypeInferenceContext::IsAssignableInternal( // that has a prospective type assignment. to.kind() == TypeKind::kTypeParam && prospective_substitutions.contains(to.GetTypeParam().name())) { + // Types have special unification: type(A) and type(B) unify to type(dyn) + // if A and B are not co-assignable (at any depth, e.g. list(type(int)) and + // list(type(uint)) unify to list(type(dyn))). This is the only widening + // other than for union types. + SubstitutionMap join_subs = prospective_substitutions; + if (std::optional joined = + JoinTypeTypes(from_subs, to_subs, join_subs); + joined.has_value() && + !OccursWithin(to.GetTypeParam().name(), *joined, join_subs)) { + join_subs[to.GetTypeParam().name()] = *std::move(joined); + prospective_substitutions = std::move(join_subs); + return true; + } SubstitutionMap prospective_subs_cpy = prospective_substitutions; if (CompareGenerality(from_subs, to_subs, prospective_subs_cpy) == RelativeGenerality::kMoreGeneral) { @@ -285,8 +324,17 @@ bool TypeInferenceContext::IsAssignableInternal( } } + // Concrete type values are interchangeable for declared (non type param) + // `type(...)` parameters, e.g. type(1) is accepted where type(string) is + // declared. + if (from_subs.IsType() && to_subs.IsType() && + to.kind() != TypeKind::kTypeParam && !HasTypeParam(from_subs) && + !HasTypeParam(to_subs)) { + return true; + } + // Type is as concrete as it can be under current substitutions. - if (absl::optional wrapped_type = WrapperToPrimitive(to_subs); + if (std::optional wrapped_type = WrapperToPrimitive(to_subs); wrapped_type.has_value()) { return from_subs.IsNull() || IsAssignableInternal(*wrapped_type, from_subs, @@ -312,13 +360,6 @@ bool TypeInferenceContext::IsAssignableInternal( } } - if (from_subs.kind() == TypeKind::kType && - to_subs.kind() == TypeKind::kType) { - // Types are always assignable to themselves (even if differently - // parameterized). - return true; - } - if (to_subs.kind() == TypeKind::kEnum && from_subs.kind() == TypeKind::kInt) { return true; } @@ -371,9 +412,75 @@ Type TypeInferenceContext::Substitute( } break; } + // A parameterless `type` is equivalent to `type(dyn)`. Normalize so + // structural comparisons treat them alike. + if (subs.IsType() && subs.GetParameters().empty()) { + return TypeOfDyn(); + } return subs; } +std::optional TypeInferenceContext::JoinTypeTypes( + const Type& from, const Type& to, + SubstitutionMap& prospective_substitutions) { + Type from_subs = Substitute(from, prospective_substitutions); + Type to_subs = Substitute(to, prospective_substitutions); + if (from_subs.IsType() && to_subs.IsType()) { + SubstitutionMap scratch = prospective_substitutions; + if (IsAssignableInternal(from_subs.GetType().GetType(), + to_subs.GetType().GetType(), scratch)) { + // Co-assignable: ordinary unification applies. + return std::nullopt; + } + return TypeOfDyn(); + } + if (from_subs.kind() != to_subs.kind() || + from_subs.name() != to_subs.name()) { + return std::nullopt; + } + TypeParameters from_params = from_subs.GetParameters(); + TypeParameters to_params = to_subs.GetParameters(); + if (to_params.empty() || from_params.size() != to_params.size()) { + return std::nullopt; + } + SubstitutionMap scratch = prospective_substitutions; + bool changed = false; + std::vector joined; + joined.reserve(to_params.size()); + for (size_t i = 0; i < to_params.size(); ++i) { + std::optional param = + JoinTypeTypes(from_params[i], to_params[i], scratch); + if (param.has_value()) { + changed = true; + joined.push_back(*std::move(param)); + } else if (IsAssignableInternal(from_params[i], to_params[i], scratch)) { + // Positions that don't need widening must still be assignable. + joined.push_back(to_params[i]); + } else { + return std::nullopt; + } + } + if (!changed) { + return std::nullopt; + } + std::optional result; + switch (to_subs.kind()) { + case TypeKind::kList: + result = ListType(arena_, joined[0]); + break; + case TypeKind::kMap: + result = MapType(arena_, joined[0], joined[1]); + break; + case TypeKind::kOpaque: + result = OpaqueType(arena_, to_subs.GetOpaque().name(), joined); + break; + default: + return std::nullopt; + } + prospective_substitutions = std::move(scratch); + return result; +} + TypeInferenceContext::RelativeGenerality TypeInferenceContext::CompareGenerality( const Type& from, const Type& to, @@ -467,7 +574,8 @@ bool TypeInferenceContext::OccursWithin( // T3 -> T2 -> null_type; Type substitution = type; while (substitution.kind() == TypeKind::kTypeParam) { - absl::string_view param_name = substitution.AsTypeParam()->name(); + TypeParamType type_param = substitution.GetTypeParam(); + absl::string_view param_name = type_param.name(); if (param_name == var_name) { return true; } @@ -507,7 +615,8 @@ bool TypeInferenceContext::IsAssignableWithConstraints( } if (to.kind() == TypeKind::kTypeParam) { - absl::string_view name = to.AsTypeParam()->name(); + TypeParamType to_param = to.GetTypeParam(); + absl::string_view name = to_param.name(); if (!OccursWithin(name, from, prospective_substitutions)) { prospective_substitutions[name] = from; return true; @@ -515,7 +624,8 @@ bool TypeInferenceContext::IsAssignableWithConstraints( } if (from.kind() == TypeKind::kTypeParam) { - absl::string_view name = from.AsTypeParam()->name(); + TypeParamType from_param = from.GetTypeParam(); + absl::string_view name = from_param.name(); if (!OccursWithin(name, to, prospective_substitutions)) { prospective_substitutions[name] = to; return true; @@ -531,27 +641,34 @@ bool TypeInferenceContext::IsAssignableWithConstraints( return false; } -absl::optional +std::optional TypeInferenceContext::ResolveOverload(const FunctionDecl& decl, absl::Span argument_types, bool is_receiver) { - absl::optional result_type; + std::optional result_type; + + bool is_logical_op = (decl.name() == cel::StandardFunctions::kAnd || + decl.name() == cel::StandardFunctions::kOr) && + argument_types.size() >= 2; std::vector matching_overloads; for (const auto& ovl : decl.overloads()) { if (ovl.member() != is_receiver || - argument_types.size() != ovl.args().size()) { + (!is_logical_op && argument_types.size() != ovl.args().size())) { continue; } auto call_type_instance = InstantiateFunctionOverload(*this, ovl); - ABSL_DCHECK_EQ(argument_types.size(), - call_type_instance.param_types.size()); + if (!is_logical_op) { + ABSL_DCHECK_EQ(argument_types.size(), + call_type_instance.param_types.size()); + } bool is_match = true; AssignabilityContext assignability_context = CreateAssignabilityContext(); for (int i = 0; i < argument_types.size(); ++i) { + int param_index = is_logical_op ? 0 : i; if (!assignability_context.IsAssignable( - argument_types[i], call_type_instance.param_types[i])) { + argument_types[i], call_type_instance.param_types[param_index])) { is_match = false; break; } @@ -571,7 +688,7 @@ TypeInferenceContext::ResolveOverload(const FunctionDecl& decl, } if (!result_type.has_value() || matching_overloads.empty()) { - return absl::nullopt; + return std::nullopt; } return OverloadResolution{ .result_type = FullySubstitute(*result_type, /*free_to_dyn=*/false), @@ -629,11 +746,12 @@ Type TypeInferenceContext::FullySubstitute(const Type& type, return MapType(arena_, key, value); } case TypeKind::kOpaque: { + OpaqueType opaque_type = type.GetOpaque(); std::vector types; - for (const auto& param : type.AsOpaque()->GetParameters()) { + for (const auto& param : opaque_type.GetParameters()) { types.push_back(FullySubstitute(param, free_to_dyn)); } - return OpaqueType(arena_, type.AsOpaque()->name(), types); + return OpaqueType(arena_, opaque_type.name(), types); } default: return type; @@ -649,14 +767,14 @@ bool TypeInferenceContext::AssignabilityContext::IsAssignable(const Type& from, std::string TypeInferenceContext::DebugString() const { return absl::StrCat( "type_parameter_bindings: ", - absl::StrJoin( - type_parameter_bindings_, "\n ", - [](std::string* out, const auto& binding) { - absl::StrAppend( - out, binding.first, " (", binding.second.name, ") -> ", - checker_internal::FormatTypeName( - binding.second.type.value_or(Type(TypeParamType("none"))))); - })); + absl::StrJoin(type_parameter_bindings_, "\n ", + [](std::string* out, const auto& binding) { + absl::StrAppend( + out, binding.first, " (", binding.second.name, + ") -> ", + cel::FormatTypeName(binding.second.type.value_or( + Type(TypeParamType("none"))))); + })); } void TypeInferenceContext::AssignabilityContext:: diff --git a/checker/internal/type_inference_context.h b/checker/internal/type_inference_context.h index 1a1043047..c042a7990 100644 --- a/checker/internal/type_inference_context.h +++ b/checker/internal/type_inference_context.h @@ -16,6 +16,7 @@ #define THIRD_PARTY_CEL_CPP_CHECKER_INTERNAL_TYPE_INFERENCE_CONTEXT_H_ #include +#include #include #include @@ -199,6 +200,18 @@ class TypeInferenceContext { Type Substitute(const Type& type, const SubstitutionMap& substitutions) const; + // Types have special unification: type(A) and type(B) unify to type(dyn) if + // A and B are not co-assignable. This applies at any depth of matching + // parameterized types (e.g. list(type(int)) and list(type(uint)) unify to + // list(type(dyn))). + // + // Returns the joined type, or nullopt if no type(...) position needs + // widening or the types do not otherwise unify (every position that is not + // widened must be assignable). On success, any type parameter bindings + // needed for the join are added to `prospective_substitutions`. + std::optional JoinTypeTypes(const Type& from, const Type& to, + SubstitutionMap& prospective_substitutions); + bool OccursWithin(absl::string_view var_name, const Type& type, const SubstitutionMap& substitutions) const; diff --git a/checker/internal/type_inference_context_test.cc b/checker/internal/type_inference_context_test.cc index d1bf7fa6d..de241e119 100644 --- a/checker/internal/type_inference_context_test.cc +++ b/checker/internal/type_inference_context_test.cc @@ -18,8 +18,10 @@ #include #include "absl/log/absl_check.h" +#include "absl/status/statusor.h" #include "absl/types/optional.h" #include "common/decl.h" +#include "common/format_type_name.h" #include "common/type.h" #include "common/type_kind.h" #include "internal/testing.h" @@ -291,7 +293,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadBasic) { MakeOverloadDecl("add_double", DoubleType(), DoubleType(), DoubleType()))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {IntType(), IntType()}, false); ASSERT_TRUE(resolution.has_value()); EXPECT_THAT(resolution->result_type, IsTypeKind(TypeKind::kInt)); @@ -309,7 +311,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadFails) { MakeOverloadDecl("add_double", DoubleType(), DoubleType(), DoubleType()))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {IntType(), DoubleType()}, false); ASSERT_FALSE(resolution.has_value()); } @@ -324,7 +326,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithParamsNoMatch) { "_==_", MakeOverloadDecl("equals", BoolType(), TypeParamType("A"), TypeParamType("A")))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {IntType(), DoubleType()}, false); ASSERT_FALSE(resolution.has_value()); } @@ -341,7 +343,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithMixedParamsMatch) { "_==_", MakeOverloadDecl("equals", BoolType(), TypeParamType("A"), TypeParamType("A")))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {list_of_a, list_of_a}, false); ASSERT_TRUE(resolution.has_value()) << context.DebugString(); } @@ -359,7 +361,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithMixedParamsMatch2) { "_==_", MakeOverloadDecl("equals", BoolType(), TypeParamType("A"), TypeParamType("A")))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {list_of_a, list_of_int}, false); ASSERT_TRUE(resolution.has_value()) << context.DebugString(); EXPECT_THAT(resolution->overloads, ElementsAre(IsOverloadDecl("equals"))); @@ -375,7 +377,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithParamsMatches) { "_==_", MakeOverloadDecl("equals", BoolType(), TypeParamType("A"), TypeParamType("A")))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {IntType(), IntType()}, false); ASSERT_TRUE(resolution.has_value()); EXPECT_TRUE(resolution->result_type.IsBool()); @@ -394,7 +396,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithNestedParamsMatch) { Type list_of_a_instance = context.InstantiateTypeParams(list_of_a); - absl::optional resolution = + std::optional resolution = context.ResolveOverload( decl, {list_of_a_instance, ListType(&arena, IntType())}, false); ASSERT_TRUE(resolution.has_value()); @@ -407,7 +409,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithNestedParamsMatch) { EXPECT_THAT(resolution->overloads, ElementsAre(IsOverloadDecl("add_list"))); - absl::optional resolution2 = + std::optional resolution2 = context.ResolveOverload( decl, {ListType(&arena, IntType()), list_of_a_instance}, false); ASSERT_TRUE(resolution2.has_value()); @@ -433,7 +435,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithNestedParamsNoMatch) { Type list_of_a_instance = context.InstantiateTypeParams(list_of_a); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {list_of_a_instance, IntType()}, false); EXPECT_FALSE(resolution.has_value()); } @@ -450,13 +452,13 @@ TEST(TypeInferenceContextTest, InferencesAccumulate) { Type list_of_a_instance = context.InstantiateTypeParams(list_of_a); - absl::optional resolution1 = + std::optional resolution1 = context.ResolveOverload(decl, {list_of_a_instance, list_of_a_instance}, false); ASSERT_TRUE(resolution1.has_value()); EXPECT_TRUE(resolution1->result_type.IsList()); - absl::optional resolution2 = + std::optional resolution2 = context.ResolveOverload( decl, {resolution1->result_type, ListType(&arena, IntType())}, false); ASSERT_TRUE(resolution2.has_value()); @@ -480,7 +482,7 @@ TEST(TypeInferenceContextTest, DebugString) { MakeFunctionDecl("_+_", MakeOverloadDecl("add_list", list_of_a, list_of_a, list_of_a))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {list_of_int, list_of_int}, false); ASSERT_TRUE(resolution.has_value()); EXPECT_TRUE(resolution->result_type.IsList()); @@ -517,7 +519,7 @@ class TypeInferenceContextWrapperTypesTest TEST_P(TypeInferenceContextWrapperTypesTest, ResolvePrimitiveArg) { const TypeInferenceContextWrapperTypesTestCase& test_case = GetParam(); - absl::optional resolution = + std::optional resolution = context_.ResolveOverload(ternary_decl_, {BoolType(), test_case.wrapper_type, test_case.wrapped_primitive_type}, @@ -534,7 +536,7 @@ TEST_P(TypeInferenceContextWrapperTypesTest, ResolvePrimitiveArg) { TEST_P(TypeInferenceContextWrapperTypesTest, ResolveWrapperArg) { const TypeInferenceContextWrapperTypesTestCase& test_case = GetParam(); - absl::optional resolution = + std::optional resolution = context_.ResolveOverload( ternary_decl_, {BoolType(), test_case.wrapper_type, test_case.wrapper_type}, false); @@ -550,7 +552,7 @@ TEST_P(TypeInferenceContextWrapperTypesTest, ResolveWrapperArg) { TEST_P(TypeInferenceContextWrapperTypesTest, ResolveNullArg) { const TypeInferenceContextWrapperTypesTestCase& test_case = GetParam(); - absl::optional resolution = + std::optional resolution = context_.ResolveOverload(ternary_decl_, {BoolType(), test_case.wrapper_type, NullType()}, false); @@ -566,7 +568,7 @@ TEST_P(TypeInferenceContextWrapperTypesTest, ResolveNullArg) { TEST_P(TypeInferenceContextWrapperTypesTest, NullWidens) { const TypeInferenceContextWrapperTypesTestCase& test_case = GetParam(); - absl::optional resolution = + std::optional resolution = context_.ResolveOverload(ternary_decl_, {BoolType(), NullType(), test_case.wrapper_type}, false); @@ -582,7 +584,7 @@ TEST_P(TypeInferenceContextWrapperTypesTest, NullWidens) { TEST_P(TypeInferenceContextWrapperTypesTest, PrimitiveWidens) { const TypeInferenceContextWrapperTypesTestCase& test_case = GetParam(); - absl::optional resolution = + std::optional resolution = context_.ResolveOverload(ternary_decl_, {BoolType(), test_case.wrapped_primitive_type, test_case.wrapper_type}, @@ -622,7 +624,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithUnionTypePromotion) { /*result_type=*/TypeParamType("A"), BoolType(), TypeParamType("A"), TypeParamType("A")))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {BoolType(), NullType(), IntWrapperType()}, false); ASSERT_TRUE(resolution.has_value()); @@ -648,7 +650,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithTypeType) { TypeType(&arena, TypeParamType("A")), TypeParamType("A")))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {StringType()}, false); ASSERT_TRUE(resolution.has_value()); @@ -680,7 +682,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithInferredTypeType) { BoolType(), TypeParamType("A"), TypeParamType("A")))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(to_type_decl, {StringType()}, false); ASSERT_TRUE(resolution.has_value()); @@ -846,5 +848,428 @@ TEST(TypeInferenceContextTest, AssignabilityContextReset) { IsTypeKind(TypeKind::kDouble)); } +TEST(TypeInferenceContextTest, + TypeTypeAssignability_ConcreteTypes_Coassignable) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type int_type = TypeType(&arena, IntType()); + Type string_type = TypeType(&arena, StringType()); + + EXPECT_TRUE(context.IsAssignable(int_type, string_type)); + EXPECT_TRUE(context.IsAssignable(string_type, int_type)); +} + +TEST(TypeInferenceContextTest, TypeTypeAssignability_MapContainerErasure) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type map_int_uint = TypeType(&arena, MapType(&arena, IntType(), UintType())); + Type map_dyn_dyn = TypeType(&arena, MapType(&arena, DynType(), DynType())); + + EXPECT_TRUE(context.IsAssignable(map_int_uint, map_dyn_dyn)); +} + +TEST(TypeInferenceContextTest, TypeTypeAssignability_ListContainerErasure) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type list_int = TypeType(&arena, ListType(&arena, IntType())); + Type list_dyn = TypeType(&arena, ListType(&arena, DynType())); + + EXPECT_TRUE(context.IsAssignable(list_int, list_dyn)); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_TypeParamTarget_BindsConcreteType) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type type_param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + Type from_type = TypeType(&arena, IntType()); + + EXPECT_TRUE(context.IsAssignable(from_type, type_param_t)); + + Type resolved_type = context.FinalizeType(type_param_t); + ASSERT_THAT(resolved_type, IsTypeKind(TypeKind::kType)); + EXPECT_THAT(resolved_type.AsType()->GetParameters(), + ElementsAre(IsTypeKind(TypeKind::kInt))); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_TypeParamSource_BindsConcreteType) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type type_param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + Type to_type = TypeType(&arena, IntType()); + + EXPECT_TRUE(context.IsAssignable(type_param_t, to_type)); + + Type resolved_type = context.FinalizeType(type_param_t); + ASSERT_THAT(resolved_type, IsTypeKind(TypeKind::kType)); + EXPECT_THAT(resolved_type.AsType()->GetParameters(), + ElementsAre(IsTypeKind(TypeKind::kInt))); +} + +TEST(TypeInferenceContextTest, TypeTypeAssignability_NestedTypeParam_Unifies) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type type_param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + Type type_param_r = context.InstantiateTypeParams( + TypeType(&arena, TypeType(&arena, TypeParamType("R")))); + + EXPECT_TRUE(context.IsAssignable(type_param_t, type_param_r)); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_DeeplyNestedTypeParam_BindsConcreteType) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type from_type = TypeType(&arena, TypeType(&arena, IntType())); + Type to_type = context.InstantiateTypeParams( + TypeType(&arena, TypeType(&arena, TypeParamType("T")))); + + EXPECT_TRUE(context.IsAssignable(from_type, to_type)); + + Type resolved_type = context.FinalizeType(to_type); + ASSERT_THAT(resolved_type, IsTypeKind(TypeKind::kType)); + Type inner_type = resolved_type.AsType()->GetType(); + ASSERT_THAT(inner_type, IsTypeKind(TypeKind::kType)); + EXPECT_THAT(inner_type.AsType()->GetParameters(), + ElementsAre(IsTypeKind(TypeKind::kInt))); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_CompositeListTypeParam_BindsConcreteType) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type from_type = TypeType(&arena, ListType(&arena, IntType())); + Type to_type = context.InstantiateTypeParams( + TypeType(&arena, ListType(&arena, TypeParamType("T")))); + + EXPECT_TRUE(context.IsAssignable(from_type, to_type)); + + Type resolved_type = context.FinalizeType(to_type); + ASSERT_THAT(resolved_type, IsTypeKind(TypeKind::kType)); + Type inner_type = resolved_type.AsType()->GetType(); + ASSERT_THAT(inner_type, IsTypeKind(TypeKind::kList)); + EXPECT_THAT(inner_type.AsList()->GetElement(), IsTypeKind(TypeKind::kInt)); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_CompositeMapTypeParam_BindsConcreteTypes) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type from_type = TypeType(&arena, MapType(&arena, StringType(), IntType())); + Type to_type = context.InstantiateTypeParams(TypeType( + &arena, MapType(&arena, TypeParamType("K"), TypeParamType("V")))); + + EXPECT_TRUE(context.IsAssignable(from_type, to_type)); + + Type resolved_type = context.FinalizeType(to_type); + ASSERT_THAT(resolved_type, IsTypeKind(TypeKind::kType)); + Type inner_type = resolved_type.AsType()->GetType(); + ASSERT_THAT(inner_type, IsTypeKind(TypeKind::kMap)); + EXPECT_THAT(inner_type.AsMap()->GetKey(), IsTypeKind(TypeKind::kString)); + EXPECT_THAT(inner_type.AsMap()->GetValue(), IsTypeKind(TypeKind::kInt)); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_OptionalTypeParam_Unifies) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type from_type = TypeType(&arena, OptionalType(&arena, IntType())); + Type to_type = context.InstantiateTypeParams( + TypeType(&arena, OptionalType(&arena, TypeParamType("T")))); + + EXPECT_TRUE(context.IsAssignable(from_type, to_type)); + + Type resolved_type = context.FinalizeType(to_type); + ASSERT_THAT(resolved_type, IsTypeKind(TypeKind::kType)); + Type inner_type = resolved_type.AsType()->GetType(); + ASSERT_THAT(inner_type, IsTypeKind(TypeKind::kOpaque)); + EXPECT_THAT(inner_type.AsOpaque()->GetParameters(), + ElementsAre(IsTypeKind(TypeKind::kInt))); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_IncompatibleTypeParams_ReturnsFalse) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type from_type = context.InstantiateTypeParams( + TypeType(&arena, ListType(&arena, TypeParamType("T")))); + Type to_type = TypeType(&arena, IntType()); + + EXPECT_FALSE(context.IsAssignable(from_type, to_type)); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_ConflictingBoundTypeParam_ReturnsFalse) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + EXPECT_TRUE(context.IsAssignable(TypeType(&arena, StringType()), param_t)); + EXPECT_FALSE(context.IsAssignable(param_t, TypeType(&arena, IntType()))); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_OccursCheck_FailsOnSelfReference) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + Type to_type = TypeType(&arena, param_t); + + EXPECT_FALSE(context.IsAssignable(param_t, to_type)); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_OccursCheck_FailsOnTransitiveCycle) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + Type param_r = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("R"))); + + EXPECT_TRUE(context.IsAssignable(param_t, TypeType(&arena, param_r))); + EXPECT_FALSE(context.IsAssignable(param_r, TypeType(&arena, param_t))); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_OccursCheck_MapTypeParamToTypeParam_ReturnsFalse) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + Type t_var = param_t.AsType()->GetType(); + Type map_type = TypeType(&arena, MapType(&arena, StringType(), t_var)); + + EXPECT_FALSE(context.IsAssignable(map_type, param_t)); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_OccursCheck_TypeParamToMapTypeParam_ReturnsFalse) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + Type t_var = param_t.AsType()->GetType(); + Type map_type = TypeType(&arena, MapType(&arena, StringType(), t_var)); + + EXPECT_FALSE(context.IsAssignable(param_t, map_type)); +} + +TEST(TypeInferenceContextTest, + TypeTypeOverloadResolution_TypeParamInTypeType_ResolvesReturnTypeInt) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + ASSERT_OK_AND_ASSIGN( + FunctionDecl decl, + MakeFunctionDecl("cast", + MakeOverloadDecl("cast_t", TypeParamType("T"), DynType(), + TypeType(&arena, TypeParamType("T"))))); + + std::optional resolution = + context.ResolveOverload(decl, {StringType(), TypeType(&arena, IntType())}, + false); + ASSERT_TRUE(resolution.has_value()); + EXPECT_THAT(context.FinalizeType(resolution->result_type), + IsTypeKind(TypeKind::kInt)); +} + +TEST(TypeInferenceContextTest, + TypeTypeOverloadResolution_TypeParamInTypeType_ResolvesReturnTypeString) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + ASSERT_OK_AND_ASSIGN( + FunctionDecl decl, + MakeFunctionDecl("cast", + MakeOverloadDecl("cast_t", TypeParamType("T"), DynType(), + TypeType(&arena, TypeParamType("T"))))); + + std::optional resolution = + context.ResolveOverload(decl, {IntType(), TypeType(&arena, StringType())}, + false); + ASSERT_TRUE(resolution.has_value()); + EXPECT_THAT(context.FinalizeType(resolution->result_type), + IsTypeKind(TypeKind::kString)); +} + +TEST(TypeInferenceContextTest, + TypeTypeOverloadResolution_TypeParamInComposite_ResolvesReturnType) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + ASSERT_OK_AND_ASSIGN( + FunctionDecl decl, + MakeFunctionDecl( + "first_elem_type", + MakeOverloadDecl( + "first_elem_type_overload", TypeParamType("T"), DynType(), + TypeType(&arena, ListType(&arena, TypeParamType("T")))))); + + std::optional resolution = + context.ResolveOverload( + decl, {StringType(), TypeType(&arena, ListType(&arena, IntType()))}, + false); + ASSERT_TRUE(resolution.has_value()); + EXPECT_THAT(context.FinalizeType(resolution->result_type), + IsTypeKind(TypeKind::kInt)); +} + +absl::StatusOr MakeSameTypeDecl() { + return MakeFunctionDecl( + "same", MakeOverloadDecl("same", TypeParamType("A"), TypeParamType("A"), + TypeParamType("A"))); +} + +TEST(TypeInferenceContextTest, TypeTypeJoinToTypeDyn) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + ASSERT_OK_AND_ASSIGN(FunctionDecl decl, MakeSameTypeDecl()); + + std::optional resolution = + context.ResolveOverload( + decl, {TypeType(&arena, IntType()), TypeType(&arena, UintType())}, + false); + ASSERT_TRUE(resolution.has_value()); + EXPECT_EQ(FormatTypeName(context.FinalizeType(resolution->result_type)), + "type(dyn)"); +} + +TEST(TypeInferenceContextTest, TypeTypeJoinNested) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + ASSERT_OK_AND_ASSIGN(FunctionDecl decl, MakeSameTypeDecl()); + + std::optional resolution = + context.ResolveOverload(decl, + {ListType(&arena, TypeType(&arena, IntType())), + ListType(&arena, TypeType(&arena, UintType()))}, + false); + ASSERT_TRUE(resolution.has_value()); + EXPECT_EQ(FormatTypeName(context.FinalizeType(resolution->result_type)), + "list(type(dyn))"); +} + +TEST(TypeInferenceContextTest, TypeTypeJoinSameType) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + ASSERT_OK_AND_ASSIGN(FunctionDecl decl, MakeSameTypeDecl()); + + std::optional resolution = + context.ResolveOverload( + decl, {TypeType(&arena, IntType()), TypeType(&arena, IntType())}, + false); + ASSERT_TRUE(resolution.has_value()); + EXPECT_EQ(FormatTypeName(context.FinalizeType(resolution->result_type)), + "type(int)"); +} + +TEST(TypeInferenceContextTest, TypeTypeJoinCoAssignableBindsTypeParam) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + ASSERT_OK_AND_ASSIGN(FunctionDecl decl, MakeSameTypeDecl()); + + Type type_of_list_t = context.InstantiateTypeParams( + TypeType(&arena, ListType(&arena, TypeParamType("T")))); + std::optional resolution = + context.ResolveOverload( + decl, {type_of_list_t, TypeType(&arena, ListType(&arena, IntType()))}, + false); + ASSERT_TRUE(resolution.has_value()); + EXPECT_EQ(FormatTypeName(context.FinalizeType(resolution->result_type)), + "type(list(int))"); + EXPECT_EQ(FormatTypeName(context.FinalizeType(type_of_list_t)), + "type(list(int))"); +} + +TEST(TypeInferenceContextTest, TypeTypeJoinMismatchedSiblingFails) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + ASSERT_OK_AND_ASSIGN(FunctionDecl decl, MakeSameTypeDecl()); + + Type int_map = MapType(&arena, IntType(), TypeType(&arena, IntType())); + Type string_map = MapType(&arena, StringType(), TypeType(&arena, UintType())); + // The type(...) values would decay to type(dyn), but the keys don't unify. + EXPECT_FALSE( + context.ResolveOverload(decl, {int_map, string_map}, false).has_value()); + EXPECT_FALSE( + context.ResolveOverload(decl, {string_map, int_map}, false).has_value()); +} + +TEST(TypeInferenceContextTest, TypeTypeJoinNestedMismatchedSiblingFails) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + ASSERT_OK_AND_ASSIGN(FunctionDecl decl, MakeSameTypeDecl()); + + Type int_map = + ListType(&arena, MapType(&arena, IntType(), TypeType(&arena, IntType()))); + Type string_map = ListType( + &arena, MapType(&arena, StringType(), TypeType(&arena, UintType()))); + EXPECT_FALSE( + context.ResolveOverload(decl, {int_map, string_map}, false).has_value()); +} + +TEST(TypeInferenceContextTest, TypeTypeJoinSiblingBindsTypeParam) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + ASSERT_OK_AND_ASSIGN(FunctionDecl decl, MakeSameTypeDecl()); + + Type map_of_k = context.InstantiateTypeParams( + MapType(&arena, TypeParamType("K"), TypeType(&arena, IntType()))); + std::optional resolution = + context.ResolveOverload(decl, + {map_of_k, MapType(&arena, StringType(), + TypeType(&arena, UintType()))}, + false); + ASSERT_TRUE(resolution.has_value()); + EXPECT_EQ(FormatTypeName(context.FinalizeType(resolution->result_type)), + "map(string, type(dyn))"); + EXPECT_EQ(FormatTypeName(context.FinalizeType(map_of_k)), + "map(string, type(int))"); +} + +TEST(TypeInferenceContextTest, TypeTypeJoinParameterlessType) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + ASSERT_OK_AND_ASSIGN(FunctionDecl decl, MakeSameTypeDecl()); + + // A parameterless `type` is equivalent to type(dyn). + std::optional resolution = + context.ResolveOverload(decl, {TypeType(), TypeType(&arena, IntType())}, + false); + ASSERT_TRUE(resolution.has_value()); + EXPECT_TRUE(context.FinalizeType(resolution->result_type).IsType()); +} + +TEST(TypeInferenceContextTest, ConcreteTypeTypeParamsAreInterchangeable) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + // Declared (non type param) type(...) parameters accept any concrete type. + EXPECT_TRUE(context.IsAssignable(TypeType(&arena, IntType()), + TypeType(&arena, StringType()))); +} + } // namespace } // namespace cel::checker_internal diff --git a/checker/optional_test.cc b/checker/optional_test.cc index 87c14f0cd..49ec84887 100644 --- a/checker/optional_test.cc +++ b/checker/optional_test.cc @@ -335,5 +335,41 @@ INSTANTIATE_TEST_SUITE_P( "== null", _, "no matching overload for '_==_'"})); +class OptionalListTypePermutationsTest + : public testing::TestWithParam {}; + +TEST_P(OptionalListTypePermutationsTest, ResolvesToListTypeDyn) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr builder, + CreateTypeCheckerBuilder(GetSharedTestingDescriptorPool())); + ASSERT_THAT(builder->AddLibrary(StandardCheckerLibrary()), IsOk()); + ASSERT_THAT(builder->AddLibrary(OptionalCheckerLibrary()), IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr checker, + std::move(*builder).Build()); + + const std::string& expr = GetParam(); + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst(expr)); + ASSERT_OK_AND_ASSIGN(auto result, checker->Check(std::move(ast))); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + ASSERT_OK_AND_ASSIGN(auto checked_ast, result.ReleaseAst()); + // type(...) values with non co-assignable parameters unify to type(dyn). + EXPECT_EQ(checked_ast->GetTypeOrDyn(checked_ast->root_expr().id()), + TypeSpec(ListTypeSpec(std::make_unique( + TypeSpec(std::make_unique(DynTypeSpec())))))); + ASSERT_EQ(checked_ast->root_expr().list_expr().elements().size(), 3); + for (const auto& elem : checked_ast->root_expr().list_expr().elements()) { + EXPECT_TRUE(checked_ast->GetTypeOrDyn(elem.expr().id()).has_type()); + } +} + +INSTANTIATE_TEST_SUITE_P( + OptionalTests, OptionalListTypePermutationsTest, + ::testing::Values("[type([]), int, type(optional.none())]", + "[type([]), type(optional.none()), int]", + "[int, type([]), type(optional.none())]", + "[int, type(optional.none()), type([])]", + "[type(optional.none()), type([]), int]", + "[type(optional.none()), int, type([])]")); + } // namespace } // namespace cel diff --git a/checker/standard_library.cc b/checker/standard_library.cc index 744a171ef..0162db003 100644 --- a/checker/standard_library.cc +++ b/checker/standard_library.cc @@ -112,13 +112,13 @@ Type TypeDynType() { Type TypeListType() { static absl::NoDestructor kInstance( - TypeType(BuiltinsArena(), ListOfA())); + TypeType(BuiltinsArena(), ListType(BuiltinsArena(), DynType()))); return *kInstance; } Type TypeMapType() { - static absl::NoDestructor kInstance( - TypeType(BuiltinsArena(), MapOfAB())); + static absl::NoDestructor kInstance(TypeType( + BuiltinsArena(), MapType(BuiltinsArena(), DynType(), DynType()))); return *kInstance; } diff --git a/checker/type_checker.cc b/checker/type_checker.cc new file mode 100644 index 000000000..6d59e144d --- /dev/null +++ b/checker/type_checker.cc @@ -0,0 +1,36 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "checker/type_checker.h" + +namespace cel { +absl::StatusOr TypeChecker::Check( + std::unique_ptr ast) const { + return CheckImpl(std::move(ast), nullptr); +} + +absl::StatusOr TypeChecker::Check( + std::unique_ptr ast, google::protobuf::Arena* arena) const { + return CheckImpl(std::move(ast), arena); +} + +absl::StatusOr TypeChecker::Check(const Ast& ast) const { + return CheckImpl(std::make_unique(ast), nullptr); +} + +absl::StatusOr TypeChecker::Check( + const Ast& ast, google::protobuf::Arena* arena) const { + return CheckImpl(std::make_unique(ast), arena); +} +} // namespace cel diff --git a/checker/type_checker.h b/checker/type_checker.h index e47b7dca6..edb6cc91f 100644 --- a/checker/type_checker.h +++ b/checker/type_checker.h @@ -16,10 +16,13 @@ #define THIRD_PARTY_CEL_CPP_CHECKER_TYPE_CHECKER_H_ #include +#include +#include "absl/base/nullability.h" #include "absl/status/statusor.h" #include "checker/validation_result.h" #include "common/ast.h" +#include "google/protobuf/arena.h" namespace cel { @@ -42,13 +45,19 @@ class TypeChecker { // A non-ok status is returned if type checking can't reasonably complete // (e.g. if an internal precondition is violated or an extension returns an // error). - virtual absl::StatusOr Check( - std::unique_ptr ast) const = 0; + absl::StatusOr Check(std::unique_ptr ast) const; + absl::StatusOr Check(std::unique_ptr ast, + google::protobuf::Arena* arena) const; + absl::StatusOr Check(const Ast& ast) const; + absl::StatusOr Check(const Ast& ast, + google::protobuf::Arena* arena) const; // Returns a builder initialized with the configuration of this type checker. virtual std::unique_ptr ToBuilder() const = 0; - // TODO(uncreated-issue/73): add overload for cref AST. + private: + virtual absl::StatusOr CheckImpl( + std::unique_ptr ast, google::protobuf::Arena* absl_nullable arena) const = 0; }; } // namespace cel diff --git a/checker/type_checker_builder.h b/checker/type_checker_builder.h index 5dd1f5256..c2d0cbf7b 100644 --- a/checker/type_checker_builder.h +++ b/checker/type_checker_builder.h @@ -17,6 +17,7 @@ #include #include +#include #include "absl/base/nullability.h" #include "absl/functional/any_invocable.h" @@ -51,7 +52,7 @@ struct CheckerLibrary { // Represents a declaration to only use a subset of a library. struct TypeCheckerSubset { using FunctionPredicate = absl::AnyInvocable; + absl::string_view function, const OverloadDecl& overload) const>; // The id of the library to subset. Only one subset can be applied per // library id. @@ -102,6 +103,27 @@ class TypeCheckerBuilder { // Note: only protobuf backed struct types are supported at this time. virtual absl::Status AddContextDeclaration(absl::string_view type) = 0; + // Declares struct type by fully qualified name as a context declaration. + // + // This version accepts a mask in terms of field selections from the + // context type. The mask specifies which fields are visible on the + // struct and its members. The visible fields for a type accumulate + // across calls. This is a lightweight way to adjust the type checking + // behavior for a group of related types. + // + // Context declarations are a way to declare a group of variables based on the + // definition of a struct type. Each top level field of the struct that is + // also the first field name in a field path is declared as an individual + // variable of the field type. + // + // It is an error if the type contains a field that overlaps with another + // declared variable. It is an error if the input field paths is the empty + // set. + // + // Note: only protobuf backed struct types are supported at this time. + virtual absl::Status AddContextDeclarationWithProtoTypeMask( + absl::string_view type, std::vector field_paths) = 0; + // Adds a function declaration that may be referenced in expressions checked // with the resulting TypeChecker. virtual absl::Status AddFunction(const FunctionDecl& decl) = 0; diff --git a/checker/type_checker_builder_factory_test.cc b/checker/type_checker_builder_factory_test.cc index 38430de5f..40406948d 100644 --- a/checker/type_checker_builder_factory_test.cc +++ b/checker/type_checker_builder_factory_test.cc @@ -235,8 +235,8 @@ TEST(TypeCheckerBuilderTest, AddLibraryIncludeSubset) { ASSERT_THAT( builder->AddLibrarySubset( {"testlib", - [](absl::string_view /*function*/, absl::string_view overload_id) { - return (overload_id == "add_int" || overload_id == "sub_int"); + [](absl::string_view /*function*/, const OverloadDecl& overload) { + return (overload.id() == "add_int" || overload.id() == "sub_int"); }}), IsOk()); ASSERT_OK_AND_ASSIGN(auto checker, builder->Build()); @@ -274,9 +274,8 @@ TEST(TypeCheckerBuilderTest, AddLibraryExcludeSubset) { ASSERT_THAT( builder->AddLibrarySubset( {"testlib", - [](absl::string_view /*function*/, absl::string_view overload_id) { - return (overload_id != "add_int" && overload_id != "sub_int"); - ; + [](absl::string_view /*function*/, const OverloadDecl& overload) { + return (overload.id() != "add_int" && overload.id() != "sub_int"); }}), IsOk()); ASSERT_OK_AND_ASSIGN(auto checker, builder->Build()); @@ -313,7 +312,7 @@ TEST(TypeCheckerBuilderTest, AddLibrarySubsetRemoveAllOvl) { ASSERT_THAT(builder->AddLibrary(SubsetTestlib()), IsOk()); ASSERT_THAT(builder->AddLibrarySubset({"testlib", [](absl::string_view function, - absl::string_view /*overload_id*/) { + const OverloadDecl& /*overload*/) { return function != "add"; }}), IsOk()); @@ -352,12 +351,12 @@ TEST(TypeCheckerBuilderTest, AddLibraryOneSubsetPerLibraryId) { ASSERT_THAT( builder->AddLibrarySubset( {"testlib", [](absl::string_view function, - absl::string_view /*overload_id*/) { return true; }}), + const OverloadDecl& /*overload*/) { return true; }}), IsOk()); EXPECT_THAT( builder->AddLibrarySubset( {"testlib", [](absl::string_view function, - absl::string_view /*overload_id*/) { return true; }}), + const OverloadDecl& /*overload*/) { return true; }}), StatusIs(absl::StatusCode::kAlreadyExists)); } @@ -369,7 +368,7 @@ TEST(TypeCheckerBuilderTest, AddLibrarySubsetLibraryIdRequireds) { ASSERT_THAT(builder->AddLibrary(SubsetTestlib()), IsOk()); EXPECT_THAT(builder->AddLibrarySubset({"", [](absl::string_view function, - absl::string_view /*overload_id*/) { + const OverloadDecl& /*overload*/) { return function == "add"; }}), StatusIs(absl::StatusCode::kInvalidArgument)); @@ -396,6 +395,27 @@ TEST(TypeCheckerBuilderTest, AddContextDeclaration) { EXPECT_TRUE(result.IsValid()); } +TEST(TypeCheckerBuilderTest, AddContextDeclarationWithProtoTypeMask) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr builder, + CreateTypeCheckerBuilder(GetSharedTestingDescriptorPool())); + + ASSERT_OK_AND_ASSIGN( + auto fn_decl, + MakeFunctionDecl("increment", MakeOverloadDecl("increment_int", IntType(), + IntType()))); + + ASSERT_THAT(builder->AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"single_int64"}), + IsOk()); + ASSERT_THAT(builder->AddFunction(fn_decl), IsOk()); + + ASSERT_OK_AND_ASSIGN(auto checker, builder->Build()); + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst("increment(single_int64)")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, checker->Check(std::move(ast))); + EXPECT_TRUE(result.IsValid()); +} + TEST(TypeCheckerBuilderTest, WellKnownTypeContextDeclarationError) { ASSERT_OK_AND_ASSIGN( std::unique_ptr builder, @@ -428,6 +448,32 @@ TEST(TypeCheckerBuilderTest, AllowWellKnownTypeContextDeclaration) { ASSERT_TRUE(result.IsValid()); } +TEST(TypeCheckerBuilderTest, + AllowWellKnownTypeContextDeclarationWithProtoTypeMask) { + CheckerOptions options; + options.allow_well_known_type_context_declarations = true; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr builder, + CreateTypeCheckerBuilder(GetSharedTestingDescriptorPool(), options)); + + ASSERT_THAT(builder->AddContextDeclarationWithProtoTypeMask( + "google.protobuf.Any", {"value"}), + IsOk()); + ASSERT_THAT(builder->AddLibrary(StandardCheckerLibrary()), IsOk()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr type_checker, + builder->Build()); + // Visible field: value + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst("value")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + type_checker->Check(std::move(ast))); + ASSERT_TRUE(result.IsValid()); + // Not visible field: type_url + ASSERT_OK_AND_ASSIGN(ast, MakeTestParsedAst("type_url")); + ASSERT_OK_AND_ASSIGN(result, type_checker->Check(std::move(ast))); + ASSERT_FALSE(result.IsValid()); +} + TEST(TypeCheckerBuilderTest, AllowWellKnownTypeContextDeclarationStruct) { CheckerOptions options; options.allow_well_known_type_context_declarations = true; diff --git a/checker/type_checker_subset_factory.cc b/checker/type_checker_subset_factory.cc index 6a05ce220..1b146c5a5 100644 --- a/checker/type_checker_subset_factory.cc +++ b/checker/type_checker_subset_factory.cc @@ -21,14 +21,21 @@ #include "absl/strings/string_view.h" #include "absl/types/span.h" #include "checker/type_checker_builder.h" +#include "common/decl.h" +#include "common/signature.h" namespace cel { TypeCheckerSubset::FunctionPredicate IncludeOverloadsByIdPredicate( absl::flat_hash_set overload_ids) { return [overload_ids = std::move(overload_ids)]( - absl::string_view /*function*/, absl::string_view overload_id) { - return overload_ids.contains(overload_id); + absl::string_view function, const OverloadDecl& overload) { + if (overload_ids.contains(overload.id())) { + return true; + } + auto signature = + MakeOverloadSignature(function, overload.args(), overload.member()); + return signature.ok() && overload_ids.contains(*signature); }; } @@ -41,8 +48,13 @@ TypeCheckerSubset::FunctionPredicate IncludeOverloadsByIdPredicate( TypeCheckerSubset::FunctionPredicate ExcludeOverloadsByIdPredicate( absl::flat_hash_set overload_ids) { return [overload_ids = std::move(overload_ids)]( - absl::string_view /*function*/, absl::string_view overload_id) { - return !overload_ids.contains(overload_id); + absl::string_view function, const OverloadDecl& overload) { + if (overload_ids.contains(overload.id())) { + return false; + } + auto signature = + MakeOverloadSignature(function, overload.args(), overload.member()); + return !signature.ok() || !overload_ids.contains(*signature); }; } diff --git a/checker/type_checker_subset_factory_test.cc b/checker/type_checker_subset_factory_test.cc index fa38e1c0d..5b644ec7c 100644 --- a/checker/type_checker_subset_factory_test.cc +++ b/checker/type_checker_subset_factory_test.cc @@ -43,6 +43,8 @@ TEST(TypeCheckerSubsetFactoryTest, IncludeOverloadsByIdPredicate) { StandardOverloadIds::kEquals, StandardOverloadIds::kNotEquals, StandardOverloadIds::kNotStrictlyFalse, + "matches(string,string)", + "string.matches(string)", }; ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); ASSERT_THAT(builder->GetCheckerBuilder().AddLibrarySubset({ @@ -65,15 +67,19 @@ TEST(TypeCheckerSubsetFactoryTest, IncludeOverloadsByIdPredicate) { EXPECT_TRUE(r.IsValid()); + // Allowed by signature. + ASSERT_OK_AND_ASSIGN(r, compiler->Compile("r'foo.*'.matches('foobar')")); + EXPECT_TRUE(r.IsValid()); + + ASSERT_OK_AND_ASSIGN(r, compiler->Compile("matches(r'foo.*', 'foobar')")); + EXPECT_TRUE(r.IsValid()); + // Not in allowlist. ASSERT_OK_AND_ASSIGN(r, compiler->Compile("1 + 2 < 3")); EXPECT_FALSE(r.IsValid()); ASSERT_OK_AND_ASSIGN(r, compiler->Compile("'abc' + 'def'")); EXPECT_FALSE(r.IsValid()); - - ASSERT_OK_AND_ASSIGN(r, compiler->Compile("r'foo.*'.matches('foobar')")); - EXPECT_FALSE(r.IsValid()); } TEST(TypeCheckerSubsetFactoryTest, ExcludeOverloadsByIdPredicate) { @@ -83,6 +89,8 @@ TEST(TypeCheckerSubsetFactoryTest, ExcludeOverloadsByIdPredicate) { absl::string_view exclude_list[] = { StandardOverloadIds::kMatches, StandardOverloadIds::kMatchesMember, + "size(string)", + "string.size()", }; ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); ASSERT_THAT(builder->GetCheckerBuilder().AddLibrarySubset({ @@ -105,18 +113,35 @@ TEST(TypeCheckerSubsetFactoryTest, ExcludeOverloadsByIdPredicate) { EXPECT_TRUE(r.IsValid()); - // Not in allowlist. + // Allowed. ASSERT_OK_AND_ASSIGN(r, compiler->Compile("1 + 2 < 3")); EXPECT_TRUE(r.IsValid()); ASSERT_OK_AND_ASSIGN(r, compiler->Compile("'abc' + 'def'")); EXPECT_TRUE(r.IsValid()); + // Excluded by ID. ASSERT_OK_AND_ASSIGN(r, compiler->Compile("r'foo.*'.matches('foobar')")); EXPECT_FALSE(r.IsValid()); ASSERT_OK_AND_ASSIGN(r, compiler->Compile("matches(r'foo.*', 'foobar')")); EXPECT_FALSE(r.IsValid()); + + // Excluded by signature (top-level function). + ASSERT_OK_AND_ASSIGN(r, compiler->Compile("size('abc')")); + EXPECT_FALSE(r.IsValid()); + + // Allowed (other overloads of size). + ASSERT_OK_AND_ASSIGN(r, compiler->Compile("size([1, 2, 3])")); + EXPECT_TRUE(r.IsValid()); + + // Excluded by signature (member function). + ASSERT_OK_AND_ASSIGN(r, compiler->Compile("'abc'.size()")); + EXPECT_FALSE(r.IsValid()); + + // Allowed (other overloads of size member). + ASSERT_OK_AND_ASSIGN(r, compiler->Compile("[1, 2, 3].size()")); + EXPECT_TRUE(r.IsValid()); } } // namespace diff --git a/checker/validation_result.h b/checker/validation_result.h index 8c84a84da..7417e9969 100644 --- a/checker/validation_result.h +++ b/checker/validation_result.h @@ -15,26 +15,32 @@ #ifndef THIRD_PARTY_CEL_CPP_CHECKER_VALIDATION_RESULT_H_ #define THIRD_PARTY_CEL_CPP_CHECKER_VALIDATION_RESULT_H_ +#include #include #include #include #include #include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/types/span.h" #include "checker/type_check_issue.h" #include "common/ast.h" +#include "common/decl.h" #include "common/source.h" +#include "common/type.h" namespace cel { -// ValidationResult holds the result of TypeChecking. +// ValidationResult holds the result of type checking. // // Error states are captured as type check issues where possible. class ValidationResult { public: + using TypeMap = absl::flat_hash_map; + ValidationResult(std::unique_ptr ast, std::vector issues) : ast_(std::move(ast)), issues_(std::move(issues)) {} @@ -71,6 +77,18 @@ class ValidationResult { return std::move(source_); } + // Returns the resolved type map for the AST. + // + // Only populated if the AST was checked with an explicit arena. + // + // The type entries may have storage in the arena or reference type + // information from the type checker that produced the AST. This means the map + // is only valid as long as both the type checker and the arena are valid. + const TypeMap& GetResolvedTypeMap() const { return resolved_type_map_; } + void SetResolvedTypeMap(TypeMap resolved_type_map) { + resolved_type_map_ = std::move(resolved_type_map); + } + // Returns a string representation of the issues in the result suitable for // display. // @@ -89,6 +107,7 @@ class ValidationResult { private: absl_nullable std::unique_ptr ast_; + TypeMap resolved_type_map_; std::vector issues_; absl_nullable std::unique_ptr source_; }; diff --git a/ci/ubuntu_floor/Dockerfile b/ci/ubuntu_floor/Dockerfile new file mode 100644 index 000000000..dd81bd45e --- /dev/null +++ b/ci/ubuntu_floor/Dockerfile @@ -0,0 +1,70 @@ +# This Dockerfile is used to create a container with compilers and tools +# building the CEL C++ library on GitHub. +# +# To update a new version of this container, use gcloud. You may need to run +# `gcloud auth login` and `gcloud auth configure-docker` first. +# +# Note, if you need to run docker using `sudo` use the following commands +# instead: +# +# sudo gcloud auth login --no-launch-browser +# sudo gcloud auth configure-docker +# +# Run the following command from the root of the CEL repository: +# +# gcloud builds submit --region=us -t gcr.io/cel-analysis/cel-cpp/ubuntu_floor . +# +# Once complete get the sha256 digest from the output using the following +# command: +# +# gcloud artifacts versions list --package=cel-cpp/ubuntu_floor --repository=gcr.io \ +# --location=us +# +# The cloudbuild.yaml file must be updated to use the new digest like so: +# +# - name: 'gcr.io/cel-analysis/cel-cpp/ubuntu_floor@' +FROM gcr.io/cloud-marketplace/google/ubuntu2204:latest + +# Install Bazel prerequesites and required tools. +# See https://docs.bazel.build/versions/master/install-ubuntu.html +# See https://github.com/google/oss-policies-info/blob/main/foundational-cxx-support-matrix.md +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ + git \ + libssl-dev \ + make \ + pkg-config \ + python3 \ + unzip \ + wget \ + zip \ + zlib1g-dev \ + default-jdk-headless \ + clang-14 \ + gcc-10 g++-10 \ + tzdata \ + && apt-get clean + +# Install Bazelisk. +# https://github.com/bazelbuild/bazelisk/releases +ARG BAZELISK_URL="https://github.com/bazelbuild/bazelisk/releases/download/v1.27.0/bazelisk-amd64.deb" +ARG BAZELISK_CHKSUM="d8b00ea975c823e15263c80200ac42979e17368547fbff4ab177af035badfa83" +ADD ${BAZELISK_URL} /tmp/bazelisk.deb + +ENV BAZELISK_CHKSUM=${BAZELISK_CHKSUM} +RUN echo "${BAZELISK_CHKSUM} */tmp/bazelisk.deb" | sha256sum --check + +RUN apt-get install /tmp/bazelisk.deb + +RUN mkdir -p /workspace +RUN mkdir -p /bazel + +RUN USE_BAZEL_VERSION=8.7.0 bazelisk help +RUN USE_BAZEL_VERSION=7.3.2 bazelisk help + +ENV CC=gcc-10 +ENV CXX=g++-10 + +ENTRYPOINT ["/usr/bin/bazelisk"] diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 8272378f6..fe78c7a2b 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,5 +1,5 @@ steps: -- name: 'gcr.io/cel-analysis/gcc9@sha256:4d5ff2e55224398807235a44b57e9c5793e922ac46e9ff428536bb8f8e5790ce' +- name: 'gcr.io/cel-analysis/cel-cpp/ubuntu_floor@sha256:fbcc9edb13aa855f435993bf12ec4fd59f8b72e7d715bbc4c72e7a2298f0bcc2' args: - '--output_base=/bazel' # This is mandatory to avoid steps accidently sharing data. - 'test' @@ -14,18 +14,23 @@ steps: - '--local_ram_resources=HOST_RAM*.4' - '--remote_cache=https://storage.googleapis.com/cel-cpp-remote-cache' - '--google_default_credentials' - id: gcc-9 + id: gcc-10 waitFor: ['-'] -- name: 'gcr.io/cel-analysis/gcc9@sha256:4d5ff2e55224398807235a44b57e9c5793e922ac46e9ff428536bb8f8e5790ce' +- name: 'gcr.io/cel-analysis/cel-cpp/ubuntu_floor@sha256:fbcc9edb13aa855f435993bf12ec4fd59f8b72e7d715bbc4c72e7a2298f0bcc2' env: - - 'CC=clang-11' - - 'CXX=clang++-11' + - 'CC=clang-14' + - 'CXX=clang++-14' args: - '--output_base=/bazel' # This is mandatory to avoid steps accidently sharing data. - 'test' - '...' - '--enable_bzlmod' - '--copt=-Wno-deprecated-declarations' + - '--copt=-Wno-unknown-warning-option' +# some older versions of clang (<16) complain about some template metaprogramming in +# absl in this release but otherwise it builds fine. Should be fixed in the next release. +# https://github.com/abseil/abseil-cpp/issues/2073 + - '--copt=-Wno-invalid-partial-specialization' - '--compilation_mode=fastbuild' - '--test_output=errors' - '--show_timestamps' @@ -34,7 +39,7 @@ steps: - '--local_ram_resources=HOST_RAM*.4' - '--remote_cache=https://storage.googleapis.com/cel-cpp-remote-cache' - '--google_default_credentials' - id: clang-11 + id: clang-14 waitFor: ['-'] timeout: 1h options: diff --git a/codelab/BUILD b/codelab/BUILD index 69c2825e2..f61b586eb 100644 --- a/codelab/BUILD +++ b/codelab/BUILD @@ -44,18 +44,22 @@ cc_library( "nobuilder", ], deps = [ - "//eval/public:activation", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_expression", - "//eval/public:cel_options", - "//eval/public:cel_value", + "//checker:validation_result", + "//common:ast", + "//common:minimal_descriptor_pool", + "//common:value", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", "//internal:status_macros", - "//parser", + "//runtime", + "//runtime:activation", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", ], ) @@ -76,22 +80,25 @@ cc_library( srcs = ["exercise2.cc"], hdrs = ["exercise2.h"], deps = [ - ":cel_compiler", + "//checker:type_checker_builder", + "//checker:validation_result", + "//common:ast", + "//common:decl", + "//common:type", + "//common:value", "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", - "//eval/public:activation", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_expression", - "//eval/public:cel_options", - "//eval/public:cel_value", "//internal:status_macros", + "//runtime", + "//runtime:activation", + "//runtime:bind_proto_to_activation", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", - "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", "@com_google_protobuf//:protobuf", ], @@ -125,42 +132,30 @@ cc_test( ) cc_library( - name = "cel_compiler", - hdrs = ["cel_compiler.h"], + name = "checked_expr_conversion_example", + hdrs = ["checked_expr_conversion_example.h"], deps = [ - "//checker:validation_result", + "//common:ast", "//common:ast_proto", - "//compiler", "//internal:status_macros", - "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", ], ) cc_test( - name = "cel_compiler_test", - srcs = ["cel_compiler_test.cc"], + name = "checked_expr_conversion_example_test", + srcs = ["checked_expr_conversion_example_test.cc"], deps = [ - ":cel_compiler", - "//common:decl", - "//common:type", + ":checked_expr_conversion_example", + "//checker:validation_result", + "//common:ast", "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", - "//eval/public:activation", - "//eval/public:activation_bind_helper", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_function_adapter", - "//eval/public:cel_value", - "//eval/public/testing:matchers", "//internal:testing", - "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", - "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", + "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", "@com_google_protobuf//:protobuf", ], ) @@ -170,22 +165,27 @@ cc_library( srcs = ["exercise4.cc"], hdrs = ["exercise4.h"], deps = [ - ":cel_compiler", + "//checker:type_checker_builder", + "//checker:validation_result", + "//common:ast", + "//common:decl", + "//common:type", + "//common:value", "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", - "//eval/public:activation", - "//eval/public:activation_bind_helper", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_expression", - "//eval/public:cel_options", - "//eval/public:cel_value", "//internal:status_macros", + "//runtime", + "//runtime:activation", + "//runtime:bind_proto_to_activation", + "//runtime:function_adapter", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", + "@com_google_absl//absl/types:optional", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", "@com_google_protobuf//:protobuf", ], @@ -270,6 +270,7 @@ cc_library( deps = [ ":network_functions", "//checker:validation_result", + "//common:ast", "//common:decl", "//common:minimal_descriptor_pool", "//common:type", @@ -277,6 +278,7 @@ cc_library( "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", + "//internal:status_macros", "//runtime", "//runtime:activation", "//runtime:runtime_builder", diff --git a/codelab/Dockerfile b/codelab/Dockerfile index c98a08f39..19040f2de 100644 --- a/codelab/Dockerfile +++ b/codelab/Dockerfile @@ -1,17 +1,17 @@ -ARG DEBIAN_IMAGE="marketplace.gcr.io/google/debian11:latest" -FROM ${DEBIAN_IMAGE} +ARG UBUNTU_IMAGE="ubuntu:22.04" +FROM ${UBUNTU_IMAGE} ARG BAZELISK_RELEASE="https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-amd64.deb" -RUN apt update && apt upgrade -y && apt install -y gcc-9 g++-9 clang-13 git curl bash openjdk-11-jdk-headless +RUN apt update && apt upgrade -y && apt install -y gcc-10 g++-10 clang-14 git curl bash openjdk-11-jdk-headless RUN curl -L ${BAZELISK_RELEASE} > ./bazelisk.deb RUN apt install ./bazelisk.deb RUN git clone https://github.com/google/cel-cpp.git -ENV CXX=clang++-13 -ENV CC=clang-13 +ENV CXX=g++-10 +ENV CC=gcc-10 WORKDIR /cel-cpp # not generally recommended to cache the bazel build in the image, diff --git a/codelab/README.md b/codelab/README.md index 96f7598ba..c751d5678 100644 --- a/codelab/README.md +++ b/codelab/README.md @@ -1,42 +1,65 @@ # What is CEL? -Common Expression Language (CEL) is an expression language that’s fast, portable, and safe to execute in performance-critical applications. CEL is designed to be embedded in an application, with application-specific extensions, and is ideal for extending declarative configurations that your applications might already use. + +Common Expression Language (CEL) is an expression language that’s fast, +portable, and safe to execute in performance-critical applications. CEL is +designed to be embedded in an application, with application-specific extensions, +and is ideal for extending declarative configurations that your applications +might already use. ## What is covered in this Codelab? -This codelab is aimed at developers who would like to learn CEL to use services that already support CEL. This Codelab covers common use cases. This codelab doesn't cover how to integrate CEL into your own project. For a more in-depth look at the language, semantics, and features see the [CEL Language Definition on GitHub](https://github.com/google/cel-spec). + +This codelab is aimed at developers who would like to learn CEL to use services +that already support CEL. This Codelab covers common use cases. This codelab +doesn't cover how to integrate CEL into your own project. For a more in-depth +look at the language, semantics, and features see the +[CEL Language Definition on GitHub](https://github.com/google/cel-spec). Some key areas covered are: -* [Hello, World: Using CEL to evaluate a String](#hello-world) -* [Creating variables](#creating-variables) -* [Commutative logical AND/OR](#logical-andor) -* [Adding custom functions](#custom-functions) +* [Hello, World: Using CEL to evaluate a String](#hello-world) +* [Creating variables](#creating-variables) +* [Commutative logical AND/OR](#logical-andor) +* [Adding custom functions](#custom-functions) ### Prerequisites -This codelab builds upon a basic understanding of Protocol Buffers and C++. -If you're not familiar with Protocol Buffers, the first exercise will give you a sense of how CEL works, but because the more advanced examples use Protocol Buffers as the input into CEL, they may be harder to understand. Consider working through one of these tutorials, first. See the devsite for [Protocol Buffers](https://protobuf.dev). +This codelab builds upon a basic understanding of Protocol Buffers and C++. -Notes on portability: Protocol Buffers are not required to use CEL -generally, but the C++ implementation has a hard dependency on the library -and some APIs reference protobuf types directly. Automated builds test -against gcc9 and clang11 on linux. We accept requests for portability -fixes for other OSes and compilers, but don't actively maintain support at -this time. A simple Docker file is provided as a reference for a known good -environment configuration for running the codelab solutions. +If you're not familiar with Protocol Buffers, the first exercise will give you a +sense of how CEL works, but because the more advanced examples use Protocol +Buffers as the input into CEL, they may be harder to understand. Consider +working through one of these tutorials, first. See the devsite for +[Protocol Buffers](https://protobuf.dev). + +If you're not familiar with Bazel, Bazel is an open-source build and test tool +that supports multi-language projects and manages dependencies. This codelab and +`cel-cpp` are built using Bazel. To learn more about building C++ projects with +Bazel, see the official [Bazel C++ Tutorial](https://bazel.build/tutorials/cpp) +and the [Bazel Documentation](https://bazel.build/docs). + +Notes on portability: Protocol Buffers are not required to use CEL generally, +but the C++ implementation has a hard dependency on the library and some APIs +reference protobuf types directly. Automated builds test against gcc10 and +clang14 on linux, a recent visual code version on Windows, and a recent xcode +version on MacOS. We accept requests for portability fixes for other OSes and +compilers, but don't actively maintain support at this time. A simple Docker +file is provided as a reference for a known good environment configuration for +running the codelab solutions. What you'll need: -- Git -- Bazel -- C/C++ Compiler (GCC, Clang, Visual Studio). -- Optional: bazelisk is a wrapper around bazel that simplifies version - management. If using, substitute all bazel commands below with `bazelisk`. +- Git +- Bazel +- C/C++ Compiler (GCC, Clang, Visual Studio). +- Optional: bazelisk is a wrapper around bazel that simplifies version + management. If using, substitute all bazel commands below with `bazelisk`. ## GitHub Setup GitHub Repo: -The code for this codelab lives in the `codelab` folder of the cel-cpp repo. The solution is available in the `codelab/solution` folder of the same repo. +The code for this codelab lives in the `codelab` folder of the cel-cpp repo. The +solutions are available in the `codelab/solution` folder of the same repo. Clone and cd into the repo: @@ -51,69 +74,131 @@ Make sure everything is working by building the codelab: bazel build //codelab:all ``` +## Setting up a Bazel Project with cel-cpp + +If you want to integrate `cel-cpp` into your own standalone C++ project using +[Bazel](https://bazel.build/docs), you can manage it as an external dependency +with [Bzlmod](https://bazel.build/external/overview#bzlmod). + +### MODULE.bazel + +In your project's root `MODULE.bazel` file, declare a dependency on `cel-cpp` +from the +[Bazel Central Registry (BCR)](https://registry.bazel.build/modules/cel-cpp): + +```python +bazel_dep(name = "cel-cpp", version = "") +``` + +Alternatively, if you want to depend directly on a specific commit from the +GitHub repository, you can use `git_override`: + +```python +bazel_dep(name = "cel-cpp", version = "0.0.0") +git_override( + module_name = "cel-cpp", + remote = "https://github.com/google/cel-cpp.git", + commit = "", +) +``` + +### BUILD + +In your project's `BUILD` (or `BUILD.bazel`) file, reference `cel-cpp` targets +using `@cel-cpp//...`. For example, to compile a C++ binary that compiles and +evaluates CEL expressions using the modern `Compiler` and `Runtime` APIs shown +in this codelab: + +```python +cc_binary( + name = "my_cel_app", + srcs = ["main.cc"], + deps = [ + "@cel-cpp//checker:validation_result", + "@cel-cpp//common:ast", + "@cel-cpp//common:minimal_descriptor_pool", + "@cel-cpp//common:value", + "@cel-cpp//compiler", + "@cel-cpp//compiler:compiler_factory", + "@cel-cpp//compiler:standard_library", + "@cel-cpp//runtime", + "@cel-cpp//runtime:activation", + "@cel-cpp//runtime:runtime_builder", + "@cel-cpp//runtime:runtime_options", + "@cel-cpp//runtime:standard_runtime_builder_factory", + ], +) +``` + ## Hello, World -In the tried and true tradition of all programming languages, let's start with "Hello, World!". + +In the tried and true tradition of all programming languages, let's start with +"Hello, World!". Update exercise1.cc with the following: Using declarations: ```c++ -using ::google::api::expr::parser::Parse; -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::CelExpression; -using ::google::api::expr::runtime::CelExpressionBuilder; -using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::InterpreterOptions; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; +using ::cel::Activation; +using ::cel::Compiler; +using ::cel::CompilerBuilder; +using ::cel::Program; +using ::cel::Runtime; +using ::cel::RuntimeBuilder; +using ::cel::RuntimeOptions; +using ::cel::ValidationResult; +using ::cel::Value; ``` Implementation: ```c++ -absl::StatusOr ParseAndEvaluate(absl::string_view cel_expr) -{ +absl::StatusOr ParseAndEvaluate(absl::string_view cel_expr) { // === Start Codelab === - // Setup a default environment for building expressions. - InterpreterOptions options; - std::unique_ptr builder = - CreateCelExpressionBuilder(options); - + // Setup a default compiler for compiling expressions. + CEL_ASSIGN_OR_RETURN( + std::unique_ptr compiler_builder, + cel::NewCompilerBuilder(cel::GetMinimalDescriptorPool())); CEL_RETURN_IF_ERROR( - RegisterBuiltinFunctions(builder->GetRegistry(), options)); - - // Parse the expression. This is fine for codelabs, but this skips the type - // checking phase. It won't check that functions and variables are available - // in the environment, and it won't handle certain ambiguous identifier - // expressions (e.g. container lookup vs namespaced name, packaged function - // vs. receiver call style function). - ParsedExpr parsed_expr; - CEL_ASSIGN_OR_RETURN(parsed_expr, Parse(cel_expr)); - - // The evaluator uses a proto Arena for incidental allocations during - // evaluation. + compiler_builder->AddLibrary(cel::StandardCompilerLibrary())); + CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, + std::move(compiler_builder)->Build()); + + // Compile the expression. + CEL_ASSIGN_OR_RETURN(ValidationResult validation_result, + compiler->Compile(cel_expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } + + // Setup a standard runtime for evaluating expressions. + RuntimeOptions options; + CEL_ASSIGN_OR_RETURN( + RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder(cel::GetMinimalDescriptorPool(), + options)); + CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + + // Build the executable program from the compiled AST. + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + // The evaluator uses a proto Arena for allocations during evaluation. proto2::Arena arena; - // The activation provides variables and functions that are bound into the + // The activation provides variables and functions bound into the // expression environment. In this example, there's no context expected, so - // we just provide an empty one to the evaluator. + // we provide an empty activation. Activation activation; - // Build the expression plan. This assumes that the source expression AST and - // the expression builder outlives the CelExpression object. - CEL_ASSIGN_OR_RETURN(std::unique_ptr expression_plan, - builder->CreateExpression(&parsed_expr.expr(), - &parsed_expr.source_info())); - - // Actually run the expression plan. We don't support any environment - // variables at the moment so just use an empty activation. - CEL_ASSIGN_OR_RETURN(CelValue result, - expression_plan->Evaluate(activation, &arena)); - - // Convert the result to a c++ string. CelValues may reference instances from - // either the input expression, or objects allocated on the arena, so we need - // to pass ownership (in this case by copying to a new instance and returning - // that). + // Run the program. + CEL_ASSIGN_OR_RETURN(Value result, + program->Evaluate(&arena, activation)); + + // Convert the result to a C++ string. return ConvertResult(result); // === End Codelab === } @@ -129,151 +214,178 @@ You can add additional test cases or experiment with different return types. Hello, World! Now, let's break down what's happening. - ### Setup the Environment -CEL applications evaluate an expression against an environment. -The standard CEL environment supports all of the types, operators, functions, and macros defined within the language spec. The environment can be customized by providing options to disable macros, declare custom variables and functions, etc. +CEL applications compile and evaluate an expression against an environment. + +The standard CEL environment supports all of the types, operators, functions, +and macros defined within the language spec. The compiler and runtime objects +can be customized with other variables, types, and functions. -An ExpressionBuilder maintains C++ evaluation environment. This creates a builder with the standard environment. +A `CompilerBuilder` configures the compilation environment, while a +`RuntimeBuilder` configures the execution environment: ```c++ -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_options.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" ... -// Setup a default environment for building expressions. - -// Breaking behavior changes and optional features are controlled by -// InterpreterOptions. -InterpreterOptions options; - -// Environment used for planning and evaluating expressions is managed by an -// ExpressionBuilder. -std::unique_ptr builder = - CreateCelExpressionBuilder(options); - -// Add standard function bindings e.g. for +,-,==,||,&& operators. -// Custom functions (implementing the CelFunction interface) can be added to the -// registry similarly. +// Create a compiler configured with the standard CEL library. +CEL_ASSIGN_OR_RETURN( + std::unique_ptr compiler_builder, + cel::NewCompilerBuilder(cel::GetMinimalDescriptorPool())); CEL_RETURN_IF_ERROR( - RegisterBuiltinFunctions(builder->GetRegistry(), options)); + compiler_builder->AddLibrary(cel::StandardCompilerLibrary())); +CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, + std::move(compiler_builder)->Build()); + +// Create a standard runtime. +cel::RuntimeOptions options; +CEL_ASSIGN_OR_RETURN( + cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder(cel::GetMinimalDescriptorPool(), + options)); +CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); ``` -### Parse -After the environment is configured, you can parse and check the expressions: +### Compile + +After the compiler is configured, you can compile source expressions into a +checked `cel::Ast`: ```c++ -#include "google/api/expr/syntax.proto.h" -#include "parser/parser.h" +#include "checker/validation_result.h" // ... -ASSIGN_OR_RETURN(google::api::expr::ParsedExpr parsed_expr, google::api::expr::parser::Parse(cel_expr)); +CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(cel_expr)); +if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); +} ``` -The C++ parser is a stand-alone utility. It's not aware of the evaluation environment and does not perform any semantic checks on the expression. A status is returned if the input string isn't a syntactically valid CEL expression or if it exceeds the configured complexity limits (see cel::ParserOptions and default limits). +`compiler->Compile(...)` parses and type-checks the expression in one step, +returning a `cel::ValidationResult`. If the expression is valid +(`validation_result.IsValid()`), the resulting `cel::Ast` can be extracted using +`validation_result.ReleaseAst()`. + +> Note: If you need to interface with external systems that store or transmit +> serialized protobuf ASTs (`google::api::expr::CheckedExpr`), conversion +> examples are available in `checked_expr_conversion_example.h` +> (`cel::AstToCheckedExpr` and `cel::CreateAstFromCheckedExpr`). ### Evaluate -After the expressions have been parsed and checked into an AST representation, it can be converted into an evaluable program whose function bindings and evaluation modes can be customized depending on the stack you are using. -Once a CEL expression is planned, it can be evaluated against an evaluation context (an activation). The evaluation result will be either a value or an error state. -The InterpreterOptions to create the expression plan are honored at evaluation. C++ uses the proto representation of either a parsed `google.api.expr.ParsedExpr` or parsed and type-checked `google.api.expr.CheckedExpr` AST directly. -Once a CEL program is planned (represented by a `google::api::expr::runtime::CelExpression`), it can be evaluated against an `google::api::expr::runtime::Activation`. The Activation provides per-evaluation bindings for variables and functions in the expression's environment. + +Once a `cel::Ast` is compiled, `runtime->CreateProgram(...)` creates an +executable `cel::Program`. The program is then evaluated against a +`cel::Activation`, which provides per-evaluation bindings for variables and +functions in the environment: ```c++ +#include "common/ast.h" +#include "runtime/activation.h" #include "third_party/protobuf/arena.h" -#include "eval/public/activation.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_value.h" -#include "internal/status_macros.h" -#include "parser/parser.h" ... -// The evaluator uses a proto Arena for incidental allocations during -// evaluation. +// Build the executable program from the compiled AST. +CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); +CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + +// Run the program against an activation and proto Arena. +cel::Activation activation; proto2::Arena arena; -// The activation provides variables and functions that are bound into the -// expression environment. In this example, there's no context expected, so -// we just provide an empty one to the evaluator. -Activation activation; - -// Build the expression plan. This assumes that the source expression AST and -// the expression builder outlives the CelExpression object. -CEL_ASSIGN_OR_RETURN(std::unique_ptr expression_plan, - builder->CreateExpression(&parsed_expr.expr(), - &parsed_expr.source_info())); - -// Actually run the expression plan. We don't support any environment -// variables at the moment so just use an empty activation. -CEL_ASSIGN_OR_RETURN(CelValue result, - expression_plan->Evaluate(activation, &arena)); - -// Convert the result to a C++ string. CelValues may reference instances from -// either the input expression, or objects allocated on the arena, so we need -// to pass ownership (in this case by copying to a new instance and returning -// that). +CEL_ASSIGN_OR_RETURN(cel::Value result, + program->Evaluate(&arena, activation)); + return ConvertResult(result); ``` ## Creating variables -Most CEL applications will declare variables that can be referenced within expressions. Variables declarations specify a name and a type. A variable's type may either be a CEL builtin type, a protocol buffer well-known type, or any protobuf message type so long as its descriptor is also provided to CEL. -At runtime, the hosting program binds instances of variables to the evaluation context (using the variable name as a key). +Most CEL applications declare variables that can be referenced within +expressions. Variable declarations specify a name and a type. A variable's type +may be a CEL built-in type, a protocol buffer well-known type, or any protobuf +message type so long as its descriptor is registered. -For the C++ evaluator at runtime, the values are managed by the `google::api::expr::runtime::CelValue` type, a variant over the C++ representations of supported CEL types. +At runtime, the hosting program binds variable values into `cel::Activation`. + +For the C++ evaluator at runtime, values are represented by `cel::Value`. Update exercise2.cc: ```c++ -// The Variables exercise shows how to declare and use variables in expressions. -// There are two overloads for preparing an expression either granularly for -// individual variables or using a helper to bind a context proto. - -// The first overload shows manually populating individual variables in the -// evaluation environment. This allows cel_expr to reference 'bool_var'. -absl::StatusOr ParseAndEvaluate(absl::string_view cel_expr, - bool bool_var) { - Activation activation; +absl::StatusOr CompileAndEvaluateWithBoolVar(absl::string_view cel_expr, + bool bool_var) { + CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, + MakeCelCompiler()); + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(cel_expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } + + cel::Activation activation; proto2::Arena arena; // === Start Codelab === - activation.InsertValue("bool_var", CelValue::CreateBool(bool_var)); - // === End Codelab === - - return ParseAndEvaluate(cel_expr, activation, &arena); + activation.InsertOrAssignValue("bool_var", cel::BoolValue(bool_var)); + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return EvalAst(std::move(ast), activation, &arena); } ``` -Run the following to check your work. You should have fixed the first two test cases in exercise2_test.cc. +Run the following to check your work. You should have fixed the first two test +cases in exercise2_test.cc. ``` bazel test //codelab:exercise2_test ``` -The second overload uses a protocol buffer message to represent the environment variables. For this use case, there is a helper to automatically bind in fields from a top level message (see `google::api::expr::runtime::BindProtoToActivation`). In this example, we assume that unset fields should be bound to default values. +The second overload uses a protocol buffer message (`AttributeContext`) to +represent the environment variables. For this use case, +`cel::BindProtoToActivation` automatically binds fields from a top-level message +into the activation (`#include +"runtime/bind_proto_to_activation.h"`): ```c++ -#include "eval/public/activation_bind_helper.h" -// ... -using ::google::api::expr::runtime::ProtoUnsetFieldOptions; +#include "runtime/bind_proto_to_activation.h" // ... -absl::StatusOr ParseAndEvaluate(absl::string_view cel_expr, - const AttributeContext& context) { - Activation activation; - google::protobuf::Arena arena; +absl::StatusOr CompileAndEvaluateWithContext( + absl::string_view cel_expr, const AttributeContext& context) { + // ... compile cel_expr ... + cel::Activation activation; + proto2::Arena arena; // === Start Codelab === - - CEL_RETURN_IF_ERROR(BindProtoToActivation( - &context, &arena, &activation, ProtoUnsetFieldOptions::kBindDefault)); + CEL_RETURN_IF_ERROR(cel::BindProtoToActivation( + context, cel::BindProtoUnsetFieldBehavior::kBindDefaultValue, + proto2::DescriptorPool::generated_pool(), + proto2::MessageFactory::generated_factory(), &arena, &activation)); // === End Codelab === - return ParseAndEvaluate(cel_expr, activation, &arena); + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return EvalAst(std::move(ast), activation, &arena); } ``` -Note: You can experiment with unset values and the alternative bind option for BindProtoToActivation. With ProtoUnsetFieldOptions::kSkip unset values will not be bound at all, and accesses in expressions will cause errors. +Note: You can experiment with unset values using +`cel::BindProtoUnsetFieldBehavior::kBindDefaultValue` or `kSkip`. With `kSkip`, +unset protobuf fields will not be bound at all, and accesses in expressions will +result in errors. ## Logical And/Or -One of CEL's more distinctive features is its use of commutative logical operators. Either side of a conditional branch can short-circuit the evaluation, even in the face of errors or partial input. -Note: If you are skipping ahead, copy the solution for exercise2 -- we'll be using it to test the behavior of some simple expressions. -exercise3_test.cc lists truth tables for simple expressions using the 'or', 'and', and 'ternary' operators. +One of CEL's more distinctive features is its use of commutative logical +operators. Either side of a conditional branch can short-circuit the evaluation, +even in the face of errors or partial input. Note: If you are skipping ahead, +copy the solution for exercise2 -- we'll be using it to test the behavior of +some simple expressions. + +exercise3_test.cc lists truth tables for simple expressions using the 'or', +'and', and 'ternary' operators. Running the following should result in some failing expectations. @@ -309,7 +421,8 @@ TEST(Exercise3Var, LogicalOr) { } ``` -Updating the two failing cases "true || (1 / 0 > 2)" and "(1 / 0 > 2) || true" should fix this test: +Updating the two failing cases "true || (1 / 0 > 2)" and "(1 / 0 > 2) || true" +should fix this test: ```c++ // ... @@ -323,6 +436,15 @@ Updating the two failing cases "true || (1 / 0 > 2)" and "(1 / 0 > 2) || true" s IsOkAndHolds(true)); ``` -You can examine the other tests for other cases for corresponding behavior for the 'and' and ternary operators. +You can examine the other tests for other cases for corresponding behavior for +the 'and' and ternary operators. + +CEL finds an evaluation order which gives results whenever possible, ignoring +errors or even missing data that might occur in other evaluation orders. +Applications like IAM conditions rely on this property to minimize the cost of +evaluation, deferring the gathering of expensive inputs when a result can be +reached without them. + +## Adding custom functions {#custom-functions} -CEL finds an evaluation order which gives results whenever possible, ignoring errors or even missing data that might occur in other evaluation orders. Applications like IAM conditions rely on this property to minimize the cost of evaluation, deferring the gathering of expensive inputs when a result can be reached without them. +See `exercise4.h` and `exercise4.cc`. Documentation to be added later. diff --git a/codelab/cel_compiler.h b/codelab/cel_compiler.h deleted file mode 100644 index 0ff2f699b..000000000 --- a/codelab/cel_compiler.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef THIRD_PARTY_CEL_CPP_CODELAB_COMPILER_H_ -#define THIRD_PARTY_CEL_CPP_CODELAB_COMPILER_H_ - -#include "cel/expr/checked.pb.h" -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/strings/string_view.h" -#include "checker/validation_result.h" -#include "common/ast_proto.h" -#include "compiler/compiler.h" -#include "internal/status_macros.h" - -namespace cel_codelab { - -// Helper for compiling expression and converting to proto. -// -// Simplifies error handling for brevity in the codelab. -inline absl::StatusOr CompileToCheckedExpr( - const cel::Compiler& compiler, absl::string_view expr) { - CEL_ASSIGN_OR_RETURN(cel::ValidationResult result, compiler.Compile(expr)); - - if (!result.IsValid() || result.GetAst() == nullptr) { - return absl::InvalidArgumentError(result.FormatError()); - } - - cel::expr::CheckedExpr pb; - CEL_RETURN_IF_ERROR(cel::AstToCheckedExpr(*result.GetAst(), &pb)); - return pb; -}; - -} // namespace cel_codelab - -#endif // THIRD_PARTY_CEL_CPP_CODELAB_COMPILER_H_ diff --git a/codelab/cel_compiler_test.cc b/codelab/cel_compiler_test.cc deleted file mode 100644 index 635b4d54d..000000000 --- a/codelab/cel_compiler_test.cc +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "codelab/cel_compiler.h" - -#include -#include - -#include "google/rpc/context/attribute_context.pb.h" -#include "absl/log/absl_check.h" -#include "absl/status/status.h" -#include "absl/status/status_matchers.h" -#include "common/decl.h" -#include "common/type.h" -#include "compiler/compiler.h" -#include "compiler/compiler_factory.h" -#include "compiler/standard_library.h" -#include "eval/public/activation.h" -#include "eval/public/activation_bind_helper.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_function_adapter.h" -#include "eval/public/cel_value.h" -#include "eval/public/testing/matchers.h" -#include "internal/testing.h" -#include "google/protobuf/arena.h" -#include "google/protobuf/descriptor.h" -#include "google/protobuf/message.h" - -namespace cel_codelab { -namespace { - -using ::absl_testing::IsOk; -using ::absl_testing::StatusIs; -using ::cel::BoolType; -using ::cel::MakeFunctionDecl; -using ::cel::MakeOverloadDecl; -using ::cel::MakeVariableDecl; -using ::cel::StringType; -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::BindProtoToActivation; -using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::FunctionAdapter; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; -using ::google::api::expr::runtime::test::IsCelBool; -using ::google::rpc::context::AttributeContext; -using ::testing::HasSubstr; - -std::unique_ptr MakeDefaultCompilerBuilder() { - google::protobuf::LinkMessageReflection(); - auto builder = - cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool()); - ABSL_CHECK_OK(builder.status()); - - ABSL_CHECK_OK((*builder)->AddLibrary(cel::StandardCompilerLibrary())); - ABSL_CHECK_OK((*builder)->GetCheckerBuilder().AddContextDeclaration( - "google.rpc.context.AttributeContext")); - - return std::move(builder).value(); -} - -TEST(DefaultCompiler, Basic) { - ASSERT_OK_AND_ASSIGN(auto compiler, MakeDefaultCompilerBuilder()->Build()); - EXPECT_THAT(compiler->Compile("1 < 2").status(), IsOk()); -} - -TEST(DefaultCompiler, AddFunctionDecl) { - auto builder = MakeDefaultCompilerBuilder(); - ASSERT_OK_AND_ASSIGN( - cel::FunctionDecl decl, - MakeFunctionDecl("IpMatch", - MakeOverloadDecl("IpMatch_string_string", BoolType(), - StringType(), StringType()))); - EXPECT_THAT(builder->GetCheckerBuilder().AddFunction(decl), IsOk()); - - ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); - - EXPECT_THAT(CompileToCheckedExpr( - *compiler, "IpMatch('255.255.255.255', '255.255.255.255')") - .status(), - IsOk()); - EXPECT_THAT( - CompileToCheckedExpr(*compiler, "IpMatch('255.255.255.255', 123436)") - .status(), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("no matching overload"))); -} - -TEST(DefaultCompiler, EndToEnd) { - google::protobuf::Arena arena; - - auto compiler_builder = MakeDefaultCompilerBuilder(); - ASSERT_OK_AND_ASSIGN( - cel::FunctionDecl func_decl, - MakeFunctionDecl("MyFunc", MakeOverloadDecl("MyFunc", BoolType()))); - ASSERT_THAT(compiler_builder->GetCheckerBuilder().AddFunction(func_decl), - IsOk()); - - ASSERT_THAT(compiler_builder->GetCheckerBuilder().AddVariable( - MakeVariableDecl("my_var", BoolType())), - IsOk()); - - ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); - - ASSERT_OK_AND_ASSIGN( - auto expr, - CompileToCheckedExpr( - *compiler, - "(my_var || MyFunc()) && request.host == 'www.google.com'")); - - auto builder = - CreateCelExpressionBuilder(google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory()); - ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry()), IsOk()); - ASSERT_THAT(FunctionAdapter::CreateAndRegister( - "MyFunc", false, [](google::protobuf::Arena*) { return true; }, - builder->GetRegistry()), - IsOk()); - - ASSERT_OK_AND_ASSIGN(auto plan, builder->CreateExpression(&expr)); - - AttributeContext context; - context.mutable_request()->set_host("www.google.com"); - Activation activation; - ASSERT_THAT(BindProtoToActivation(&context, &arena, &activation), IsOk()); - activation.InsertValue("my_var", CelValue::CreateBool(false)); - - ASSERT_OK_AND_ASSIGN(CelValue result, plan->Evaluate(activation, &arena)); - - EXPECT_THAT(result, IsCelBool(true)); -} - -} // namespace -} // namespace cel_codelab diff --git a/codelab/checked_expr_conversion_example.h b/codelab/checked_expr_conversion_example.h new file mode 100644 index 000000000..fc3783ddd --- /dev/null +++ b/codelab/checked_expr_conversion_example.h @@ -0,0 +1,59 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_CODELAB_CHECKED_EXPR_CONVERSION_EXAMPLE_H_ +#define THIRD_PARTY_CEL_CPP_CODELAB_CHECKED_EXPR_CONVERSION_EXAMPLE_H_ + +#include + +#include "cel/expr/checked.pb.h" +#include "absl/status/statusor.h" +#include "common/ast.h" +#include "common/ast_proto.h" +#include "internal/status_macros.h" + +namespace cel_codelab { + +// Examples demonstrating how to convert between protobuf-based AST +// representations (cel::expr::CheckedExpr) and modern runtime AST +// representations (cel::Ast). +// +// When evaluating expressions with modern `cel::Runtime`, compilation via +// `cel::Compiler::Compile` returns a `cel::ValidationResult` holding a +// `cel::Ast` that can be passed directly to `runtime->CreateProgram(...)`. +// +// However, when working with external services or storage layers that persist +// or transmit serialized protobuf ASTs (`cel::expr::CheckedExpr`), +// conversion functions in `common/ast_proto.h` can be used +// to move between `cel::Ast` and `cel::expr::CheckedExpr`. + +// Example: Convert a runtime `cel::Ast` to a `cel::expr::CheckedExpr` +// proto. +inline absl::StatusOr ConvertAstToCheckedExpr( + const cel::Ast& ast) { + cel::expr::CheckedExpr checked_expr; + CEL_RETURN_IF_ERROR(cel::AstToCheckedExpr(ast, &checked_expr)); + return checked_expr; +} + +// Example: Convert a `cel::expr::CheckedExpr` proto to a runtime +// `cel::Ast`. +inline absl::StatusOr> ConvertCheckedExprToAst( + const cel::expr::CheckedExpr& checked_expr) { + return cel::CreateAstFromCheckedExpr(checked_expr); +} + +} // namespace cel_codelab + +#endif // THIRD_PARTY_CEL_CPP_CODELAB_CHECKED_EXPR_CONVERSION_EXAMPLE_H_ diff --git a/codelab/checked_expr_conversion_example_test.cc b/codelab/checked_expr_conversion_example_test.cc new file mode 100644 index 000000000..31fb41a55 --- /dev/null +++ b/codelab/checked_expr_conversion_example_test.cc @@ -0,0 +1,60 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "codelab/checked_expr_conversion_example.h" + +#include +#include + +#include "cel/expr/checked.pb.h" +#include "absl/status/status_matchers.h" +#include "checker/validation_result.h" +#include "common/ast.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" +#include "internal/testing.h" +#include "google/protobuf/descriptor.h" + +namespace cel_codelab { +namespace { + +using ::absl_testing::IsOk; + +TEST(CheckedExprConversionExampleTest, ConvertAstToCheckedExprAndBack) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(cel::StandardCompilerLibrary()), IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + std::move(builder)->Build()); + + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler->Compile("1 + 2 == 3")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + validation_result.ReleaseAst()); + + // Convert cel::Ast to CheckedExpr proto + ASSERT_OK_AND_ASSIGN(cel::expr::CheckedExpr checked_expr, + ConvertAstToCheckedExpr(*ast)); + + // Convert CheckedExpr proto back to cel::Ast + ASSERT_OK_AND_ASSIGN(std::unique_ptr roundtrip_ast, + ConvertCheckedExprToAst(checked_expr)); + EXPECT_NE(roundtrip_ast, nullptr); +} + +} // namespace +} // namespace cel_codelab diff --git a/codelab/exercise1.cc b/codelab/exercise1.cc index de7ccf6e0..7ca81451b 100644 --- a/codelab/exercise1.cc +++ b/codelab/exercise1.cc @@ -14,69 +14,74 @@ #include "codelab/exercise1.h" -#include +#include // IWYU pragma: keep #include +#include // IWYU pragma: keep -#include "cel/expr/syntax.pb.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "eval/public/activation.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_options.h" -#include "eval/public/cel_value.h" -#include "internal/status_macros.h" -#include "parser/parser.h" +#include "checker/validation_result.h" // IWYU pragma: keep, needed for codelab solution +#include "common/ast.h" // IWYU pragma: keep +#include "common/minimal_descriptor_pool.h" // IWYU pragma: keep +#include "common/value.h" +#include "compiler/compiler.h" // IWYU pragma: keep +#include "compiler/compiler_factory.h" // IWYU pragma: keep +#include "compiler/standard_library.h" // IWYU pragma: keep +#include "internal/status_macros.h" // IWYU pragma: keep +#include "runtime/activation.h" +#include "runtime/runtime.h" // IWYU pragma: keep +#include "runtime/runtime_builder.h" // IWYU pragma: keep +#include "runtime/runtime_options.h" // IWYU pragma: keep +#include "runtime/standard_runtime_builder_factory.h" // IWYU pragma: keep #include "google/protobuf/arena.h" namespace cel_codelab { namespace { -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::CelValue; - -// Convert the CelResult to a C++ string if it is string typed. Otherwise, -// return invalid argument error. This takes a copy to avoid lifecycle concerns -// (the evaluator may represent strings as stringviews backed by the input -// expression). -absl::StatusOr ConvertResult(const CelValue& value) { - if (CelValue::StringHolder inner_value; value.GetValue(&inner_value)) { - return std::string(inner_value.value()); - } else { - return absl::InvalidArgumentError(absl::StrCat( - "expected string result got '", CelValue::TypeName(value.type()), "'")); +// Convert the cel::Value to a C++ string if it is string typed. Otherwise, +// return invalid argument error. +absl::StatusOr ConvertResult(const cel::Value& value) { + if (value.IsString()) { + return value.GetString().ToString(); } + return absl::InvalidArgumentError( + absl::StrCat("expected string result got '", value.GetTypeName(), "'")); } + } // namespace absl::StatusOr ParseAndEvaluate(absl::string_view cel_expr) { // === Start Codelab === - // Parse the expression using ::google::api::expr::parser::Parse; - // This will return a cel::expr::ParsedExpr message. + // 1. Setup a default compiler for compiling expressions: + // Use cel::NewCompilerBuilder(cel::GetMinimalDescriptorPool()) and add + // cel::StandardCompilerLibrary(). Build the cel::Compiler. + + // 2. Compile the expression using compiler->Compile(cel_expr). + // Check that the resulting validation_result.IsValid(). - // Setup a default environment for building expressions. - // std::unique_ptr builder = - // CreateCelExpressionBuilder(options); + // 3. Setup a standard runtime for evaluating expressions: + // Use cel::CreateStandardRuntimeBuilder(cel::GetMinimalDescriptorPool(), + // options) and build the cel::Runtime. - // Register standard functions. - // CEL_RETURN_IF_ERROR( - // RegisterBuiltinFunctions(builder->GetRegistry(), options)); + // 4. Create an executable program from the compiled AST: + // Use CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + // validation_result.ReleaseAst()) and + // runtime->CreateProgram(std::move(ast)). - // The evaluator uses a proto Arena for incidental allocations during - // evaluation. + // The evaluator uses a proto Arena for allocations during evaluation. google::protobuf::Arena arena; - // The activation provides variables and functions that are bound into the + // The activation provides variables and functions bound into the // expression environment. In this example, there's no context expected, so - // we just provide an empty one to the evaluator. - Activation activation; + // we provide an empty activation. + cel::Activation activation; + (void)arena; + (void)activation; - // Using the CelExpressionBuilder and the ParseExpr, create an execution plan - // (google::api::expr::runtime::CelExpression), evaluate, and return the - // result. Use the provided helper function ConvertResult to copy the value - // for return. + // 5. Evaluate the program and convert the result: + // Call program->Evaluate(&arena, activation) and pass the resulting + // cel::Value to ConvertResult. return absl::UnimplementedError("Not yet implemented"); // === End Codelab === } diff --git a/codelab/exercise10.cc b/codelab/exercise10.cc index 37eaa7642..bbe202cb3 100644 --- a/codelab/exercise10.cc +++ b/codelab/exercise10.cc @@ -23,6 +23,7 @@ #include "absl/strings/string_view.h" #include "checker/validation_result.h" #include "codelab/network_functions.h" +#include "common/ast.h" #include "common/decl.h" #include "common/minimal_descriptor_pool.h" #include "common/type.h" @@ -30,6 +31,7 @@ #include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "compiler/standard_library.h" +#include "internal/status_macros.h" #include "runtime/activation.h" #include "runtime/runtime.h" #include "runtime/runtime_builder.h" @@ -95,8 +97,9 @@ absl::StatusOr CompileAndEvaluateExercise10(absl::string_view expression, return absl::InvalidArgumentError(checked->FormatError()); } + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, checked->ReleaseAst()); absl::StatusOr> program = - (*runtime)->CreateProgram(checked->ReleaseAst().value()); + (*runtime)->CreateProgram(std::move(ast)); if (!program.ok()) { return std::move(program).status(); diff --git a/codelab/exercise2.cc b/codelab/exercise2.cc index 373f63365..dd73dae62 100644 --- a/codelab/exercise2.cc +++ b/codelab/exercise2.cc @@ -15,25 +15,29 @@ #include "codelab/exercise2.h" #include +#include -#include "cel/expr/checked.pb.h" -#include "cel/expr/syntax.pb.h" #include "google/rpc/context/attribute_context.pb.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "codelab/cel_compiler.h" +#include "checker/type_checker_builder.h" // IWYU pragma: keep, needed for codelab solution +#include "checker/validation_result.h" +#include "common/ast.h" +#include "common/decl.h" // IWYU pragma: keep +#include "common/type.h" // IWYU pragma: keep +#include "common/value.h" #include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "compiler/standard_library.h" -#include "eval/public/activation.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_options.h" -#include "eval/public/cel_value.h" #include "internal/status_macros.h" +#include "runtime/activation.h" +#include "runtime/bind_proto_to_activation.h" // IWYU pragma: keep +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -41,15 +45,6 @@ namespace cel_codelab { namespace { -using ::cel::expr::CheckedExpr; -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::CelError; -using ::google::api::expr::runtime::CelExpression; -using ::google::api::expr::runtime::CelExpressionBuilder; -using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::InterpreterOptions; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; using ::google::rpc::context::AttributeContext; absl::StatusOr> MakeCelCompiler() { @@ -65,45 +60,41 @@ absl::StatusOr> MakeCelCompiler() { // === Start Codelab === // Add 'AttributeContext' as a context message to the type checker and a // boolean variable 'bool_var'. Relevant functions are on the - // TypeCheckerBuilder class (see CompilerBuilder::GetCheckerBuilder). + // cel::TypeCheckerBuilder class (see builder->GetCheckerBuilder()). // // We're reusing the same compiler for both evaluation paths here for brevity, // but it's likely a better fit to configure a separate compiler per use case. // === End Codelab === - return builder->Build(); + return std::move(builder)->Build(); } -// Parse a cel expression and evaluate it against the given activation and -// arena. -absl::StatusOr EvalCheckedExpr(const CheckedExpr& checked_expr, - const Activation& activation, - google::protobuf::Arena* arena) { - // Setup a default environment for building expressions. - InterpreterOptions options; - std::unique_ptr builder = CreateCelExpressionBuilder( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory(), options); - CEL_RETURN_IF_ERROR( - RegisterBuiltinFunctions(builder->GetRegistry(), options)); - - // Note, the expression_plan below is reusable for different inputs, but we - // create one just in time for evaluation here. - CEL_ASSIGN_OR_RETURN(std::unique_ptr expression_plan, - builder->CreateExpression(&checked_expr)); - - CEL_ASSIGN_OR_RETURN(CelValue result, - expression_plan->Evaluate(activation, arena)); - - if (bool value; result.GetValue(&value)) { - return value; - } else if (const CelError * value; result.GetValue(&value)) { - return *value; - } else { - return absl::InvalidArgumentError(absl::StrCat( - "expected 'bool' result got '", result.DebugString(), "'")); +// Evaluate a runtime cel::Ast against the given activation and arena. +absl::StatusOr EvalAst(std::unique_ptr ast, + const cel::Activation& activation, + google::protobuf::Arena* arena) { + cel::RuntimeOptions options; + CEL_ASSIGN_OR_RETURN(cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder( + google::protobuf::DescriptorPool::generated_pool(), options)); + CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + CEL_ASSIGN_OR_RETURN(cel::Value result, program->Evaluate(arena, activation)); + + if (result.IsBool()) { + return result.GetBool().NativeValue(); } + if (result.IsError()) { + return result.GetError().ToStatus(); + } + return absl::InvalidArgumentError( + absl::StrCat("expected 'bool' result got '", result.GetTypeName(), "'")); } + } // namespace absl::StatusOr CompileAndEvaluateWithBoolVar(absl::string_view cel_expr, @@ -111,16 +102,21 @@ absl::StatusOr CompileAndEvaluateWithBoolVar(absl::string_view cel_expr, CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, MakeCelCompiler()); - CEL_ASSIGN_OR_RETURN(CheckedExpr checked_expr, - CompileToCheckedExpr(*compiler, cel_expr)); + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(cel_expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } - Activation activation; + cel::Activation activation; google::protobuf::Arena arena; // === Start Codelab === - // Update the activation to bind the bool argument to 'bool_var' + // Update the activation to bind the bool argument to 'bool_var': + // activation.InsertOrAssignValue("bool_var", cel::BoolValue(bool_var)); // === End Codelab === - - return EvalCheckedExpr(checked_expr, activation, &arena); + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return EvalAst(std::move(ast), activation, &arena); } absl::StatusOr CompileAndEvaluateWithContext( @@ -128,16 +124,22 @@ absl::StatusOr CompileAndEvaluateWithContext( CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, MakeCelCompiler()); - CEL_ASSIGN_OR_RETURN(CheckedExpr checked_expr, - CompileToCheckedExpr(*compiler, cel_expr)); + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(cel_expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } - Activation activation; + cel::Activation activation; google::protobuf::Arena arena; // === Start Codelab === - // Update the activation to bind the AttributeContext. + // Update the activation to bind the AttributeContext using + // cel::BindProtoToActivation (in + // "runtime/bind_proto_to_activation.h"). // === End Codelab === - - return EvalCheckedExpr(checked_expr, activation, &arena); + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return EvalAst(std::move(ast), activation, &arena); } } // namespace cel_codelab diff --git a/codelab/exercise2.h b/codelab/exercise2.h index d4836dc2b..7e0b3b8c7 100644 --- a/codelab/exercise2.h +++ b/codelab/exercise2.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_CODELAB_EXERCISE1_H_ -#define THIRD_PARTY_CEL_CPP_CODELAB_EXERCISE1_H_ +#ifndef THIRD_PARTY_CEL_CPP_CODELAB_EXERCISE2_H_ +#define THIRD_PARTY_CEL_CPP_CODELAB_EXERCISE2_H_ #include "google/rpc/context/attribute_context.pb.h" #include "absl/status/statusor.h" @@ -37,4 +37,4 @@ absl::StatusOr CompileAndEvaluateWithContext( } // namespace cel_codelab -#endif // THIRD_PARTY_CEL_CPP_CODELAB_EXERCISE1_H_ +#endif // THIRD_PARTY_CEL_CPP_CODELAB_EXERCISE2_H_ diff --git a/codelab/exercise4.cc b/codelab/exercise4.cc index cf02a88bd..ff3e38ece 100644 --- a/codelab/exercise4.cc +++ b/codelab/exercise4.cc @@ -15,25 +15,31 @@ #include "codelab/exercise4.h" #include +#include -#include "cel/expr/checked.pb.h" #include "google/rpc/context/attribute_context.pb.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "codelab/cel_compiler.h" +#include "absl/types/optional.h" // IWYU pragma: keep, needed for codelab solution +#include "checker/type_checker_builder.h" +#include "checker/validation_result.h" +#include "common/ast.h" +#include "common/decl.h" // IWYU pragma: keep +#include "common/type.h" // IWYU pragma: keep +#include "common/value.h" #include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "compiler/standard_library.h" -#include "eval/public/activation.h" -#include "eval/public/activation_bind_helper.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_options.h" -#include "eval/public/cel_value.h" #include "internal/status_macros.h" +#include "runtime/activation.h" +#include "runtime/bind_proto_to_activation.h" +#include "runtime/function_adapter.h" // IWYU pragma: keep +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -41,21 +47,25 @@ namespace cel_codelab { namespace { -using ::cel::expr::CheckedExpr; -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::BindProtoToActivation; -using ::google::api::expr::runtime::CelError; -using ::google::api::expr::runtime::CelExpressionBuilder; -using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::InterpreterOptions; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; using ::google::rpc::context::AttributeContext; +absl::StatusOr ContainsExtensionFunction( + const cel::MapValue& map, const cel::StringValue& key, + const cel::Value& value, const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, google::protobuf::Arena* arena) { + (void)map; + (void)key; + (void)value; + (void)descriptor_pool; + (void)message_factory; + (void)arena; + // Implement the contains function: check if key is in map and if its value + // matches `value`. + return absl::UnimplementedError("Not yet implemented"); +} + absl::StatusOr> MakeConfiguredCompiler() { - // Setup for handling for protobuf types. - // Using the generated descriptor pool is simpler to configure, but often - // adds more types than necessary. + // Setup for handling protobuf types. google::protobuf::LinkMessageReflection(); CEL_ASSIGN_OR_RETURN( std::unique_ptr builder, @@ -67,48 +77,54 @@ absl::StatusOr> MakeConfiguredCompiler() { // Codelab part 1: // Add a declaration for the map.contains(string, V) function. - // Hint: use cel::MakeFunctionDecl and cel::TypeCheckerBuilder::MergeFunction. - return builder->Build(); + // Hint: use cel::MakeFunctionDecl and cel::MakeMemberOverloadDecl. + return std::move(builder)->Build(); } class Evaluator { public: - Evaluator() { - builder_ = CreateCelExpressionBuilder( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory(), options_); - } + Evaluator() = default; absl::Status SetupEvaluatorEnvironment() { - CEL_RETURN_IF_ERROR(RegisterBuiltinFunctions(builder_->GetRegistry())); + cel::RuntimeOptions options; + CEL_ASSIGN_OR_RETURN( + cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder( + google::protobuf::DescriptorPool::generated_pool(), options)); // Codelab part 2: // Register the map.contains(string, value) function. - // Hint: use `CelFunctionAdapter::CreateAndRegister` to adapt from a free - // function ContainsExtensionFunction. + // Hint: use `TernaryFunctionAdapter::RegisterMemberOverload` to adapt + // from a free function ContainsExtensionFunction. + CEL_ASSIGN_OR_RETURN(runtime_, std::move(runtime_builder).Build()); return absl::OkStatus(); } - absl::StatusOr Evaluate(const CheckedExpr& expr, + absl::StatusOr Evaluate(std::unique_ptr ast, const AttributeContext& context) { - Activation activation; - CEL_RETURN_IF_ERROR(BindProtoToActivation(&context, &arena_, &activation)); - CEL_ASSIGN_OR_RETURN(auto plan, builder_->CreateExpression(&expr)); - CEL_ASSIGN_OR_RETURN(CelValue result, plan->Evaluate(activation, &arena_)); - - if (bool value; result.GetValue(&value)) { - return value; - } else if (const CelError * value; result.GetValue(&value)) { - return *value; - } else { - return absl::InvalidArgumentError( - absl::StrCat("unexpected return type: ", result.DebugString())); + cel::Activation activation; + CEL_RETURN_IF_ERROR(cel::BindProtoToActivation( + context, cel::BindProtoUnsetFieldBehavior::kBindDefaultValue, + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), &arena_, &activation)); + + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime_->CreateProgram(std::move(ast))); + CEL_ASSIGN_OR_RETURN(cel::Value result, + program->Evaluate(&arena_, activation)); + + if (result.IsBool()) { + return result.GetBool().NativeValue(); } + if (result.IsError()) { + return result.GetError().ToStatus(); + } + return absl::InvalidArgumentError( + absl::StrCat("unexpected return type: ", result.GetTypeName())); } private: google::protobuf::Arena arena_; - std::unique_ptr builder_; - InterpreterOptions options_; + std::unique_ptr runtime_; }; } // namespace @@ -118,15 +134,20 @@ absl::StatusOr EvaluateWithExtensionFunction( // Prepare a checked expression. CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, MakeConfiguredCompiler()); - CEL_ASSIGN_OR_RETURN(auto checked_expr, - CompileToCheckedExpr(*compiler, expr)); + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } // Prepare an evaluation environment. Evaluator evaluator; CEL_RETURN_IF_ERROR(evaluator.SetupEvaluatorEnvironment()); - // Evaluate a checked expression against a particular activation - return evaluator.Evaluate(checked_expr, context); + // Evaluate the checked AST against a particular activation. + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return evaluator.Evaluate(std::move(ast), context); } } // namespace cel_codelab diff --git a/codelab/network_functions.cc b/codelab/network_functions.cc index f4f729827..170e4243b 100644 --- a/codelab/network_functions.cc +++ b/codelab/network_functions.cc @@ -213,8 +213,7 @@ absl::Status NetworkAddressRepEqual( return absl::OkStatus(); } const NetworkAddressRep rep = content.To(); - absl::optional other_rep = - NetworkAddressRep::Unwrap(other); + std::optional other_rep = NetworkAddressRep::Unwrap(other); ABSL_DCHECK(other_rep.has_value()); *result = cel::BoolValue(rep.IsEqualTo(*other_rep)); return absl::OkStatus(); @@ -311,9 +310,10 @@ cel::Value parseAddress( google::protobuf::Arena* absl_nonnull arena) { std::string buf; absl::string_view addr = str.ToStringView(&buf); - absl::optional rep = NetworkAddressRep::Parse(addr); + std::optional rep = NetworkAddressRep::Parse(addr); if (!rep.has_value()) { - return cel::ErrorValue(absl::InvalidArgumentError("invalid address")); + return cel::ErrorValue::From(absl::InvalidArgumentError("invalid address"), + arena); } return NetworkAddressRep::MakeValue(*rep); } @@ -321,7 +321,7 @@ cel::Value parseAddress( cel::Value parseAddressOrZero(const cel::StringValue& str) { std::string buf; absl::string_view addr = str.ToStringView(&buf); - absl::optional rep = NetworkAddressRep::Parse(addr); + std::optional rep = NetworkAddressRep::Parse(addr); static const NetworkAddressRep kZero; if (!rep.has_value()) { return NetworkAddressRep::MakeValue(kZero); @@ -336,24 +336,27 @@ cel::Value parseAddressMatcher( google::protobuf::Arena* absl_nonnull arena) { std::string buf; absl::string_view addr = str.ToStringView(&buf); - absl::optional rep = - NetworkAddressMatcher::Parse(addr); + std::optional rep = NetworkAddressMatcher::Parse(addr); if (!rep.has_value()) { - return cel::ErrorValue( - absl::InvalidArgumentError("invalid address matcher")); + return cel::ErrorValue::From( + absl::InvalidArgumentError("invalid address matcher"), arena); } return NetworkAddressMatcher::MakeValue(arena, std::move(rep).value()); } -cel::Value containsAddress(const cel::OpaqueValue& matcher, - const cel::OpaqueValue& addr) { +cel::Value containsAddress( + const cel::OpaqueValue& matcher, const cel::OpaqueValue& addr, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) { const auto* matcher_rep = NetworkAddressMatcher::Unwrap(matcher); auto addr_rep = NetworkAddressRep::Unwrap(addr); if (matcher_rep == nullptr || !addr_rep.has_value()) { // dispatcher should catch this, but right now only distiguishes at the // kind level. - return cel::ErrorValue(absl::InvalidArgumentError("no matching overload")); + return cel::ErrorValue::From( + absl::InvalidArgumentError("no matching overload"), arena); } return cel::BoolValue(matcher_rep->Match(*addr_rep)); } @@ -365,12 +368,12 @@ cel::Value NetworkAddressRep::MakeValue(const NetworkAddressRep& rep) { cel::OpaqueValueContent::From(rep)); } -absl::optional NetworkAddressRep::Unwrap( +std::optional NetworkAddressRep::Unwrap( const cel::Value& value) { auto opaque = value.AsOpaque(); if (!opaque.has_value() || opaque->GetTypeId() != cel::TypeId()) { - return absl::nullopt; + return std::nullopt; } // Note: safety depends on: @@ -381,16 +384,16 @@ absl::optional NetworkAddressRep::Unwrap( return opaque->content().To(); } -absl::optional NetworkAddressRep::Parse( +std::optional NetworkAddressRep::Parse( absl::string_view str) { uint32_t ipv4 = 0; char ipv6[16]; auto version = ParseAddressImpl(str, &ipv4, ipv6); if (!version.ok()) { - return absl::nullopt; + return std::nullopt; } if (*version != IpVersion::kIPv4) { - return absl::nullopt; + return std::nullopt; } NetworkAddressRep rep; rep.version_ = *version; @@ -418,13 +421,13 @@ bool NetworkAddressRep::IsLessThan(const NetworkAddressRep& other) const { return false; } -absl::optional NetworkAddressMatcher::Parse( +std::optional NetworkAddressMatcher::Parse( absl::string_view str) { // range style addr-addr int dash_pos = str.find('-'); if (dash_pos == absl::string_view::npos) { // TODO(uncreated-issue/86): CIDR style addr/prefix-length - return absl::nullopt; + return std::nullopt; } absl::string_view min_str = str.substr(0, dash_pos); absl::string_view max_str = str.substr(dash_pos + 1); @@ -433,23 +436,23 @@ absl::optional NetworkAddressMatcher::Parse( NetworkRangev6 v6; auto min_parse = ParseAddressImpl(min_str, &v4.min_incl, v6.min_incl); if (!min_parse.ok()) { - return absl::nullopt; + return std::nullopt; } auto max_parse = ParseAddressImpl(max_str, &v4.max_incl, v6.max_incl); if (!max_parse.ok()) { - return absl::nullopt; + return std::nullopt; } if (*min_parse != *max_parse) { - return absl::nullopt; + return std::nullopt; } NetworkAddressMatcher rep; if (*min_parse == IpVersion::kIPv4) { if (v4.min_incl > v4.max_incl) { - return absl::nullopt; + return std::nullopt; } rep.ranges_v4_.push_back(v4); } else if (*min_parse == IpVersion::kIPv6) { - return absl::nullopt; + return std::nullopt; } return rep; diff --git a/codelab/solutions/BUILD b/codelab/solutions/BUILD index a1597e182..d86ae8fe5 100644 --- a/codelab/solutions/BUILD +++ b/codelab/solutions/BUILD @@ -24,18 +24,22 @@ cc_library( srcs = ["exercise1.cc"], hdrs = ["//codelab:exercise1.h"], deps = [ - "//eval/public:activation", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_expression", - "//eval/public:cel_options", - "//eval/public:cel_value", + "//checker:validation_result", + "//common:ast", + "//common:minimal_descriptor_pool", + "//common:value", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", "//internal:status_macros", - "//parser", + "//runtime", + "//runtime:activation", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", ], ) @@ -56,25 +60,24 @@ cc_library( hdrs = ["//codelab:exercise2.h"], deps = [ "//checker:type_checker_builder", - "//codelab:cel_compiler", + "//checker:validation_result", + "//common:ast", "//common:decl", "//common:type", + "//common:value", "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", - "//eval/public:activation", - "//eval/public:activation_bind_helper", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_expression", - "//eval/public:cel_options", - "//eval/public:cel_value", "//internal:status_macros", + "//runtime", + "//runtime:activation", + "//runtime:bind_proto_to_activation", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", - "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", "@com_google_protobuf//:protobuf", ], @@ -110,26 +113,27 @@ cc_library( srcs = ["exercise4.cc"], hdrs = ["//codelab:exercise4.h"], deps = [ - "//codelab:cel_compiler", + "//checker:type_checker_builder", + "//checker:validation_result", + "//common:ast", "//common:decl", "//common:type", + "//common:value", "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", - "//eval/public:activation", - "//eval/public:activation_bind_helper", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_expression", - "//eval/public:cel_function_adapter", - "//eval/public:cel_options", - "//eval/public:cel_value", "//internal:status_macros", + "//runtime", + "//runtime:activation", + "//runtime:bind_proto_to_activation", + "//runtime:function_adapter", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:optional", - "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", "@com_google_protobuf//:protobuf", ], @@ -154,6 +158,7 @@ cc_library( deps = [ "//checker:validation_result", "//codelab:network_functions", + "//common:ast", "//common:decl", "//common:minimal_descriptor_pool", "//common:type", @@ -161,6 +166,7 @@ cc_library( "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", + "//internal:status_macros", "//runtime", "//runtime:activation", "//runtime:runtime_builder", diff --git a/codelab/solutions/exercise1.cc b/codelab/solutions/exercise1.cc index aef6c0efe..e2fc4443a 100644 --- a/codelab/solutions/exercise1.cc +++ b/codelab/solutions/exercise1.cc @@ -16,90 +16,86 @@ #include #include +#include -#include "cel/expr/syntax.pb.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "eval/public/activation.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_options.h" -#include "eval/public/cel_value.h" +#include "checker/validation_result.h" +#include "common/ast.h" +#include "common/minimal_descriptor_pool.h" +#include "common/value.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" #include "internal/status_macros.h" -#include "parser/parser.h" +#include "runtime/activation.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" #include "google/protobuf/arena.h" namespace cel_codelab { namespace { -using ::cel::expr::ParsedExpr; -using ::google::api::expr::parser::Parse; -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::CelExpression; -using ::google::api::expr::runtime::CelExpressionBuilder; -using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::InterpreterOptions; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; - -// Convert the CelResult to a C++ string if it is string typed. Otherwise, -// return invalid argument error. This takes a copy to avoid lifecycle concerns -// (the evaluator may represent strings as stringviews backed by the input -// expression). -absl::StatusOr ConvertResult(const CelValue& value) { - if (CelValue::StringHolder inner_value; value.GetValue(&inner_value)) { - return std::string(inner_value.value()); - } else { - return absl::InvalidArgumentError(absl::StrCat( - "expected string result got '", CelValue::TypeName(value.type()), "'")); +// Convert the cel::Value to a C++ string if it is string typed. Otherwise, +// return invalid argument error. +absl::StatusOr ConvertResult(const cel::Value& value) { + if (value.IsString()) { + return value.GetString().ToString(); } + return absl::InvalidArgumentError( + absl::StrCat("expected string result got '", value.GetTypeName(), "'")); } + } // namespace absl::StatusOr ParseAndEvaluate(absl::string_view cel_expr) { // === Start Codelab === - // Setup a default environment for building expressions. - InterpreterOptions options; - std::unique_ptr builder = - CreateCelExpressionBuilder(options); - + // Setup a default compiler for compiling expressions. + CEL_ASSIGN_OR_RETURN( + std::unique_ptr compiler_builder, + cel::NewCompilerBuilder(cel::GetMinimalDescriptorPool())); CEL_RETURN_IF_ERROR( - RegisterBuiltinFunctions(builder->GetRegistry(), options)); + compiler_builder->AddLibrary(cel::StandardCompilerLibrary())); + CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, + std::move(compiler_builder)->Build()); - // Parse the expression. This is fine for codelabs, but this skips the type - // checking phase. It won't check that functions and variables are available - // in the environment, and it won't handle certain ambiguous identifier - // expressions (e.g. container lookup vs namespaced name, packaged function - // vs. receiver call style function). - ParsedExpr parsed_expr; - CEL_ASSIGN_OR_RETURN(parsed_expr, Parse(cel_expr)); + // Compile the expression. + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(cel_expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } - // The evaluator uses a proto Arena for incidental allocations during - // evaluation. + // Setup a standard runtime for evaluating expressions. + cel::RuntimeOptions options; + CEL_ASSIGN_OR_RETURN(cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder( + cel::GetMinimalDescriptorPool(), options)); + CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + + // Build the executable program from the compiled AST. + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + // The evaluator uses a proto Arena for allocations during evaluation. google::protobuf::Arena arena; - // The activation provides variables and functions that are bound into the + // The activation provides variables and functions bound into the // expression environment. In this example, there's no context expected, so - // we just provide an empty one to the evaluator. - Activation activation; - - // Build the expression plan. This assumes that the source expression AST and - // the expression builder outlive the CelExpression object. - CEL_ASSIGN_OR_RETURN(std::unique_ptr expression_plan, - builder->CreateExpression(&parsed_expr.expr(), - &parsed_expr.source_info())); + // we provide an empty activation. + cel::Activation activation; - // Actually run the expression plan. We don't support any environment - // variables at the moment so just use an empty activation. - CEL_ASSIGN_OR_RETURN(CelValue result, - expression_plan->Evaluate(activation, &arena)); + // Run the program. + CEL_ASSIGN_OR_RETURN(cel::Value result, + program->Evaluate(&arena, activation)); - // Convert the result to a c++ string. CelValues may reference instances from - // either the input expression, or objects allocated on the arena, so we need - // to pass ownership (in this case by copying to a new instance and returning - // that). + // Convert the result to a C++ string. return ConvertResult(result); // === End Codelab === } diff --git a/codelab/solutions/exercise10.cc b/codelab/solutions/exercise10.cc index 0d2c197d6..6d13652da 100644 --- a/codelab/solutions/exercise10.cc +++ b/codelab/solutions/exercise10.cc @@ -23,6 +23,7 @@ #include "absl/strings/string_view.h" #include "checker/validation_result.h" #include "codelab/network_functions.h" +#include "common/ast.h" #include "common/decl.h" #include "common/minimal_descriptor_pool.h" #include "common/type.h" @@ -30,6 +31,7 @@ #include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "compiler/standard_library.h" +#include "internal/status_macros.h" #include "runtime/activation.h" #include "runtime/runtime.h" #include "runtime/runtime_builder.h" @@ -105,8 +107,9 @@ absl::StatusOr CompileAndEvaluateExercise10(absl::string_view expression, return absl::InvalidArgumentError(checked->FormatError()); } + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, checked->ReleaseAst()); absl::StatusOr> program = - (*runtime)->CreateProgram(checked->ReleaseAst().value()); + (*runtime)->CreateProgram(std::move(ast)); if (!program.ok()) { return std::move(program).status(); diff --git a/codelab/solutions/exercise2.cc b/codelab/solutions/exercise2.cc index d07645aed..9a2958e7d 100644 --- a/codelab/solutions/exercise2.cc +++ b/codelab/solutions/exercise2.cc @@ -15,29 +15,29 @@ #include "codelab/exercise2.h" #include +#include -#include "cel/expr/checked.pb.h" -#include "cel/expr/syntax.pb.h" #include "google/rpc/context/attribute_context.pb.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "checker/type_checker_builder.h" -#include "codelab/cel_compiler.h" +#include "checker/validation_result.h" +#include "common/ast.h" #include "common/decl.h" #include "common/type.h" +#include "common/value.h" #include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "compiler/standard_library.h" -#include "eval/public/activation.h" -#include "eval/public/activation_bind_helper.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_options.h" -#include "eval/public/cel_value.h" #include "internal/status_macros.h" +#include "runtime/activation.h" +#include "runtime/bind_proto_to_activation.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -45,16 +45,6 @@ namespace cel_codelab { namespace { -using ::cel::expr::CheckedExpr; -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::CelError; -using ::google::api::expr::runtime::CelExpression; -using ::google::api::expr::runtime::CelExpressionBuilder; -using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::InterpreterOptions; -using ::google::api::expr::runtime::ProtoUnsetFieldOptions; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; using ::google::rpc::context::AttributeContext; absl::StatusOr> MakeCelCompiler() { @@ -75,39 +65,38 @@ absl::StatusOr> MakeCelCompiler() { AttributeContext::descriptor()->full_name())); // === End Codelab === - return builder->Build(); + return std::move(builder)->Build(); } -// Parse a cel expression and evaluate it against the given activation and -// arena. -absl::StatusOr EvalCheckedExpr(const CheckedExpr& checked_expr, - const Activation& activation, - google::protobuf::Arena* arena) { - // Setup a default environment for building expressions. - InterpreterOptions options; - std::unique_ptr builder = CreateCelExpressionBuilder( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory(), options); - CEL_RETURN_IF_ERROR( - RegisterBuiltinFunctions(builder->GetRegistry(), options)); - - // Note, the expression_plan below is reusable for different inputs, but we +// Evaluate a runtime cel::Ast against the given activation and arena. +absl::StatusOr EvalAst(std::unique_ptr ast, + const cel::Activation& activation, + google::protobuf::Arena* arena) { + // Setup a default standard runtime. + cel::RuntimeOptions options; + CEL_ASSIGN_OR_RETURN(cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder( + google::protobuf::DescriptorPool::generated_pool(), options)); + CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + + // Note, the program plan below is reusable across different inputs, but we // create one just in time for evaluation here. - CEL_ASSIGN_OR_RETURN(std::unique_ptr expression_plan, - builder->CreateExpression(&checked_expr)); - - CEL_ASSIGN_OR_RETURN(CelValue result, - expression_plan->Evaluate(activation, arena)); - - if (bool value; result.GetValue(&value)) { - return value; - } else if (const CelError * value; result.GetValue(&value)) { - return *value; - } else { - return absl::InvalidArgumentError(absl::StrCat( - "expected 'bool' result got '", result.DebugString(), "'")); + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + CEL_ASSIGN_OR_RETURN(cel::Value result, program->Evaluate(arena, activation)); + + if (result.IsBool()) { + return result.GetBool().NativeValue(); + } + if (result.IsError()) { + return result.GetError().ToStatus(); } + return absl::InvalidArgumentError( + absl::StrCat("expected 'bool' result got '", result.GetTypeName(), "'")); } + } // namespace absl::StatusOr CompileAndEvaluateWithBoolVar(absl::string_view cel_expr, @@ -115,16 +104,21 @@ absl::StatusOr CompileAndEvaluateWithBoolVar(absl::string_view cel_expr, CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, MakeCelCompiler()); - CEL_ASSIGN_OR_RETURN(CheckedExpr checked_expr, - CompileToCheckedExpr(*compiler, cel_expr)); + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(cel_expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } - Activation activation; + cel::Activation activation; google::protobuf::Arena arena; // === Start Codelab === - activation.InsertValue("bool_var", CelValue::CreateBool(bool_var)); + activation.InsertOrAssignValue("bool_var", cel::BoolValue(bool_var)); // === End Codelab === - return EvalCheckedExpr(checked_expr, activation, &arena); + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return EvalAst(std::move(ast), activation, &arena); } absl::StatusOr CompileAndEvaluateWithContext( @@ -132,17 +126,24 @@ absl::StatusOr CompileAndEvaluateWithContext( CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, MakeCelCompiler()); - CEL_ASSIGN_OR_RETURN(CheckedExpr checked_expr, - CompileToCheckedExpr(*compiler, cel_expr)); + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(cel_expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } - Activation activation; + cel::Activation activation; google::protobuf::Arena arena; // === Start Codelab === - CEL_RETURN_IF_ERROR(BindProtoToActivation( - &context, &arena, &activation, ProtoUnsetFieldOptions::kBindDefault)); + CEL_RETURN_IF_ERROR(cel::BindProtoToActivation( + context, cel::BindProtoUnsetFieldBehavior::kBindDefaultValue, + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), &arena, &activation)); // === End Codelab === - return EvalCheckedExpr(checked_expr, activation, &arena); + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return EvalAst(std::move(ast), activation, &arena); } } // namespace cel_codelab diff --git a/codelab/solutions/exercise4.cc b/codelab/solutions/exercise4.cc index 244fdac05..bbeaff83b 100644 --- a/codelab/solutions/exercise4.cc +++ b/codelab/solutions/exercise4.cc @@ -15,29 +15,31 @@ #include "codelab/exercise4.h" #include +#include -#include "cel/expr/checked.pb.h" #include "google/rpc/context/attribute_context.pb.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "codelab/cel_compiler.h" +#include "checker/type_checker_builder.h" +#include "checker/validation_result.h" +#include "common/ast.h" #include "common/decl.h" #include "common/type.h" +#include "common/value.h" #include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "compiler/standard_library.h" -#include "eval/public/activation.h" -#include "eval/public/activation_bind_helper.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_function_adapter.h" -#include "eval/public/cel_options.h" -#include "eval/public/cel_value.h" #include "internal/status_macros.h" +#include "runtime/activation.h" +#include "runtime/bind_proto_to_activation.h" +#include "runtime/function_adapter.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -45,40 +47,24 @@ namespace cel_codelab { namespace { -using ::cel::expr::CheckedExpr; -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::BindProtoToActivation; -using ::google::api::expr::runtime::CelError; -using ::google::api::expr::runtime::CelExpressionBuilder; -using ::google::api::expr::runtime::CelMap; -using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::FunctionAdapter; -using ::google::api::expr::runtime::InterpreterOptions; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; using ::google::rpc::context::AttributeContext; -// Handle the parametric type overload with a single generic CelValue overload. -absl::StatusOr ContainsExtensionFunction(google::protobuf::Arena* arena, - const CelMap* map, - CelValue::StringHolder key, - const CelValue& value) { - absl::optional entry = (*map)[CelValue::CreateString(key)]; +absl::StatusOr ContainsExtensionFunction( + const cel::MapValue& map, const cel::StringValue& key, + const cel::Value& value, const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, google::protobuf::Arena* arena) { + CEL_ASSIGN_OR_RETURN(absl::optional entry, + map.Find(key, descriptor_pool, message_factory, arena)); if (!entry.has_value()) { return false; } - if (value.IsInt64() && entry->IsInt64()) { - return value.Int64OrDie() == entry->Int64OrDie(); - } else if (value.IsString() && entry->IsString()) { - return value.StringOrDie().value() == entry->StringOrDie().value(); - } - return false; + CEL_ASSIGN_OR_RETURN(cel::Value equal, entry->Equal(value, descriptor_pool, + message_factory, arena)); + return equal.IsBool() && equal.GetBool().NativeValue(); } absl::StatusOr> MakeConfiguredCompiler() { - // Setup for handling for protobuf types. - // Using the generated descriptor pool is simpler to configure, but often - // adds more types than necessary. + // Setup for handling protobuf types. google::protobuf::LinkMessageReflection(); CEL_ASSIGN_OR_RETURN( std::unique_ptr builder, @@ -99,59 +85,66 @@ absl::StatusOr> MakeConfiguredCompiler() { cel::MakeFunctionDecl( "contains", cel::MakeMemberOverloadDecl( - "map_contains_string_string", cel::BoolType(), + "map_contains_string_value", cel::BoolType(), cel::MapType(checker_builder.arena(), cel::StringType(), cel::TypeParamType("V")), cel::StringType(), cel::TypeParamType("V")))); // Note: we use MergeFunction instead of AddFunction because we are adding // an overload to an already declared function with the same name. CEL_RETURN_IF_ERROR(checker_builder.MergeFunction(decl)); - return builder->Build(); + return std::move(builder)->Build(); } class Evaluator { public: - Evaluator() { - builder_ = CreateCelExpressionBuilder( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory(), options_); - } + Evaluator() = default; absl::Status SetupEvaluatorEnvironment() { - CEL_RETURN_IF_ERROR(RegisterBuiltinFunctions(builder_->GetRegistry())); + cel::RuntimeOptions options; + CEL_ASSIGN_OR_RETURN( + cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder( + google::protobuf::DescriptorPool::generated_pool(), options)); // Codelab part 2: - // Register the map.contains(string, string) function. - // Hint: use `FunctionAdapter::CreateAndRegister` to adapt from a free - // function ContainsExtensionFunction. - using AdapterT = FunctionAdapter, const CelMap*, - CelValue::StringHolder, CelValue>; - CEL_RETURN_IF_ERROR(AdapterT::CreateAndRegister( - "contains", /*receiver_style=*/true, &ContainsExtensionFunction, - builder_->GetRegistry())); + // Register the map.contains(string, value) function. + // Hint: use `TernaryFunctionAdapter::RegisterMemberOverload` to adapt + // from a free function ContainsExtensionFunction. + using AdapterT = + cel::TernaryFunctionAdapter, const cel::MapValue&, + const cel::StringValue&, const cel::Value&>; + CEL_RETURN_IF_ERROR( + AdapterT::RegisterMemberOverload("contains", &ContainsExtensionFunction, + runtime_builder.function_registry())); + CEL_ASSIGN_OR_RETURN(runtime_, std::move(runtime_builder).Build()); return absl::OkStatus(); } - absl::StatusOr Evaluate(const CheckedExpr& expr, + absl::StatusOr Evaluate(std::unique_ptr ast, const AttributeContext& context) { - Activation activation; - CEL_RETURN_IF_ERROR(BindProtoToActivation(&context, &arena_, &activation)); - CEL_ASSIGN_OR_RETURN(auto plan, builder_->CreateExpression(&expr)); - CEL_ASSIGN_OR_RETURN(CelValue result, plan->Evaluate(activation, &arena_)); - - if (bool value; result.GetValue(&value)) { - return value; - } else if (const CelError* value; result.GetValue(&value)) { - return *value; - } else { - return absl::InvalidArgumentError( - absl::StrCat("unexpected return type: ", result.DebugString())); + cel::Activation activation; + CEL_RETURN_IF_ERROR(cel::BindProtoToActivation( + context, cel::BindProtoUnsetFieldBehavior::kBindDefaultValue, + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), &arena_, &activation)); + + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime_->CreateProgram(std::move(ast))); + CEL_ASSIGN_OR_RETURN(cel::Value result, + program->Evaluate(&arena_, activation)); + + if (result.IsBool()) { + return result.GetBool().NativeValue(); } + if (result.IsError()) { + return result.GetError().ToStatus(); + } + return absl::InvalidArgumentError( + absl::StrCat("unexpected return type: ", result.GetTypeName())); } private: google::protobuf::Arena arena_; - std::unique_ptr builder_; - InterpreterOptions options_; + std::unique_ptr runtime_; }; } // namespace @@ -161,15 +154,20 @@ absl::StatusOr EvaluateWithExtensionFunction( // Prepare a checked expression. CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, MakeConfiguredCompiler()); - CEL_ASSIGN_OR_RETURN(auto checked_expr, - CompileToCheckedExpr(*compiler, expr)); + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } // Prepare an evaluation environment. Evaluator evaluator; CEL_RETURN_IF_ERROR(evaluator.SetupEvaluatorEnvironment()); - // Evaluate a checked expression against a particular activation - return evaluator.Evaluate(checked_expr, context); + // Evaluate the checked AST against a particular activation. + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return evaluator.Evaluate(std::move(ast), context); } } // namespace cel_codelab diff --git a/common/BUILD b/common/BUILD index 0ead8b15a..c3a91034b 100644 --- a/common/BUILD +++ b/common/BUILD @@ -34,6 +34,39 @@ cc_library( ], ) +cc_library( + name = "expr_printer", + srcs = ["expr_printer.cc"], + hdrs = ["expr_printer.h"], + deps = [ + ":ast", + ":ast_proto", + ":constant", + ":expr", + "//internal:strings", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", + ], +) + +cc_test( + name = "expr_printer_test", + srcs = ["expr_printer_test.cc"], + deps = [ + ":expr", + ":expr_printer", + "//internal:testing", + "//parser", + "//parser:options", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/strings", + ], +) + cc_test( name = "ast_test", srcs = ["ast_test.cc"], @@ -46,6 +79,109 @@ cc_test( ], ) +cc_library( + name = "type_spec_resolver", + srcs = ["type_spec_resolver.cc"], + hdrs = ["type_spec_resolver.h"], + deps = [ + ":ast", + ":descriptor_pool_type_introspector", + ":type", + ":type_kind", + "//internal:status_macros", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "type_spec_resolver_test", + srcs = ["type_spec_resolver_test.cc"], + deps = [ + ":ast", + ":type", + ":type_kind", + ":type_spec_resolver", + "//internal:testing", + "//internal:testing_descriptor_pool", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_protobuf//:protobuf", + ], +) + +cc_library( + name = "descriptor_pool_type_introspector", + srcs = ["descriptor_pool_type_introspector.cc"], + hdrs = ["descriptor_pool_type_introspector.h"], + deps = [ + ":type", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "descriptor_pool_type_introspector_test", + srcs = ["descriptor_pool_type_introspector_test.cc"], + deps = [ + ":descriptor_pool_type_introspector", + ":type", + "//internal:testing", + "//internal:testing_descriptor_pool", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_library( + name = "signature", + srcs = ["signature.cc"], + hdrs = ["signature.h"], + deps = [ + ":ast", + ":type", + ":type_spec_resolver", + "//internal:status_macros", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "signature_test", + srcs = ["signature_test.cc"], + deps = [ + ":ast", + ":signature", + ":type", + ":type_kind", + ":type_spec_resolver", + "//internal:testing", + "//internal:testing_descriptor_pool", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_protobuf//:protobuf", + ], +) + cc_library( name = "expr", srcs = ["expr.cc"], @@ -112,15 +248,15 @@ cc_library( hdrs = ["decl.h"], deps = [ ":constant", + ":signature", ":type", ":type_kind", - "//common/internal:signature", "//internal:status_macros", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/hash", - "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -211,7 +347,6 @@ cc_library( ":constant", ":expr", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/types:variant", ], ) @@ -327,15 +462,6 @@ cc_test( ], ) -cc_library( - name = "casting", - hdrs = ["casting.h"], - deps = [ - "//common/internal:casting", - "@com_google_absl//absl/base:core_headers", - ], -) - cc_library( name = "json", hdrs = ["json.h"], @@ -368,16 +494,8 @@ cc_library( hdrs = ["memory.h"], deps = [ ":allocator", - ":arena", - ":data", - ":native_type", - ":reference_count", - "//common/internal:metadata", - "//common/internal:reference_count", - "//internal:exceptions", "//internal:to_address", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/numeric:bits", @@ -385,25 +503,6 @@ cc_library( ], ) -cc_test( - name = "memory_test", - srcs = ["memory_test.cc"], - deps = [ - ":allocator", - ":data", - ":memory", - ":native_type", - "//common/internal:reference_count", - "//internal:testing", - "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/debugging:leak_check", - "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/types:optional", - "@com_google_protobuf//:protobuf", - "@com_google_protobuf//:struct_cc_proto", - ], -) - cc_library( name = "memory_testing", testonly = True, @@ -458,6 +557,7 @@ cc_test( "//internal:testing", "@com_google_absl//absl/status", "@com_google_absl//absl/time", + "@com_google_protobuf//:protobuf", ], ) @@ -509,8 +609,9 @@ cc_test( deps = [ ":source", "//internal:testing", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings:cord", - "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/strings:string_view", ], ) @@ -551,7 +652,6 @@ cc_library( "//internal:string_pool", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -574,6 +674,17 @@ cc_library( ], ) +cc_library( + name = "format_type_name", + srcs = ["format_type_name.cc"], + hdrs = ["format_type_name.h"], + deps = [ + ":type", + ":type_kind", + "@com_google_absl//absl/strings", + ], +) + cc_test( name = "type_test", srcs = glob([ @@ -596,6 +707,18 @@ cc_test( ], ) +cc_test( + name = "format_type_name_test", + srcs = ["format_type_name_test.cc"], + deps = [ + ":format_type_name", + ":type", + "//internal:testing", + "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", + "@com_google_protobuf//:protobuf", + ], +) + cc_library( name = "value", srcs = glob( @@ -625,7 +748,6 @@ cc_library( ":allocator", ":any", ":arena", - ":casting", ":kind", ":memory", ":native_type", @@ -635,17 +757,17 @@ cc_library( ":unknown", ":value_kind", "//base:attributes", + "//base:function_result", + "//base:function_result_set", "//common/internal:byte_string", - "//common/internal:reference_count", + "//common/internal:unknowns", "//eval/internal:cel_value_equal", "//eval/public:cel_value", "//eval/public:message_wrapper", - "//eval/public/containers:field_backed_list_impl", - "//eval/public/containers:field_backed_map_impl", - "//eval/public/structs:cel_proto_wrap_util", - "//eval/public/structs:legacy_type_adapter", + "//eval/public/structs:cel_proto_wrap_value_to_message", "//eval/public/structs:legacy_type_info_apis", "//eval/public/structs:proto_message_type_adapter", + "//eval/public/structs:trivial_legacy_type_info_internal", "//extensions/protobuf/internal:map_reflection", "//extensions/protobuf/internal:qualify", "//internal:casts", @@ -680,6 +802,7 @@ cc_library( "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/time", "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:source_location", "@com_google_absl//absl/types:span", "@com_google_absl//absl/types:variant", "@com_google_absl//absl/utility", @@ -703,7 +826,6 @@ cc_test( "value_test.cc", ], deps = [ - ":casting", ":memory", ":native_type", ":type", @@ -711,6 +833,8 @@ cc_test( ":value_kind", ":value_testing", "//base:attributes", + "//eval/public:cel_value", + "//eval/public/structs:proto_message_type_adapter", "//internal:parse_text_proto", "//internal:status_macros", "//internal:testing", @@ -732,7 +856,9 @@ cc_test( "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/time", "@com_google_absl//absl/types:optional", + "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", + "@com_google_protobuf//:field_mask_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:struct_cc_proto", "@com_google_protobuf//:type_cc_proto", @@ -760,18 +886,11 @@ cc_library( ], ) -cc_library( - name = "reference_count", - hdrs = ["reference_count.h"], - deps = ["//common/internal:reference_count"], -) - cc_library( name = "allocator", hdrs = ["allocator.h"], deps = [ ":arena", - ":data", "//internal:new", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", @@ -793,29 +912,6 @@ cc_test( ], ) -cc_library( - name = "data", - hdrs = ["data.h"], - deps = [ - "//common/internal:metadata", - "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/log:absl_check", - "@com_google_protobuf//:protobuf", - ], -) - -cc_test( - name = "data_test", - srcs = ["data_test.cc"], - deps = [ - ":data", - "//common/internal:reference_count", - "//internal:testing", - "@com_google_absl//absl/base:nullability", - "@com_google_protobuf//:protobuf", - ], -) - cc_library( name = "optional_ref", hdrs = ["optional_ref.h"], @@ -942,8 +1038,10 @@ cc_library( "function_descriptor.h", ], deps = [ + ":ast", ":kind", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/functional:overload", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:span", ], @@ -992,9 +1090,6 @@ cc_library( deps = [ ":decl", ":decl_proto", - ":type", - ":type_proto", - "//internal:status_macros", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -1162,6 +1257,5 @@ cc_test( ":container", "//internal:testing", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:status_matchers", ], ) diff --git a/common/allocator.h b/common/allocator.h index 81d56b096..a6453d3f9 100644 --- a/common/allocator.h +++ b/common/allocator.h @@ -27,7 +27,6 @@ #include "absl/log/die_if_null.h" #include "absl/numeric/bits.h" #include "common/arena.h" -#include "common/data.h" #include "internal/new.h" #include "google/protobuf/arena.h" @@ -291,9 +290,6 @@ class ArenaAllocator { using U = std::remove_const_t; U* object; if constexpr (google::protobuf::Arena::is_arena_constructable::value) { - // Classes derived from `cel::Data` are manually allocated and constructed - // as those class support determining whether the destructor is skippable - // at runtime. object = google::protobuf::Arena::Create(arena(), std::forward(args)...); } else { if constexpr (ArenaTraits<>::constructible()) { @@ -309,8 +305,7 @@ class ArenaAllocator { } } } - if constexpr (google::protobuf::Arena::is_arena_constructable::value || - std::is_base_of_v) { + if constexpr (google::protobuf::Arena::is_arena_constructable::value) { ABSL_DCHECK_EQ(object->GetArena(), arena()); } return object; @@ -322,8 +317,7 @@ class ArenaAllocator { void delete_object(T* p) noexcept { using U = std::remove_const_t; ABSL_DCHECK(p != nullptr); - if constexpr (google::protobuf::Arena::is_arena_constructable::value || - std::is_base_of_v) { + if constexpr (google::protobuf::Arena::is_arena_constructable::value) { ABSL_DCHECK_EQ(p->GetArena(), arena()); } } diff --git a/common/ast/constant_proto.cc b/common/ast/constant_proto.cc index c0fe1c9f6..1982c05b4 100644 --- a/common/ast/constant_proto.cc +++ b/common/ast/constant_proto.cc @@ -35,7 +35,7 @@ using ConstantProto = cel::expr::Constant; absl::Status ConstantToProto(const Constant& constant, ConstantProto* absl_nonnull proto) { return absl::visit(absl::Overload( - [proto](absl::monostate) -> absl::Status { + [proto](std::monostate) -> absl::Status { proto->clear_constant_kind(); return absl::OkStatus(); }, diff --git a/common/ast/metadata.h b/common/ast/metadata.h index 197790ff3..1a69b5b50 100644 --- a/common/ast/metadata.h +++ b/common/ast/metadata.h @@ -573,6 +573,10 @@ class TypeSpec { TypeSpecKind& mutable_type_kind() { return type_kind_; } + bool is_specified() const { + return !absl::holds_alternative(type_kind_); + } + bool has_dyn() const { return absl::holds_alternative(type_kind_); } diff --git a/common/ast_rewrite.cc b/common/ast_rewrite.cc index 14582f44f..b61e1fab6 100644 --- a/common/ast_rewrite.cc +++ b/common/ast_rewrite.cc @@ -54,7 +54,7 @@ struct ExprRecord { }; using StackRecordKind = - absl::variant; + std::variant; struct StackRecord { public: diff --git a/common/ast_rewrite_test.cc b/common/ast_rewrite_test.cc index 5417b23ac..748bd2473 100644 --- a/common/ast_rewrite_test.cc +++ b/common/ast_rewrite_test.cc @@ -495,7 +495,7 @@ TEST(AstCrawlerTest, CheckExprRewriteHandlers) { // Simple rewrite that replaces a select path with a dot-qualified identifier. class RewriterExample : public AstRewriterBase { public: - RewriterExample() {} + RewriterExample() = default; bool PostVisitRewrite(Expr& expr) override { if (target_.has_value() && expr.id() == *target_) { expr.mutable_ident_expr().set_name("com.google.Identifier"); @@ -554,7 +554,7 @@ TEST(AstRewrite, SelectRewriteExample) { // both passes. class PreRewriterExample : public AstRewriterBase { public: - PreRewriterExample() {} + PreRewriterExample() = default; bool PreVisitRewrite(Expr& expr) override { if (expr.ident_expr().name() == "x") { expr.mutable_ident_expr().set_name("y"); diff --git a/common/ast_traverse.cc b/common/ast_traverse.cc index a6ba0d1ba..bfbabf01b 100644 --- a/common/ast_traverse.cc +++ b/common/ast_traverse.cc @@ -17,7 +17,6 @@ #include #include -#include "absl/log/absl_log.h" #include "absl/types/variant.h" #include "common/ast_visitor.h" #include "common/constant.h" @@ -53,7 +52,7 @@ struct ExprRecord { }; using StackRecordKind = - absl::variant; + std::variant; struct StackRecord { public: @@ -158,9 +157,7 @@ struct PostVisitor { void operator()(const ComprehensionExpr& comprehension) { visitor->PostVisitComprehension(*expr, expr->comprehension_expr()); } - void operator()(const UnspecifiedExpr&) { - ABSL_LOG(ERROR) << "Unsupported Expr kind"; - } + void operator()(const UnspecifiedExpr&) {} } handler{visitor, record.expr}; absl::visit(handler, record.expr->kind()); diff --git a/common/ast_traverse.h b/common/ast_traverse.h index 004727e49..21272a898 100644 --- a/common/ast_traverse.h +++ b/common/ast_traverse.h @@ -43,8 +43,6 @@ struct TraversalOptions { // while(!traversal.IsDone()) { // traversal.Step(visitor); // } -// -// This class is thread-hostile and should only be used in synchronous code. class AstTraversal { public: static AstTraversal Create(const cel::Expr& ast ABSL_ATTRIBUTE_LIFETIME_BOUND, diff --git a/common/casting.h b/common/casting.h deleted file mode 100644 index 69074d4d9..000000000 --- a/common/casting.h +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef THIRD_PARTY_CEL_CPP_COMMON_CASTING_H_ -#define THIRD_PARTY_CEL_CPP_COMMON_CASTING_H_ - -#include "absl/base/attributes.h" -#include "common/internal/casting.h" - -namespace cel { - -// `InstanceOf(const From&)` determines whether `From` holds or is `To`. -// -// `To` must be a plain non-union class type that is not qualified. -// -// We expose `InstanceOf` this way to avoid ADL. -// -// Example: -// -// if (InstanceOf(superclass)) { -// Cast(superclass).SomeMethod(); -// } -template -ABSL_DEPRECATED("Use Is member functions instead.") -inline constexpr common_internal::InstanceOfImpl InstanceOf{}; - -// `Cast(From)` is a "checked cast". In debug builds an assertion is emitted -// which verifies `From` is an instance-of `To`. In non-debug builds, invalid -// casts are undefined behavior. -// -// We expose `Cast` this way to avoid ADL. -// -// Example: -// -// if (InstanceOf(superclass)) { -// Cast(superclass).SomeMethod(); -// } -template -ABSL_DEPRECATED( - "Use explicit conversion functions instead through static_cast.") -inline constexpr common_internal::CastImpl Cast{}; - -// `As(From)` is a "checking cast". The result is explicitly convertible to -// `bool`, such that it can be used with `if` statements. The result can be -// accessed with `operator*` or `operator->`. The return type should be treated -// as an implementation detail, with no assumptions on the concrete type. You -// should use `auto`. -// -// `As` is analogous to the paradigm `if (InstanceOf(a)) Cast(a)`. -// -// We expose `As` this way to avoid ADL. -// -// Example: -// -// if (auto subclass = As(superclass); subclass) { -// subclass->SomeMethod(); -// } -template -ABSL_DEPRECATED("Use As member functions instead.") -inline constexpr common_internal::AsImpl As{}; - -} // namespace cel - -#endif // THIRD_PARTY_CEL_CPP_COMMON_INSTANCE_OF_H_ diff --git a/common/container.cc b/common/container.cc index f69f0cc80..e1db8f86c 100644 --- a/common/container.cc +++ b/common/container.cc @@ -19,6 +19,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "internal/lexis.h" @@ -92,9 +93,11 @@ absl::Status ExpressionContainer::SetContainer(absl::string_view name) { } absl::Status ExpressionContainer::AddAbbreviation(absl::string_view abrev) { + abrev = absl::StripAsciiWhitespace(abrev); if (!IsValidQualifiedName(abrev)) { return absl::InvalidArgumentError( - absl::StrCat("invalid qualified name: ", abrev)); + absl::StrCat("invalid qualified name: ", abrev, + ", wanted name of the form 'qualified.name'")); } auto pos = abrev.rfind('.'); diff --git a/common/data.h b/common/data.h deleted file mode 100644 index cefc21fa4..000000000 --- a/common/data.h +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef THIRD_PARTY_CEL_CPP_COMMON_DATA_H_ -#define THIRD_PARTY_CEL_CPP_COMMON_DATA_H_ - -#include -#include - -#include "absl/base/nullability.h" -#include "absl/log/absl_check.h" -#include "common/internal/metadata.h" -#include "google/protobuf/arena.h" - -namespace cel { - -class Data; -template -struct Ownable; -template -struct Borrowable; - -namespace common_internal { - -class ReferenceCount; - -void SetDataReferenceCount(const Data* absl_nonnull data, - const ReferenceCount* absl_nonnull refcount); - -const ReferenceCount* absl_nullable GetDataReferenceCount( - const Data* absl_nonnull data); - -} // namespace common_internal - -// `Data` is one of the base classes of objects that can be managed by -// `MemoryManager`, the other is `google::protobuf::MessageLite`. -class Data { - public: - Data(const Data&) = default; - Data(Data&&) = default; - ~Data() = default; - Data& operator=(const Data&) = default; - Data& operator=(Data&&) = default; - - google::protobuf::Arena* absl_nullable GetArena() const { - return (owner_ & kOwnerBits) == kOwnerArenaBit - ? reinterpret_cast(owner_ & kOwnerPointerMask) - : nullptr; - } - - protected: - // At this point, the reference count has not been created. So we create it - // unowned and set the reference count after. In theory we could create the - // reference count ahead of time and then update it with the data it has to - // delete, but that is a bit counter intuitive. Doing it this way is also - // similar to how std::enable_shared_from_this works. - Data() = default; - - Data(std::nullptr_t) = delete; - - explicit Data(google::protobuf::Arena* absl_nullable arena) - : owner_(reinterpret_cast(arena) | - (arena != nullptr ? kOwnerArenaBit : kOwnerNone)) {} - - private: - static constexpr uintptr_t kOwnerNone = common_internal::kMetadataOwnerNone; - static constexpr uintptr_t kOwnerReferenceCountBit = - common_internal::kMetadataOwnerReferenceCountBit; - static constexpr uintptr_t kOwnerArenaBit = - common_internal::kMetadataOwnerArenaBit; - static constexpr uintptr_t kOwnerBits = common_internal::kMetadataOwnerBits; - static constexpr uintptr_t kOwnerPointerMask = - common_internal::kMetadataOwnerPointerMask; - - friend void common_internal::SetDataReferenceCount( - const Data* absl_nonnull data, - const common_internal::ReferenceCount* absl_nonnull refcount); - friend const common_internal::ReferenceCount* absl_nullable - common_internal::GetDataReferenceCount(const Data* absl_nonnull data); - template - friend struct Ownable; - template - friend struct Borrowable; - - mutable uintptr_t owner_ = kOwnerNone; -}; - -namespace common_internal { - -inline void SetDataReferenceCount(const Data* absl_nonnull data, - const ReferenceCount* absl_nonnull refcount) { - ABSL_DCHECK_EQ(data->owner_, Data::kOwnerNone); - data->owner_ = - reinterpret_cast(refcount) | Data::kOwnerReferenceCountBit; -} - -inline const ReferenceCount* absl_nullable GetDataReferenceCount( - const Data* absl_nonnull data) { - return (data->owner_ & Data::kOwnerBits) == Data::kOwnerReferenceCountBit - ? reinterpret_cast(data->owner_ & - Data::kOwnerPointerMask) - : nullptr; -} - -} // namespace common_internal - -} // namespace cel - -#endif // THIRD_PARTY_CEL_CPP_COMMON_DATA_H_ diff --git a/common/data_test.cc b/common/data_test.cc deleted file mode 100644 index a6b2a788f..000000000 --- a/common/data_test.cc +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This header contains primitives for reference counting, roughly equivalent to -// the primitives used to implement `std::shared_ptr`. These primitives should -// not be used directly in most cases, instead `cel::ManagedMemory` should be -// used instead. - -#include "common/data.h" - -#include "absl/base/nullability.h" -#include "common/internal/reference_count.h" -#include "internal/testing.h" -#include "google/protobuf/arena.h" - -namespace cel { -namespace { - -using ::testing::IsNull; - -class DataTest final : public Data { - public: - DataTest() noexcept : Data() {} - - explicit DataTest(google::protobuf::Arena* absl_nullable arena) noexcept - : Data(arena) {} -}; - -class DataReferenceCount final : public common_internal::ReferenceCounted { - public: - explicit DataReferenceCount(const Data* data) : data_(data) {} - - private: - void Finalize() noexcept override { delete data_; } - - const Data* data_; -}; - -TEST(Data, Arena) { - google::protobuf::Arena arena; - DataTest data(&arena); - EXPECT_EQ(data.GetArena(), &arena); - EXPECT_THAT(common_internal::GetDataReferenceCount(&data), IsNull()); -} - -TEST(Data, ReferenceCount) { - auto* data = new DataTest(); - EXPECT_THAT(data->GetArena(), IsNull()); - auto* refcount = new DataReferenceCount(data); - common_internal::SetDataReferenceCount(data, refcount); - EXPECT_EQ(common_internal::GetDataReferenceCount(data), refcount); - common_internal::StrongUnref(refcount); -} - -} // namespace -} // namespace cel diff --git a/common/decl.cc b/common/decl.cc index b338bfd4f..858e6fb49 100644 --- a/common/decl.cc +++ b/common/decl.cc @@ -20,13 +20,13 @@ #include #include +#include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" -#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "common/internal/signature.h" +#include "common/signature.h" #include "common/type.h" #include "common/type_kind.h" @@ -109,43 +109,45 @@ bool SignaturesOverlap(const OverloadDecl& lhs, const OverloadDecl& rhs) { template void AddOverloadInternal(std::string_view function_name, std::vector& insertion_order, - OverloadDeclHashSet& overloads, Overload&& overload, - absl::Status& status) { + absl::flat_hash_map& by_id, + absl::flat_hash_map& by_signature, + Overload&& overload, absl::Status& status) { if (!status.ok()) { return; } - if (overload.id().empty()) { - OverloadDecl overload_decl = overload; - absl::StatusOr overload_id = - common_internal::MakeOverloadSignature( - function_name, overload_decl.args(), overload_decl.member()); - if (!overload_id.ok()) { - status = overload_id.status(); - return; - } - overload_decl.set_id(*overload_id); - AddOverloadInternal(function_name, insertion_order, overloads, - std::move(overload_decl), status); + absl::StatusOr signature = + MakeOverloadSignature(function_name, overload.args(), overload.member()); + if (!signature.ok()) { + status = signature.status(); return; } - if (auto it = overloads.find(overload.id()); it != overloads.end()) { + OverloadDecl mutable_overload = std::forward(overload); + + if (mutable_overload.id().empty()) { + mutable_overload.set_id(*signature); + } + + if (auto it = by_id.find(mutable_overload.id()); it != by_id.end()) { status = absl::AlreadyExistsError( - absl::StrCat("overload already exists: ", overload.id())); + absl::StrCat("overload exists: ", mutable_overload.id())); return; } - for (const auto& existing : overloads) { - if (SignaturesOverlap(overload, existing)) { + + for (const auto& existing : insertion_order) { + if (SignaturesOverlap(mutable_overload, existing)) { status = absl::InvalidArgumentError( absl::StrCat("overload signature collision: ", existing.id(), - " collides with ", overload.id())); + " collides with ", mutable_overload.id())); return; } } - const auto inserted = overloads.insert(std::forward(overload)); - ABSL_DCHECK(inserted.second); - insertion_order.push_back(*inserted.first); + + size_t index = insertion_order.size(); + by_id[mutable_overload.id()] = index; + by_signature[*signature] = index; + insertion_order.push_back(std::move(mutable_overload)); } void CollectTypeParams(absl::flat_hash_set& type_params, @@ -195,14 +197,25 @@ absl::flat_hash_set OverloadDecl::GetTypeParams() const { void FunctionDecl::AddOverloadImpl(const OverloadDecl& overload, absl::Status& status) { - AddOverloadInternal(name_, overloads_.insertion_order, overloads_.set, - overload, status); + AddOverloadInternal(name_, overloads_.insertion_order, overloads_.by_id, + overloads_.by_signature, overload, status); } void FunctionDecl::AddOverloadImpl(OverloadDecl&& overload, absl::Status& status) { - AddOverloadInternal(name_, overloads_.insertion_order, overloads_.set, - std::move(overload), status); + AddOverloadInternal(name_, overloads_.insertion_order, overloads_.by_id, + overloads_.by_signature, std::move(overload), status); +} + +const OverloadDecl* FunctionDecl::FindOverloadById(absl::string_view id) const { + if (auto it = overloads_.by_id.find(id); it != overloads_.by_id.end()) { + return &overloads_.insertion_order[it->second]; + } + if (auto it = overloads_.by_signature.find(id); + it != overloads_.by_signature.end()) { + return &overloads_.insertion_order[it->second]; + } + return nullptr; } } // namespace cel diff --git a/common/decl.h b/common/decl.h index 22ee8cbf0..e894189c2 100644 --- a/common/decl.h +++ b/common/decl.h @@ -22,11 +22,11 @@ #include "absl/algorithm/container.h" #include "absl/base/attributes.h" +#include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/hash/hash.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "absl/types/span.h" @@ -264,39 +264,6 @@ OverloadDecl MakeMemberOverloadDecl(absl::string_view id, Type result, return overload_decl; } -struct OverloadDeclHash { - using is_transparent = void; - - size_t operator()(const OverloadDecl& overload_decl) const { - return (*this)(overload_decl.id()); - } - - size_t operator()(absl::string_view id) const { return absl::HashOf(id); } -}; - -struct OverloadDeclEqualTo { - using is_transparent = void; - - bool operator()(const OverloadDecl& lhs, const OverloadDecl& rhs) const { - return (*this)(lhs.id(), rhs.id()); - } - - bool operator()(const OverloadDecl& lhs, absl::string_view rhs) const { - return (*this)(lhs.id(), rhs); - } - - bool operator()(absl::string_view lhs, const OverloadDecl& rhs) const { - return (*this)(lhs, rhs.id()); - } - - bool operator()(absl::string_view lhs, absl::string_view rhs) const { - return lhs == rhs; - } -}; - -using OverloadDeclHashSet = - absl::flat_hash_set; - template absl::StatusOr MakeFunctionDecl(std::string name, Overloads&&... overloads); @@ -346,21 +313,27 @@ class FunctionDecl final { return overloads_.insertion_order; } + ABSL_MUST_USE_RESULT const OverloadDecl* FindOverloadById( + absl::string_view id) const; + std::vector release_overloads() { std::vector released = std::move(overloads_.insertion_order); overloads_.insertion_order.clear(); - overloads_.set.clear(); + overloads_.by_id.clear(); + overloads_.by_signature.clear(); return released; } private: struct Overloads { std::vector insertion_order; - OverloadDeclHashSet set; + absl::flat_hash_map by_id; + absl::flat_hash_map by_signature; void Reserve(size_t size) { insertion_order.reserve(size); - set.reserve(size); + by_id.reserve(size); + by_signature.reserve(size); } }; @@ -405,6 +378,74 @@ bool TypeIsAssignable(const Type& to, const Type& from); } // namespace common_internal +struct VariableDeclEqualTo { + using is_transparent = void; + + bool operator()(const cel::VariableDecl& lhs, + const cel::VariableDecl& rhs) const { + return (*this)(lhs.name(), rhs.name()); + } + + bool operator()(const cel::VariableDecl& lhs, std::string_view rhs) const { + return (*this)(lhs.name(), rhs); + } + + bool operator()(std::string_view lhs, const cel::VariableDecl& rhs) const { + return (*this)(lhs, rhs.name()); + } + + bool operator()(std::string_view lhs, std::string_view rhs) const { + return lhs == rhs; + } +}; + +struct VariableDeclHash { + using is_transparent = void; + + size_t operator()(const cel::VariableDecl& decl) const { + return (*this)(decl.name()); + } + + size_t operator()(std::string_view name) const { return absl::HashOf(name); } +}; + +using VariableDeclSet = absl::flat_hash_set; + +struct FunctionDeclEqualTo { + using is_transparent = void; + + bool operator()(const cel::FunctionDecl& lhs, + const cel::FunctionDecl& rhs) const { + return (*this)(lhs.name(), rhs.name()); + } + + bool operator()(const cel::FunctionDecl& lhs, std::string_view rhs) const { + return (*this)(lhs.name(), rhs); + } + + bool operator()(std::string_view lhs, const cel::FunctionDecl& rhs) const { + return (*this)(lhs, rhs.name()); + } + + bool operator()(std::string_view lhs, std::string_view rhs) const { + return lhs == rhs; + } +}; + +struct FunctionDeclHash { + using is_transparent = void; + + size_t operator()(const cel::FunctionDecl& decl) const { + return (*this)(decl.name()); + } + + size_t operator()(std::string_view name) const { return absl::HashOf(name); } +}; + +using FunctionDeclSet = absl::flat_hash_set; + } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_DECL_H_ diff --git a/common/decl_proto.cc b/common/decl_proto.cc index 89f7f4453..098c5068c 100644 --- a/common/decl_proto.cc +++ b/common/decl_proto.cc @@ -69,7 +69,7 @@ absl::StatusOr FunctionDeclFromProto( return decl; } -absl::StatusOr> DeclFromProto( +absl::StatusOr> DeclFromProto( const cel::expr::Decl& decl, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::Arena* absl_nonnull arena) { diff --git a/common/decl_proto_test.cc b/common/decl_proto_test.cc index 62215f07f..d72d97e09 100644 --- a/common/decl_proto_test.cc +++ b/common/decl_proto_test.cc @@ -49,7 +49,7 @@ TEST_P(DeclFromProtoTest, FromProtoWorks) { cel::expr::Decl decl_pb; ASSERT_TRUE( google::protobuf::TextFormat::ParseFromString(test_case.proto_decl, &decl_pb)); - absl::StatusOr> decl_or = + absl::StatusOr> decl_or = DeclFromProto(decl_pb, descriptor_pool, &arena); switch (test_case.decl_type) { case DeclType::kVariable: { @@ -79,7 +79,7 @@ TEST_P(DeclFromProtoTest, FromV1Alpha1ProtoWorks) { google::api::expr::v1alpha1::Decl decl_pb; ASSERT_TRUE( google::protobuf::TextFormat::ParseFromString(test_case.proto_decl, &decl_pb)); - absl::StatusOr> decl_or = + absl::StatusOr> decl_or = DeclFromV1Alpha1Proto(decl_pb, descriptor_pool, &arena); switch (test_case.decl_type) { case DeclType::kVariable: { diff --git a/common/decl_proto_v1alpha1.cc b/common/decl_proto_v1alpha1.cc index 2c6cfb6e4..a8d73e5c2 100644 --- a/common/decl_proto_v1alpha1.cc +++ b/common/decl_proto_v1alpha1.cc @@ -52,7 +52,7 @@ absl::StatusOr FunctionDeclFromV1Alpha1Proto( return FunctionDeclFromProto(name, unversioned, descriptor_pool, arena); } -absl::StatusOr> DeclFromV1Alpha1Proto( +absl::StatusOr> DeclFromV1Alpha1Proto( const google::api::expr::v1alpha1::Decl& decl, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::Arena* absl_nonnull arena) { diff --git a/common/decl_test.cc b/common/decl_test.cc index 510cd5017..72e7f1b93 100644 --- a/common/decl_test.cc +++ b/common/decl_test.cc @@ -165,6 +165,53 @@ TEST(FunctionDecl, Overloads) { StatusIs(absl::StatusCode::kInvalidArgument)); } +TEST(FunctionDecl, AddOverloadInvalidSignature) { + FunctionDecl function_decl; + function_decl.set_name("foo"); + // Member overload must have at least one argument (the receiver). + // This should fail to add because signature generation fails. + EXPECT_THAT(function_decl.AddOverload(MakeMemberOverloadDecl(StringType{})), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +TEST(FunctionDecl, AddOverloadDuplicateId) { + ASSERT_OK_AND_ASSIGN( + auto function_decl, + MakeFunctionDecl("hello", + MakeOverloadDecl("foo", StringType{}, StringType{}))); + // Adding another overload with the same ID "foo" should fail. + EXPECT_THAT( + function_decl.AddOverload(MakeOverloadDecl("foo", IntType{}, IntType{})), + StatusIs(absl::StatusCode::kAlreadyExists)); +} + +TEST(FunctionDecl, FindOverload) { + ASSERT_OK_AND_ASSIGN( + auto function_decl, + MakeFunctionDecl( + "hello", MakeOverloadDecl("foo", StringType{}, StringType{}), + MakeMemberOverloadDecl("bar", StringType{}, StringType{}), + MakeOverloadDecl(IntType{}, IntType{}))); + + // Find by explicit ID + const OverloadDecl* overload = function_decl.FindOverloadById("foo"); + ASSERT_NE(overload, nullptr); + EXPECT_EQ(overload->id(), "foo"); + + // Find by ID fallback to signature + overload = function_decl.FindOverloadById("hello(string)"); + ASSERT_NE(overload, nullptr); + EXPECT_EQ(overload->id(), "foo"); + + // Find implicit overload (where ID == signature) + overload = function_decl.FindOverloadById("hello(int)"); + ASSERT_NE(overload, nullptr); + EXPECT_EQ(overload->id(), "hello(int)"); + + // Non-existent + EXPECT_EQ(function_decl.FindOverloadById("non_existent"), nullptr); +} + TEST(FunctionDecl, OverloadId) { google::protobuf::Arena arena; const auto* descriptor = diff --git a/checker/internal/descriptor_pool_type_introspector.cc b/common/descriptor_pool_type_introspector.cc similarity index 90% rename from checker/internal/descriptor_pool_type_introspector.cc rename to common/descriptor_pool_type_introspector.cc index f6001e947..980b2d25d 100644 --- a/checker/internal/descriptor_pool_type_introspector.cc +++ b/common/descriptor_pool_type_introspector.cc @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "checker/internal/descriptor_pool_type_introspector.h" +#include "common/descriptor_pool_type_introspector.h" #include +#include #include #include @@ -24,28 +25,26 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" -#include "absl/types/optional.h" #include "absl/types/span.h" #include "common/type.h" #include "common/type_introspector.h" #include "google/protobuf/descriptor.h" -namespace cel::checker_internal { +namespace cel { namespace { // Standard implementation for field lookups. // Avoids building a FieldTable and just checks the DescriptorPool directly. -absl::StatusOr> +absl::StatusOr> FindStructTypeFieldByNameDirectly( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, absl::string_view type, absl::string_view name) { const google::protobuf::Descriptor* absl_nullable descriptor = descriptor_pool->FindMessageTypeByName(type); if (descriptor == nullptr) { - return absl::nullopt; + return std::nullopt; } - const google::protobuf::FieldDescriptor* absl_nullable field = - descriptor->FindFieldByName(name); + const google::protobuf::FieldDescriptor* field = descriptor->FindFieldByName(name); if (field != nullptr) { return StructTypeField(MessageTypeField(field)); } @@ -54,20 +53,20 @@ FindStructTypeFieldByNameDirectly( if (field != nullptr) { return StructTypeField(MessageTypeField(field)); } - return absl::nullopt; + return std::nullopt; } // Standard implementation for listing fields. // Avoids building a FieldTable and just checks the DescriptorPool directly. absl::StatusOr< - absl::optional>> + std::optional>> ListStructTypeFieldsDirectly( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, absl::string_view type) { const google::protobuf::Descriptor* absl_nullable descriptor = descriptor_pool->FindMessageTypeByName(type); if (descriptor == nullptr) { - return absl::nullopt; + return std::nullopt; } std::vector extensions; @@ -88,7 +87,7 @@ ListStructTypeFieldsDirectly( using Field = DescriptorPoolTypeIntrospector::Field; -absl::StatusOr> +absl::StatusOr> DescriptorPoolTypeIntrospector::FindTypeImpl(absl::string_view name) const { const google::protobuf::Descriptor* absl_nullable descriptor = descriptor_pool_->FindMessageTypeByName(name); @@ -100,10 +99,10 @@ DescriptorPoolTypeIntrospector::FindTypeImpl(absl::string_view name) const { if (enum_descriptor != nullptr) { return Type::Enum(enum_descriptor); } - return absl::nullopt; + return std::nullopt; } -absl::StatusOr> +absl::StatusOr> DescriptorPoolTypeIntrospector::FindEnumConstantImpl( absl::string_view type, absl::string_view value) const { const google::protobuf::EnumDescriptor* absl_nullable enum_descriptor = @@ -112,7 +111,7 @@ DescriptorPoolTypeIntrospector::FindEnumConstantImpl( const google::protobuf::EnumValueDescriptor* absl_nullable enum_value_descriptor = enum_descriptor->FindValueByName(value); if (enum_value_descriptor == nullptr) { - return absl::nullopt; + return std::nullopt; } return EnumConstant{ .type = Type::Enum(enum_descriptor), @@ -121,10 +120,10 @@ DescriptorPoolTypeIntrospector::FindEnumConstantImpl( .number = enum_value_descriptor->number(), }; } - return absl::nullopt; + return std::nullopt; } -absl::StatusOr> +absl::StatusOr> DescriptorPoolTypeIntrospector::FindStructTypeFieldByNameImpl( absl::string_view type, absl::string_view name) const { if (!use_json_name_) { @@ -134,7 +133,7 @@ DescriptorPoolTypeIntrospector::FindStructTypeFieldByNameImpl( const FieldTable* field_table = GetFieldTable(type); if (field_table == nullptr) { - return absl::nullopt; + return std::nullopt; } if (auto it = field_table->json_name_map.find(name); @@ -147,11 +146,11 @@ DescriptorPoolTypeIntrospector::FindStructTypeFieldByNameImpl( return field_table->fields[it->second].field; } - return absl::nullopt; + return std::nullopt; } absl::StatusOr< - absl::optional>> + std::optional>> DescriptorPoolTypeIntrospector::ListFieldsForStructTypeImpl( absl::string_view type) const { if (!use_json_name_) { @@ -160,7 +159,7 @@ DescriptorPoolTypeIntrospector::ListFieldsForStructTypeImpl( const FieldTable* field_table = GetFieldTable(type); if (field_table == nullptr) { - return absl::nullopt; + return std::nullopt; } std::vector fields; fields.reserve(field_table->non_extensions.size()); @@ -242,4 +241,4 @@ DescriptorPoolTypeIntrospector::CreateFieldTable( return result; } -} // namespace cel::checker_internal +} // namespace cel diff --git a/checker/internal/descriptor_pool_type_introspector.h b/common/descriptor_pool_type_introspector.h similarity index 91% rename from checker/internal/descriptor_pool_type_introspector.h rename to common/descriptor_pool_type_introspector.h index 8a970ea00..e7d6091d2 100644 --- a/checker/internal/descriptor_pool_type_introspector.h +++ b/common/descriptor_pool_type_introspector.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_CHECKER_INTERNAL_DESCRIPTOR_POOL_TYPE_INTROSPECTOR_H_ -#define THIRD_PARTY_CEL_CPP_CHECKER_INTERNAL_DESCRIPTOR_POOL_TYPE_INTROSPECTOR_H_ +#ifndef THIRD_PARTY_CEL_CPP_COMMON_DESCRIPTOR_POOL_TYPE_INTROSPECTOR_H_ +#define THIRD_PARTY_CEL_CPP_COMMON_DESCRIPTOR_POOL_TYPE_INTROSPECTOR_H_ #include #include @@ -30,14 +30,15 @@ #include "common/type_introspector.h" #include "google/protobuf/descriptor.h" -namespace cel::checker_internal { +namespace cel { // Implementation of `TypeIntrospector` that uses a `google::protobuf::DescriptorPool`. // // This is used by the type checker to resolve protobuf types and their fields // and apply any options like using JSON names. // -// Neither copyable nor movable. Should be managed by a TypeCheckEnv. +// Neither copyable nor movable. Should be managed by a TypeCheckEnv or a +// runtime Environment. class DescriptorPoolTypeIntrospector : public TypeIntrospector { public: struct Field { @@ -100,6 +101,6 @@ class DescriptorPoolTypeIntrospector : public TypeIntrospector { const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool_; }; -} // namespace cel::checker_internal +} // namespace cel -#endif // THIRD_PARTY_CEL_CPP_CHECKER_INTERNAL_DESCRIPTOR_POOL_TYPE_INTROSPECTOR_H_ +#endif // THIRD_PARTY_CEL_CPP_COMMON_DESCRIPTOR_POOL_TYPE_INTROSPECTOR_H_ diff --git a/checker/internal/descriptor_pool_type_introspector_test.cc b/common/descriptor_pool_type_introspector_test.cc similarity index 92% rename from checker/internal/descriptor_pool_type_introspector_test.cc rename to common/descriptor_pool_type_introspector_test.cc index e2fdc9d40..94acd317f 100644 --- a/checker/internal/descriptor_pool_type_introspector_test.cc +++ b/common/descriptor_pool_type_introspector_test.cc @@ -12,19 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "checker/internal/descriptor_pool_type_introspector.h" +#include "common/descriptor_pool_type_introspector.h" +#include #include #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" -#include "absl/types/optional.h" #include "common/type.h" #include "common/type_introspector.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" -namespace cel::checker_internal { +namespace cel { namespace { using ::absl_testing::IsOkAndHolds; @@ -47,7 +47,7 @@ TEST(DescriptorPoolTypeIntrospectorTest, FindType) { "cel.expr.conformance.proto3.TestAllTypes.NestedEnum"), IsOkAndHolds(Optional(Property(&Type::IsEnum, true)))); EXPECT_THAT(introspector.FindType("non.existent.Type"), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST(DescriptorPoolTypeIntrospectorTest, FindEnumConstant) { @@ -84,7 +84,7 @@ TEST(DescriptorPoolTypeIntrospectorTest, auto field = introspector.FindStructTypeFieldByName( "cel.expr.conformance.proto3.TestAllTypes", "singleInt64"); - EXPECT_THAT(field, IsOkAndHolds(Eq(absl::nullopt))); + EXPECT_THAT(field, IsOkAndHolds(Eq(std::nullopt))); } TEST(DescriptorPoolTypeIntrospectorTest, FindExtension) { @@ -108,7 +108,7 @@ TEST(DescriptorPoolTypeIntrospectorTest, FindStructTypeFieldByNameWithJsonOpt) { auto field = introspector.FindStructTypeFieldByName( "cel.expr.conformance.proto3.TestAllTypes", "single_int64"); - ASSERT_THAT(field, IsOkAndHolds(Eq(absl::nullopt))); + ASSERT_THAT(field, IsOkAndHolds(Eq(std::nullopt))); } TEST(DescriptorPoolTypeIntrospectorTest, @@ -117,7 +117,7 @@ TEST(DescriptorPoolTypeIntrospectorTest, internal::GetTestingDescriptorPool()); introspector.set_use_json_name(true); - absl::StatusOr> field = + absl::StatusOr> field = introspector.FindStructTypeFieldByName( "cel.expr.conformance.proto3.TestAllTypes", "singleInt64"); @@ -132,7 +132,7 @@ TEST(DescriptorPoolTypeIntrospectorTest, ListFieldsForStructType) { DescriptorPoolTypeIntrospector introspector( internal::GetTestingDescriptorPool()); absl::StatusOr< - absl::optional>> + std::optional>> fields = introspector.ListFieldsForStructType( "cel.expr.conformance.proto3.TestAllTypes"); ASSERT_THAT(fields, IsOkAndHolds(Optional(SizeIs(260)))); @@ -168,8 +168,8 @@ TEST(DescriptorPoolTypeIntrospectorTest, ListFieldsForStructTypeNotFound) { internal::GetTestingDescriptorPool()); auto fields = introspector.ListFieldsForStructType( "cel.expr.conformance.proto3.SomeOtherType"); - EXPECT_THAT(fields, IsOkAndHolds(Eq(absl::nullopt))); + EXPECT_THAT(fields, IsOkAndHolds(Eq(std::nullopt))); } } // namespace -} // namespace cel::checker_internal +} // namespace cel diff --git a/common/expr.h b/common/expr.h index 9c6f508c6..cc887a537 100644 --- a/common/expr.h +++ b/common/expr.h @@ -45,7 +45,9 @@ class MapExprEntry; class MapExpr; class ComprehensionExpr; -inline constexpr absl::string_view kAccumulatorVariableName = "__result__"; +inline constexpr absl::string_view kAccumulatorVariableName = "@result"; +inline constexpr absl::string_view kDeprecatedAccumulatorVariableName = + "__result__"; bool operator==(const Expr& lhs, const Expr& rhs); @@ -856,6 +858,7 @@ class ComprehensionExpr final { inline bool operator==(const ComprehensionExpr& lhs, const ComprehensionExpr& rhs) { return lhs.iter_var() == rhs.iter_var() && + lhs.iter_var2() == rhs.iter_var2() && lhs.iter_range() == rhs.iter_range() && lhs.accu_var() == rhs.accu_var() && lhs.accu_init() == rhs.accu_init() && @@ -1171,6 +1174,7 @@ inline std::unique_ptr SelectExpr::release( inline void ComprehensionExpr::Clear() { iter_var_.clear(); + iter_var2_.clear(); iter_range_.reset(); accu_var_.clear(); accu_init_.reset(); @@ -1376,7 +1380,7 @@ inline Expr& Expr::operator=(Expr&&) = default; inline void Expr::set_kind(ExprKind kind) { kind_ = std::move(kind); } -inline ABSL_MUST_USE_RESULT ExprKind Expr::release_kind() { +[[nodiscard]] inline ExprKind Expr::release_kind() { ExprKind kind = std::move(kind_); kind_.emplace(); return kind; @@ -1386,7 +1390,7 @@ inline void Expr::set_call_expr(CallExpr call_expr) { try_emplace_kind() = std::move(call_expr); } -inline ABSL_MUST_USE_RESULT CallExpr Expr::release_call_expr() { +[[nodiscard]] inline CallExpr Expr::release_call_expr() { return release_kind(); } @@ -1497,12 +1501,12 @@ inline void ListExprElement::Clear() { optional_ = false; } -inline ABSL_MUST_USE_RESULT const Expr& ListExprElement::expr() const +[[nodiscard]] inline const Expr& ListExprElement::expr() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return has_expr() ? *expr_ : Expr::default_instance(); } -inline ABSL_MUST_USE_RESULT Expr& ListExprElement::mutable_expr() +[[nodiscard]] inline Expr& ListExprElement::mutable_expr() ABSL_ATTRIBUTE_LIFETIME_BOUND { if (!has_expr()) { expr_ = std::make_unique(); @@ -1518,7 +1522,7 @@ inline void ListExprElement::set_expr(std::unique_ptr expr) { expr_ = std::move(expr); } -inline ABSL_MUST_USE_RESULT Expr ListExprElement::release_expr() { +[[nodiscard]] inline Expr ListExprElement::release_expr() { return release(expr_); } @@ -1568,12 +1572,12 @@ inline void StructExprField::Clear() { optional_ = false; } -inline ABSL_MUST_USE_RESULT const Expr& StructExprField::value() const +[[nodiscard]] inline const Expr& StructExprField::value() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return has_value() ? *value_ : Expr::default_instance(); } -inline ABSL_MUST_USE_RESULT Expr& StructExprField::mutable_value() +[[nodiscard]] inline Expr& StructExprField::mutable_value() ABSL_ATTRIBUTE_LIFETIME_BOUND { if (!has_value()) { value_ = std::make_unique(); @@ -1589,7 +1593,7 @@ inline void StructExprField::set_value(std::unique_ptr value) { value_ = std::move(value); } -inline ABSL_MUST_USE_RESULT Expr StructExprField::release_value() { +[[nodiscard]] inline Expr StructExprField::release_value() { return release(value_); } @@ -1644,12 +1648,12 @@ inline void MapExprEntry::Clear() { optional_ = false; } -inline ABSL_MUST_USE_RESULT const Expr& MapExprEntry::key() const +[[nodiscard]] inline const Expr& MapExprEntry::key() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return has_key() ? *key_ : Expr::default_instance(); } -inline ABSL_MUST_USE_RESULT Expr& MapExprEntry::mutable_key() +[[nodiscard]] inline Expr& MapExprEntry::mutable_key() ABSL_ATTRIBUTE_LIFETIME_BOUND { if (!has_key()) { key_ = std::make_unique(); @@ -1663,16 +1667,14 @@ inline void MapExprEntry::set_key(std::unique_ptr key) { key_ = std::move(key); } -inline ABSL_MUST_USE_RESULT Expr MapExprEntry::release_key() { - return release(key_); -} +[[nodiscard]] inline Expr MapExprEntry::release_key() { return release(key_); } -inline ABSL_MUST_USE_RESULT const Expr& MapExprEntry::value() const +[[nodiscard]] inline const Expr& MapExprEntry::value() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return has_value() ? *value_ : Expr::default_instance(); } -inline ABSL_MUST_USE_RESULT Expr& MapExprEntry::mutable_value() +[[nodiscard]] inline Expr& MapExprEntry::mutable_value() ABSL_ATTRIBUTE_LIFETIME_BOUND { if (!has_value()) { value_ = std::make_unique(); @@ -1688,7 +1690,7 @@ inline void MapExprEntry::set_value(std::unique_ptr value) { value_ = std::move(value); } -inline ABSL_MUST_USE_RESULT Expr MapExprEntry::release_value() { +[[nodiscard]] inline Expr MapExprEntry::release_value() { return release(value_); } diff --git a/common/expr_factory.h b/common/expr_factory.h index c8a9b831f..371361948 100644 --- a/common/expr_factory.h +++ b/common/expr_factory.h @@ -32,6 +32,16 @@ namespace cel { class MacroExprFactory; class ParserMacroExprFactory; +class OptimizerExprFactory; + +namespace tools { +class ProtoToPredicateBuilder; +} + +namespace parser_internal { +template +class AstFactoryInterface; +} class ExprFactory { protected: @@ -352,12 +362,37 @@ class ExprFactory { return expr; } + template ::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>> + Expr NewBind(NextIdFunc next_id, BindVar bind_var, BindExpr bind_expr, + RestExpr rest_expr) { + Expr expr; + expr.set_id(next_id()); + auto& comprehension_expr = expr.mutable_comprehension_expr(); + comprehension_expr.set_iter_var("#unused"); + comprehension_expr.set_iter_range( + NewList(next_id(), std::vector{})); + comprehension_expr.set_accu_var(bind_var); + comprehension_expr.set_accu_init(std::move(bind_expr)); + comprehension_expr.set_loop_condition(NewBoolConst(next_id(), false)); + comprehension_expr.set_loop_step(NewIdent(next_id(), bind_var)); + comprehension_expr.set_result(std::move(rest_expr)); + return expr; + } + private: friend class MacroExprFactory; friend class ParserMacroExprFactory; + friend class OptimizerExprFactory; + friend class tools::ProtoToPredicateBuilder; + friend class parser_internal::AstFactoryInterface; ExprFactory() : accu_var_(kAccumulatorVariableName) {} - explicit ExprFactory(absl::string_view accu_var) : accu_var_(accu_var) {} std::string accu_var_; }; diff --git a/testutil/expr_printer.cc b/common/expr_printer.cc similarity index 98% rename from testutil/expr_printer.cc rename to common/expr_printer.cc index 40dea3c33..8a80777ea 100644 --- a/testutil/expr_printer.cc +++ b/common/expr_printer.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "testutil/expr_printer.h" +#include "common/expr_printer.h" #include #include @@ -29,7 +29,7 @@ #include "common/expr.h" #include "internal/strings.h" -namespace cel::test { +namespace cel { namespace { class EmptyAdornerImpl : public ExpressionAdorner { @@ -328,4 +328,4 @@ std::string ExprPrinter::Print(const Expr& expr) const { return w.Print(expr); } -} // namespace cel::test +} // namespace cel diff --git a/testutil/expr_printer.h b/common/expr_printer.h similarity index 88% rename from testutil/expr_printer.h rename to common/expr_printer.h index 6b0a8c161..9304276ef 100644 --- a/testutil/expr_printer.h +++ b/common/expr_printer.h @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_TESTUTIL_EXPR_PRINTER_H_ -#define THIRD_PARTY_CEL_CPP_TESTUTIL_EXPR_PRINTER_H_ +#ifndef THIRD_PARTY_CEL_CPP_COMMON_EXPR_PRINTER_H_ +#define THIRD_PARTY_CEL_CPP_COMMON_EXPR_PRINTER_H_ #include #include "cel/expr/syntax.pb.h" #include "common/expr.h" -namespace cel::test { +namespace cel { // Interface for adding additional information to an expression during // printing. @@ -52,6 +52,6 @@ class ExprPrinter { const ExpressionAdorner& adorner_; }; -} // namespace cel::test +} // namespace cel -#endif // THIRD_PARTY_CEL_CPP_TESTUTIL_EXPR_PRINTER_H_ +#endif // THIRD_PARTY_CEL_CPP_COMMON_EXPR_PRINTER_H_ diff --git a/testutil/expr_printer_test.cc b/common/expr_printer_test.cc similarity index 99% rename from testutil/expr_printer_test.cc rename to common/expr_printer_test.cc index 9b1e7ca37..36646bfef 100644 --- a/testutil/expr_printer_test.cc +++ b/common/expr_printer_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "testutil/expr_printer.h" +#include "common/expr_printer.h" #include diff --git a/common/expr_test.cc b/common/expr_test.cc index 4f30dbd6f..efdc17d3b 100644 --- a/common/expr_test.cc +++ b/common/expr_test.cc @@ -321,10 +321,10 @@ TEST(MapExprEntry, Optional) { } TEST(MapExprEntry, Equality) { - EXPECT_EQ(StructExprField{}, StructExprField{}); - StructExprField field; - field.set_optional(true); - EXPECT_NE(StructExprField{}, field); + EXPECT_EQ(MapExprEntry{}, MapExprEntry{}); + MapExprEntry entry; + entry.set_optional(true); + EXPECT_NE(MapExprEntry{}, entry); } TEST(MapExpr, Entries) { @@ -358,6 +358,16 @@ TEST(ComprehensionExpr, IterVar) { EXPECT_THAT(comprehension_expr.iter_var(), IsEmpty()); } +TEST(ComprehensionExpr, IterVar2) { + ComprehensionExpr comprehension_expr; + EXPECT_THAT(comprehension_expr.iter_var2(), IsEmpty()); + comprehension_expr.set_iter_var2("foo"); + EXPECT_THAT(comprehension_expr.iter_var2(), Eq("foo")); + std::string iter_var2 = comprehension_expr.release_iter_var2(); + EXPECT_THAT(iter_var2, Eq("foo")); + EXPECT_THAT(comprehension_expr.iter_var2(), IsEmpty()); +} + TEST(ComprehensionExpr, IterRange) { ComprehensionExpr comprehension_expr; EXPECT_THAT(comprehension_expr.has_iter_range(), IsFalse()); @@ -433,6 +443,12 @@ TEST(ComprehensionExpr, Equality) { ComprehensionExpr comprehension_expr; comprehension_expr.set_result(MakeUnspecifiedExpr(1)); EXPECT_NE(ComprehensionExpr{}, comprehension_expr); + + ComprehensionExpr iter_var2_expr; + iter_var2_expr.set_iter_var2("v"); + EXPECT_NE(ComprehensionExpr{}, iter_var2_expr); + iter_var2_expr.Clear(); + EXPECT_EQ(ComprehensionExpr{}, iter_var2_expr); } TEST(Expr, Unspecified) { diff --git a/checker/internal/format_type_name.cc b/common/format_type_name.cc similarity index 97% rename from checker/internal/format_type_name.cc rename to common/format_type_name.cc index 7cd17251f..4bd6c2e61 100644 --- a/checker/internal/format_type_name.cc +++ b/common/format_type_name.cc @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include "checker/internal/format_type_name.h" +#include "common/format_type_name.h" #include #include @@ -20,7 +20,7 @@ #include "common/type.h" #include "common/type_kind.h" -namespace cel::checker_internal { +namespace cel { namespace { struct FormatImplRecord { @@ -177,4 +177,4 @@ std::string FormatTypeName(const Type& type) { return out; } -} // namespace cel::checker_internal +} // namespace cel diff --git a/checker/internal/format_type_name.h b/common/format_type_name.h similarity index 74% rename from checker/internal/format_type_name.h rename to common/format_type_name.h index c31e1c4d0..723ac20fd 100644 --- a/checker/internal/format_type_name.h +++ b/common/format_type_name.h @@ -12,19 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_CHECKER_INTERNAL_FORMAT_TYPE_NAME_H_ -#define THIRD_PARTY_CEL_CPP_CHECKER_INTERNAL_FORMAT_TYPE_NAME_H_ +#ifndef THIRD_PARTY_CEL_CPP_COMMON_FORMAT_TYPE_NAME_H_ +#define THIRD_PARTY_CEL_CPP_COMMON_FORMAT_TYPE_NAME_H_ #include #include "common/type.h" -namespace cel::checker_internal { +namespace cel { // Format the type name for presentation in error messages. Matches the // formatting used in github.com/cel-spec. std::string FormatTypeName(const Type& type); -} // namespace cel::checker_internal +} // namespace cel -#endif // THIRD_PARTY_CEL_CPP_CHECKER_INTERNAL_FORMAT_TYPE_NAME_H_ +#endif // THIRD_PARTY_CEL_CPP_COMMON_FORMAT_TYPE_NAME_H_ diff --git a/checker/internal/format_type_name_test.cc b/common/format_type_name_test.cc similarity index 97% rename from checker/internal/format_type_name_test.cc rename to common/format_type_name_test.cc index ff04e04d2..ca63f60b0 100644 --- a/checker/internal/format_type_name_test.cc +++ b/common/format_type_name_test.cc @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "checker/internal/format_type_name.h" +#include "common/format_type_name.h" #include "common/type.h" #include "internal/testing.h" #include "cel/expr/conformance/proto2/test_all_types.pb.h" #include "google/protobuf/arena.h" -namespace cel::checker_internal { +namespace cel { namespace { using ::cel::expr::conformance::proto2::GlobalEnum_descriptor; @@ -115,4 +115,4 @@ TEST(FormatTypeNameTest, ArbitraryNesting) { #endif } // namespace -} // namespace cel::checker_internal +} // namespace cel diff --git a/common/function_descriptor.cc b/common/function_descriptor.cc index be32e8616..a48f9c211 100644 --- a/common/function_descriptor.cc +++ b/common/function_descriptor.cc @@ -16,9 +16,14 @@ #include #include +#include +#include +#include #include "absl/base/macros.h" +#include "absl/functional/overload.h" #include "absl/types/span.h" +#include "common/ast.h" #include "common/kind.h" namespace cel { @@ -45,17 +50,12 @@ bool FunctionDescriptor::ShapeMatches(bool receiver_style, } bool FunctionDescriptor::operator==(const FunctionDescriptor& other) const { - return impl_.get() == other.impl_.get() || - (name() == other.name() && - receiver_style() == other.receiver_style() && - types().size() == other.types().size() && - std::equal(types().begin(), types().end(), other.types().begin())); + return name() == other.name() && receiver_style() == other.receiver_style() && + types().size() == other.types().size() && + std::equal(types().begin(), types().end(), other.types().begin()); } bool FunctionDescriptor::operator<(const FunctionDescriptor& other) const { - if (impl_.get() == other.impl_.get()) { - return false; - } if (name() < other.name()) { return true; } @@ -95,4 +95,86 @@ bool FunctionDescriptor::operator<(const FunctionDescriptor& other) const { return false; } +namespace { + +[[nodiscard]] +Kind TypeSpecToKind(const TypeSpec& spec) { + return std::visit( + absl::Overload( + [](UnsetTypeSpec) -> Kind { return Kind::kError; }, + [](const DynTypeSpec& spec) -> Kind { return Kind::kDyn; }, + [](const NullTypeSpec& spec) -> Kind { return Kind::kNull; }, + [](const PrimitiveType& spec) -> Kind { + switch (spec) { + case PrimitiveType::kPrimitiveTypeUnspecified: + return Kind::kError; + case PrimitiveType::kBool: + return Kind::kBool; + case PrimitiveType::kInt64: + return Kind::kInt; + case PrimitiveType::kUint64: + return Kind::kUint; + case PrimitiveType::kDouble: + return Kind::kDouble; + case PrimitiveType::kString: + return Kind::kString; + case PrimitiveType::kBytes: + return Kind::kBytes; + } + }, + [](const PrimitiveTypeWrapper& spec) -> Kind { + switch (spec.type()) { + case PrimitiveType::kPrimitiveTypeUnspecified: + return Kind::kError; + case PrimitiveType::kBool: + return Kind::kBoolWrapper; + case PrimitiveType::kInt64: + return Kind::kIntWrapper; + case PrimitiveType::kUint64: + return Kind::kUintWrapper; + case PrimitiveType::kDouble: + return Kind::kDoubleWrapper; + case PrimitiveType::kString: + return Kind::kStringWrapper; + case PrimitiveType::kBytes: + return Kind::kBytesWrapper; + } + }, + [](const WellKnownTypeSpec& spec) -> Kind { + switch (spec) { + case cel::WellKnownTypeSpec::kWellKnownTypeUnspecified: + return Kind::kError; + case cel::WellKnownTypeSpec::kDuration: + return Kind::kError; + case cel::WellKnownTypeSpec::kTimestamp: + return Kind::kTimestamp; + case cel::WellKnownTypeSpec::kAny: + return Kind::kAny; + } + }, + [](const ListTypeSpec& spec) -> Kind { return Kind::kList; }, + [](const MapTypeSpec& spec) -> Kind { return Kind::kMap; }, + [](const FunctionTypeSpec& spec) -> Kind { return Kind::kFunction; }, + [](const MessageTypeSpec& spec) -> Kind { return Kind::kStruct; }, + [](const ParamTypeSpec& spec) -> Kind { return Kind::kTypeParam; }, + [](const std::unique_ptr& spec) -> Kind { + return Kind::kType; + }, + [](const ErrorTypeSpec& spec) -> Kind { return Kind::kError; }, + [](const AbstractType& spec) -> Kind { return Kind::kOpaque; }), + spec.type_kind()); +} + +} // namespace + +std::vector FunctionDescriptor::FunctionTypeSpecToArgumentKinds( + const FunctionTypeSpec& spec) { + std::vector kinds; + kinds.reserve(spec.arg_types().size()); + for (const auto& arg : spec.arg_types()) { + kinds.push_back(TypeSpecToKind(arg)); + } + return kinds; +} + } // namespace cel diff --git a/common/function_descriptor.h b/common/function_descriptor.h index 75c61e13a..2f3b6ffc8 100644 --- a/common/function_descriptor.h +++ b/common/function_descriptor.h @@ -15,13 +15,14 @@ #ifndef THIRD_PARTY_CEL_CPP_COMMON_FUNCTION_DESCRIPTOR_H_ #define THIRD_PARTY_CEL_CPP_COMMON_FUNCTION_DESCRIPTOR_H_ -#include #include #include #include +#include "absl/base/attributes.h" #include "absl/strings/string_view.h" #include "absl/types/span.h" +#include "common/ast.h" #include "common/kind.h" namespace cel { @@ -43,43 +44,76 @@ struct FunctionDescriptorOptions { // Coarsely describes a function for the purpose of runtime resolution of // overloads. -class FunctionDescriptor final { +class FunctionDescriptor { public: + // Creates a FunctionDescriptor for a global function. + [[nodiscard]] + static FunctionDescriptor Global(absl::string_view name, + absl::string_view overload_id, + const FunctionTypeSpec& type, + FunctionDescriptorOptions options = {}) { + return FunctionDescriptor(name, overload_id, type, options, + /*is_receiver_style=*/false); + } + + // Creates a FunctionDescriptor for a member function. + [[nodiscard]] + static FunctionDescriptor Member(absl::string_view name, + absl::string_view overload_id, + const FunctionTypeSpec& type, + FunctionDescriptorOptions options = {}) { + return FunctionDescriptor(name, overload_id, type, options, + /*is_receiver_style=*/true); + } + + FunctionDescriptor(const FunctionDescriptor&) = default; + FunctionDescriptor(FunctionDescriptor&&) = default; + FunctionDescriptor& operator=(const FunctionDescriptor&) = default; + FunctionDescriptor& operator=(FunctionDescriptor&&) = default; + FunctionDescriptor(absl::string_view name, bool receiver_style, std::vector types, bool is_strict) - : impl_(std::make_shared( - name, std::move(types), receiver_style, - FunctionDescriptorOptions{is_strict, - /*is_contextual=*/false})) {} + : FunctionDescriptor(name, receiver_style, std::move(types), + FunctionDescriptorOptions{is_strict, + /*is_contextual=*/false}) { + } FunctionDescriptor(absl::string_view name, bool receiver_style, std::vector types, bool is_strict, bool is_contextual) - : impl_(std::make_shared( - name, std::move(types), receiver_style, - FunctionDescriptorOptions{is_strict, is_contextual})) {} + : FunctionDescriptor( + name, receiver_style, std::move(types), + FunctionDescriptorOptions{is_strict, is_contextual}) {} FunctionDescriptor(absl::string_view name, bool is_receiver_style, std::vector types, FunctionDescriptorOptions options = {}) - : impl_(std::make_shared(name, std::move(types), is_receiver_style, - options)) {} + : name_(name), + kinds_(std::move(types)), + is_receiver_style_(is_receiver_style), + options_(std::move(options)) {} // Function name. - const std::string& name() const { return impl_->name; } + const std::string& name() const { return name_; } + + // Function overload ID. This may be empty for legacy code. + [[nodiscard]] + absl::string_view overload_id() const ABSL_ATTRIBUTE_LIFETIME_BOUND { + return overload_id_; + } // Whether function is receiver style i.e. true means arg0.name(args[1:]...). - bool receiver_style() const { return impl_->is_receiver_style; } + bool receiver_style() const { return is_receiver_style_; } // The argument types the function accepts. // // TODO(uncreated-issue/17): make this kinds - const std::vector& types() const { return impl_->types; } + const std::vector& types() const { return kinds_; } // if true (strict, default), error or unknown arguments are propagated // instead of calling the function. if false (non-strict), the function may // receive error or unknown values as arguments. - bool is_strict() const { return impl_->options.is_strict; } + bool is_strict() const { return options_.is_strict; } // Whether the function is contextual (impure). // @@ -87,7 +121,7 @@ class FunctionDescriptor final { // the CEL expression evaluation or have visible side effects. This breaks // some of the assumptions of CEL. This flag is used as a hint to the planner // that some optimizations are not safe or not effective. - bool is_contextual() const { return impl_->options.is_contextual; } + bool is_contextual() const { return options_.is_contextual; } // Helper for matching a descriptor. This tests that the shape is the same -- // |other| accepts the same number and types of arguments and is the same call @@ -102,21 +136,24 @@ class FunctionDescriptor final { bool operator<(const FunctionDescriptor& other) const; private: - struct Impl final { - Impl(absl::string_view name, std::vector types, - bool is_receiver_style, FunctionDescriptorOptions options) - : name(name), - types(std::move(types)), - is_receiver_style(is_receiver_style), - options(options) {} - - std::string name; - std::vector types; - bool is_receiver_style; - FunctionDescriptorOptions options; - }; - - std::shared_ptr impl_; + [[nodiscard]] + static std::vector FunctionTypeSpecToArgumentKinds( + const FunctionTypeSpec& spec); + + FunctionDescriptor(absl::string_view name, absl::string_view overload_id, + const FunctionTypeSpec& type, + FunctionDescriptorOptions options, bool is_receiver_style) + : name_(name), + overload_id_(overload_id), + kinds_(FunctionTypeSpecToArgumentKinds(type)), + is_receiver_style_(is_receiver_style), + options_(std::move(options)) {} + + std::string name_; + std::string overload_id_; + std::vector kinds_; + bool is_receiver_style_; + FunctionDescriptorOptions options_; }; } // namespace cel diff --git a/common/internal/BUILD b/common/internal/BUILD index 10084b685..7a877ac53 100644 --- a/common/internal/BUILD +++ b/common/internal/BUILD @@ -18,27 +18,36 @@ load("@rules_cc//cc:cc_test.bzl", "cc_test") package(default_visibility = ["//visibility:public"]) cc_library( - name = "casting", - hdrs = ["casting.h"], + name = "attribute_trail", + srcs = ["attribute_trail.cc"], + hdrs = ["attribute_trail.h"], deps = [ - "//common:native_type", - "//internal:casts", + ":unknowns", + "//base:attribute_matcher", + "//base:attributes", + "//common:value", + "//runtime/internal:errors", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/container:btree", "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/meta:type_traits", - "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", ], ) cc_library( - name = "reference_count", - srcs = ["reference_count.cc"], - hdrs = ["reference_count.h"], + name = "unknowns", + srcs = ["unknowns.cc"], + hdrs = ["unknowns.h"], deps = [ - "//common:data", - "//internal:new", + "//base:attribute_matcher", + "//base:attributes", + "//internal:arena_tree", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/container:inlined_vector", + "@com_google_absl//absl/functional:overload", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings:string_view", "@com_google_protobuf//:protobuf", @@ -46,41 +55,30 @@ cc_library( ) cc_test( - name = "reference_count_test", - srcs = ["reference_count_test.cc"], + name = "unknowns_test", + srcs = ["unknowns_test.cc"], deps = [ - ":reference_count", - "//common:data", + ":unknowns", + "//base:attribute_matcher", + "//base:attributes", "//internal:testing", - "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/status:status_matchers", "@com_google_protobuf//:protobuf", - "@com_google_protobuf//:struct_cc_proto", ], ) -cc_library( - name = "metadata", - hdrs = ["metadata.h"], - deps = ["@com_google_protobuf//:protobuf"], -) - cc_library( name = "byte_string", srcs = ["byte_string.cc"], hdrs = ["byte_string.h"], deps = [ - ":metadata", - ":reference_count", - "//common:allocator", - "//common:arena", - "//common:memory", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/functional:overload", - "@com_google_absl//absl/hash", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:cord", + "@com_google_absl//absl/strings:resize_and_overwrite", "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/types:optional", "@com_google_protobuf//:protobuf", @@ -92,16 +90,12 @@ cc_test( srcs = ["byte_string_test.cc"], deps = [ ":byte_string", - ":reference_count", - "//common:allocator", - "//common:memory", "//internal:testing", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/hash", "@com_google_absl//absl/strings:cord", "@com_google_absl//absl/strings:cord_test_helpers", "@com_google_absl//absl/strings:string_view", - "@com_google_absl//absl/types:optional", "@com_google_protobuf//:protobuf", ], ) @@ -137,34 +131,3 @@ cc_library( "@com_google_protobuf//src/google/protobuf/io", ], ) - -cc_library( - name = "signature", - srcs = ["signature.cc"], - hdrs = ["signature.h"], - deps = [ - "//common:type", - "//common:type_kind", - "//internal:status_macros", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/strings:string_view", - ], -) - -cc_test( - name = "signature_test", - srcs = ["signature_test.cc"], - deps = [ - ":signature", - "//common:type", - "//internal:testing", - "//internal:testing_descriptor_pool", - "@com_google_absl//absl/base:no_destructor", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_protobuf//:protobuf", - ], -) diff --git a/common/internal/attribute_trail.cc b/common/internal/attribute_trail.cc new file mode 100644 index 000000000..9e68cce8d --- /dev/null +++ b/common/internal/attribute_trail.cc @@ -0,0 +1,190 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/internal/attribute_trail.h" + +#include +#include +#include + +#include "absl/base/optimization.h" +#include "absl/container/btree_set.h" +#include "absl/log/absl_check.h" +#include "common/internal/unknowns.h" +#include "common/value.h" + +namespace cel::common_internal { + +void UnknownAccumulator::Add(const UnknownSet& value) { + empty_ = false; + if (value.root == nullptr) { + // Ugh, there is some client code intentionally using empty unknowns to + // trigger unknown handling and we also have some tests which do evil + // things. We should fix them. + ABSL_DCHECK(value.attributes == nullptr); + ABSL_DCHECK(value.functions == nullptr); + return; + } + if (ABSL_PREDICT_FALSE(value.root != tree_->Root())) { + // Slow path. Somebody returned an unknown value from a separate evaluation. + // This is technically not supported but handling it is better than + // crashing. + Add(tree_->Graft(value)); + return; + } + // We perform adapting merging for performance reasons. If only two unknown + // sets (attribute or function) are added to the accumulator, we can trigger a + // faster path in UnknownAccumulator::Accumulate. This complexity exists to + // enable that. + if (value.attributes != nullptr) { + // Keep track of the largest set. If after merging the resulting set is the + // same size as the largest, they are equal. + if (largest_unknown_attribute_set_ != nullptr) { + if (value.attributes->size > largest_unknown_attribute_set_->size) { + largest_unknown_attribute_set_ = value.attributes; + } + } else { + largest_unknown_attribute_set_ = value.attributes; + } + // The adaptive merging shenanigans mentioned above. Keep track if we have + // one or two sets so we can trigger faster path handling. + if (unknown_attribute_set_pair_[0] == nullptr) { + unknown_attribute_set_pair_[0] = value.attributes; + } else if (unknown_attribute_set_pair_[1] == nullptr) { + unknown_attribute_set_pair_[1] = value.attributes; + } else { + if (!added_unknown_attribute_set_pair_) { + Add(unknown_attribute_set_pair_[0]); + Add(unknown_attribute_set_pair_[1]); + added_unknown_attribute_set_pair_ = true; + } + Add(value.attributes); + } + } + if (value.functions != nullptr) { + // Keep track of the largest set. If after merging the resulting set is the + // same size as the largest, they are equal. + if (largest_unknown_function_set_ != nullptr) { + if (value.functions->size > largest_unknown_function_set_->size) { + largest_unknown_function_set_ = value.functions; + } + } else { + largest_unknown_function_set_ = value.functions; + } + // The adaptive merging shenanigans mentioned above. Keep track if we have + // one or two sets so we can trigger faster path handling. + if (unknown_function_set_pair_[0] == nullptr) { + unknown_function_set_pair_[0] = value.functions; + } else if (unknown_function_set_pair_[1] == nullptr) { + unknown_function_set_pair_[1] = value.functions; + } else { + if (!added_unknown_function_set_pair_) { + Add(unknown_function_set_pair_[0]); + Add(unknown_function_set_pair_[1]); + added_unknown_function_set_pair_ = true; + } + Add(value.functions); + } + } +} + +std::optional UnknownAccumulator::Accumulate() && { + if (IsEmpty()) { + return std::nullopt; + } + const UnknownAttributeSet* unknown_attribute_set; + if (unknown_attributes_.empty()) { + if (unknown_attribute_set_pair_[0] == nullptr) { + ABSL_DCHECK(unknown_attribute_set_pair_[1] == nullptr); + unknown_attribute_set = nullptr; + } else if (unknown_attribute_set_pair_[1] == nullptr) { + unknown_attribute_set = unknown_attribute_set_pair_[0]; + } else { + // Two way merger can be done with no additional memory. + unknown_attribute_set = MergeUnknownAttributeSets( + unknown_attribute_set_pair_[0], unknown_attribute_set_pair_[1], + tree_->GetArena()); + } + } else { + if (largest_unknown_attribute_set_ != nullptr && + largest_unknown_attribute_set_->size == unknown_attributes_.size()) { + // We have more than 2 sets and the resulting set would have the same + // amount of entries as the largest we encountered, they are the same. + // Just return it. + unknown_attribute_set = largest_unknown_attribute_set_; + } else { + UnknownAttributeSet* mutable_unknown_attribute_set = + UnknownAttributeSet::Allocate(unknown_attributes_.size(), + tree_->GetArena()); + mutable_unknown_attribute_set->max_depth = 0; + uint32_t i = 0; + for (UnknownAttributeNode* unknown_attribute : unknown_attributes_) { + mutable_unknown_attribute_set->nodes[i++] = unknown_attribute; + mutable_unknown_attribute_set->max_depth = + std::max(mutable_unknown_attribute_set->max_depth, + static_cast(unknown_attribute->depth())); + } + unknown_attribute_set = mutable_unknown_attribute_set; + } + unknown_attributes_.clear(); + } + const UnknownFunctionSet* unknown_function_set; + if (unknown_functions_.empty()) { + if (unknown_function_set_pair_[0] == nullptr) { + ABSL_DCHECK(unknown_function_set_pair_[1] == nullptr); + unknown_function_set = nullptr; + } else if (unknown_function_set_pair_[1] == nullptr) { + unknown_function_set = unknown_function_set_pair_[0]; + } else { + // Two way merger can be done with no additional memory. + unknown_function_set = MergeUnknownFunctionSets( + unknown_function_set_pair_[0], unknown_function_set_pair_[1], + tree_->GetArena()); + } + } else { + if (largest_unknown_function_set_ != nullptr && + largest_unknown_function_set_->size == unknown_functions_.size()) { + // We have more than 2 sets and the resulting set would have the same + // amount of entries as the largest we encountered, they are the same. + // Just return it. + unknown_function_set = largest_unknown_function_set_; + } else { + UnknownFunctionSet* mutable_unknown_function_set = + UnknownFunctionSet::Allocate(unknown_functions_.size(), + tree_->GetArena()); + uint32_t i = 0; + for (UnknownFunctionNode* unknown_function : unknown_functions_) { + mutable_unknown_function_set->nodes[i++] = unknown_function; + } + unknown_function_set = mutable_unknown_function_set; + } + unknown_functions_.clear(); + } + return common_internal::MakeUnknownValue(tree_->Root(), unknown_attribute_set, + unknown_function_set); +} + +void UnknownAccumulator::Add(const UnknownAttributeSet* set) { + for (uint32_t i = 0; i < set->size; ++i) { + unknown_attributes_.insert(set->nodes[i]); + } +} + +void UnknownAccumulator::Add(const UnknownFunctionSet* set) { + for (uint32_t i = 0; i < set->size; ++i) { + unknown_functions_.insert(set->nodes[i]); + } +} + +} // namespace cel::common_internal diff --git a/common/internal/attribute_trail.h b/common/internal/attribute_trail.h new file mode 100644 index 000000000..46244398b --- /dev/null +++ b/common/internal/attribute_trail.h @@ -0,0 +1,551 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This header should ideally be moved to the runtime. Its here temporarily. + +#ifndef THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_ATTRIBUTE_TRAIL_H_ +#define THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_ATTRIBUTE_TRAIL_H_ + +#include +#include + +#include "absl/base/macros.h" +#include "absl/base/optimization.h" +#include "absl/container/btree_set.h" +#include "absl/log/absl_check.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "base/attribute.h" +#include "base/attribute_matcher.h" +#include "common/internal/unknowns.h" +#include "common/value.h" +#include "runtime/internal/errors.h" + +namespace cel::common_internal { + +[[nodiscard]] +inline UnknownValue AttributeMatchToUnknownValue( + const AttributeMatcherNode* node, UnknownTree* tree) { + ABSL_DCHECK(!node->key().IsWildcard()); + ABSL_DCHECK(tree != nullptr); + UnknownAttributeNode* unknown = tree->Convert(node); + return common_internal::MakeUnknownValue(UnknownSet{ + .root = tree->Root(), + .attributes = + common_internal::CreateUnknownAttributeSet(unknown, tree->GetArena()), + .functions = nullptr, + }); +} + +[[nodiscard]] +inline UnknownValue AttributeMatchToUnknownValue(AttributeMatch match, + UnknownTree* tree) { + return common_internal::AttributeMatchToUnknownValue( + common_internal::GetAttributeMatcherNode(match), tree); +} + +[[nodiscard]] +inline UnknownValue AttributeMatchToUnknownValue( + const AttributeMatcherNode* node, const AttributeQualifierView& key, + UnknownTree* tree) { + ABSL_DCHECK(node->key().IsWildcard() || node->key() == key); + ABSL_DCHECK(tree != nullptr); + UnknownAttributeNode* unknown = tree->Convert(node, key); + return common_internal::MakeUnknownValue(UnknownSet{ + .root = tree->Root(), + .attributes = + common_internal::CreateUnknownAttributeSet(unknown, tree->GetArena()), + .functions = nullptr, + }); +} + +[[nodiscard]] +inline UnknownValue AttributeMatchToUnknownValue( + AttributeMatch match, const AttributeQualifierView& key, + UnknownTree* tree) { + return common_internal::AttributeMatchToUnknownValue( + common_internal::GetAttributeMatcherNode(match), key, tree); +} + +[[nodiscard]] +inline UnknownValue AttributeMatchToUnknownValue(AttributeMatch match, + absl::string_view key, + UnknownTree* tree) { + return common_internal::AttributeMatchToUnknownValue( + common_internal::GetAttributeMatcherNode(match), + AttributeQualifierView::OfString(key), tree); +} + +[[nodiscard]] +inline ErrorValue AttributeMatchToMissingValue( + const AttributeMatcherNode* node, const AttributeQualifierView& key, + UnknownTree* tree) { + ABSL_DCHECK(node->IsFull()); + ABSL_DCHECK(node->key().IsWildcard() || node->key() == key); + ABSL_DCHECK(tree != nullptr); + // We do not need to convert matcher. + return ErrorValue::From( + runtime_internal::CreateMissingAttributeError(tree->ToString(node, key)), + tree->GetArena()); +} + +[[nodiscard]] +inline ErrorValue AttributeMatchToMissingValue( + AttributeMatch match, const AttributeQualifierView& key, + UnknownTree* tree) { + return common_internal::AttributeMatchToMissingValue( + common_internal::GetAttributeMatcherNode(match), key, tree); +} + +[[nodiscard]] +inline ErrorValue AttributeMatchToMissingValue(AttributeMatch match, + absl::string_view key, + UnknownTree* tree) { + return common_internal::AttributeMatchToMissingValue( + common_internal::GetAttributeMatcherNode(match), + AttributeQualifierView::OfString(key), tree); +} + +class AttributeTrail; +class UnknownAccumulator; + +// AttributeLeadMatch and AttributeTrail::Match make up the core of the +// unknown/known/missing attribute matching algorithm in the runtime. The former +// is used by identifier lookup and the later is used by selects and container +// accesses. + +// Entry point into attribute matching for unknowns, knowns, and missing. It +// should be used when looking up variables from the embedder. All subsequent +// "stepping" is done through AttributeTrail::Match. The trail will filled out +// with any partial matches and the value will be set with an unknown value or +// error value if there are any full matches. The function returns true if a +// full match was encountered and the value is set, false otherwise. +[[nodiscard]] +bool AttributeLeadMatch(const AttributeMatcher& unknown_attributes, + const AttributeMatcher& known_attributes, + const AttributeMatcher& missing_attributes, + absl::string_view variable, Value& value, + AttributeTrail& trail, UnknownTree* unknown_tree); + +class AttributeTrail { + public: + enum MatchMode { + kFull = 0, + kPartial, + }; + + AttributeTrail() = default; + AttributeTrail(const AttributeTrail&) = default; + AttributeTrail& operator=(const AttributeTrail&) = default; + + // Called to advance attribute matching for unknowns, knowns, and missing. It + // should be called before performing field access, map key accesses, list + // indexing, and related operations. The trail will be filled out with any + // partial matches and the value will be set with an unknown value or error + // value if there are any full matches. The function returns true if a + // full match was encountered and the value is set, false otherwise. + template + [[nodiscard]] + bool Match(const AttributeQualifierView& qualifier, Value& value, + AttributeTrail& trail, UnknownTree* unknown_tree) const { + // !!!WARNING!!! + // trail can alias *this be careful when updating this code. + // !!!WARNING!!! + static_assert(MissingMode == MatchMode::kFull, + "we do not support partial matching for missing attributes"); + if (known_.IsNone()) { + // Known is not being used or the default outcome for the trail is + // unknown. + ABSL_DCHECK(!unknown_.IsFull()); + trail.unknown_ = unknown_.MatchQualifier(qualifier); + if (KnownUnknownMode == MatchMode::kPartial ? !trail.unknown_.IsNone() + : trail.unknown_.IsFull()) { + trail.missing_ = AttributeMatch(); + value = AttributeMatchToUnknownValue( + std::exchange(trail.unknown_, AttributeMatch()), qualifier, + unknown_tree); + return true; + } + } else { + // Known is partial or full. + if (unknown_.IsNone()) { + // Unknown is none, it does not matter what known is the outcome is + // known. + trail.known_ = trail.unknown_ = AttributeMatch(); + } else { + if (MatchSlow(qualifier, value, trail, + unknown_tree)) { + return true; + } + } + } + trail.missing_ = missing_.MatchQualifier(qualifier); + if (trail.missing_.IsFull()) { + trail.known_ = trail.unknown_ = AttributeMatch(); + value = AttributeMatchToMissingValue( + std::exchange(trail.missing_, AttributeMatch()), qualifier, + unknown_tree); + return true; + } + return false; + } + + // Save as above, except this attribute trail is updated. + template + [[nodiscard]] + bool Match(const AttributeQualifierView& qualifier, Value& value, + UnknownTree* unknown_tree) { + return Match(qualifier, value, *this, + unknown_tree); + } + + ABSL_DEPRECATE_AND_INLINE() + const AttributeTrail& attribute() const { return *this; } + + [[nodiscard]] + bool IsNone() const { + return missing_.IsNone() && unknown_.IsNone(); + } + + [[nodiscard]] + bool IsFullMatch() const { + return missing_.IsFull() || (unknown_.IsFull() && known_.IsNone()); + } + + [[nodiscard]] + std::optional PartialUnknownMatch( + UnknownTree* unknown_tree) const { + ABSL_DCHECK(!known_.IsFull()); + if (!unknown_.IsNone()) { + if (known_.IsPartialWildcard()) { + return std::nullopt; + } + if (known_.IsPartial()) { + return AttributeMatchToUnknownValue(known_, unknown_tree); + } + return AttributeMatchToUnknownValue(unknown_, unknown_tree); + } + return std::nullopt; + } + + void Clear() { missing_ = known_ = unknown_ = AttributeMatch(); } + + private: + friend class UnknownAccumulator; + friend bool AttributeLeadMatch(const AttributeMatcher& unknown_attributes, + const AttributeMatcher& known_attributes, + const AttributeMatcher& missing_attributes, + absl::string_view variable, Value& value, + AttributeTrail& trail, + UnknownTree* unknown_tree); + + // Out of line remainder of the logic for Match to allow the compiler to more + // easily inline Match. + template + [[nodiscard]] + bool MatchSlow(const AttributeQualifierView& qualifier, Value& value, + AttributeTrail& trail, UnknownTree* unknown_tree) const { + static_assert(MissingMode == MatchMode::kFull, + "we do not support partial matching for missing attributes"); + // Unknown is partial or full. + if (known_.IsFull()) { + // The default from here on out is known. + trail.known_ = trail.unknown_ = AttributeMatch(); + return false; + } + // Partial known. + ABSL_DCHECK(known_.IsPartial()); + if (unknown_.IsFull()) { + // The default from here on out is unknown unless we get full known. + // To signal this we leave unknown as full and only match known. + trail.unknown_ = unknown_; + AttributeMatch prev_known = known_; + trail.known_ = known_.MatchQualifier(qualifier); + switch (trail.known_.GetType()) { + case AttributeMatch::Type::NONE: { + // Result is unknown. + trail.missing_ = AttributeMatch(); + trail.unknown_ = AttributeMatch(); + UnknownAttributeNode* match = + unknown_tree->Convert(prev_known)->Step(unknown_tree, qualifier); + value = cel::common_internal::MakeUnknownValue( + cel::common_internal::UnknownSet{ + .root = unknown_tree->Root(), + .attributes = cel::common_internal::CreateUnknownAttributeSet( + match, unknown_tree->GetArena()), + .functions = nullptr}); + return true; + } + case AttributeMatch::Type::PARTIAL: + // If we have a known attribute pattern `a.b.*` and we just + // advanced to `a.b`, we do not consider that partial unknown we + // consider that full known. + if (!trail.known_.IsPartialWildcard()) { + if (KnownUnknownMode != MatchMode::kPartial) { + break; + } + // Result is unknown. + trail.unknown_ = AttributeMatch(); + trail.missing_ = AttributeMatch(); + value = AttributeMatchToUnknownValue( + std::exchange(trail.known_, AttributeMatch()), qualifier, + unknown_tree); + return true; + } + [[fallthrough]]; + case AttributeMatch::Type::FULL: + // Result is known. + trail.known_ = trail.unknown_ = AttributeMatch(); + break; + } + return false; + } + ABSL_DCHECK(unknown_.IsPartial()); + // Partial known and unknown. + trail.known_ = known_.MatchQualifier(qualifier); + trail.unknown_ = unknown_.MatchQualifier(qualifier); + if (trail.known_.IsNone()) { + if (KnownUnknownMode == MatchMode::kPartial ? !trail.unknown_.IsNone() + : trail.unknown_.IsFull()) { + trail.missing_ = AttributeMatch(); + value = AttributeMatchToUnknownValue( + std::exchange(trail.unknown_, AttributeMatch()), qualifier, + unknown_tree); + return true; + } + } else { + if (trail.unknown_.IsNone()) { + trail.known_ = AttributeMatch(); + } else if (trail.known_.IsFull() || trail.known_.IsPartialWildcard()) { + trail.known_ = AttributeMatch(); + trail.unknown_ = AttributeMatch(); + } + } + return false; + } + + // Should only be called from when the unknown accumulator. + AttributeMatch GetNotFullUnknownMatch() const { + if (known_.IsPartialWildcard()) { + // If we have a known attribute pattern `a.b.*` and we just + // advanced to `a.b`, we do not consider that partial. + return AttributeMatch(); + } + ABSL_DCHECK(!unknown_.IsFull()); + ABSL_DCHECK(!known_.IsFull()); + if (known_.IsPartial()) { + return known_; + } + return unknown_; + } + + // Due to the complicated logic required to correctly perform attribute + // matching with missing, known, and unknown attributes we keep all this an + // implementation detail and return only what the caller needs to see. + + // Current match in the attribute matcher used for missing attributes. + AttributeMatch missing_; + // Current match in the attribute matcher used for known attributes. + AttributeMatch known_; + // Current match in the attribute matcher used for unknown attributes. + AttributeMatch unknown_; +}; + +[[nodiscard]] +inline bool AttributeLeadMatch(const AttributeMatcher& unknown_attributes, + const AttributeMatcher& known_attributes, + const AttributeMatcher& missing_attributes, + absl::string_view variable, Value& value, + AttributeTrail& trail, + UnknownTree* unknown_tree) { + trail.known_ = known_attributes.MatchVariable(variable); + trail.unknown_ = unknown_attributes.MatchVariable(variable); + if (trail.known_.IsNone()) { + if (trail.unknown_.IsFull()) { + trail.missing_ = AttributeMatch(); + value = AttributeMatchToUnknownValue( + std::exchange(trail.unknown_, AttributeMatch()), variable, + unknown_tree); + return true; + } + } else { + if (trail.unknown_.IsNone()) { + trail.known_ = AttributeMatch(); + } else if (trail.known_.IsFull() || trail.known_.IsPartialWildcard()) { + trail.known_ = trail.unknown_ = AttributeMatch(); + } + } + trail.missing_ = missing_attributes.MatchVariable(variable); + if (trail.missing_.IsFull()) { + trail.known_ = trail.unknown_ = AttributeMatch(); + value = AttributeMatchToMissingValue( + std::exchange(trail.missing_, AttributeMatch()), variable, + unknown_tree); + return true; + } + return false; +} + +class UnknownAccumulator { + public: + UnknownAccumulator(const UnknownAccumulator&) = delete; + UnknownAccumulator(UnknownAccumulator&&) = delete; + UnknownAccumulator& operator=(const UnknownAccumulator&) = delete; + UnknownAccumulator& operator=(UnknownAccumulator&&) = delete; + + explicit UnknownAccumulator(UnknownTree* tree) : tree_(tree) { + ABSL_DCHECK(tree != nullptr); + } + + void Add(const UnknownValue& value) { + Add(common_internal::GetUnknownValueRep(value)); + } + + void Add(const UnknownSet& value); + + void Add(AttributeMatch match) { + ABSL_DCHECK(!match.IsNone()); + unknown_attributes_.insert( + tree_->Convert(common_internal::GetAttributeMatcherNode(match))); + empty_ = false; + } + + void MaybeAdd(AttributeMatch match) { + if (!match.IsNone()) { + Add(match); + } + } + + void MaybeAdd(const Value& value) { + if (auto unknown_value = value.AsUnknown(); unknown_value.has_value()) { + Add(*unknown_value); + } + } + + void MaybeAdd(const AttributeTrail& trail) { + MaybeAdd(trail.GetNotFullUnknownMatch()); + } + + void MaybeAdd(const Value& value, AttributeMatch match) { + if (auto unknown_value = value.AsUnknown(); unknown_value.has_value()) { + Add(*unknown_value); + } else { + MaybeAdd(match); + } + } + + void MaybeAdd(const Value& value, const AttributeTrail& trail) { + MaybeAdd(value, trail.GetNotFullUnknownMatch()); + } + + [[nodiscard]] + bool IsEmpty() const { + return empty_; + } + + [[nodiscard]] + std::optional Accumulate() &&; + + [[nodiscard]] + UnknownValue Build() && { + return std::move(*this).Accumulate().value_or(UnknownValue()); + } + + private: + void Add(const UnknownAttributeSet* set); + + void Add(const UnknownFunctionSet* set); + + UnknownTree* const tree_; + // BTrees used to perform merges when more than two unknown sets are + // encountered. Merges between two sets are performed without additional + // memory. + absl::btree_set unknown_attributes_; + absl::btree_set unknown_functions_; + const UnknownAttributeSet* largest_unknown_attribute_set_ = nullptr; + const UnknownFunctionSet* largest_unknown_function_set_ = nullptr; + const UnknownAttributeSet* unknown_attribute_set_pair_[2] = {nullptr, + nullptr}; + const UnknownFunctionSet* unknown_function_set_pair_[2] = {nullptr, nullptr}; + bool added_unknown_attribute_set_pair_ = false; + bool added_unknown_function_set_pair_ = false; + bool empty_ = true; +}; + +[[nodiscard]] +inline std::optional PartiallyIdentifyAndMergeUnknowns( + absl::Span values, absl::Span attributes, + UnknownTree* tree) { + ABSL_DCHECK_EQ(values.size(), attributes.size()); + ABSL_DCHECK(tree != nullptr); + UnknownAccumulator accumulator(tree); + for (const Value& value : values) { + accumulator.MaybeAdd(value); + } + // There should not be any full unknown matches in attributes, so only inspect + // it if we want partials. + for (const AttributeTrail& attribute : attributes) { + accumulator.MaybeAdd(attribute); + } + return std::move(accumulator).Accumulate(); +} + +[[nodiscard]] +inline std::optional FullyIdentifyAndMergeUnknowns( + absl::Span values, UnknownTree* tree) { + ABSL_DCHECK(tree != nullptr); + UnknownAccumulator accumulator(tree); + for (const Value& value : values) { + accumulator.MaybeAdd(value); + } + return std::move(accumulator).Accumulate(); +} + +[[nodiscard]] +inline UnknownValue GraftUnknowns(const UnknownValue& value, + UnknownTree* tree) { + ABSL_DCHECK(tree != nullptr); + return common_internal::MakeUnknownValue( + tree->Graft(common_internal::GetUnknownValueRep(value))); +} + +[[nodiscard]] +inline Value GraftUnknowns(const Value& value, UnknownTree* tree) { + ABSL_DCHECK(tree != nullptr); + if (auto unknown_value = value.AsUnknown(); + ABSL_PREDICT_FALSE(unknown_value.has_value())) { + return common_internal::GraftUnknowns(*unknown_value, tree); + } + return value; +} + +inline void GraftUnknownsAt(UnknownValue& value, UnknownTree* tree) { + ABSL_DCHECK(tree != nullptr); + tree->GraftAt(common_internal::GetUnknownValueRep(value)); +} + +inline void GraftUnknownsAt(Value& value, UnknownTree* tree) { + ABSL_DCHECK(tree != nullptr); + if (auto unknown_value = value.AsUnknown(); + ABSL_PREDICT_FALSE(unknown_value.has_value())) { + tree->GraftAt(common_internal::GetUnknownValueRep(*unknown_value)); + } +} + +} // namespace cel::common_internal + +#endif // THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_ATTRIBUTE_TRAIL_H_ diff --git a/common/internal/byte_string.cc b/common/internal/byte_string.cc index 304104a87..dcc8479ca 100644 --- a/common/internal/byte_string.cc +++ b/common/internal/byte_string.cc @@ -14,48 +14,70 @@ #include "common/internal/byte_string.h" +#include #include -#include #include #include -#include #include #include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/functional/overload.h" -#include "absl/hash/hash.h" #include "absl/log/absl_check.h" #include "absl/strings/cord.h" -#include "absl/strings/match.h" +#include "absl/strings/resize_and_overwrite.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/allocator.h" -#include "common/internal/metadata.h" -#include "common/internal/reference_count.h" -#include "common/memory.h" #include "google/protobuf/arena.h" namespace cel::common_internal { -namespace { - -char* CopyCordToArray(const absl::Cord& cord, char* data) { - for (auto chunk : cord.Chunks()) { - std::memcpy(data, chunk.data(), chunk.size()); - data += chunk.size(); +ByteString ByteString::From(absl::string_view value, + google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK(arena != nullptr); + ByteString result(UninitializedTag{}); + if (value.size() <= kSmallByteStringCapacity) { + result.SetSmall(value); + } else { + char* arena_value = + reinterpret_cast(arena->AllocateAligned(value.size())); + std::memcpy(arena_value, value.data(), value.size()); + result.SetMedium(arena, absl::string_view(arena_value, value.size())); } - return data; + return result; } -template -T ConsumeAndDestroy(T& object) { - T consumed = std::move(object); - object.~T(); // NOLINT(bugprone-use-after-move) - return consumed; +ByteString ByteString::From(absl::Cord value, + google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK(arena != nullptr); + ByteString result(UninitializedTag{}); + if (value.size() <= kSmallByteStringCapacity) { + result.SetSmall(value); + } else { + result.SetLarge(arena, + google::protobuf::Arena::Create(arena, std::move(value))); + } + return result; } -} // namespace +ByteString ByteString::From(std::string&& value, + google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK(arena != nullptr); + ByteString result(UninitializedTag{}); + if (value.size() <= kSmallByteStringCapacity) { + result.SetSmall(value); + } else if (value.size() > sizeof(std::string)) { + value.shrink_to_fit(); + result.SetMedium( + arena, google::protobuf::Arena::Create(arena, std::move(value))); + } else { + char* arena_value = + reinterpret_cast(arena->AllocateAligned(value.size())); + std::memcpy(arena_value, value.data(), value.size()); + result.SetMedium(arena, absl::string_view(arena_value, value.size())); + } + return result; +} ByteString ByteString::Concat(const ByteString& lhs, const ByteString& rhs, google::protobuf::Arena* absl_nonnull arena) { @@ -72,19 +94,19 @@ ByteString ByteString::Concat(const ByteString& lhs, const ByteString& rhs, rhs.GetKind() == ByteStringKind::kLarge) { // If either the left or right are absl::Cord, use absl::Cord. absl::Cord result; - result.Append(lhs.ToCord()); - result.Append(rhs.ToCord()); - return ByteString(std::move(result)); + lhs.AppendToCord(&result); + rhs.AppendToCord(&result); + return From(result, arena); } const size_t lhs_size = lhs.size(); const size_t rhs_size = rhs.size(); const size_t result_size = lhs_size + rhs_size; - ByteString result; + ByteString result(UninitializedTag{}); if (result_size <= kSmallByteStringCapacity) { // If the resulting string fits in inline storage, do it. + result.rep_.header.kind = ByteStringKind::kSmall; result.rep_.small.size = result_size; - result.rep_.small.arena = arena; lhs.CopyToArray(result.rep_.small.data); rhs.CopyToArray(result.rep_.small.data + lhs_size); } else { @@ -93,212 +115,29 @@ ByteString ByteString::Concat(const ByteString& lhs, const ByteString& rhs, reinterpret_cast(arena->AllocateAligned(result_size)); lhs.CopyToArray(result_data); rhs.CopyToArray(result_data + lhs_size); + result.rep_.header.kind = ByteStringKind::kMedium; result.rep_.medium.data = result_data; result.rep_.medium.size = result_size; - result.rep_.medium.owner = - reinterpret_cast(arena) | kMetadataOwnerArenaBit; - result.rep_.header.kind = ByteStringKind::kMedium; + result.rep_.medium.arena = arena; } return result; } -ByteString::ByteString(Allocator<> allocator, absl::string_view string) { - ABSL_DCHECK_LE(string.size(), max_size()); - auto* arena = allocator.arena(); - if (string.size() <= kSmallByteStringCapacity) { - SetSmall(arena, string); - } else { - SetMedium(arena, string); - } -} - -ByteString::ByteString(Allocator<> allocator, const std::string& string) { - ABSL_DCHECK_LE(string.size(), max_size()); - auto* arena = allocator.arena(); - if (string.size() <= kSmallByteStringCapacity) { - SetSmall(arena, string); - } else { - SetMedium(arena, string); - } -} - -ByteString::ByteString(Allocator<> allocator, std::string&& string) { - ABSL_DCHECK_LE(string.size(), max_size()); - auto* arena = allocator.arena(); - if (string.size() <= kSmallByteStringCapacity) { - SetSmall(arena, string); - } else { - SetMedium(arena, std::move(string)); - } -} - -ByteString::ByteString(Allocator<> allocator, const absl::Cord& cord) { - ABSL_DCHECK_LE(cord.size(), max_size()); - auto* arena = allocator.arena(); - if (cord.size() <= kSmallByteStringCapacity) { - SetSmall(arena, cord); - } else if (arena != nullptr) { - SetMedium(arena, cord); - } else { - SetLarge(cord); - } -} - -ByteString ByteString::Borrowed(Borrower borrower, absl::string_view string) { - ABSL_DCHECK(borrower != Borrower::None()) << "Borrowing from Owner::None()"; - auto* arena = borrower.arena(); - if (string.size() <= kSmallByteStringCapacity || arena != nullptr) { - return ByteString(arena, string); - } - const auto* refcount = BorrowerRelease(borrower); - // A nullptr refcount indicates somebody called us to borrow something that - // has no owner. If this is the case, we fallback to assuming operator - // new/delete and convert it to a reference count. - if (refcount == nullptr) { - std::tie(refcount, string) = MakeReferenceCountedString(string); - } else { - StrongRef(*refcount); - } - return ByteString(refcount, string); -} - -ByteString ByteString::Borrowed(Borrower borrower, const absl::Cord& cord) { - ABSL_DCHECK(borrower != Borrower::None()) << "Borrowing from Owner::None()"; - return ByteString(borrower.arena(), cord); -} - -ByteString::ByteString(const ReferenceCount* absl_nonnull refcount, - absl::string_view string) { - ABSL_DCHECK_LE(string.size(), max_size()); - SetMedium(string, reinterpret_cast(refcount) | - kMetadataOwnerReferenceCountBit); -} - -ByteString::ByteString(ByteString::ExternalStringTag, - absl::string_view string) { - if (string.size() <= kSmallByteStringCapacity) { - SetSmall(nullptr, string); - } else { - SetExternalMedium(string); - } -} - -ByteString ByteString::FromExternal(absl::string_view string) { - return ByteString(ExternalStringTag{}, string); -} - -google::protobuf::Arena* absl_nullable ByteString::GetArena() const { - switch (GetKind()) { - case ByteStringKind::kSmall: - return GetSmallArena(); - case ByteStringKind::kMedium: - return GetMediumArena(); - case ByteStringKind::kLarge: - return nullptr; - } -} - -bool ByteString::empty() const { - switch (GetKind()) { - case ByteStringKind::kSmall: - return rep_.small.size == 0; - case ByteStringKind::kMedium: - return rep_.medium.size == 0; - case ByteStringKind::kLarge: - return GetLarge().empty(); - } -} - -size_t ByteString::size() const { - switch (GetKind()) { - case ByteStringKind::kSmall: - return rep_.small.size; - case ByteStringKind::kMedium: - return rep_.medium.size; - case ByteStringKind::kLarge: - return GetLarge().size(); - } -} - -absl::string_view ByteString::Flatten() { - switch (GetKind()) { - case ByteStringKind::kSmall: - return GetSmall(); - case ByteStringKind::kMedium: - return GetMedium(); - case ByteStringKind::kLarge: - return GetLarge().Flatten(); - } -} - absl::optional ByteString::TryFlat() const { switch (GetKind()) { case ByteStringKind::kSmall: return GetSmall(); case ByteStringKind::kMedium: return GetMedium(); - case ByteStringKind::kLarge: - return GetLarge().TryFlat(); + case ByteStringKind::kLarge: { + if (auto flat = rep_.large.data->TryFlat(); flat.has_value()) { + return flat->substr(rep_.large.offset, rep_.large.size); + } + return std::nullopt; + } } } -bool ByteString::Equals(absl::string_view rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> bool { return lhs == rhs; }, - [&rhs](const absl::Cord& lhs) -> bool { return lhs == rhs; })); -} - -bool ByteString::Equals(const absl::Cord& rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> bool { return lhs == rhs; }, - [&rhs](const absl::Cord& lhs) -> bool { return lhs == rhs; })); -} - -int ByteString::Compare(absl::string_view rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> int { return lhs.compare(rhs); }, - [&rhs](const absl::Cord& lhs) -> int { return lhs.Compare(rhs); })); -} - -int ByteString::Compare(const absl::Cord& rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> int { return -rhs.Compare(lhs); }, - [&rhs](const absl::Cord& lhs) -> int { return lhs.Compare(rhs); })); -} - -bool ByteString::StartsWith(absl::string_view rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> bool { - return absl::StartsWith(lhs, rhs); - }, - [&rhs](const absl::Cord& lhs) -> bool { return lhs.StartsWith(rhs); })); -} - -bool ByteString::StartsWith(const absl::Cord& rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> bool { - return lhs.size() >= rhs.size() && lhs.substr(0, rhs.size()) == rhs; - }, - [&rhs](const absl::Cord& lhs) -> bool { return lhs.StartsWith(rhs); })); -} - -bool ByteString::EndsWith(absl::string_view rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> bool { - return absl::EndsWith(lhs, rhs); - }, - [&rhs](const absl::Cord& lhs) -> bool { return lhs.EndsWith(rhs); })); -} - -bool ByteString::EndsWith(const absl::Cord& rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> bool { - return lhs.size() >= rhs.size() && - lhs.substr(lhs.size() - rhs.size()) == rhs; - }, - [&rhs](const absl::Cord& lhs) -> bool { return lhs.EndsWith(rhs); })); -} - absl::optional ByteString::Find(absl::string_view needle, size_t pos) const { ABSL_DCHECK_LE(pos, size()); @@ -307,7 +146,7 @@ absl::optional ByteString::Find(absl::string_view needle, [&needle, pos](absl::string_view lhs) -> absl::optional { absl::string_view::size_type i = lhs.find(needle, pos); if (i == absl::string_view::npos) { - return absl::nullopt; + return std::nullopt; } return i; }, @@ -315,7 +154,7 @@ absl::optional ByteString::Find(absl::string_view needle, absl::Cord cord = lhs.Subcord(pos, lhs.size() - pos); absl::Cord::CharIterator it = cord.Find(needle); if (it == cord.char_end()) { - return absl::nullopt; + return std::nullopt; } return pos + static_cast(absl::Cord::Distance(cord.char_begin(), it)); @@ -331,14 +170,14 @@ absl::optional ByteString::Find(const absl::Cord& needle, if (auto flat_needle = needle.TryFlat(); flat_needle) { absl::string_view::size_type i = lhs.find(*flat_needle, pos); if (i == absl::string_view::npos) { - return absl::nullopt; + return std::nullopt; } return i; } // Needle is fragmented, we have to do a linear scan. const size_t needle_size = needle.size(); if (pos + needle_size > lhs.size()) { - return absl::nullopt; + return std::nullopt; } if (ABSL_PREDICT_FALSE(needle_size == 0)) { return pos; @@ -354,7 +193,7 @@ absl::optional ByteString::Find(const absl::Cord& needle, size_t found_pos = lhs.find(first_chunk, current_pos); if (found_pos == absl::string_view::npos || found_pos > lhs.size() - needle_size) { - return absl::nullopt; + return std::nullopt; } if (lhs.substr(found_pos + first_chunk.size(), rest_of_needle.size()) == rest_of_needle) { @@ -367,7 +206,7 @@ absl::optional ByteString::Find(const absl::Cord& needle, absl::Cord cord = lhs.Subcord(pos, lhs.size() - pos); absl::Cord::CharIterator it = cord.Find(needle); if (it == cord.char_end()) { - return absl::nullopt; + return std::nullopt; } return pos + static_cast(absl::Cord::Distance(cord.char_begin(), it)); @@ -380,22 +219,20 @@ ByteString ByteString::Substring(size_t pos, size_t npos) const { switch (GetKind()) { case ByteStringKind::kSmall: { - ByteString result; - result.rep_.header.kind = ByteStringKind::kSmall; - result.rep_.small.size = npos - pos; - std::memcpy(result.rep_.small.data, rep_.small.data + pos, - result.rep_.small.size); - result.rep_.small.arena = GetSmallArena(); + ByteString result(UninitializedTag{}); + result.SetSmall(GetSmall().substr(pos, npos - pos)); return result; } case ByteStringKind::kMedium: { - ByteString result(*this); - result.rep_.medium.data += pos; - result.rep_.medium.size = npos - pos; + ByteString result(UninitializedTag{}); + result.SetMedium(GetMediumArena(), GetMedium().substr(pos, npos - pos)); return result; } case ByteStringKind::kLarge: - return ByteString(GetLarge().Subcord(pos, npos - pos)); + ByteString result(UninitializedTag{}); + result.SetLarge(GetLargeArena(), rep_.large.data, rep_.large.offset + pos, + npos - pos); + return result; } } @@ -407,29 +244,16 @@ void ByteString::RemovePrefix(size_t n) { switch (GetKind()) { case ByteStringKind::kSmall: std::memmove(rep_.small.data, rep_.small.data + n, rep_.small.size - n); - rep_.small.size -= n; + rep_.small.size = rep_.small.size - n; break; case ByteStringKind::kMedium: - rep_.medium.data += n; - rep_.medium.size -= n; - if (rep_.medium.size <= kSmallByteStringCapacity) { - const auto* refcount = GetMediumReferenceCount(); - SetSmall(GetMediumArena(), GetMedium()); - StrongUnref(refcount); - } + rep_.medium.data = rep_.medium.data + n; + rep_.medium.size = rep_.medium.size - n; + break; + case ByteStringKind::kLarge: + rep_.large.offset = rep_.large.offset + n; + rep_.large.size = rep_.large.size - n; break; - case ByteStringKind::kLarge: { - auto& large = GetLarge(); - const auto large_size = large.size(); - const auto new_large_pos = n; - const auto new_large_size = large_size - n; - large = large.Subcord(new_large_pos, new_large_size); - if (new_large_size <= kSmallByteStringCapacity) { - auto large_copy = std::move(large); - DestroyLarge(); - SetSmall(nullptr, large_copy); - } - } break; } } @@ -440,34 +264,19 @@ void ByteString::RemoveSuffix(size_t n) { } switch (GetKind()) { case ByteStringKind::kSmall: - rep_.small.size -= n; + rep_.small.size = rep_.small.size - n; break; case ByteStringKind::kMedium: - rep_.medium.size -= n; - if (rep_.medium.size <= kSmallByteStringCapacity) { - const auto* refcount = GetMediumReferenceCount(); - SetSmall(GetMediumArena(), GetMedium()); - StrongUnref(refcount); - } + rep_.medium.size = rep_.medium.size - n; + break; + case ByteStringKind::kLarge: + rep_.large.size = rep_.large.size - n; break; - case ByteStringKind::kLarge: { - auto& large = GetLarge(); - const auto large_size = large.size(); - const auto new_large_pos = 0; - const auto new_large_size = large_size - n; - large = large.Subcord(new_large_pos, new_large_size); - if (new_large_size <= kSmallByteStringCapacity) { - auto large_copy = std::move(large); - DestroyLarge(); - SetSmall(nullptr, large_copy); - } - } break; } } void ByteString::CopyToArray(char* absl_nonnull out) const { ABSL_DCHECK(out != nullptr); - switch (GetKind()) { case ByteStringKind::kSmall: { absl::string_view small = GetSmall(); @@ -478,8 +287,8 @@ void ByteString::CopyToArray(char* absl_nonnull out) const { std::memcpy(out, medium.data(), medium.size()); } break; case ByteStringKind::kLarge: { - const absl::Cord& large = GetLarge(); - (CopyCordToArray)(large, out); + (CopyCordToArray)(*rep_.large.data, rep_.large.offset, rep_.large.size, + out); } break; } } @@ -490,14 +299,22 @@ std::string ByteString::ToString() const { return std::string(GetSmall()); case ByteStringKind::kMedium: return std::string(GetMedium()); - case ByteStringKind::kLarge: - return static_cast(GetLarge()); + case ByteStringKind::kLarge: { + std::string result; + absl::StringResizeAndOverwrite( + result, rep_.large.size, + [this](char* buffer, size_t buffer_size) -> size_t { + (CopyCordToArray)(*rep_.large.data, rep_.large.offset, + rep_.large.size, buffer); + return rep_.large.size; + }); + return result; + } } } void ByteString::CopyToString(std::string* absl_nonnull out) const { ABSL_DCHECK(out != nullptr); - switch (GetKind()) { case ByteStringKind::kSmall: out->assign(GetSmall()); @@ -506,14 +323,19 @@ void ByteString::CopyToString(std::string* absl_nonnull out) const { out->assign(GetMedium()); break; case ByteStringKind::kLarge: - absl::CopyCordToString(GetLarge(), out); + absl::StringResizeAndOverwrite( + *out, rep_.large.size, + [this](char* buffer, size_t buffer_size) -> size_t { + CopyCordToArray(*rep_.large.data, rep_.large.offset, + rep_.large.size, buffer); + return rep_.large.size; + }); break; } } void ByteString::AppendToString(std::string* absl_nonnull out) const { ABSL_DCHECK(out != nullptr); - switch (GetKind()) { case ByteStringKind::kSmall: out->append(GetSmall()); @@ -522,53 +344,18 @@ void ByteString::AppendToString(std::string* absl_nonnull out) const { out->append(GetMedium()); break; case ByteStringKind::kLarge: - absl::AppendCordToString(GetLarge(), out); + AppendCordToString(*rep_.large.data, rep_.large.offset, rep_.large.size, + *out); break; } } -namespace { - -struct ReferenceCountReleaser { - const ReferenceCount* absl_nonnull refcount; - - void operator()() const { StrongUnref(*refcount); } -}; - -} // namespace - -absl::Cord ByteString::ToCord() const& { - switch (GetKind()) { - case ByteStringKind::kSmall: - return absl::Cord(GetSmall()); - case ByteStringKind::kMedium: { - const auto* refcount = GetMediumReferenceCount(); - if (refcount != nullptr) { - StrongRef(*refcount); - return absl::MakeCordFromExternal(GetMedium(), - ReferenceCountReleaser{refcount}); - } - return absl::Cord(GetMedium()); - } - case ByteStringKind::kLarge: - return GetLarge(); - } -} - -absl::Cord ByteString::ToCord() && { +absl::Cord ByteString::ToCord() const { switch (GetKind()) { case ByteStringKind::kSmall: return absl::Cord(GetSmall()); - case ByteStringKind::kMedium: { - const auto* refcount = GetMediumReferenceCount(); - if (refcount != nullptr) { - auto medium = GetMedium(); - SetSmallEmpty(nullptr); - return absl::MakeCordFromExternal(medium, - ReferenceCountReleaser{refcount}); - } + case ByteStringKind::kMedium: return absl::Cord(GetMedium()); - } case ByteStringKind::kLarge: return GetLarge(); } @@ -576,21 +363,13 @@ absl::Cord ByteString::ToCord() && { void ByteString::CopyToCord(absl::Cord* absl_nonnull out) const { ABSL_DCHECK(out != nullptr); - switch (GetKind()) { case ByteStringKind::kSmall: *out = absl::Cord(GetSmall()); break; - case ByteStringKind::kMedium: { - const auto* refcount = GetMediumReferenceCount(); - if (refcount != nullptr) { - StrongRef(*refcount); - *out = absl::MakeCordFromExternal(GetMedium(), - ReferenceCountReleaser{refcount}); - } else { - *out = absl::Cord(GetMedium()); - } - } break; + case ByteStringKind::kMedium: + *out = absl::Cord(GetMedium()); + break; case ByteStringKind::kLarge: *out = GetLarge(); break; @@ -599,21 +378,13 @@ void ByteString::CopyToCord(absl::Cord* absl_nonnull out) const { void ByteString::AppendToCord(absl::Cord* absl_nonnull out) const { ABSL_DCHECK(out != nullptr); - switch (GetKind()) { case ByteStringKind::kSmall: out->Append(GetSmall()); break; - case ByteStringKind::kMedium: { - const auto* refcount = GetMediumReferenceCount(); - if (refcount != nullptr) { - StrongRef(*refcount); - out->Append(absl::MakeCordFromExternal( - GetMedium(), ReferenceCountReleaser{refcount})); - } else { - out->Append(GetMedium()); - } - } break; + case ByteStringKind::kMedium: + out->Append(GetMedium()); + break; case ByteStringKind::kLarge: out->Append(GetLarge()); break; @@ -623,419 +394,83 @@ void ByteString::AppendToCord(absl::Cord* absl_nonnull out) const { absl::string_view ByteString::ToStringView( std::string* absl_nonnull scratch) const { ABSL_DCHECK(scratch != nullptr); - switch (GetKind()) { case ByteStringKind::kSmall: return GetSmall(); case ByteStringKind::kMedium: return GetMedium(); case ByteStringKind::kLarge: - if (auto flat = GetLarge().TryFlat(); flat) { - return *flat; + if (auto flat = rep_.large.data->TryFlat(); flat.has_value()) { + return flat->substr(rep_.large.offset, rep_.large.size); } - absl::CopyCordToString(GetLarge(), scratch); + absl::StringResizeAndOverwrite( + *scratch, rep_.large.size, + [this](char* buffer, size_t buffer_size) -> size_t { + CopyCordToArray(*rep_.large.data, rep_.large.offset, + rep_.large.size, buffer); + return rep_.large.size; + }); return absl::string_view(*scratch); } } -absl::string_view ByteString::AsStringView() const { - const ByteStringKind kind = GetKind(); - ABSL_CHECK(kind == ByteStringKind::kSmall || // Crash OK - kind == ByteStringKind::kMedium); - switch (kind) { - case ByteStringKind::kSmall: - return GetSmall(); - case ByteStringKind::kMedium: - return GetMedium(); - case ByteStringKind::kLarge: - ABSL_UNREACHABLE(); - } -} - -google::protobuf::Arena* absl_nullable ByteString::GetMediumArena( - const MediumByteStringRep& rep) { - if ((rep.owner & kMetadataOwnerBits) == kMetadataOwnerArenaBit) { - return reinterpret_cast(rep.owner & - kMetadataOwnerPointerMask); - } - return nullptr; -} - -const ReferenceCount* absl_nullable ByteString::GetMediumReferenceCount( - const MediumByteStringRep& rep) { - if ((rep.owner & kMetadataOwnerBits) == kMetadataOwnerReferenceCountBit) { - return reinterpret_cast(rep.owner & - kMetadataOwnerPointerMask); - } - return nullptr; -} - -void ByteString::Construct(const ByteString& other, - absl::optional> allocator) { - switch (other.GetKind()) { - case ByteStringKind::kSmall: - rep_.small = other.rep_.small; - if (allocator.has_value()) { - rep_.small.arena = allocator->arena(); - } - break; - case ByteStringKind::kMedium: - if (allocator.has_value() && - allocator->arena() != other.GetMediumArena()) { - SetMedium(allocator->arena(), other.GetMedium()); - } else { - rep_.medium = other.rep_.medium; - StrongRef(GetMediumReferenceCount()); - } - break; - case ByteStringKind::kLarge: - if (allocator.has_value() && allocator->arena() != nullptr) { - SetMedium(allocator->arena(), other.GetLarge()); - } else { - SetLarge(other.GetLarge()); - } - break; - } -} - -void ByteString::Construct(ByteString& other, - absl::optional> allocator) { - switch (other.GetKind()) { - case ByteStringKind::kSmall: - rep_.small = other.rep_.small; - if (allocator.has_value()) { - rep_.small.arena = allocator->arena(); - } - break; - case ByteStringKind::kMedium: - if (allocator.has_value() && - allocator->arena() != other.GetMediumArena()) { - SetMedium(allocator->arena(), other.GetMedium()); - } else { - rep_.medium = other.rep_.medium; - other.rep_.medium.owner = 0; - } - break; - case ByteStringKind::kLarge: - if (allocator.has_value() && allocator->arena() != nullptr) { - SetMedium(allocator->arena(), other.GetLarge()); - } else { - SetLarge(std::move(other.GetLarge())); - } - break; - } -} - -void ByteString::CopyFrom(const ByteString& other) { - ABSL_DCHECK_NE(&other, this); - - switch (other.GetKind()) { - case ByteStringKind::kSmall: - switch (GetKind()) { - case ByteStringKind::kSmall: - break; - case ByteStringKind::kMedium: - DestroyMedium(); - break; - case ByteStringKind::kLarge: - DestroyLarge(); - break; - } - rep_.small = other.rep_.small; - break; - case ByteStringKind::kMedium: - switch (GetKind()) { - case ByteStringKind::kSmall: - rep_.medium = other.rep_.medium; - StrongRef(GetMediumReferenceCount()); - break; - case ByteStringKind::kMedium: - StrongRef(other.GetMediumReferenceCount()); - DestroyMedium(); - rep_.medium = other.rep_.medium; - break; - case ByteStringKind::kLarge: - DestroyLarge(); - rep_.medium = other.rep_.medium; - StrongRef(GetMediumReferenceCount()); - break; - } - break; - case ByteStringKind::kLarge: - switch (GetKind()) { - case ByteStringKind::kSmall: - SetLarge(other.GetLarge()); - break; - case ByteStringKind::kMedium: - DestroyMedium(); - SetLarge(other.GetLarge()); - break; - case ByteStringKind::kLarge: - GetLarge() = other.GetLarge(); - break; - } - break; - } -} - -void ByteString::MoveFrom(ByteString& other) { - ABSL_DCHECK_NE(&other, this); - - switch (other.GetKind()) { - case ByteStringKind::kSmall: - switch (GetKind()) { - case ByteStringKind::kSmall: - break; - case ByteStringKind::kMedium: - DestroyMedium(); - break; - case ByteStringKind::kLarge: - DestroyLarge(); - break; - } - rep_.small = other.rep_.small; - break; - case ByteStringKind::kMedium: - switch (GetKind()) { - case ByteStringKind::kSmall: - rep_.medium = other.rep_.medium; - break; - case ByteStringKind::kMedium: - DestroyMedium(); - rep_.medium = other.rep_.medium; - break; - case ByteStringKind::kLarge: - DestroyLarge(); - rep_.medium = other.rep_.medium; - break; - } - other.rep_.medium.owner = 0; - break; - case ByteStringKind::kLarge: - switch (GetKind()) { - case ByteStringKind::kSmall: - SetLarge(std::move(other.GetLarge())); - break; - case ByteStringKind::kMedium: - DestroyMedium(); - SetLarge(std::move(other.GetLarge())); - break; - case ByteStringKind::kLarge: - GetLarge() = std::move(other.GetLarge()); - break; - } - break; - } -} - ByteString ByteString::Clone(google::protobuf::Arena* absl_nonnull arena) const { ABSL_DCHECK(arena != nullptr); - switch (GetKind()) { - case ByteStringKind::kSmall: - return ByteString(arena, GetSmall()); + case ByteStringKind::kSmall: { + ByteString result(UninitializedTag{}); + result.SetSmall(GetSmall()); + return result; + } case ByteStringKind::kMedium: { google::protobuf::Arena* absl_nullable other_arena = GetMediumArena(); - if (arena != nullptr) { - if (arena == other_arena) { - return *this; - } - return ByteString(arena, GetMedium()); + if (other_arena != arena) { + return From(GetMedium(), arena); } - if (other_arena != nullptr) { - return ByteString(arena, GetMedium()); + return *this; + } + case ByteStringKind::kLarge: { + google::protobuf::Arena* absl_nullable other_arena = GetLargeArena(); + if (other_arena != arena) { + return From(GetLarge(), arena); } return *this; } - case ByteStringKind::kLarge: - return ByteString(arena, GetLarge()); } } -void ByteString::HashValue(absl::HashState state) const { - switch (GetKind()) { - case ByteStringKind::kSmall: - absl::HashState::combine(std::move(state), GetSmall()); - break; - case ByteStringKind::kMedium: - absl::HashState::combine(std::move(state), GetMedium()); - break; - case ByteStringKind::kLarge: - absl::HashState::combine(std::move(state), GetLarge()); - break; - } -} - -void ByteString::Swap(ByteString& other) { - ABSL_DCHECK_NE(&other, this); - using std::swap; - - switch (other.GetKind()) { - case ByteStringKind::kSmall: - switch (GetKind()) { - case ByteStringKind::kSmall: - // small <=> small - swap(rep_.small, other.rep_.small); - break; - case ByteStringKind::kMedium: - // medium <=> small - swap(rep_, other.rep_); - break; - case ByteStringKind::kLarge: { - absl::Cord cord = std::move(GetLarge()); - DestroyLarge(); - rep_ = other.rep_; - other.SetLarge(std::move(cord)); - } break; - } - break; - case ByteStringKind::kMedium: - switch (GetKind()) { - case ByteStringKind::kSmall: - swap(rep_, other.rep_); - break; - case ByteStringKind::kMedium: - swap(rep_.medium, other.rep_.medium); - break; - case ByteStringKind::kLarge: { - absl::Cord cord = std::move(GetLarge()); - DestroyLarge(); - rep_ = other.rep_; - other.SetLarge(std::move(cord)); - } break; - } +void ByteString::CopyCordToArray(const absl::Cord& cord, size_t offset, + size_t size, char* data) { + for (auto chunk : cord.Chunks()) { + if (size == 0) { break; - case ByteStringKind::kLarge: - switch (GetKind()) { - case ByteStringKind::kSmall: { - absl::Cord cord = std::move(other.GetLarge()); - other.DestroyLarge(); - other.rep_.small = rep_.small; - SetLarge(std::move(cord)); - } break; - case ByteStringKind::kMedium: { - absl::Cord cord = std::move(other.GetLarge()); - other.DestroyLarge(); - other.rep_.medium = rep_.medium; - SetLarge(std::move(cord)); - } break; - case ByteStringKind::kLarge: - swap(GetLarge(), other.GetLarge()); - break; + } + if (offset > 0) { + size_t min_offset = std::min(chunk.size(), offset); + offset -= min_offset; + if (offset > 0) { + continue; } - break; - } -} - -void ByteString::Destroy() { - switch (GetKind()) { - case ByteStringKind::kSmall: - break; - case ByteStringKind::kMedium: - DestroyMedium(); - break; - case ByteStringKind::kLarge: - DestroyLarge(); - break; - } -} - -void ByteString::SetSmall(google::protobuf::Arena* absl_nullable arena, - absl::string_view string) { - ABSL_DCHECK_LE(string.size(), kSmallByteStringCapacity); - rep_.header.kind = ByteStringKind::kSmall; - rep_.small.size = string.size(); - rep_.small.arena = arena; - std::memcpy(rep_.small.data, string.data(), rep_.small.size); -} - -void ByteString::SetSmall(google::protobuf::Arena* absl_nullable arena, - const absl::Cord& cord) { - ABSL_DCHECK_LE(cord.size(), kSmallByteStringCapacity); - rep_.header.kind = ByteStringKind::kSmall; - rep_.small.size = cord.size(); - rep_.small.arena = arena; - (CopyCordToArray)(cord, rep_.small.data); -} - -void ByteString::SetMedium(google::protobuf::Arena* absl_nullable arena, - absl::string_view string) { - ABSL_DCHECK_GT(string.size(), kSmallByteStringCapacity); - rep_.header.kind = ByteStringKind::kMedium; - rep_.medium.size = string.size(); - if (arena != nullptr) { - char* data = static_cast( - arena->AllocateAligned(rep_.medium.size, alignof(char))); - std::memcpy(data, string.data(), rep_.medium.size); - rep_.medium.data = data; - rep_.medium.owner = - reinterpret_cast(arena) | kMetadataOwnerArenaBit; - } else { - auto pair = MakeReferenceCountedString(string); - rep_.medium.data = pair.second.data(); - rep_.medium.owner = reinterpret_cast(pair.first) | - kMetadataOwnerReferenceCountBit; - } -} - -void ByteString::SetExternalMedium(absl::string_view string) { - ABSL_DCHECK_GT(string.size(), kSmallByteStringCapacity); - rep_.header.kind = ByteStringKind::kMedium; - rep_.medium.size = string.size(); - rep_.medium.data = string.data(); - rep_.medium.owner = 0; -} - -void ByteString::SetMedium(google::protobuf::Arena* absl_nullable arena, - std::string&& string) { - ABSL_DCHECK_GT(string.size(), kSmallByteStringCapacity); - rep_.header.kind = ByteStringKind::kMedium; - rep_.medium.size = string.size(); - if (arena != nullptr) { - auto* data = google::protobuf::Arena::Create(arena, std::move(string)); - rep_.medium.data = data->data(); - rep_.medium.owner = - reinterpret_cast(arena) | kMetadataOwnerArenaBit; - } else { - auto pair = MakeReferenceCountedString(std::move(string)); - rep_.medium.data = pair.second.data(); - rep_.medium.owner = reinterpret_cast(pair.first) | - kMetadataOwnerReferenceCountBit; - } -} - -void ByteString::SetMedium(google::protobuf::Arena* absl_nonnull arena, - const absl::Cord& cord) { - ABSL_DCHECK_GT(cord.size(), kSmallByteStringCapacity); - rep_.header.kind = ByteStringKind::kMedium; - rep_.medium.size = cord.size(); - char* data = static_cast( - arena->AllocateAligned(rep_.medium.size, alignof(char))); - (CopyCordToArray)(cord, data); - rep_.medium.data = data; - rep_.medium.owner = - reinterpret_cast(arena) | kMetadataOwnerArenaBit; -} - -void ByteString::SetMedium(absl::string_view string, uintptr_t owner) { - ABSL_DCHECK_GT(string.size(), kSmallByteStringCapacity); - ABSL_DCHECK_NE(owner, 0); - rep_.header.kind = ByteStringKind::kMedium; - rep_.medium.size = string.size(); - rep_.medium.data = string.data(); - rep_.medium.owner = owner; -} - -void ByteString::SetLarge(const absl::Cord& cord) { - ABSL_DCHECK_GT(cord.size(), kSmallByteStringCapacity); - rep_.header.kind = ByteStringKind::kLarge; - ::new (static_cast(&rep_.large.data[0])) absl::Cord(cord); -} - -void ByteString::SetLarge(absl::Cord&& cord) { - ABSL_DCHECK_GT(cord.size(), kSmallByteStringCapacity); - rep_.header.kind = ByteStringKind::kLarge; - ::new (static_cast(&rep_.large.data[0])) absl::Cord(std::move(cord)); + chunk.remove_prefix(min_offset); + } + size_t min_size = std::min(size, chunk.size()); + std::memcpy(data, chunk.data(), min_size); + data += min_size; + size -= min_size; + } +} + +void ByteString::AppendCordToString(const absl::Cord& cord, size_t offset, + size_t size, std::string& data) { + size_t original_size = data.size(); + size_t target_size = original_size + size; + absl::StringResizeAndOverwrite( + data, original_size + size, + [&cord, offset, size, original_size, target_size]( + char* buf, size_t buf_size) -> size_t { + CopyCordToArray(cord, offset, size, buf + original_size); + return target_size; + }); } absl::string_view LegacyByteString(const ByteString& string, bool stable, diff --git a/common/internal/byte_string.h b/common/internal/byte_string.h index c576e5634..b9fb222a8 100644 --- a/common/internal/byte_string.h +++ b/common/internal/byte_string.h @@ -17,25 +17,20 @@ #include #include -#include +#include +#include #include #include -#include #include #include "absl/base/attributes.h" #include "absl/base/nullability.h" -#include "absl/base/optimization.h" #include "absl/functional/overload.h" -#include "absl/hash/hash.h" #include "absl/log/absl_check.h" #include "absl/strings/cord.h" +#include "absl/strings/match.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/allocator.h" -#include "common/arena.h" -#include "common/internal/reference_count.h" -#include "common/memory.h" #include "google/protobuf/arena.h" namespace cel { @@ -46,15 +41,7 @@ class StringValue; namespace common_internal { -// absl::Cord is trivially relocatable IFF we are not using ASan or MSan. When -// using ASan or MSan absl::Cord will poison/unpoison its inline storage. -#if defined(ABSL_HAVE_ADDRESS_SANITIZER) || defined(ABSL_HAVE_MEMORY_SANITIZER) -#define CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI -#else -#define CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI ABSL_ATTRIBUTE_TRIVIAL_ABI -#endif - -class CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI [[nodiscard]] ByteString; +class [[nodiscard]] ByteString; struct ByteStringTestFriend; @@ -76,19 +63,18 @@ inline std::ostream& operator<<(std::ostream& out, ByteStringKind kind) { } // Representation of small strings in ByteString, which are stored in place. -struct CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI SmallByteStringRep final { +struct SmallByteStringRep final { #ifdef _MSC_VER #pragma pack(push, 1) #endif - struct ABSL_ATTRIBUTE_PACKED CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI { + struct ABSL_ATTRIBUTE_PACKED { std::uint8_t kind : 2; std::uint8_t size : 6; }; #ifdef _MSC_VER #pragma pack(pop) #endif - char data[23 - sizeof(google::protobuf::Arena*)]; - google::protobuf::Arena* absl_nullable arena; + char data[23]; }; inline constexpr size_t kSmallByteStringCapacity = @@ -98,6 +84,9 @@ inline constexpr size_t kMediumByteStringSizeBits = sizeof(size_t) * 8 - 2; inline constexpr size_t kMediumByteStringMaxSize = (size_t{1} << kMediumByteStringSizeBits) - 1; +inline constexpr size_t kLargeByteStringMaxSize = + (size_t{1} << kMediumByteStringSizeBits) - 1; + inline constexpr size_t kByteStringViewSizeBits = sizeof(size_t) * 8 - 1; inline constexpr size_t kByteStringViewMaxSize = (size_t{1} << kByteStringViewSizeBits) - 1; @@ -105,43 +94,45 @@ inline constexpr size_t kByteStringViewMaxSize = // Representation of medium strings in ByteString. These are either owned by an // arena or managed by a reference count. This is encoded in `owner` following // the same semantics as `cel::Owner`. -struct CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI MediumByteStringRep final { +struct MediumByteStringRep final { #ifdef _MSC_VER #pragma pack(push, 1) #endif - struct ABSL_ATTRIBUTE_PACKED CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI { + struct ABSL_ATTRIBUTE_PACKED { size_t kind : 2; size_t size : kMediumByteStringSizeBits; }; #ifdef _MSC_VER #pragma pack(pop) #endif - const char* data; - uintptr_t owner; + const char* absl_nullability_unknown data; + google::protobuf::Arena* absl_nullable arena; }; // Representation of large strings in ByteString. These are stored as -// `absl::Cord` and never owned by an arena. -struct CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI LargeByteStringRep final { +// a pointer to `absl::Cord`. +struct LargeByteStringRep final { #ifdef _MSC_VER #pragma pack(push, 1) #endif - struct ABSL_ATTRIBUTE_PACKED CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI { + struct ABSL_ATTRIBUTE_PACKED { size_t kind : 2; - size_t padding : kMediumByteStringSizeBits; + size_t offset : kMediumByteStringSizeBits / 2; + size_t size : kMediumByteStringSizeBits / 2; }; #ifdef _MSC_VER #pragma pack(pop) #endif - alignas(absl::Cord) std::byte data[sizeof(absl::Cord)]; + const absl::Cord* absl_nullability_unknown data; + google::protobuf::Arena* absl_nullable arena; }; // Representation of ByteString. -union CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI ByteStringRep final { +union ByteStringRep final { #ifdef _MSC_VER #pragma pack(push, 1) #endif - struct ABSL_ATTRIBUTE_PACKED CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI { + struct ABSL_ATTRIBUTE_PACKED { ByteStringKind kind : 2; } header; #ifdef _MSC_VER @@ -166,100 +157,94 @@ absl::string_view LegacyByteString(const ByteString& string, bool stable, // string is constructed the allocator will not and cannot change. Copying and // moving between different allocators is supported and dealt with // transparently by copying. -class CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI [[nodiscard]] -ByteString final { +class [[nodiscard]] ByteString final { public: - static ByteString Concat(const ByteString& lhs, const ByteString& rhs, - google::protobuf::Arena* absl_nonnull arena); - - ByteString() : ByteString(NewDeleteAllocator()) {} - - explicit ByteString(const char* absl_nullable string) - : ByteString(NewDeleteAllocator(), string) {} - - explicit ByteString(absl::string_view string) - : ByteString(NewDeleteAllocator(), string) {} - - explicit ByteString(const std::string& string) - : ByteString(NewDeleteAllocator(), string) {} - - explicit ByteString(std::string&& string) - : ByteString(NewDeleteAllocator(), std::move(string)) {} - - explicit ByteString(const absl::Cord& cord) - : ByteString(NewDeleteAllocator(), cord) {} - - ByteString(const ByteString& other) noexcept { - Construct(other, /*allocator=*/absl::nullopt); + static ByteString From(const char* absl_nullable value, + google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return From(absl::NullSafeStringView(value), arena); } - - ByteString(ByteString&& other) noexcept { - Construct(other, /*allocator=*/absl::nullopt); + static ByteString From(absl::string_view value, + google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND); + static ByteString From(absl::Cord value, google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND); + static ByteString From(std::string&& value, + google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND); + + static ByteString Wrap(absl::string_view value, + google::protobuf::Arena* absl_nullable arena + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + ByteString result(UninitializedTag{}); + result.SetMedium(arena, value); + return result; } - - explicit ByteString(Allocator<> allocator) { - SetSmallEmpty(allocator.arena()); + static ByteString Wrap( + const absl::Cord* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND, + google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return Wrap(value, 0, value->size(), arena); } - - ByteString(Allocator<> allocator, const char* absl_nullable string) - : ByteString(allocator, absl::NullSafeStringView(string)) {} - - ByteString(Allocator<> allocator, absl::string_view string); - - ByteString(Allocator<> allocator, const std::string& string); - - ByteString(Allocator<> allocator, std::string&& string); - - ByteString(Allocator<> allocator, const absl::Cord& cord); - - ByteString(Allocator<> allocator, const ByteString& other) { - Construct(other, allocator); + static ByteString Wrap( + const absl::Cord* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND, + size_t offset, size_t size, + google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND) { + ByteString result(UninitializedTag{}); + result.SetLarge(arena, value, offset, size); + return result; } + static ByteString Wrap(std::nullptr_t, google::protobuf::Arena*) = delete; + static ByteString Wrap(std::nullptr_t, size_t, size_t, + google::protobuf::Arena*) = delete; + static ByteString Wrap(std::string&& value, google::protobuf::Arena*) = delete; - ByteString(Allocator<> allocator, ByteString&& other) { - Construct(other, allocator); + static ByteString WrapUnsafe(absl::string_view value) { + return Wrap(value, static_cast(nullptr)); } + static ByteString WrapUnsafe(const absl::Cord* absl_nonnull value) { + return WrapUnsafe(value, 0, value->size()); + } + static ByteString WrapUnsafe(const absl::Cord* absl_nonnull value, + size_t offset, size_t size) { + return Wrap(value, offset, size, static_cast(nullptr)); + } + static ByteString WrapUnsafe(std::nullptr_t) = delete; + static ByteString WrapUnsafe(std::nullptr_t, size_t, size_t) = delete; - ByteString(Borrower borrower, - const char* absl_nullable string ABSL_ATTRIBUTE_LIFETIME_BOUND) - : ByteString(borrower, absl::NullSafeStringView(string)) {} - - ByteString(Borrower borrower, - absl::string_view string ABSL_ATTRIBUTE_LIFETIME_BOUND) - : ByteString(Borrowed(borrower, string)) {} - - ByteString(Borrower borrower, - const absl::Cord& cord ABSL_ATTRIBUTE_LIFETIME_BOUND) - : ByteString(Borrowed(borrower, cord)) {} + static ByteString Concat(const ByteString& lhs, const ByteString& rhs, + google::protobuf::Arena* absl_nonnull arena); - // Creates a medium byte string that is backed by an external string. Should - // only be called from explicit 'Unsafe' factories. - static ByteString FromExternal(absl::string_view string); + ByteString() noexcept { SetSmallEmpty(); } - ~ByteString() { Destroy(); } + ByteString(const ByteString&) = default; + ByteString(ByteString&&) = default; + ByteString& operator=(const ByteString&) = default; + ByteString& operator=(ByteString&&) = default; - ByteString& operator=(const ByteString& other) noexcept { - if (ABSL_PREDICT_TRUE(this != &other)) { - CopyFrom(other); + bool empty() const { + switch (GetKind()) { + case ByteStringKind::kSmall: + return rep_.small.size == 0; + case ByteStringKind::kMedium: + return rep_.medium.size == 0; + case ByteStringKind::kLarge: + return rep_.large.size == 0; } - return *this; } - ByteString& operator=(ByteString&& other) noexcept { - if (ABSL_PREDICT_TRUE(this != &other)) { - MoveFrom(other); + size_t size() const { + switch (GetKind()) { + case ByteStringKind::kSmall: + return rep_.small.size; + case ByteStringKind::kMedium: + return rep_.medium.size; + case ByteStringKind::kLarge: + return rep_.large.size; } - return *this; } - bool empty() const; - - size_t size() const; - size_t max_size() const { return kByteStringViewMaxSize; } - absl::string_view Flatten() ABSL_ATTRIBUTE_LIFETIME_BOUND; - absl::optional TryFlat() const ABSL_ATTRIBUTE_LIFETIME_BOUND; @@ -307,9 +292,7 @@ ByteString final { void AppendToString(std::string* absl_nonnull out) const; - absl::Cord ToCord() const&; - - absl::Cord ToCord() &&; + absl::Cord ToCord() const; void CopyToCord(absl::Cord* absl_nonnull out) const; @@ -319,36 +302,40 @@ ByteString final { std::string* absl_nonnull scratch ABSL_ATTRIBUTE_LIFETIME_BOUND) const ABSL_ATTRIBUTE_LIFETIME_BOUND; - absl::string_view AsStringView() const ABSL_ATTRIBUTE_LIFETIME_BOUND; - - google::protobuf::Arena* absl_nullable GetArena() const; - - ByteString Clone(google::protobuf::Arena* absl_nonnull arena) const; - - void HashValue(absl::HashState state) const; - - template - decltype(auto) Visit(Visitor&& visitor) const { + // Returns the arena which owns the underling data for this byte string. + // Returns null when the data is not owned by an arena. + google::protobuf::Arena* absl_nullable GetArena() const { switch (GetKind()) { case ByteStringKind::kSmall: - return std::forward(visitor)(GetSmall()); + return nullptr; case ByteStringKind::kMedium: - return std::forward(visitor)(GetMedium()); + return GetMediumArena(); case ByteStringKind::kLarge: - return std::forward(visitor)(GetLarge()); + return GetLargeArena(); } } - friend void swap(ByteString& lhs, ByteString& rhs) { - if (&lhs != &rhs) { - lhs.Swap(rhs); + ByteString Clone(google::protobuf::Arena* absl_nonnull arena) const; + + template + decltype(auto) Visit(Visitor&& visitor) const { + if (auto string_view = AsStringView(); string_view) { + return std::forward(visitor)(*string_view); } + return std::forward(visitor)(GetLarge()); + } + + friend void swap(ByteString& lhs, ByteString& rhs) noexcept { + using std::swap; + swap(lhs.rep_, rhs.rep_); } template friend H AbslHashValue(H state, const ByteString& byte_string) { - byte_string.HashValue(absl::HashState::Create(&state)); - return state; + if (auto string_view = byte_string.AsStringView(); string_view) { + return H::combine(std::move(state), *string_view); + } + return H::combine(std::move(state), byte_string.GetLarge()); } private: @@ -360,21 +347,12 @@ ByteString final { friend absl::string_view LegacyByteString(const ByteString& string, bool stable, google::protobuf::Arena* absl_nonnull arena); - friend struct cel::ArenaTraits; - - struct ExternalStringTag {}; - - static ByteString Borrowed(Borrower borrower, - absl::string_view string - ABSL_ATTRIBUTE_LIFETIME_BOUND); - static ByteString Borrowed( - Borrower borrower, const absl::Cord& cord ABSL_ATTRIBUTE_LIFETIME_BOUND); - - ByteString(const ReferenceCount* absl_nonnull refcount, - absl::string_view string); + struct UninitializedTag { + explicit UninitializedTag() = default; + }; - ByteString(ExternalStringTag, absl::string_view string); + explicit ByteString(UninitializedTag) {} constexpr ByteStringKind GetKind() const { return rep_.header.kind; } @@ -396,135 +374,190 @@ ByteString final { return absl::string_view(rep.data, rep.size); } - google::protobuf::Arena* absl_nullable GetSmallArena() const { - ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kSmall); - return GetSmallArena(rep_.small); - } - - static google::protobuf::Arena* absl_nullable GetSmallArena( - const SmallByteStringRep& rep) { - return rep.arena; - } - google::protobuf::Arena* absl_nullable GetMediumArena() const { ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kMedium); return GetMediumArena(rep_.medium); } static google::protobuf::Arena* absl_nullable GetMediumArena( - const MediumByteStringRep& rep); - - const ReferenceCount* absl_nullable GetMediumReferenceCount() const { - ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kMedium); - return GetMediumReferenceCount(rep_.medium); + const MediumByteStringRep& rep) { + return rep.arena; } - static const ReferenceCount* absl_nullable GetMediumReferenceCount( - const MediumByteStringRep& rep); - - uintptr_t GetMediumOwner() const { - ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kMedium); - return rep_.medium.owner; + static absl::Cord GetLarge( + const LargeByteStringRep& rep ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return rep.data->Subcord(rep.offset, rep.size); } - absl::Cord& GetLarge() ABSL_ATTRIBUTE_LIFETIME_BOUND { + absl::Cord GetLarge() const ABSL_ATTRIBUTE_LIFETIME_BOUND { ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kLarge); return GetLarge(rep_.large); } - static absl::Cord& GetLarge( - LargeByteStringRep& rep ABSL_ATTRIBUTE_LIFETIME_BOUND) { - return *std::launder(reinterpret_cast(&rep.data[0])); - } - - const absl::Cord& GetLarge() const ABSL_ATTRIBUTE_LIFETIME_BOUND { + google::protobuf::Arena* absl_nullable GetLargeArena() const { ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kLarge); - return GetLarge(rep_.large); + return GetLargeArena(rep_.large); } - static const absl::Cord& GetLarge( + static google::protobuf::Arena* absl_nullable GetLargeArena( const LargeByteStringRep& rep ABSL_ATTRIBUTE_LIFETIME_BOUND) { - return *std::launder(reinterpret_cast(&rep.data[0])); + return rep.arena; } - void SetSmallEmpty(google::protobuf::Arena* absl_nullable arena) { + void SetSmallEmpty() { rep_.header.kind = ByteStringKind::kSmall; rep_.small.size = 0; - rep_.small.arena = arena; } - void SetSmall(google::protobuf::Arena* absl_nullable arena, absl::string_view string); - - void SetSmall(google::protobuf::Arena* absl_nullable arena, const absl::Cord& cord); - - void SetMedium(google::protobuf::Arena* absl_nullable arena, absl::string_view string); - - // This is used to create a medium byte string that is backed by an external - // string. Should only be called from explicit 'Unsafe' factories. - void SetExternalMedium(absl::string_view string); - - void SetMedium(google::protobuf::Arena* absl_nullable arena, std::string&& string); - - void SetMedium(google::protobuf::Arena* absl_nonnull arena, const absl::Cord& cord); - - void SetMedium(absl::string_view string, uintptr_t owner); - - void SetLarge(const absl::Cord& cord); - - void SetLarge(absl::Cord&& cord); - - void Swap(ByteString& other); - - void Construct(const ByteString& other, - absl::optional> allocator); - - void Construct(ByteString& other, absl::optional> allocator); - - void CopyFrom(const ByteString& other); + void SetSmall(absl::string_view string) { + ABSL_DCHECK_LE(string.size(), kSmallByteStringCapacity); + rep_.header.kind = ByteStringKind::kSmall; + rep_.small.size = string.size(); + if (!string.empty()) { + std::memcpy(rep_.small.data, string.data(), rep_.small.size); + } + } - void MoveFrom(ByteString& other); + void SetSmall(const absl::Cord& cord) { + ABSL_DCHECK_LE(cord.size(), kSmallByteStringCapacity); + rep_.header.kind = ByteStringKind::kSmall; + rep_.small.size = cord.size(); + CopyCordToArray(cord, rep_.small.data); + } - void Destroy(); + void SetMedium(google::protobuf::Arena* absl_nullable arena, absl::string_view string) { + rep_.header.kind = ByteStringKind::kMedium; + rep_.medium.size = string.size(); + rep_.medium.data = string.data(); + rep_.medium.arena = arena; + } - void DestroyMedium() { - ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kMedium); - DestroyMedium(rep_.medium); + void SetMedium(google::protobuf::Arena* absl_nullable arena, + const std::string* absl_nonnull string) { + SetMedium(arena, absl::string_view(*string)); } - static void DestroyMedium(const MediumByteStringRep& rep) { - StrongUnref(GetMediumReferenceCount(rep)); + void SetLarge(google::protobuf::Arena* absl_nullable arena, + const absl::Cord* absl_nonnull cord, size_t offset = 0, + size_t size = static_cast(-1)) { + ABSL_DCHECK_LE(offset, cord->size()); + ABSL_DCHECK_LE(offset, kLargeByteStringMaxSize); + rep_.header.kind = ByteStringKind::kLarge; + rep_.large.offset = offset; + if (size == static_cast(-1)) { + size = cord->size() - offset; + } + ABSL_DCHECK_LE(size, cord->size() - offset); + ABSL_DCHECK_LE(size, kLargeByteStringMaxSize); + rep_.large.size = size; + rep_.large.data = cord; + rep_.large.arena = arena; } - void DestroyLarge() { - ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kLarge); - DestroyLarge(rep_.large); + static void CopyCordToArray(const absl::Cord& cord, char* data) { + for (absl::string_view chunk : cord.Chunks()) { + std::memcpy(data, chunk.data(), chunk.size()); + data += chunk.size(); + } } - static void DestroyLarge(LargeByteStringRep& rep) { GetLarge(rep).~Cord(); } + static void CopyCordToArray(const absl::Cord& cord, size_t offset, + size_t size, char* data); + + static void AppendCordToString(const absl::Cord& cord, size_t offset, + size_t size, std::string& data); void CopyToArray(char* absl_nonnull out) const; + std::optional AsStringView() const { + switch (GetKind()) { + case ByteStringKind::kSmall: + return GetSmall(); + case ByteStringKind::kMedium: + return GetMedium(); + case ByteStringKind::kLarge: + return std::nullopt; + } + } + ByteStringRep rep_; }; +inline bool ByteString::Equals(absl::string_view rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> bool { return lhs == rhs; }, + [&rhs](const absl::Cord& lhs) -> bool { return lhs == rhs; })); +} + +inline bool ByteString::Equals(const absl::Cord& rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> bool { return lhs == rhs; }, + [&rhs](const absl::Cord& lhs) -> bool { return lhs == rhs; })); +} + inline bool ByteString::Equals(const ByteString& rhs) const { return rhs.Visit(absl::Overload( [this](absl::string_view rhs) -> bool { return Equals(rhs); }, [this](const absl::Cord& rhs) -> bool { return Equals(rhs); })); } +inline int ByteString::Compare(absl::string_view rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> int { return lhs.compare(rhs); }, + [&rhs](const absl::Cord& lhs) -> int { return lhs.Compare(rhs); })); +} + +inline int ByteString::Compare(const absl::Cord& rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> int { return -rhs.Compare(lhs); }, + [&rhs](const absl::Cord& lhs) -> int { return lhs.Compare(rhs); })); +} + inline int ByteString::Compare(const ByteString& rhs) const { return rhs.Visit(absl::Overload( [this](absl::string_view rhs) -> int { return Compare(rhs); }, [this](const absl::Cord& rhs) -> int { return Compare(rhs); })); } +inline bool ByteString::StartsWith(absl::string_view rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> bool { + return absl::StartsWith(lhs, rhs); + }, + [&rhs](const absl::Cord& lhs) -> bool { return lhs.StartsWith(rhs); })); +} + +inline bool ByteString::StartsWith(const absl::Cord& rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> bool { + return lhs.size() >= rhs.size() && lhs.substr(0, rhs.size()) == rhs; + }, + [&rhs](const absl::Cord& lhs) -> bool { return lhs.StartsWith(rhs); })); +} + inline bool ByteString::StartsWith(const ByteString& rhs) const { return rhs.Visit(absl::Overload( [this](absl::string_view rhs) -> bool { return StartsWith(rhs); }, [this](const absl::Cord& rhs) -> bool { return StartsWith(rhs); })); } +inline bool ByteString::EndsWith(absl::string_view rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> bool { + return absl::EndsWith(lhs, rhs); + }, + [&rhs](const absl::Cord& lhs) -> bool { return lhs.EndsWith(rhs); })); +} + +inline bool ByteString::EndsWith(const absl::Cord& rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> bool { + return lhs.size() >= rhs.size() && + lhs.substr(lhs.size() - rhs.size()) == rhs; + }, + [&rhs](const absl::Cord& lhs) -> bool { return lhs.EndsWith(rhs); })); +} + inline bool ByteString::EndsWith(const ByteString& rhs) const { return rhs.Visit(absl::Overload( [this](absl::string_view rhs) -> bool { return EndsWith(rhs); }, @@ -662,27 +695,8 @@ inline bool operator>=(const absl::Cord& lhs, const ByteString& rhs) { return -rhs.Compare(lhs) >= 0; } -#undef CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI - } // namespace common_internal -template <> -struct ArenaTraits { - using constructible = std::true_type; - - static bool trivially_destructible( - const common_internal::ByteString& byte_string) { - switch (byte_string.GetKind()) { - case common_internal::ByteStringKind::kSmall: - return true; - case common_internal::ByteStringKind::kMedium: - return byte_string.GetMediumReferenceCount() == nullptr; - case common_internal::ByteStringKind::kLarge: - return false; - } - } -}; - } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_BYTE_STRING_H_ diff --git a/common/internal/byte_string_test.cc b/common/internal/byte_string_test.cc index 553c9c13a..b0cdee814 100644 --- a/common/internal/byte_string_test.cc +++ b/common/internal/byte_string_test.cc @@ -24,10 +24,6 @@ #include "absl/strings/cord.h" #include "absl/strings/cord_test_helpers.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "common/allocator.h" -#include "common/internal/reference_count.h" -#include "common/memory.h" #include "internal/testing.h" #include "google/protobuf/arena.h" @@ -44,10 +40,10 @@ namespace { using ::testing::_; using ::testing::Eq; using ::testing::IsEmpty; +using ::testing::IsNull; using ::testing::Not; using ::testing::Optional; using ::testing::SizeIs; -using ::testing::TestWithParam; TEST(ByteStringKind, Ostream) { { @@ -67,17 +63,9 @@ TEST(ByteStringKind, Ostream) { } } -class ByteStringTest : public TestWithParam, - public ByteStringTestFriend { +class ByteStringTest : public ByteStringTestFriend, public ::testing::Test { public: - Allocator<> GetAllocator() { - switch (GetParam()) { - case AllocatorKind::kNewDelete: - return NewDeleteAllocator<>{}; - case AllocatorKind::kArena: - return ArenaAllocator<>(&arena_); - } - } + google::protobuf::Arena* GetArena() { return &arena_; } private: google::protobuf::Arena arena_; @@ -117,680 +105,165 @@ const absl::Cord& GetMediumOrLargeFragmentedCord() { return *medium_or_large; } -TEST_P(ByteStringTest, Default) { - ByteString byte_string = ByteString(GetAllocator(), ""); +TEST_F(ByteStringTest, Default) { + ByteString byte_string = ByteString(); EXPECT_THAT(byte_string, SizeIs(0)); EXPECT_THAT(byte_string, IsEmpty()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); } -TEST_P(ByteStringTest, ConstructSmallCString) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallString().c_str()); +TEST_F(ByteStringTest, ConstructNullDataStringView) { + ByteString byte_string = ByteString::From(absl::string_view(), GetArena()); + EXPECT_THAT(byte_string, IsEmpty()); + EXPECT_THAT(byte_string.GetArena(), IsNull()); +} + +TEST_F(ByteStringTest, ConstructSmallCString) { + ByteString byte_string = + ByteString::From(GetSmallString().c_str(), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetSmallStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetSmallStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_THAT(byte_string.GetArena(), IsNull()); } -TEST_P(ByteStringTest, ConstructMediumCString) { +TEST_F(ByteStringTest, ConstructMediumCString) { ByteString byte_string = - ByteString(GetAllocator(), GetMediumString().c_str()); + ByteString::From(GetMediumString().c_str(), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetMediumStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetMediumStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_EQ(byte_string.GetArena(), GetArena()); } -TEST_P(ByteStringTest, ConstructSmallRValueString) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallString()); +TEST_F(ByteStringTest, ConstructSmallRValueString) { + ByteString byte_string = ByteString::From(GetSmallString(), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetSmallStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetSmallStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_THAT(byte_string.GetArena(), IsNull()); } -TEST_P(ByteStringTest, ConstructSmallLValueString) { - ByteString byte_string = ByteString( - GetAllocator(), static_cast(GetSmallString())); +TEST_F(ByteStringTest, ConstructSmallLValueString) { + ByteString byte_string = ByteString::From( + static_cast(GetSmallString()), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetSmallStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetSmallStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_THAT(byte_string.GetArena(), IsNull()); } -TEST_P(ByteStringTest, ConstructMediumRValueString) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumString()); +TEST_F(ByteStringTest, ConstructMediumRValueString) { + ByteString byte_string = ByteString::From(GetMediumString(), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetMediumStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetMediumStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_EQ(byte_string.GetArena(), GetArena()); } -TEST_P(ByteStringTest, ConstructMediumLValueString) { - ByteString byte_string = ByteString( - GetAllocator(), static_cast(GetMediumString())); +TEST_F(ByteStringTest, ConstructMediumLValueString) { + ByteString byte_string = ByteString::From( + static_cast(GetMediumString()), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetMediumStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetMediumStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_EQ(byte_string.GetArena(), GetArena()); } -TEST_P(ByteStringTest, ConstructSmallCord) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallCord()); +TEST_F(ByteStringTest, ConstructSmallCord) { + ByteString byte_string = ByteString::From(GetSmallCord(), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetSmallStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetSmallStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_THAT(byte_string.GetArena(), IsNull()); } -TEST_P(ByteStringTest, ConstructMediumOrLargeCord) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, ConstructMediumOrLargeCord) { + ByteString byte_string = ByteString::From(GetMediumOrLargeCord(), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetMediumStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetMediumStringView()); - if (GetAllocator().arena() == nullptr) { - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); - } else { - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); - } - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); + EXPECT_EQ(byte_string.GetArena(), GetArena()); } -TEST(ByteStringTest, BorrowedUnownedString) { -#ifdef NDEBUG - ByteString byte_string = ByteString(Owner::None(), GetMediumStringView()); +TEST_F(ByteStringTest, BorrowedArenaSmallString) { + ByteString byte_string = ByteString::Wrap(GetSmallStringView(), GetArena()); EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), ByteStringKind::kMedium); - EXPECT_EQ(byte_string.GetArena(), nullptr); - EXPECT_EQ(byte_string, GetMediumStringView()); -#else - EXPECT_DEBUG_DEATH( - static_cast(ByteString(Owner::None(), GetMediumStringView())), - ::testing::_); -#endif -} - -TEST(ByteStringTest, BorrowedUnownedCord) { -#ifdef NDEBUG - ByteString byte_string = ByteString(Owner::None(), GetMediumOrLargeCord()); - EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), ByteStringKind::kLarge); - EXPECT_EQ(byte_string.GetArena(), nullptr); - EXPECT_EQ(byte_string, GetMediumOrLargeCord()); -#else - EXPECT_DEBUG_DEATH( - static_cast(ByteString(Owner::None(), GetMediumOrLargeCord())), - ::testing::_); -#endif -} - -TEST(ByteStringTest, BorrowedReferenceCountSmallString) { - auto* refcount = new ReferenceCounted(); - Owner owner = Owner::ReferenceCount(refcount); - StrongUnref(refcount); - ByteString byte_string = ByteString(owner, GetSmallStringView()); - EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), nullptr); + EXPECT_EQ(byte_string.GetArena(), GetArena()); EXPECT_EQ(byte_string, GetSmallStringView()); } -TEST(ByteStringTest, BorrowedReferenceCountMediumString) { - auto* refcount = new ReferenceCounted(); - Owner owner = Owner::ReferenceCount(refcount); - StrongUnref(refcount); - ByteString byte_string = ByteString(owner, GetMediumStringView()); +TEST_F(ByteStringTest, BorrowedArenaMediumString) { + ByteString byte_string = ByteString::Wrap(GetMediumStringView(), GetArena()); EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), ByteStringKind::kMedium); - EXPECT_EQ(byte_string.GetArena(), nullptr); + EXPECT_EQ(byte_string.GetArena(), GetArena()); EXPECT_EQ(byte_string, GetMediumStringView()); } -TEST(ByteStringTest, BorrowedArenaSmallString) { - google::protobuf::Arena arena; - ByteString byte_string = - ByteString(Owner::Arena(&arena), GetSmallStringView()); - EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), &arena); - EXPECT_EQ(byte_string, GetSmallStringView()); -} - -TEST(ByteStringTest, BorrowedArenaMediumString) { - google::protobuf::Arena arena; +TEST_F(ByteStringTest, BorrowedArenaCord) { ByteString byte_string = - ByteString(Owner::Arena(&arena), GetMediumStringView()); - EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), - ByteStringKind::kMedium); - EXPECT_EQ(byte_string.GetArena(), &arena); - EXPECT_EQ(byte_string, GetMediumStringView()); -} - -TEST(ByteStringTest, BorrowedReferenceCountCord) { - auto* refcount = new ReferenceCounted(); - Owner owner = Owner::ReferenceCount(refcount); - StrongUnref(refcount); - ByteString byte_string = ByteString(owner, GetMediumOrLargeCord()); + ByteString::Wrap(&GetMediumOrLargeCord(), GetArena()); EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), ByteStringKind::kLarge); - EXPECT_EQ(byte_string.GetArena(), nullptr); - EXPECT_EQ(byte_string, GetMediumOrLargeCord()); -} - -TEST(ByteStringTest, BorrowedArenaCord) { - google::protobuf::Arena arena; - Owner owner = Owner::Arena(&arena); - ByteString byte_string = ByteString(owner, GetMediumOrLargeCord()); - EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), - ByteStringKind::kMedium); - EXPECT_EQ(byte_string.GetArena(), &arena); + EXPECT_EQ(byte_string.GetArena(), GetArena()); EXPECT_EQ(byte_string, GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, CopyConstruct) { - ByteString small_byte_string = - ByteString(GetAllocator(), GetSmallStringView()); - ByteString medium_byte_string = - ByteString(GetAllocator(), GetMediumStringView()); - ByteString large_byte_string = - ByteString(GetAllocator(), GetMediumOrLargeCord()); - - EXPECT_EQ(ByteString(NewDeleteAllocator(), small_byte_string), - small_byte_string); - EXPECT_EQ(ByteString(NewDeleteAllocator(), medium_byte_string), - medium_byte_string); - EXPECT_EQ(ByteString(NewDeleteAllocator(), large_byte_string), - large_byte_string); - - google::protobuf::Arena arena; - EXPECT_EQ(ByteString(ArenaAllocator(&arena), small_byte_string), - small_byte_string); - EXPECT_EQ(ByteString(ArenaAllocator(&arena), medium_byte_string), - medium_byte_string); - EXPECT_EQ(ByteString(ArenaAllocator(&arena), large_byte_string), - large_byte_string); - - EXPECT_EQ(ByteString(GetAllocator(), small_byte_string), small_byte_string); - EXPECT_EQ(ByteString(GetAllocator(), medium_byte_string), medium_byte_string); - EXPECT_EQ(ByteString(GetAllocator(), large_byte_string), large_byte_string); - - EXPECT_EQ(ByteString(small_byte_string), small_byte_string); - EXPECT_EQ(ByteString(medium_byte_string), medium_byte_string); - EXPECT_EQ(ByteString(large_byte_string), large_byte_string); -} - -TEST_P(ByteStringTest, CopyConstructFromExternal) { - ByteString small_byte_string = ByteString::FromExternal(GetSmallStringView()); - ByteString medium_byte_string = - ByteString::FromExternal(GetMediumStringView()); - - EXPECT_EQ(ByteString(NewDeleteAllocator(), small_byte_string), - small_byte_string); - EXPECT_EQ(ByteString(NewDeleteAllocator(), medium_byte_string), - medium_byte_string); - - google::protobuf::Arena arena; - EXPECT_EQ(ByteString(ArenaAllocator(&arena), small_byte_string), - small_byte_string); - EXPECT_EQ(ByteString(ArenaAllocator(&arena), medium_byte_string), - medium_byte_string); - - EXPECT_EQ(ByteString(GetAllocator(), small_byte_string), small_byte_string); - EXPECT_EQ(ByteString(GetAllocator(), medium_byte_string), medium_byte_string); - - EXPECT_EQ(ByteString(small_byte_string), small_byte_string); - EXPECT_EQ(ByteString(medium_byte_string), medium_byte_string); -} - -TEST_P(ByteStringTest, MoveConstruct) { - const auto& small_byte_string = [this]() { - return ByteString(GetAllocator(), GetSmallStringView()); - }; - const auto& medium_byte_string = [this]() { - return ByteString(GetAllocator(), GetMediumStringView()); - }; - const auto& large_byte_string = [this]() { - return ByteString(GetAllocator(), GetMediumOrLargeCord()); - }; - - EXPECT_EQ(ByteString(NewDeleteAllocator(), small_byte_string()), - small_byte_string()); - EXPECT_EQ(ByteString(NewDeleteAllocator(), medium_byte_string()), - medium_byte_string()); - EXPECT_EQ(ByteString(NewDeleteAllocator(), large_byte_string()), - large_byte_string()); - - google::protobuf::Arena arena; - EXPECT_EQ(ByteString(ArenaAllocator(&arena), small_byte_string()), - small_byte_string()); - EXPECT_EQ(ByteString(ArenaAllocator(&arena), medium_byte_string()), - medium_byte_string()); - EXPECT_EQ(ByteString(ArenaAllocator(&arena), large_byte_string()), - large_byte_string()); - - EXPECT_EQ(ByteString(GetAllocator(), small_byte_string()), - small_byte_string()); - EXPECT_EQ(ByteString(GetAllocator(), medium_byte_string()), - medium_byte_string()); - EXPECT_EQ(ByteString(GetAllocator(), large_byte_string()), - large_byte_string()); - - EXPECT_EQ(ByteString(small_byte_string()), small_byte_string()); - EXPECT_EQ(ByteString(medium_byte_string()), medium_byte_string()); - EXPECT_EQ(ByteString(large_byte_string()), large_byte_string()); -} - -TEST_P(ByteStringTest, MoveConstructFromExternal) { - const auto& small_byte_string = []() { - return ByteString::FromExternal(GetSmallStringView()); - }; - const auto& medium_byte_string = []() { - return ByteString::FromExternal(GetMediumStringView()); - }; - - EXPECT_EQ(ByteString(NewDeleteAllocator(), small_byte_string()), - small_byte_string()); - EXPECT_EQ(ByteString(NewDeleteAllocator(), medium_byte_string()), - medium_byte_string()); - - google::protobuf::Arena arena; - EXPECT_EQ(ByteString(ArenaAllocator(&arena), small_byte_string()), - small_byte_string()); - EXPECT_EQ(ByteString(ArenaAllocator(&arena), medium_byte_string()), - medium_byte_string()); - - EXPECT_EQ(ByteString(GetAllocator(), small_byte_string()), - small_byte_string()); - EXPECT_EQ(ByteString(GetAllocator(), medium_byte_string()), - medium_byte_string()); - - EXPECT_EQ(ByteString(small_byte_string()), small_byte_string()); - EXPECT_EQ(ByteString(medium_byte_string()), medium_byte_string()); -} - -TEST_P(ByteStringTest, CopyFromByteString) { - ByteString small_byte_string = - ByteString(GetAllocator(), GetSmallStringView()); - ByteString medium_byte_string = - ByteString(GetAllocator(), GetMediumStringView()); - ByteString large_byte_string = - ByteString(GetAllocator(), GetMediumOrLargeCord()); - - ByteString new_delete_byte_string(NewDeleteAllocator<>{}); - // Small <= Small - new_delete_byte_string = small_byte_string; - EXPECT_EQ(new_delete_byte_string, small_byte_string); - // Small <= Medium - new_delete_byte_string = medium_byte_string; - EXPECT_EQ(new_delete_byte_string, medium_byte_string); - // Medium <= Medium - new_delete_byte_string = medium_byte_string; - EXPECT_EQ(new_delete_byte_string, medium_byte_string); - // Medium <= Large - new_delete_byte_string = large_byte_string; - EXPECT_EQ(new_delete_byte_string, large_byte_string); - // Large <= Large - new_delete_byte_string = large_byte_string; - EXPECT_EQ(new_delete_byte_string, large_byte_string); - // Large <= Small - new_delete_byte_string = small_byte_string; - EXPECT_EQ(new_delete_byte_string, small_byte_string); - // Small <= Large - new_delete_byte_string = large_byte_string; - EXPECT_EQ(new_delete_byte_string, large_byte_string); - // Large <= Medium - new_delete_byte_string = medium_byte_string; - EXPECT_EQ(new_delete_byte_string, medium_byte_string); - // Medium <= Small - new_delete_byte_string = small_byte_string; - EXPECT_EQ(new_delete_byte_string, small_byte_string); - - google::protobuf::Arena arena; - ByteString arena_byte_string(ArenaAllocator<>{&arena}); - // Small <= Small - arena_byte_string = small_byte_string; - EXPECT_EQ(arena_byte_string, small_byte_string); - // Small <= Medium - arena_byte_string = medium_byte_string; - EXPECT_EQ(arena_byte_string, medium_byte_string); - // Medium <= Medium - arena_byte_string = medium_byte_string; - EXPECT_EQ(arena_byte_string, medium_byte_string); - // Medium <= Large - arena_byte_string = large_byte_string; - EXPECT_EQ(arena_byte_string, large_byte_string); - // Large <= Large - arena_byte_string = large_byte_string; - EXPECT_EQ(arena_byte_string, large_byte_string); - // Large <= Small - arena_byte_string = small_byte_string; - EXPECT_EQ(arena_byte_string, small_byte_string); - // Small <= Large - arena_byte_string = large_byte_string; - EXPECT_EQ(arena_byte_string, large_byte_string); - // Large <= Medium - arena_byte_string = medium_byte_string; - EXPECT_EQ(arena_byte_string, medium_byte_string); - // Medium <= Small - arena_byte_string = small_byte_string; - EXPECT_EQ(arena_byte_string, small_byte_string); - - ByteString allocator_byte_string(GetAllocator()); - // Small <= Small - allocator_byte_string = small_byte_string; - EXPECT_EQ(allocator_byte_string, small_byte_string); - // Small <= Medium - allocator_byte_string = medium_byte_string; - EXPECT_EQ(allocator_byte_string, medium_byte_string); - // Medium <= Medium - allocator_byte_string = medium_byte_string; - EXPECT_EQ(allocator_byte_string, medium_byte_string); - // Medium <= Large - allocator_byte_string = large_byte_string; - EXPECT_EQ(allocator_byte_string, large_byte_string); - // Large <= Large - allocator_byte_string = large_byte_string; - EXPECT_EQ(allocator_byte_string, large_byte_string); - // Large <= Small - allocator_byte_string = small_byte_string; - EXPECT_EQ(allocator_byte_string, small_byte_string); - // Small <= Large - allocator_byte_string = large_byte_string; - EXPECT_EQ(allocator_byte_string, large_byte_string); - // Large <= Medium - allocator_byte_string = medium_byte_string; - EXPECT_EQ(allocator_byte_string, medium_byte_string); - // Medium <= Small - allocator_byte_string = small_byte_string; - EXPECT_EQ(allocator_byte_string, small_byte_string); - - // Miscellaneous cases not covered above. - // Large <= Medium Arena String - ByteString large_new_delete_byte_string(NewDeleteAllocator<>{}, - GetMediumOrLargeCord()); - ByteString medium_arena_byte_string(ArenaAllocator<>{&arena}, - GetMediumStringView()); - large_new_delete_byte_string = medium_arena_byte_string; - EXPECT_EQ(large_new_delete_byte_string, medium_arena_byte_string); -} - -TEST_P(ByteStringTest, MoveFrom) { - const auto& small_byte_string = [this]() { - return ByteString(GetAllocator(), GetSmallStringView()); - }; - const auto& medium_byte_string = [this]() { - return ByteString(GetAllocator(), GetMediumStringView()); - }; - const auto& large_byte_string = [this]() { - return ByteString(GetAllocator(), GetMediumOrLargeCord()); - }; - - ByteString new_delete_byte_string(NewDeleteAllocator<>{}); - // Small <= Small - new_delete_byte_string = small_byte_string(); - EXPECT_EQ(new_delete_byte_string, small_byte_string()); - // Small <= Medium - new_delete_byte_string = medium_byte_string(); - EXPECT_EQ(new_delete_byte_string, medium_byte_string()); - // Medium <= Medium - new_delete_byte_string = medium_byte_string(); - EXPECT_EQ(new_delete_byte_string, medium_byte_string()); - // Medium <= Large - new_delete_byte_string = large_byte_string(); - EXPECT_EQ(new_delete_byte_string, large_byte_string()); - // Large <= Large - new_delete_byte_string = large_byte_string(); - EXPECT_EQ(new_delete_byte_string, large_byte_string()); - // Large <= Small - new_delete_byte_string = small_byte_string(); - EXPECT_EQ(new_delete_byte_string, small_byte_string()); - // Small <= Large - new_delete_byte_string = large_byte_string(); - EXPECT_EQ(new_delete_byte_string, large_byte_string()); - // Large <= Medium - new_delete_byte_string = medium_byte_string(); - EXPECT_EQ(new_delete_byte_string, medium_byte_string()); - // Medium <= Small - new_delete_byte_string = small_byte_string(); - EXPECT_EQ(new_delete_byte_string, small_byte_string()); - - google::protobuf::Arena arena; - ByteString arena_byte_string(ArenaAllocator<>{&arena}); - // Small <= Small - arena_byte_string = small_byte_string(); - EXPECT_EQ(arena_byte_string, small_byte_string()); - // Small <= Medium - arena_byte_string = medium_byte_string(); - EXPECT_EQ(arena_byte_string, medium_byte_string()); - // Medium <= Medium - arena_byte_string = medium_byte_string(); - EXPECT_EQ(arena_byte_string, medium_byte_string()); - // Medium <= Large - arena_byte_string = large_byte_string(); - EXPECT_EQ(arena_byte_string, large_byte_string()); - // Large <= Large - arena_byte_string = large_byte_string(); - EXPECT_EQ(arena_byte_string, large_byte_string()); - // Large <= Small - arena_byte_string = small_byte_string(); - EXPECT_EQ(arena_byte_string, small_byte_string()); - // Small <= Large - arena_byte_string = large_byte_string(); - EXPECT_EQ(arena_byte_string, large_byte_string()); - // Large <= Medium - arena_byte_string = medium_byte_string(); - EXPECT_EQ(arena_byte_string, medium_byte_string()); - // Medium <= Small - arena_byte_string = small_byte_string(); - EXPECT_EQ(arena_byte_string, small_byte_string()); - - ByteString allocator_byte_string(GetAllocator()); - // Small <= Small - allocator_byte_string = small_byte_string(); - EXPECT_EQ(allocator_byte_string, small_byte_string()); - // Small <= Medium - allocator_byte_string = medium_byte_string(); - EXPECT_EQ(allocator_byte_string, medium_byte_string()); - // Medium <= Medium - allocator_byte_string = medium_byte_string(); - EXPECT_EQ(allocator_byte_string, medium_byte_string()); - // Medium <= Large - allocator_byte_string = large_byte_string(); - EXPECT_EQ(allocator_byte_string, large_byte_string()); - // Large <= Large - allocator_byte_string = large_byte_string(); - EXPECT_EQ(allocator_byte_string, large_byte_string()); - // Large <= Small - allocator_byte_string = small_byte_string(); - EXPECT_EQ(allocator_byte_string, small_byte_string()); - // Small <= Large - allocator_byte_string = large_byte_string(); - EXPECT_EQ(allocator_byte_string, large_byte_string()); - // Large <= Medium - allocator_byte_string = medium_byte_string(); - EXPECT_EQ(allocator_byte_string, medium_byte_string()); - // Medium <= Small - allocator_byte_string = small_byte_string(); - EXPECT_EQ(allocator_byte_string, small_byte_string()); - - // Miscellaneous cases not covered above. - // Large <= Medium Arena String - ByteString large_new_delete_byte_string(NewDeleteAllocator<>{}, - GetMediumOrLargeCord()); - ByteString medium_arena_byte_string(ArenaAllocator<>{&arena}, - GetMediumStringView()); - large_new_delete_byte_string = std::move(medium_arena_byte_string); - EXPECT_EQ(large_new_delete_byte_string, GetMediumStringView()); -} - -TEST_P(ByteStringTest, Swap) { - using std::swap; - ByteString empty_byte_string(GetAllocator()); - ByteString small_byte_string = - ByteString(GetAllocator(), GetSmallStringView()); - ByteString medium_byte_string = - ByteString(GetAllocator(), GetMediumStringView()); - ByteString large_byte_string = - ByteString(GetAllocator(), GetMediumOrLargeCord()); - - // Small <=> Small - swap(empty_byte_string, small_byte_string); - EXPECT_EQ(empty_byte_string, GetSmallStringView()); - EXPECT_EQ(small_byte_string, ""); - swap(empty_byte_string, small_byte_string); - EXPECT_EQ(empty_byte_string, ""); - EXPECT_EQ(small_byte_string, GetSmallStringView()); - - // Small <=> Medium - swap(small_byte_string, medium_byte_string); - EXPECT_EQ(small_byte_string, GetMediumStringView()); - EXPECT_EQ(medium_byte_string, GetSmallStringView()); - swap(small_byte_string, medium_byte_string); - EXPECT_EQ(small_byte_string, GetSmallStringView()); - EXPECT_EQ(medium_byte_string, GetMediumStringView()); - - // Small <=> Large - swap(small_byte_string, large_byte_string); - EXPECT_EQ(small_byte_string, GetMediumOrLargeCord()); - EXPECT_EQ(large_byte_string, GetSmallStringView()); - swap(small_byte_string, large_byte_string); - EXPECT_EQ(small_byte_string, GetSmallStringView()); - EXPECT_EQ(large_byte_string, GetMediumOrLargeCord()); - - // Medium <=> Medium - static constexpr absl::string_view kDifferentMediumStringView = - "A different string that is too large for the small string optimization!"; - ByteString other_medium_byte_string = - ByteString(GetAllocator(), kDifferentMediumStringView); - swap(medium_byte_string, other_medium_byte_string); - EXPECT_EQ(medium_byte_string, kDifferentMediumStringView); - EXPECT_EQ(other_medium_byte_string, GetMediumStringView()); - swap(medium_byte_string, other_medium_byte_string); - EXPECT_EQ(medium_byte_string, GetMediumStringView()); - EXPECT_EQ(other_medium_byte_string, kDifferentMediumStringView); - - // Medium <=> Large - swap(medium_byte_string, large_byte_string); - EXPECT_EQ(medium_byte_string, GetMediumOrLargeCord()); - EXPECT_EQ(large_byte_string, GetMediumStringView()); - swap(medium_byte_string, large_byte_string); - EXPECT_EQ(medium_byte_string, GetMediumStringView()); - EXPECT_EQ(large_byte_string, GetMediumOrLargeCord()); - - // Large <=> Large - const absl::Cord different_medium_or_large_cord = - absl::Cord(kDifferentMediumStringView); - ByteString other_large_byte_string = - ByteString(GetAllocator(), different_medium_or_large_cord); - swap(large_byte_string, other_large_byte_string); - EXPECT_EQ(large_byte_string, different_medium_or_large_cord); - EXPECT_EQ(other_large_byte_string, GetMediumStringView()); - swap(large_byte_string, other_large_byte_string); - EXPECT_EQ(large_byte_string, GetMediumStringView()); - EXPECT_EQ(other_large_byte_string, different_medium_or_large_cord); - - // Miscellaneous cases not covered above. These do not swap a second time to - // restore state, so they are destructive. - // Small <=> Different Allocator Medium - ByteString medium_new_delete_byte_string = - ByteString(NewDeleteAllocator<>{}, kDifferentMediumStringView); - swap(empty_byte_string, medium_new_delete_byte_string); - EXPECT_EQ(empty_byte_string, kDifferentMediumStringView); - EXPECT_EQ(medium_new_delete_byte_string, ""); - // Small <=> Different Allocator Large - ByteString large_new_delete_byte_string = - ByteString(NewDeleteAllocator<>{}, GetMediumOrLargeCord()); - swap(small_byte_string, large_new_delete_byte_string); - EXPECT_EQ(small_byte_string, GetMediumOrLargeCord()); - EXPECT_EQ(large_new_delete_byte_string, GetSmallStringView()); - // Medium <=> Different Allocator Large - large_new_delete_byte_string = - ByteString(NewDeleteAllocator<>{}, different_medium_or_large_cord); - swap(medium_byte_string, large_new_delete_byte_string); - EXPECT_EQ(medium_byte_string, different_medium_or_large_cord); - EXPECT_EQ(large_new_delete_byte_string, GetMediumStringView()); - // Medium <=> Different Allocator Medium - medium_byte_string = ByteString(GetAllocator(), GetMediumStringView()); - medium_new_delete_byte_string = - ByteString(NewDeleteAllocator<>{}, kDifferentMediumStringView); - swap(medium_byte_string, medium_new_delete_byte_string); - EXPECT_EQ(medium_byte_string, kDifferentMediumStringView); - EXPECT_EQ(medium_new_delete_byte_string, GetMediumStringView()); -} - -TEST_P(ByteStringTest, FlattenSmall) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.Flatten(), GetSmallStringView()); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); -} - -TEST_P(ByteStringTest, FlattenMedium) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); - EXPECT_EQ(byte_string.Flatten(), GetMediumStringView()); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); -} - -TEST_P(ByteStringTest, FlattenLarge) { - if (GetAllocator().arena() != nullptr) { - GTEST_SKIP(); - } - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); - EXPECT_EQ(byte_string.Flatten(), GetMediumStringView()); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); -} - -TEST_P(ByteStringTest, TryFlatSmall) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); +TEST_F(ByteStringTest, TryFlatSmall) { + ByteString byte_string = ByteString::From(GetSmallStringView(), GetArena()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); EXPECT_THAT(byte_string.TryFlat(), Optional(GetSmallStringView())); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); } -TEST_P(ByteStringTest, TryFlatMedium) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); +TEST_F(ByteStringTest, TryFlatMedium) { + ByteString byte_string = ByteString::From(GetMediumStringView(), GetArena()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); EXPECT_THAT(byte_string.TryFlat(), Optional(GetMediumStringView())); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); } -TEST_P(ByteStringTest, TryFlatLarge) { - if (GetAllocator().arena() != nullptr) { +TEST_F(ByteStringTest, TryFlatLarge) { + if (GetArena() != nullptr) { GTEST_SKIP(); } ByteString byte_string = - ByteString(GetAllocator(), GetMediumOrLargeFragmentedCord()); + ByteString::From(GetMediumOrLargeFragmentedCord(), GetArena()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); - EXPECT_THAT(byte_string.TryFlat(), Eq(absl::nullopt)); + EXPECT_THAT(byte_string.TryFlat(), Eq(std::nullopt)); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); } -TEST_P(ByteStringTest, Equals) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, Equals) { + ByteString byte_string = ByteString::From(GetMediumOrLargeCord(), GetArena()); EXPECT_TRUE(byte_string.Equals(GetMediumStringView())); } -TEST_P(ByteStringTest, Compare) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, Compare) { + ByteString byte_string = ByteString::From(GetMediumOrLargeCord(), GetArena()); EXPECT_EQ(byte_string.Compare(GetMediumStringView()), 0); EXPECT_EQ(byte_string.Compare(GetMediumOrLargeCord()), 0); } -TEST_P(ByteStringTest, StartsWith) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, StartsWith) { + ByteString byte_string = ByteString::From(GetMediumOrLargeCord(), GetArena()); EXPECT_TRUE(byte_string.StartsWith( GetMediumStringView().substr(0, kSmallByteStringCapacity))); EXPECT_TRUE(byte_string.StartsWith( GetMediumOrLargeCord().Subcord(0, kSmallByteStringCapacity))); } -TEST_P(ByteStringTest, EndsWith) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, EndsWith) { + ByteString byte_string = ByteString::From(GetMediumOrLargeCord(), GetArena()); EXPECT_TRUE(byte_string.EndsWith( GetMediumStringView().substr(kSmallByteStringCapacity))); EXPECT_TRUE(byte_string.EndsWith(GetMediumOrLargeCord().Subcord( @@ -798,18 +271,18 @@ TEST_P(ByteStringTest, EndsWith) { GetMediumOrLargeCord().size() - kSmallByteStringCapacity))); } -TEST_P(ByteStringTest, Find) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); +TEST_F(ByteStringTest, Find) { + ByteString byte_string = ByteString::From(GetMediumStringView(), GetArena()); // Find string_view EXPECT_THAT(byte_string.Find("A string"), Optional(0)); EXPECT_THAT( byte_string.Find("small string optimization!"), Optional(GetMediumStringView().find("small string optimization!"))); - EXPECT_THAT(byte_string.Find("not found"), Eq(absl::nullopt)); + EXPECT_THAT(byte_string.Find("not found"), Eq(std::nullopt)); EXPECT_THAT(byte_string.Find(""), Optional(0)); EXPECT_THAT(byte_string.Find("", 3), Optional(3)); - EXPECT_THAT(byte_string.Find("A string", 1), Eq(absl::nullopt)); + EXPECT_THAT(byte_string.Find("A string", 1), Eq(std::nullopt)); // Find cord EXPECT_THAT(byte_string.Find(absl::Cord("A string")), Optional(0)); @@ -820,25 +293,25 @@ TEST_P(ByteStringTest, Find) { byte_string.Find(absl::MakeFragmentedCord( {"A string", " that is too large for the small string optimization!", " extra"})), - Eq(absl::nullopt)); + Eq(std::nullopt)); EXPECT_THAT(byte_string.Find(GetMediumOrLargeFragmentedCord()), Optional(0)); - EXPECT_THAT(byte_string.Find(absl::Cord("not found")), Eq(absl::nullopt)); + EXPECT_THAT(byte_string.Find(absl::Cord("not found")), Eq(std::nullopt)); EXPECT_THAT(byte_string.Find(absl::Cord("")), Optional(0)); EXPECT_THAT(byte_string.Find(absl::Cord(""), 3), Optional(3)); } -TEST_P(ByteStringTest, FindEdgeCases) { - ByteString empty_byte_string(GetAllocator(), ""); - EXPECT_THAT(empty_byte_string.Find("a"), Eq(absl::nullopt)); +TEST_F(ByteStringTest, FindEdgeCases) { + ByteString empty_byte_string; + EXPECT_THAT(empty_byte_string.Find("a"), Eq(std::nullopt)); EXPECT_THAT(empty_byte_string.Find(""), Optional(0)); ByteString cord_byte_string = - ByteString(GetAllocator(), GetMediumOrLargeCord()); - EXPECT_THAT(cord_byte_string.Find("not found"), Eq(absl::nullopt)); - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); + ByteString::From(GetMediumOrLargeCord(), GetArena()); + EXPECT_THAT(cord_byte_string.Find("not found"), Eq(std::nullopt)); + ByteString byte_string = ByteString::From(GetMediumStringView(), GetArena()); // Needle longer than haystack. EXPECT_THAT(byte_string.Find(std::string(byte_string.size() + 1, 'a')), - Eq(absl::nullopt)); + Eq(std::nullopt)); // Needle at the end. absl::string_view suffix = "optimization!"; @@ -846,7 +319,7 @@ TEST_P(ByteStringTest, FindEdgeCases) { Optional(byte_string.size() - suffix.size())); // pos at the end. - EXPECT_THAT(byte_string.Find("a", byte_string.size()), Eq(absl::nullopt)); + EXPECT_THAT(byte_string.Find("a", byte_string.size()), Eq(std::nullopt)); EXPECT_THAT(byte_string.Find("", byte_string.size()), Optional(byte_string.size())); @@ -869,15 +342,15 @@ TEST_P(ByteStringTest, FindEdgeCases) { // Search with fragmented cord needle on string_view backed ByteString with // partial match. - ByteString partial_match_haystack(GetAllocator(), "abababac"); + ByteString partial_match_haystack = ByteString::WrapUnsafe("abababac"); absl::Cord partial_match_needle = absl::MakeFragmentedCord({"aba", "c"}); EXPECT_THAT(partial_match_haystack.Find(partial_match_needle), Optional(4)); // Search with fragmented cord needle where first chunk is found but not // enough space for the rest. - ByteString short_haystack(GetAllocator(), "abcdefg"); + ByteString short_haystack = ByteString::WrapUnsafe("abcdefg"); absl::Cord needle_too_long = absl::MakeFragmentedCord({"ef", "gh"}); - EXPECT_THAT(short_haystack.Find(needle_too_long), Eq(absl::nullopt)); + EXPECT_THAT(short_haystack.Find(needle_too_long), Eq(std::nullopt)); // Search with a fragmented empty cord. absl::Cord fragmented_empty_cord = absl::MakeFragmentedCord({"", ""}); @@ -885,8 +358,8 @@ TEST_P(ByteStringTest, FindEdgeCases) { EXPECT_THAT(byte_string.Find(fragmented_empty_cord, 3), Optional(3)); // Search for suffix in a fragmented cord. - ByteString fragmented_cord_byte_string(GetAllocator(), - GetMediumOrLargeFragmentedCord()); + ByteString fragmented_cord_byte_string = + ByteString::From(GetMediumOrLargeFragmentedCord(), GetArena()); EXPECT_THAT(fragmented_cord_byte_string.Find(suffix_sv), Optional(fragmented_cord_byte_string.size() - suffix_sv.size())); EXPECT_THAT(fragmented_cord_byte_string.Find(absl::Cord(suffix_sv)), @@ -894,311 +367,290 @@ TEST_P(ByteStringTest, FindEdgeCases) { } #ifndef NDEBUG -TEST_P(ByteStringTest, FindOutOfBounds) { - ByteString byte_string = ByteString(GetAllocator(), "test"); +TEST_F(ByteStringTest, FindOutOfBounds) { + ByteString byte_string = ByteString::WrapUnsafe("test"); EXPECT_DEATH(byte_string.Find("t", 5), _); } #endif -TEST_P(ByteStringTest, Substring) { +TEST_F(ByteStringTest, Substring) { // small byte_string substring - ByteString small_byte_string = - ByteString(GetAllocator(), GetSmallStringView()); + ByteString small_byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(small_byte_string.Substring(1, 5), GetSmallStringView().substr(1, 4)); EXPECT_EQ(small_byte_string.Substring(0, small_byte_string.size()), GetSmallStringView()); EXPECT_EQ(small_byte_string.Substring(1, 1), ""); // medium byte_string substring - ByteString medium_byte_string = - ByteString(GetAllocator(), GetMediumStringView()); + ByteString medium_byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(medium_byte_string.Substring(2, 12), GetMediumStringView().substr(2, 10)); EXPECT_EQ(medium_byte_string.Substring(0, medium_byte_string.size()), GetMediumStringView()); // large byte_string substring ByteString large_byte_string = - ByteString(GetAllocator(), GetMediumOrLargeCord()); + ByteString::WrapUnsafe(&GetMediumOrLargeCord()); EXPECT_EQ(large_byte_string.Substring(3, 15), GetMediumOrLargeCord().Subcord(3, 12)); EXPECT_EQ(large_byte_string.Substring(0, large_byte_string.size()), GetMediumOrLargeCord()); // substring with one parameter - ByteString tacocat_byte_string = ByteString(GetAllocator(), "tacocat"); + ByteString tacocat_byte_string = ByteString::WrapUnsafe("tacocat"); EXPECT_EQ(tacocat_byte_string.Substring(4), "cat"); } -TEST_P(ByteStringTest, SubstringEdgeCases) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); +TEST_F(ByteStringTest, SubstringEdgeCases) { + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(byte_string.Substring(byte_string.size(), byte_string.size()), ""); EXPECT_EQ(byte_string.Substring(0, 0), ""); } #ifndef NDEBUG -TEST_P(ByteStringTest, SubstringOutOfBounds) { - ByteString byte_string = ByteString(GetAllocator(), "test"); +TEST_F(ByteStringTest, SubstringOutOfBounds) { + ByteString byte_string = ByteString::WrapUnsafe("test"); EXPECT_DEATH(static_cast(byte_string.Substring(5, 5)), _); EXPECT_DEATH(static_cast(byte_string.Substring(0, 5)), _); EXPECT_DEATH(static_cast(byte_string.Substring(3, 2)), _); } #endif -TEST_P(ByteStringTest, RemovePrefixSmall) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); +TEST_F(ByteStringTest, RemovePrefixSmall) { + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); byte_string.RemovePrefix(1); EXPECT_EQ(byte_string, GetSmallStringView().substr(1)); } -TEST_P(ByteStringTest, RemovePrefixMedium) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); +TEST_F(ByteStringTest, RemovePrefixMedium) { + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); byte_string.RemovePrefix(byte_string.size() - kSmallByteStringCapacity); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); + EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); EXPECT_EQ(byte_string, GetMediumStringView().substr(GetMediumStringView().size() - kSmallByteStringCapacity)); } -TEST_P(ByteStringTest, RemovePrefixMediumOrLarge) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, RemovePrefixMediumOrLarge) { + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); byte_string.RemovePrefix(byte_string.size() - kSmallByteStringCapacity); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); + EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); EXPECT_EQ(byte_string, GetMediumStringView().substr(GetMediumStringView().size() - kSmallByteStringCapacity)); } -TEST_P(ByteStringTest, RemoveSuffixSmall) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); +TEST_F(ByteStringTest, RemoveSuffixSmall) { + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); byte_string.RemoveSuffix(1); EXPECT_EQ(byte_string, GetSmallStringView().substr(0, GetSmallStringView().size() - 1)); } -TEST_P(ByteStringTest, RemoveSuffixMedium) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); +TEST_F(ByteStringTest, RemoveSuffixMedium) { + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); byte_string.RemoveSuffix(byte_string.size() - kSmallByteStringCapacity); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); + EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); EXPECT_EQ(byte_string, GetMediumStringView().substr(0, kSmallByteStringCapacity)); } -TEST_P(ByteStringTest, RemoveSuffixMediumOrLarge) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, RemoveSuffixMediumOrLarge) { + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); byte_string.RemoveSuffix(byte_string.size() - kSmallByteStringCapacity); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); + EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); EXPECT_EQ(byte_string, GetMediumStringView().substr(0, kSmallByteStringCapacity)); } -TEST_P(ByteStringTest, ToStringSmall) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); +TEST_F(ByteStringTest, ToStringSmall) { + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(byte_string.ToString(), byte_string); } -TEST_P(ByteStringTest, ToStringMedium) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); +TEST_F(ByteStringTest, ToStringMedium) { + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(byte_string.ToString(), byte_string); } -TEST_P(ByteStringTest, ToStringLarge) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, ToStringLarge) { + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); EXPECT_EQ(byte_string.ToString(), byte_string); } -TEST_P(ByteStringTest, ToStringViewSmall) { +TEST_F(ByteStringTest, ToStringViewSmall) { std::string scratch; - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(byte_string.ToStringView(&scratch), GetSmallStringView()); } -TEST_P(ByteStringTest, ToStringViewMedium) { +TEST_F(ByteStringTest, ToStringViewMedium) { std::string scratch; - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(byte_string.ToStringView(&scratch), GetMediumStringView()); } -TEST_P(ByteStringTest, ToStringViewLarge) { +TEST_F(ByteStringTest, ToStringViewLarge) { std::string scratch; - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); EXPECT_EQ(byte_string.ToStringView(&scratch), GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, AsStringViewSmall) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); - EXPECT_EQ(byte_string.AsStringView(), GetSmallStringView()); -} - -TEST_P(ByteStringTest, AsStringViewMedium) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); - EXPECT_EQ(byte_string.AsStringView(), GetMediumStringView()); -} - -TEST_P(ByteStringTest, AsStringViewLarge) { - ByteString byte_string = ByteString(GetMediumOrLargeCord()); - EXPECT_DEATH(byte_string.AsStringView(), _); -} - -TEST_P(ByteStringTest, CopyToStringSmall) { +TEST_F(ByteStringTest, CopyToStringSmall) { std::string out; - ByteString(GetAllocator(), GetSmallStringView()).CopyToString(&out); + ByteString::WrapUnsafe(GetSmallStringView()).CopyToString(&out); EXPECT_EQ(out, GetSmallStringView()); } -TEST_P(ByteStringTest, CopyToStringMedium) { +TEST_F(ByteStringTest, CopyToStringMedium) { std::string out; - ByteString(GetAllocator(), GetMediumStringView()).CopyToString(&out); + ByteString::WrapUnsafe(GetMediumStringView()).CopyToString(&out); EXPECT_EQ(out, GetMediumStringView()); } -TEST_P(ByteStringTest, CopyToStringLarge) { +TEST_F(ByteStringTest, CopyToStringLarge) { std::string out; - ByteString(GetAllocator(), GetMediumOrLargeCord()).CopyToString(&out); + ByteString::WrapUnsafe(&GetMediumOrLargeCord()).CopyToString(&out); EXPECT_EQ(out, GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, AppendToStringSmall) { +TEST_F(ByteStringTest, AppendToStringSmall) { std::string out; - ByteString(GetAllocator(), GetSmallStringView()).AppendToString(&out); + ByteString::WrapUnsafe(GetSmallStringView()).AppendToString(&out); EXPECT_EQ(out, GetSmallStringView()); } -TEST_P(ByteStringTest, AppendToStringMedium) { +TEST_F(ByteStringTest, AppendToStringMedium) { std::string out; - ByteString(GetAllocator(), GetMediumStringView()).AppendToString(&out); + ByteString::WrapUnsafe(GetMediumStringView()).AppendToString(&out); EXPECT_EQ(out, GetMediumStringView()); } -TEST_P(ByteStringTest, AppendToStringLarge) { +TEST_F(ByteStringTest, AppendToStringLarge) { std::string out; - ByteString(GetAllocator(), GetMediumOrLargeCord()).AppendToString(&out); + ByteString::WrapUnsafe(&GetMediumOrLargeCord()).AppendToString(&out); EXPECT_EQ(out, GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, ToCordSmall) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); +TEST_F(ByteStringTest, ToCordSmall) { + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(byte_string.ToCord(), byte_string); EXPECT_EQ(std::move(byte_string).ToCord(), GetSmallStringView()); } -TEST_P(ByteStringTest, ToCordMedium) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); +TEST_F(ByteStringTest, ToCordMedium) { + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(byte_string.ToCord(), byte_string); EXPECT_EQ(std::move(byte_string).ToCord(), GetMediumStringView()); } -TEST_P(ByteStringTest, ToCordLarge) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, ToCordLarge) { + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); EXPECT_EQ(byte_string.ToCord(), byte_string); EXPECT_EQ(std::move(byte_string).ToCord(), GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, CopyToCordSmall) { +TEST_F(ByteStringTest, CopyToCordSmall) { absl::Cord out; - ByteString(GetAllocator(), GetSmallStringView()).CopyToCord(&out); + ByteString::WrapUnsafe(GetSmallStringView()).CopyToCord(&out); EXPECT_EQ(out, GetSmallStringView()); } -TEST_P(ByteStringTest, CopyToCordMedium) { +TEST_F(ByteStringTest, CopyToCordMedium) { absl::Cord out; - ByteString(GetAllocator(), GetMediumStringView()).CopyToCord(&out); + ByteString::WrapUnsafe(GetMediumStringView()).CopyToCord(&out); EXPECT_EQ(out, GetMediumStringView()); } -TEST_P(ByteStringTest, CopyToCordLarge) { +TEST_F(ByteStringTest, CopyToCordLarge) { absl::Cord out; - ByteString(GetAllocator(), GetMediumOrLargeCord()).CopyToCord(&out); + ByteString::WrapUnsafe(&GetMediumOrLargeCord()).CopyToCord(&out); EXPECT_EQ(out, GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, AppendToCordSmall) { +TEST_F(ByteStringTest, AppendToCordSmall) { absl::Cord out; - ByteString(GetAllocator(), GetSmallStringView()).AppendToCord(&out); + ByteString::WrapUnsafe(GetSmallStringView()).AppendToCord(&out); EXPECT_EQ(out, GetSmallStringView()); } -TEST_P(ByteStringTest, AppendToCordMedium) { +TEST_F(ByteStringTest, AppendToCordMedium) { absl::Cord out; - ByteString(GetAllocator(), GetMediumStringView()).AppendToCord(&out); + ByteString::WrapUnsafe(GetMediumStringView()).AppendToCord(&out); EXPECT_EQ(out, GetMediumStringView()); } -TEST_P(ByteStringTest, AppendToCordLarge) { +TEST_F(ByteStringTest, AppendToCordLarge) { absl::Cord out; - ByteString(GetAllocator(), GetMediumOrLargeCord()).AppendToCord(&out); + ByteString::WrapUnsafe(&GetMediumOrLargeCord()).AppendToCord(&out); EXPECT_EQ(out, GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, CloneSmall) { +TEST_F(ByteStringTest, CloneSmall) { google::protobuf::Arena arena; - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(byte_string.Clone(&arena), byte_string); } -TEST_P(ByteStringTest, CloneMedium) { +TEST_F(ByteStringTest, CloneMedium) { google::protobuf::Arena arena; - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(byte_string.Clone(&arena), byte_string); } -TEST_P(ByteStringTest, CloneLarge) { +TEST_F(ByteStringTest, CloneLarge) { google::protobuf::Arena arena; - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); EXPECT_EQ(byte_string.Clone(&arena), byte_string); } -TEST_P(ByteStringTest, LegacyByteStringSmall) { +TEST_F(ByteStringTest, LegacyByteStringSmall) { google::protobuf::Arena arena; - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(LegacyByteString(byte_string, /*stable=*/false, &arena), GetSmallStringView()); EXPECT_EQ(LegacyByteString(byte_string, /*stable=*/true, &arena), GetSmallStringView()); } -TEST_P(ByteStringTest, LegacyByteStringMedium) { +TEST_F(ByteStringTest, LegacyByteStringMedium) { google::protobuf::Arena arena; - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(LegacyByteString(byte_string, /*stable=*/false, &arena), GetMediumStringView()); EXPECT_EQ(LegacyByteString(byte_string, /*stable=*/true, &arena), GetMediumStringView()); } -TEST_P(ByteStringTest, LegacyByteStringLarge) { +TEST_F(ByteStringTest, LegacyByteStringLarge) { google::protobuf::Arena arena; - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); EXPECT_EQ(LegacyByteString(byte_string, /*stable=*/false, &arena), GetMediumOrLargeCord()); EXPECT_EQ(LegacyByteString(byte_string, /*stable=*/true, &arena), GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, HashValue) { - EXPECT_EQ(absl::HashOf(ByteString(GetAllocator(), GetSmallStringView())), +TEST_F(ByteStringTest, HashValue) { + EXPECT_EQ(absl::HashOf(ByteString::WrapUnsafe(GetSmallStringView())), absl::HashOf(GetSmallStringView())); - EXPECT_EQ(absl::HashOf(ByteString(GetAllocator(), GetMediumStringView())), + EXPECT_EQ(absl::HashOf(ByteString::WrapUnsafe(GetMediumStringView())), absl::HashOf(GetMediumStringView())); - EXPECT_EQ(absl::HashOf(ByteString(GetAllocator(), GetMediumOrLargeCord())), + EXPECT_EQ(absl::HashOf(ByteString::WrapUnsafe(&GetMediumOrLargeCord())), absl::HashOf(GetMediumOrLargeCord())); } -INSTANTIATE_TEST_SUITE_P(ByteStringTest, ByteStringTest, - ::testing::Values(AllocatorKind::kNewDelete, - AllocatorKind::kArena)); - } // namespace } // namespace cel::common_internal diff --git a/common/internal/casting.h b/common/internal/casting.h deleted file mode 100644 index fe7d03279..000000000 --- a/common/internal/casting.h +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// IWYU pragma: private, include "common/casting.h" - -#ifndef THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_CASTING_H_ -#define THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_CASTING_H_ - -#include -#include -#include - -#include "absl/base/attributes.h" -#include "absl/meta/type_traits.h" -#include "absl/types/optional.h" -#include "internal/casts.h" - -namespace cel { - -namespace common_internal { - -template -using propagate_const_t = - std::conditional_t>, - std::add_const_t, To>; - -template -using propagate_volatile_t = - std::conditional_t>, - std::add_volatile_t, To>; - -template -using propagate_reference_t = - std::conditional_t, - std::add_lvalue_reference_t, - std::conditional_t, - std::add_rvalue_reference_t, To>>; - -template -using propagate_cvref_t = propagate_reference_t< - propagate_volatile_t, From>, From>; - -} // namespace common_internal - -namespace common_internal { - -// Implementation of `cel::InstanceOf`. -template -struct ABSL_DEPRECATED("Use Is member functions instead.") - InstanceOfImpl final { - static_assert(!std::is_pointer_v, "To must not be a pointer"); - static_assert(!std::is_array_v, "To must not be an array"); - static_assert(!std::is_lvalue_reference_v, - "To must not be a lvalue reference"); - static_assert(!std::is_rvalue_reference_v, - "To must not be a lvalue reference"); - static_assert(!std::is_const_v, "To must not be const qualified"); - static_assert(!std::is_volatile_v, "To must not be volatile qualified"); - static_assert(std::is_class_v, "To must be a non-union class"); - - explicit InstanceOfImpl() = default; - - template - ABSL_DEPRECATED("Use Is member functions instead.") - ABSL_MUST_USE_RESULT bool operator()(const From& from) const { - static_assert(!std::is_volatile_v, - "From must not be volatile qualified"); - static_assert(std::is_class_v, "From must be a non-union class"); - if constexpr (std::is_same_v, To>) { - // Same type. Separate from the next `else if` to work on in-complete - // types. - return true; - } else if constexpr (std::is_polymorphic_v && - std::is_polymorphic_v> && - std::is_base_of_v>) { - // Polymorphic upcast. - return true; - } else if constexpr (!std::is_polymorphic_v && - !std::is_polymorphic_v> && - (std::is_convertible_v || - std::is_convertible_v || - std::is_convertible_v || - std::is_convertible_v)) { - // Implicitly convertible. - return true; - } else { - // Something else. - return from.template Is(); - } - } - - template - ABSL_DEPRECATED("Use Is member functions instead.") - ABSL_MUST_USE_RESULT bool operator()(const From* from) const { - static_assert(!std::is_volatile_v, - "From must not be volatile qualified"); - static_assert(std::is_class_v, "From must be a non-union class"); - return from != nullptr && (*this)(*from); - } -}; - -// Implementation of `cel::Cast`. -template -struct ABSL_DEPRECATED( - "Use explicit conversion functions instead through static_cast.") - CastImpl final { - static_assert(!std::is_pointer_v, "To must not be a pointer"); - static_assert(!std::is_array_v, "To must not be an array"); - static_assert(!std::is_lvalue_reference_v, - "To must not be a lvalue reference"); - static_assert(!std::is_rvalue_reference_v, - "To must not be a lvalue reference"); - static_assert(!std::is_const_v, "To must not be const qualified"); - static_assert(!std::is_volatile_v, "To must not be volatile qualified"); - static_assert(std::is_class_v, "To must be a non-union class"); - - explicit CastImpl() = default; - - template - ABSL_DEPRECATED( - "Use explicit conversion functions instead through static_cast.") - ABSL_MUST_USE_RESULT decltype(auto) - operator()(From&& from) const { - static_assert(!std::is_volatile_v, - "From must not be volatile qualified"); - static_assert(std::is_class_v>, - "From must be a non-union class"); - if constexpr (std::is_polymorphic_v) { - static_assert(std::is_lvalue_reference_v, - "polymorphic casts are only possible on lvalue references"); - } - if constexpr (std::is_same_v, To>) { - // Same type. Separate from the next `else if` to work on in-complete - // types. - return static_cast>(from); - } else if constexpr (std::is_polymorphic_v && - std::is_polymorphic_v> && - std::is_base_of_v>) { - // Polymorphic upcast. - return static_cast>(from); - } else if constexpr (std::is_polymorphic_v && - std::is_polymorphic_v> && - std::is_base_of_v, To>) { - // Polymorphic downcast. - return cel::internal::down_cast>( - std::forward(from)); - } else if constexpr (std::is_convertible_v && - !std::is_polymorphic_v && - !std::is_polymorphic_v>) { - return static_cast(std::forward(from)); - } else { - // Something else. - return std::forward(from).template Get(); - } - } - - template - ABSL_DEPRECATED( - "Use explicit conversion functions instead through static_cast.") - ABSL_MUST_USE_RESULT decltype(auto) - operator()(From* from) const { - static_assert(!std::is_volatile_v, - "From must not be volatile qualified"); - static_assert(std::is_class_v, "From must be a non-union class"); - using R = decltype((*this)(*from)); - static_assert(std::is_lvalue_reference_v); - if (from == nullptr) { - return static_cast>>( - nullptr); - } - return static_cast>>( - std::addressof((*this)(*from))); - } -}; - -// Implementation of `cel::As`. -template -struct ABSL_DEPRECATED("Use As member functions instead.") AsImpl final { - static_assert(!std::is_pointer_v, "To must not be a pointer"); - static_assert(!std::is_array_v, "To must not be an array"); - static_assert(!std::is_lvalue_reference_v, - "To must not be a lvalue reference"); - static_assert(!std::is_rvalue_reference_v, - "To must not be a lvalue reference"); - static_assert(!std::is_const_v, "To must not be const qualified"); - static_assert(!std::is_volatile_v, "To must not be volatile qualified"); - static_assert(std::is_class_v, "To must be a non-union class"); - - explicit AsImpl() = default; - - template - ABSL_DEPRECATED("Use As member functions instead.") - ABSL_MUST_USE_RESULT decltype(auto) operator()(From&& from) const { - // Returns either `absl::optional` or `cel::optional_ref` - // depending on the return type of `CastTraits::Convert`. The use of these - // two types is an implementation detail. - static_assert(!std::is_volatile_v, - "From must not be volatile qualified"); - static_assert(std::is_class_v>, - "From must be a non-union class"); - return std::forward(from).template As(); - } - - // Returns a pointer. - template - ABSL_DEPRECATED("Use As member functions instead.") - ABSL_MUST_USE_RESULT decltype(auto) operator()(From* from) const { - // Returns either `absl::optional` or `To*` depending on the return type of - // `CastTraits::Convert`. The use of these two types is an implementation - // detail. - static_assert(!std::is_volatile_v, - "From must not be volatile qualified"); - static_assert(std::is_class_v, "From must be a non-union class"); - using R = decltype(from->template As()); - if (from == nullptr) { - return R{absl::nullopt}; - } - return from->template As(); - } -}; - -} // namespace common_internal - -} // namespace cel - -#endif // THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_CASTING_H_ diff --git a/common/internal/metadata.h b/common/internal/metadata.h deleted file mode 100644 index 5d2fa8322..000000000 --- a/common/internal/metadata.h +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_METADATA_H_ -#define THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_METADATA_H_ - -#include - -#include "google/protobuf/arena.h" - -namespace cel::common_internal { - -// `google::protobuf::Arena` has a minimum alignment of 8. `ReferenceCount` has a minimum -// alignment that is guaranteed to be greater than or equal to `google::protobuf::Arena`. -inline constexpr uintptr_t kMetadataOwnerNone = 0; -inline constexpr uintptr_t kMetadataOwnerReferenceCountBit = uintptr_t{1} << 0; -inline constexpr uintptr_t kMetadataOwnerArenaBit = uintptr_t{1} << 1; -inline constexpr uintptr_t kMetadataOwnerBits = alignof(google::protobuf::Arena) - 1; -inline constexpr uintptr_t kMetadataOwnerPointerMask = ~kMetadataOwnerBits; - -// Ensure kMetadataOwnerBits encompasses kMetadataOwnerReferenceCountBit and -// kMetadataOwnerArenaBit. -static_assert((kMetadataOwnerBits | kMetadataOwnerReferenceCountBit) == - kMetadataOwnerBits); -static_assert((kMetadataOwnerBits | kMetadataOwnerArenaBit) == - kMetadataOwnerBits); - -} // namespace cel::common_internal - -#endif // THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_METADATA_H_ diff --git a/common/internal/reference_count.cc b/common/internal/reference_count.cc deleted file mode 100644 index c954c685e..000000000 --- a/common/internal/reference_count.cc +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/internal/reference_count.h" - -#include -#include -#include -#include -#include -#include - -#include "absl/base/nullability.h" -#include "absl/log/absl_check.h" -#include "absl/strings/string_view.h" -#include "common/data.h" -#include "internal/new.h" -#include "google/protobuf/message_lite.h" - -namespace cel::common_internal { - -template class DeletingReferenceCount; - -namespace { - -class ReferenceCountedStdString final : public ReferenceCounted { - public: - static std::pair New( - std::string&& string) { - const auto* const refcount = - new ReferenceCountedStdString(std::move(string)); - const auto* const refcount_string = std::launder( - reinterpret_cast(&refcount->string_[0])); - return std::pair{static_cast(refcount), - absl::string_view(*refcount_string)}; - } - - explicit ReferenceCountedStdString(std::string&& string) { - (::new (static_cast(&string_[0])) std::string(std::move(string))) - ->shrink_to_fit(); - } - - private: - void Finalize() noexcept override { - std::destroy_at(std::launder(reinterpret_cast(&string_[0]))); - } - - alignas(std::string) char string_[sizeof(std::string)]; -}; - -class ReferenceCountedString final : public ReferenceCounted { - public: - static std::pair New( - absl::string_view string) { - const auto* const refcount = - ::new (internal::New(Overhead() + string.size())) - ReferenceCountedString(string); - return std::pair{static_cast(refcount), - absl::string_view(refcount->data_, refcount->size_)}; - } - - private: -// ReferenceCountedString is non-standard-layout due to having virtual functions -// from a base class. This causes compilers to warn about the use of offsetof(), -// but it still works here, so silence the warning and proceed. -#if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Winvalid-offsetof" -#endif - - static size_t Overhead() { return offsetof(ReferenceCountedString, data_); } - -#if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic pop -#endif - - explicit ReferenceCountedString(absl::string_view string) - : size_(string.size()) { - std::memcpy(data_, string.data(), size_); - } - - void Delete() noexcept override { - void* const that = this; - const auto size = size_; - std::destroy_at(this); - internal::SizedDelete(that, Overhead() + size); - } - - const size_t size_; - char data_[]; -}; - -} // namespace - -std::pair -MakeReferenceCountedString(absl::string_view value) { - ABSL_DCHECK(!value.empty()); - return ReferenceCountedString::New(value); -} - -std::pair -MakeReferenceCountedString(std::string&& value) { - ABSL_DCHECK(!value.empty()); - return ReferenceCountedStdString::New(std::move(value)); -} - -} // namespace cel::common_internal diff --git a/common/internal/reference_count.h b/common/internal/reference_count.h deleted file mode 100644 index 9c7fb5371..000000000 --- a/common/internal/reference_count.h +++ /dev/null @@ -1,406 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This header contains primitives for reference counting, roughly equivalent to -// the primitives used to implement `std::shared_ptr`. These primitives should -// not be used directly in most cases, instead `cel::Shared` should be -// used instead. - -#ifndef THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_REFERENCE_COUNT_H_ -#define THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_REFERENCE_COUNT_H_ - -#include -#include -#include -#include -#include -#include - -#include "absl/base/attributes.h" -#include "absl/base/nullability.h" -#include "absl/base/optimization.h" -#include "absl/log/absl_check.h" -#include "absl/strings/string_view.h" -#include "common/data.h" -#include "google/protobuf/arena.h" -#include "google/protobuf/message_lite.h" - -namespace cel::common_internal { - -struct AdoptRef final { - explicit AdoptRef() = default; -}; - -inline constexpr AdoptRef kAdoptRef{}; - -class ReferenceCount; -struct ReferenceCountFromThis; - -void SetReferenceCountForThat(ReferenceCountFromThis& that, - ReferenceCount* absl_nullable refcount); - -ReferenceCount* absl_nullable GetReferenceCountForThat( - const ReferenceCountFromThis& that); - -// `ReferenceCountFromThis` is similar to `std::enable_shared_from_this`. It -// allows the derived object to inspect its own reference count. It should not -// be used directly, but should be used through -// `cel::EnableManagedMemoryFromThis`. -struct ReferenceCountFromThis { - private: - friend void SetReferenceCountForThat(ReferenceCountFromThis& that, - ReferenceCount* absl_nullable refcount); - friend ReferenceCount* absl_nullable GetReferenceCountForThat( - const ReferenceCountFromThis& that); - - static constexpr uintptr_t kNullPtr = uintptr_t{0}; - static constexpr uintptr_t kSentinelPtr = ~kNullPtr; - - void* absl_nullable refcount = reinterpret_cast(kSentinelPtr); -}; - -inline void SetReferenceCountForThat(ReferenceCountFromThis& that, - ReferenceCount* absl_nullable refcount) { - ABSL_DCHECK_EQ(that.refcount, - reinterpret_cast(ReferenceCountFromThis::kSentinelPtr)); - that.refcount = static_cast(refcount); -} - -inline ReferenceCount* absl_nullable GetReferenceCountForThat( - const ReferenceCountFromThis& that) { - ABSL_DCHECK_NE(that.refcount, - reinterpret_cast(ReferenceCountFromThis::kSentinelPtr)); - return static_cast(that.refcount); -} - -void StrongRef(const ReferenceCount& refcount) noexcept; - -void StrongRef(const ReferenceCount* absl_nullable refcount) noexcept; - -void StrongUnref(const ReferenceCount& refcount) noexcept; - -void StrongUnref(const ReferenceCount* absl_nullable refcount) noexcept; - -ABSL_MUST_USE_RESULT -bool StrengthenRef(const ReferenceCount& refcount) noexcept; - -ABSL_MUST_USE_RESULT -bool StrengthenRef(const ReferenceCount* absl_nullable refcount) noexcept; - -void WeakRef(const ReferenceCount& refcount) noexcept; - -void WeakRef(const ReferenceCount* absl_nullable refcount) noexcept; - -void WeakUnref(const ReferenceCount& refcount) noexcept; - -void WeakUnref(const ReferenceCount* absl_nullable refcount) noexcept; - -ABSL_MUST_USE_RESULT -bool IsUniqueRef(const ReferenceCount& refcount) noexcept; - -ABSL_MUST_USE_RESULT -bool IsUniqueRef(const ReferenceCount* absl_nullable refcount) noexcept; - -ABSL_MUST_USE_RESULT -bool IsExpiredRef(const ReferenceCount& refcount) noexcept; - -ABSL_MUST_USE_RESULT -bool IsExpiredRef(const ReferenceCount* absl_nullable refcount) noexcept; - -// `ReferenceCount` is similar to the control block used by `std::shared_ptr`. -// It is not meant to be interacted with directly in most cases, instead -// `cel::Shared` should be used. -class alignas(8) ReferenceCount { - public: - ReferenceCount() = default; - - ReferenceCount(const ReferenceCount&) = delete; - ReferenceCount(ReferenceCount&&) = delete; - ReferenceCount& operator=(const ReferenceCount&) = delete; - ReferenceCount& operator=(ReferenceCount&&) = delete; - - virtual ~ReferenceCount() = default; - - private: - friend void StrongRef(const ReferenceCount& refcount) noexcept; - friend void StrongUnref(const ReferenceCount& refcount) noexcept; - friend bool StrengthenRef(const ReferenceCount& refcount) noexcept; - friend void WeakRef(const ReferenceCount& refcount) noexcept; - friend void WeakUnref(const ReferenceCount& refcount) noexcept; - friend bool IsUniqueRef(const ReferenceCount& refcount) noexcept; - friend bool IsExpiredRef(const ReferenceCount& refcount) noexcept; - - virtual void Finalize() noexcept = 0; - - virtual void Delete() noexcept = 0; - - mutable std::atomic strong_refcount_ = 1; - mutable std::atomic weak_refcount_ = 1; -}; - -// ReferenceCount and its derivations must be at least as aligned as -// google::protobuf::Arena. This is a requirement for the pointer tagging defined in -// common/internal/metadata.h. -static_assert(alignof(ReferenceCount) >= alignof(google::protobuf::Arena)); - -// `ReferenceCounted` is a base class for classes which should be reference -// counted. It provides default implementations for `Finalize()` and `Delete()`. -class ReferenceCounted : public ReferenceCount { - private: - void Finalize() noexcept override {} - - void Delete() noexcept override { delete this; } -}; - -// `EmplacedReferenceCount` adapts `T` to make it reference countable, by -// storing `T` inside the reference count. This only works when `T` has not yet -// been allocated. -template -class EmplacedReferenceCount final : public ReferenceCounted { - public: - static_assert(std::is_destructible_v, "T must be destructible"); - static_assert(!std::is_reference_v, "T must not be a reference"); - static_assert(!std::is_volatile_v, "T must not be volatile qualified"); - static_assert(!std::is_const_v, "T must not be const qualified"); - static_assert(!std::is_array_v, "T must not be an array"); - - template - explicit EmplacedReferenceCount(T*& value, Args&&... args) noexcept( - std::is_nothrow_constructible_v) { - value = - ::new (static_cast(&value_[0])) T(std::forward(args)...); - } - - private: - void Finalize() noexcept override { - std::destroy_at(std::launder(reinterpret_cast(&value_[0]))); - } - - // We store the instance of `T` in a char buffer and use placement new and - // direct calls to the destructor. The reason for this is `Finalize()` is - // called when the strong reference count hits 0. This allows us to destroy - // our instance of `T` once we are no longer strongly reachable and deallocate - // the memory once we are no longer weakly reachable. - alignas(T) char value_[sizeof(T)]; -}; - -// `DeletingReferenceCount` adapts `T` to make it reference countable, by taking -// ownership of `T` and deleting it. This only works when `T` has already been -// allocated and is to expensive to move or copy. -template -class DeletingReferenceCount final : public ReferenceCounted { - public: - explicit DeletingReferenceCount(const T* absl_nonnull to_delete) noexcept - : to_delete_(to_delete) {} - - private: - void Finalize() noexcept override { delete to_delete_; } - - const T* absl_nonnull const to_delete_; -}; - -extern template class DeletingReferenceCount; - -template -const ReferenceCount* absl_nonnull MakeDeletingReferenceCount( - const T* absl_nonnull to_delete) { - if constexpr (google::protobuf::Arena::is_arena_constructable::value) { - ABSL_DCHECK_EQ(to_delete->GetArena(), nullptr); - } - if constexpr (std::is_base_of_v) { - return new DeletingReferenceCount(to_delete); - } else { - auto* refcount = new DeletingReferenceCount(to_delete); - if constexpr (std::is_base_of_v) { - common_internal::SetDataReferenceCount(to_delete, refcount); - } - return refcount; - } -} - -template -std::pair -MakeEmplacedReferenceCount(Args&&... args) { - using U = std::remove_const_t; - U* pointer; - auto* const refcount = - new EmplacedReferenceCount(pointer, std::forward(args)...); - if constexpr (google::protobuf::Arena::is_arena_constructable::value) { - ABSL_DCHECK_EQ(pointer->GetArena(), nullptr); - } - if constexpr (std::is_base_of_v) { - common_internal::SetDataReferenceCount(pointer, refcount); - } - return std::pair{static_cast(pointer), - static_cast(refcount)}; -} - -template -class InlinedReferenceCount final : public ReferenceCounted { - public: - template - explicit InlinedReferenceCount(std::in_place_t, Args&&... args) - : ReferenceCounted() { - ::new (static_cast(value())) T(std::forward(args)...); - } - - ABSL_ATTRIBUTE_ALWAYS_INLINE T* absl_nonnull value() { - return reinterpret_cast(&value_[0]); - } - - ABSL_ATTRIBUTE_ALWAYS_INLINE const T* absl_nonnull value() const { - return reinterpret_cast(&value_[0]); - } - - private: - void Finalize() noexcept override { value()->~T(); } - - // We store the instance of `T` in a char buffer and use placement new and - // direct calls to the destructor. The reason for this is `Finalize()` is - // called when the strong reference count hits 0. This allows us to destroy - // our instance of `T` once we are no longer strongly reachable and deallocate - // the memory once we are no longer weakly reachable. - alignas(T) char value_[sizeof(T)]; -}; - -template -std::pair MakeReferenceCount( - Args&&... args) { - using U = std::remove_const_t; - auto* const refcount = - new InlinedReferenceCount(std::in_place, std::forward(args)...); - auto* const pointer = refcount->value(); - if constexpr (std::is_base_of_v) { - SetReferenceCountForThat(*pointer, refcount); - } - return std::make_pair(static_cast(pointer), - static_cast(refcount)); -} - -inline void StrongRef(const ReferenceCount& refcount) noexcept { - const auto count = - refcount.strong_refcount_.fetch_add(1, std::memory_order_relaxed); - ABSL_DCHECK_GT(count, 0); -} - -inline void StrongRef(const ReferenceCount* absl_nullable refcount) noexcept { - if (refcount != nullptr) { - StrongRef(*refcount); - } -} - -inline void StrongUnref(const ReferenceCount& refcount) noexcept { - const auto count = - refcount.strong_refcount_.fetch_sub(1, std::memory_order_acq_rel); - ABSL_DCHECK_GT(count, 0); - ABSL_ASSUME(count > 0); - if (ABSL_PREDICT_FALSE(count == 1)) { - const_cast(refcount).Finalize(); - WeakUnref(refcount); - } -} - -inline void StrongUnref(const ReferenceCount* absl_nullable refcount) noexcept { - if (refcount != nullptr) { - StrongUnref(*refcount); - } -} - -ABSL_MUST_USE_RESULT -inline bool StrengthenRef(const ReferenceCount& refcount) noexcept { - auto count = refcount.strong_refcount_.load(std::memory_order_relaxed); - while (true) { - ABSL_DCHECK_GE(count, 0); - ABSL_ASSUME(count >= 0); - if (count == 0) { - return false; - } - if (refcount.strong_refcount_.compare_exchange_weak( - count, count + 1, std::memory_order_release, - std::memory_order_relaxed)) { - return true; - } - } -} - -ABSL_MUST_USE_RESULT -inline bool StrengthenRef( - const ReferenceCount* absl_nullable refcount) noexcept { - return refcount != nullptr ? StrengthenRef(*refcount) : false; -} - -inline void WeakRef(const ReferenceCount& refcount) noexcept { - const auto count = - refcount.weak_refcount_.fetch_add(1, std::memory_order_relaxed); - ABSL_DCHECK_GT(count, 0); -} - -inline void WeakRef(const ReferenceCount* absl_nullable refcount) noexcept { - if (refcount != nullptr) { - WeakRef(*refcount); - } -} - -inline void WeakUnref(const ReferenceCount& refcount) noexcept { - const auto count = - refcount.weak_refcount_.fetch_sub(1, std::memory_order_acq_rel); - ABSL_DCHECK_GT(count, 0); - ABSL_ASSUME(count > 0); - if (ABSL_PREDICT_FALSE(count == 1)) { - const_cast(refcount).Delete(); - } -} - -inline void WeakUnref(const ReferenceCount* absl_nullable refcount) noexcept { - if (refcount != nullptr) { - WeakUnref(*refcount); - } -} - -ABSL_MUST_USE_RESULT -inline bool IsUniqueRef(const ReferenceCount& refcount) noexcept { - const auto count = refcount.strong_refcount_.load(std::memory_order_acquire); - ABSL_DCHECK_GT(count, 0); - ABSL_ASSUME(count > 0); - return count == 1; -} - -ABSL_MUST_USE_RESULT -inline bool IsUniqueRef(const ReferenceCount* absl_nullable refcount) noexcept { - return refcount != nullptr ? IsUniqueRef(*refcount) : false; -} - -ABSL_MUST_USE_RESULT -inline bool IsExpiredRef(const ReferenceCount& refcount) noexcept { - const auto count = refcount.strong_refcount_.load(std::memory_order_acquire); - ABSL_DCHECK_GE(count, 0); - ABSL_ASSUME(count >= 0); - return count == 0; -} - -ABSL_MUST_USE_RESULT -inline bool IsExpiredRef( - const ReferenceCount* absl_nullable refcount) noexcept { - return refcount != nullptr ? IsExpiredRef(*refcount) : false; -} - -std::pair -MakeReferenceCountedString(absl::string_view value); - -std::pair -MakeReferenceCountedString(std::string&& value); - -} // namespace cel::common_internal - -#endif // THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_REFERENCE_COUNT_H_ diff --git a/common/internal/reference_count_test.cc b/common/internal/reference_count_test.cc deleted file mode 100644 index af36fa9a5..000000000 --- a/common/internal/reference_count_test.cc +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/internal/reference_count.h" - -#include - -#include "google/protobuf/struct.pb.h" -#include "absl/base/nullability.h" -#include "common/data.h" -#include "internal/testing.h" -#include "google/protobuf/arena.h" -#include "google/protobuf/message_lite.h" - -namespace cel::common_internal { -namespace { - -using ::testing::NotNull; -using ::testing::WhenDynamicCastTo; - -class Object : public virtual ReferenceCountFromThis { - public: - explicit Object(bool& destructed) : destructed_(destructed) {} - - ~Object() { destructed_ = true; } - - private: - bool& destructed_; -}; - -class Subobject : public Object, public virtual ReferenceCountFromThis { - public: - using Object::Object; -}; - -TEST(ReferenceCount, Strong) { - bool destructed = false; - Object* object; - ReferenceCount* refcount; - std::tie(object, refcount) = MakeReferenceCount(destructed); - EXPECT_EQ(GetReferenceCountForThat(*object), refcount); - EXPECT_EQ(GetReferenceCountForThat(*static_cast(object)), - refcount); - StrongRef(refcount); - StrongUnref(refcount); - EXPECT_TRUE(IsUniqueRef(refcount)); - EXPECT_FALSE(IsExpiredRef(refcount)); - EXPECT_FALSE(destructed); - StrongUnref(refcount); - EXPECT_TRUE(destructed); -} - -TEST(ReferenceCount, Weak) { - bool destructed = false; - Object* object; - ReferenceCount* refcount; - std::tie(object, refcount) = MakeReferenceCount(destructed); - EXPECT_EQ(GetReferenceCountForThat(*object), refcount); - EXPECT_EQ(GetReferenceCountForThat(*static_cast(object)), - refcount); - WeakRef(refcount); - ASSERT_TRUE(StrengthenRef(refcount)); - StrongUnref(refcount); - EXPECT_TRUE(IsUniqueRef(refcount)); - EXPECT_FALSE(IsExpiredRef(refcount)); - EXPECT_FALSE(destructed); - StrongUnref(refcount); - EXPECT_TRUE(destructed); - EXPECT_TRUE(IsExpiredRef(refcount)); - ASSERT_FALSE(StrengthenRef(refcount)); - WeakUnref(refcount); -} - -class DataObject final : public Data { - public: - DataObject() noexcept : Data() {} - - explicit DataObject(google::protobuf::Arena* absl_nullable arena) noexcept - : Data(arena) {} - - char member_[17]; -}; - -struct OtherObject final { - char data[17]; -}; - -TEST(DeletingReferenceCount, Data) { - auto* data = new DataObject(); - const auto* refcount = MakeDeletingReferenceCount(data); - EXPECT_THAT( - refcount, - WhenDynamicCastTo*>(NotNull())); - EXPECT_EQ(common_internal::GetDataReferenceCount(data), refcount); - StrongUnref(refcount); -} - -TEST(DeletingReferenceCount, MessageLite) { - auto* message_lite = new google::protobuf::Value(); - const auto* refcount = MakeDeletingReferenceCount(message_lite); - EXPECT_THAT( - refcount, - WhenDynamicCastTo*>( - NotNull())); - StrongUnref(refcount); -} - -TEST(DeletingReferenceCount, Other) { - auto* other = new OtherObject(); - const auto* refcount = MakeDeletingReferenceCount(other); - EXPECT_THAT( - refcount, - WhenDynamicCastTo*>(NotNull())); - StrongUnref(refcount); -} - -TEST(EmplacedReferenceCount, Data) { - Data* data; - const ReferenceCount* refcount; - std::tie(data, refcount) = MakeEmplacedReferenceCount(); - EXPECT_THAT( - refcount, - WhenDynamicCastTo*>(NotNull())); - EXPECT_EQ(common_internal::GetDataReferenceCount(data), refcount); - StrongUnref(refcount); -} - -TEST(EmplacedReferenceCount, MessageLite) { - google::protobuf::Value* message_lite; - const ReferenceCount* refcount; - std::tie(message_lite, refcount) = - MakeEmplacedReferenceCount(); - EXPECT_THAT( - refcount, - WhenDynamicCastTo*>( - NotNull())); - StrongUnref(refcount); -} - -TEST(EmplacedReferenceCount, Other) { - OtherObject* other; - const ReferenceCount* refcount; - std::tie(other, refcount) = MakeEmplacedReferenceCount(); - EXPECT_THAT( - refcount, - WhenDynamicCastTo*>(NotNull())); - StrongUnref(refcount); -} - -} // namespace -} // namespace cel::common_internal diff --git a/common/internal/signature.cc b/common/internal/signature.cc deleted file mode 100644 index f63049878..000000000 --- a/common/internal/signature.cc +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/internal/signature.h" - -#include -#include -#include -#include - -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" -#include "common/type.h" -#include "common/type_kind.h" -#include "internal/status_macros.h" - -namespace cel::common_internal { - -namespace { - -void AppendEscaped(std::string* result, std::string_view str, bool escape_dot) { - for (char c : str) { - switch (c) { - case '\\': - case '(': - case ')': - case '<': - case '>': - case '"': - case ',': - case '~': - result->push_back('\\'); - break; - case '.': - if (escape_dot) { - result->push_back('\\'); - } - break; - } - result->push_back(c); - } -} - -absl::Status AppendTypeParameters(std::string* result, const Type& type); - -// Recursively appends a string representation of the given `type` to `result`. -// Type parameters are enclosed in angle brackets and separated by commas. - -// Grammar: -// TypeDesc = NamespaceIdentifier [ "<" TypeList ">" ] ; -// NamespaceIdentifier = [ "." ] Identifier { "." Identifier } ; -// TypeList = TypeElem { "," TypeElem } ; -// TypeElem = TypeDesc | TypeParam -// TypeParam = "~" Alpha ; -// Identifier = ( Alpha | "_" ) { AlphaNumeric | "_" } ; -// (* Terminals *) -// Alpha = "a"..."z" | "A"..."Z" ; -// Digit = "0"..."9" ; -// AlphaNumeric = Alpha | Digit ; -// -// For compatibility, the implementation allows unexpected characters in -// type names and parameters and escapes them with a backslash. -absl::Status AppendTypeDesc(std::string* result, const Type& type) { - switch (type.kind()) { - case TypeKind::kNull: - absl::StrAppend(result, "null"); - break; - case TypeKind::kBool: - absl::StrAppend(result, "bool"); - break; - case TypeKind::kInt: - absl::StrAppend(result, "int"); - break; - case TypeKind::kUint: - absl::StrAppend(result, "uint"); - break; - case TypeKind::kDouble: - absl::StrAppend(result, "double"); - break; - case TypeKind::kString: - absl::StrAppend(result, "string"); - break; - case TypeKind::kBytes: - absl::StrAppend(result, "bytes"); - break; - case TypeKind::kDuration: - absl::StrAppend(result, "duration"); - break; - case TypeKind::kTimestamp: - absl::StrAppend(result, "timestamp"); - break; - case TypeKind::kAny: - absl::StrAppend(result, "any"); - break; - case TypeKind::kDyn: - absl::StrAppend(result, "dyn"); - break; - case TypeKind::kBoolWrapper: - absl::StrAppend(result, "bool_wrapper"); - break; - case TypeKind::kIntWrapper: - absl::StrAppend(result, "int_wrapper"); - break; - case TypeKind::kUintWrapper: - absl::StrAppend(result, "uint_wrapper"); - break; - case TypeKind::kDoubleWrapper: - absl::StrAppend(result, "double_wrapper"); - break; - case TypeKind::kStringWrapper: - absl::StrAppend(result, "string_wrapper"); - break; - case TypeKind::kBytesWrapper: - absl::StrAppend(result, "bytes_wrapper"); - break; - case TypeKind::kList: - absl::StrAppend(result, "list"); - CEL_RETURN_IF_ERROR(AppendTypeParameters(result, type)); - break; - case TypeKind::kMap: - absl::StrAppend(result, "map"); - CEL_RETURN_IF_ERROR(AppendTypeParameters(result, type)); - break; - case TypeKind::kFunction: - absl::StrAppend(result, "function"); - CEL_RETURN_IF_ERROR(AppendTypeParameters(result, type)); - break; - case TypeKind::kType: - absl::StrAppend(result, "type"); - CEL_RETURN_IF_ERROR(AppendTypeParameters(result, type)); - break; - case TypeKind::kTypeParam: - absl::StrAppend(result, "~"); - AppendEscaped(result, type.GetTypeParam().name(), /*escape_dot=*/true); - break; - case TypeKind::kOpaque: - AppendEscaped(result, type.name(), /*escape_dot=*/false); - CEL_RETURN_IF_ERROR(AppendTypeParameters(result, type)); - break; - case TypeKind::kStruct: - AppendEscaped(result, type.name(), /*escape_dot=*/false); - CEL_RETURN_IF_ERROR(AppendTypeParameters(result, type)); - break; - default: - return absl::InvalidArgumentError( - absl::StrFormat("Type kind: %s is not supported in CEL declarations", - type.DebugString())); - } - return absl::OkStatus(); -} - -absl::Status AppendTypeParameters(std::string* result, const Type& type) { - const auto& parameters = type.GetParameters(); - if (!parameters.empty()) { - result->push_back('<'); - for (size_t i = 0; i < parameters.size(); ++i) { - CEL_RETURN_IF_ERROR(AppendTypeDesc(result, parameters[i])); - if (i < parameters.size() - 1) { - result->push_back(','); - } - } - result->push_back('>'); - } - return absl::OkStatus(); -} -} // namespace - -absl::StatusOr MakeTypeSignature(const Type& type) { - std::string result; - CEL_RETURN_IF_ERROR(AppendTypeDesc(&result, type)); - return result; -} - -absl::StatusOr MakeOverloadSignature( - std::string_view function_name, const std::vector& args, - bool is_member) { - std::string result; - if (is_member) { - if (!args.empty()) { - CEL_RETURN_IF_ERROR(AppendTypeDesc(&result, args[0])); - } else { - return absl::InvalidArgumentError("Member function with no receiver"); - } - result.push_back('.'); - } - AppendEscaped(&result, function_name, /*escape_dot=*/true); - result.push_back('('); - for (size_t i = is_member ? 1 : 0; i < args.size(); ++i) { - CEL_RETURN_IF_ERROR(AppendTypeDesc(&result, args[i])); - if (i < args.size() - 1) { - result.push_back(','); - } - } - result.push_back(')'); - - return result; -} -} // namespace cel::common_internal diff --git a/common/internal/signature.h b/common/internal/signature.h deleted file mode 100644 index 3f31d8fd1..000000000 --- a/common/internal/signature.h +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_SIGNATURE_H_ -#define THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_SIGNATURE_H_ - -#include -#include -#include - -#include "absl/status/statusor.h" -#include "common/type.h" - -namespace cel::common_internal { - -// Generates an signature for a `cel::Type`, which is a string representation of -// the type. -// -// Examples: -// -// - `int` -// - `list` -// - `list>` -absl::StatusOr MakeTypeSignature(const Type& type); - -// Generates an identifier for a function overload based on the function name -// and the types of the arguments. If `is_member` is true, the first argument -// type is used as the receiver and is prepended to the function name, followed -// by a dollar sign. -// -// Examples: -// -// - `foo()` -// - `foo(int)` -// - `bar.foo(int)` -// - `foo(int,string)` -// - `foo(list,list)` -// - `bar.foo(list,list>)` -// -// If the function name contains a period, it is escaped with a backslash, e.g. -// `foo.bar` becomes `foo\.bar`. This allows to disambiguate between a member -// function and qualified target type name. -// -absl::StatusOr MakeOverloadSignature( - std::string_view function_name, const std::vector& args, - bool is_member); - -} // namespace cel::common_internal - -#endif // THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_SIGNATURE_H_ diff --git a/common/internal/signature_test.cc b/common/internal/signature_test.cc deleted file mode 100644 index 8e41c70fb..000000000 --- a/common/internal/signature_test.cc +++ /dev/null @@ -1,249 +0,0 @@ -#include "common/internal/signature.h" -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "absl/base/no_destructor.h" -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "common/type.h" -#include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "google/protobuf/arena.h" - -namespace cel::common_internal { -namespace { - -using ::absl_testing::IsOkAndHolds; -using ::absl_testing::StatusIs; -using ::cel::internal::GetTestingDescriptorPool; -using ::testing::HasSubstr; -using ::testing::ValuesIn; - -google::protobuf::Arena* GetTestArena() { - static absl::NoDestructor arena; - return &*arena; -} - -struct TypeSignatureTestCase { - Type type; - std::string expected_signature; - std::string expected_error; -}; - -using TypeSignatureTest = testing::TestWithParam; - -TEST_P(TypeSignatureTest, TypeSignature) { - const auto& param = GetParam(); - - absl::StatusOr signature = - common_internal::MakeTypeSignature(param.type); - if (!param.expected_error.empty()) { - EXPECT_THAT(signature, StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr(param.expected_error))); - } else { - EXPECT_THAT(signature, IsOkAndHolds(param.expected_signature)); - } -} - -std::vector GetTypeSignatureTestCases() { - return { - { - .type = StringType{}, - .expected_signature = "string", - }, - { - .type = IntType{}, - .expected_signature = "int", - }, - { - .type = ListType(GetTestArena(), StringType{}), - .expected_signature = "list", - }, - { - .type = ListType(GetTestArena(), TypeParamType("A")), - .expected_signature = "list<~A>", - }, - { - .type = MapType(GetTestArena(), IntType{}, DynType{}), - .expected_signature = "map", - }, - { - .type = - MapType(GetTestArena(), TypeParamType("B"), TypeParamType("C")), - .expected_signature = "map<~B,~C>", - }, - { - .type = OpaqueType( - GetTestArena(), "bar", - {FunctionType(GetTestArena(), TypeParamType("D"), {})}), - .expected_signature = "bar>", - }, - { - .type = AnyType{}, - .expected_signature = "any", - }, - { - .type = DurationType{}, - .expected_signature = "duration", - }, - { - .type = TimestampType{}, - .expected_signature = "timestamp", - }, - { - .type = IntWrapperType{}, - .expected_signature = "int_wrapper", - }, - { - .type = MessageType(GetTestingDescriptorPool()->FindMessageTypeByName( - "cel.expr.conformance.proto3.TestAllTypes")), - .expected_signature = "cel.expr.conformance.proto3.TestAllTypes", - }, - { - .type = ListType(GetTestArena(), TypeParamType(R"(a,b..(d)\e)")), - .expected_signature = R"(list<~a\,b\.\\.\(d\)\\e>)", - }, - { - .type = UnknownType{}, - .expected_error = - "Type kind: *unknown* is not supported in CEL declarations", - }, - { - .type = ErrorType{}, - .expected_error = - "Type kind: *error* is not supported in CEL declarations", - }, - }; -} - -INSTANTIATE_TEST_SUITE_P(TypeIdTest, TypeSignatureTest, - ValuesIn(GetTypeSignatureTestCases())); - -struct OverloadSignatureTestCase { - std::string function_name = "hello"; - std::vector args; - bool is_member = false; - std::string expected_signature; - std::string expected_error; -}; - -using OverloadSignatureTest = testing::TestWithParam; - -TEST_P(OverloadSignatureTest, OverloadSignature) { - const auto& param = GetParam(); - - absl::StatusOr signature = - common_internal::MakeOverloadSignature(param.function_name, param.args, - param.is_member); - if (!param.expected_error.empty()) { - EXPECT_THAT(signature, StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr(param.expected_error))); - } else { - EXPECT_THAT(signature, IsOkAndHolds(param.expected_signature)); - } -} - -std::vector GetOverloadSignatureTestCases() { - return { - { - .args = {StringType{}}, - .expected_signature = "hello(string)", - }, - { - .args = {IntType{}, UintType{}}, - .expected_signature = "hello(int,uint)", - }, - { - .args = {ListType(GetTestArena(), StringType{})}, - .expected_signature = "hello(list)", - }, - { - .args = {ListType(GetTestArena(), TypeParamType("A"))}, - .expected_signature = "hello(list<~A>)", - }, - { - .args = {MapType(GetTestArena(), IntType{}, DynType{})}, - .expected_signature = "hello(map)", - }, - { - .args = {MapType(GetTestArena(), TypeParamType("B"), - TypeParamType("C"))}, - .expected_signature = "hello(map<~B,~C>)", - }, - { - .args = {OpaqueType( - GetTestArena(), "bar", - {FunctionType(GetTestArena(), TypeParamType("D"), {})})}, - .expected_signature = "hello(bar>)", - }, - { - .args = {AnyType{}}, - .expected_signature = "hello(any)", - }, - { - .args = {DurationType{}}, - .expected_signature = "hello(duration)", - }, - { - .args = {TimestampType{}}, - .expected_signature = "hello(timestamp)", - }, - { - .args = {IntWrapperType{}}, - .expected_signature = "hello(int_wrapper)", - }, - { - .args = {MessageType( - GetTestingDescriptorPool()->FindMessageTypeByName( - "cel.expr.conformance.proto3.TestAllTypes"))}, - .expected_signature = - "hello(cel.expr.conformance.proto3.TestAllTypes)", - }, - {.args = {}, - .is_member = true, - .expected_error = "Member function with no receiver"}, - { - .args = {StringType{}}, - .is_member = true, - .expected_signature = "string.hello()", - }, - { - .args = {StringType{}, ListType(GetTestArena(), BoolType{})}, - .is_member = true, - .expected_signature = "string.hello(list)", - }, - { - .args = {StringType{}, BoolType{}, DynType{}}, - .is_member = true, - .expected_signature = "string.hello(bool,dyn)", - }, - { - .function_name = R"(h.(e),l\o)", - .args = {StringType{}, - ListType(GetTestArena(), TypeParamType(R"(a,b..(d)\e)"))}, - .is_member = true, - .expected_signature = - R"(string.h\.\(e\)\,l\\\o(list<~a\,b\.\\.\(d\)\\e>))", - }, - }; -} - -INSTANTIATE_TEST_SUITE_P(OverloadIdTest, OverloadSignatureTest, - ValuesIn(GetOverloadSignatureTestCases())); - -} // namespace -} // namespace cel::common_internal diff --git a/common/internal/unknowns.cc b/common/internal/unknowns.cc new file mode 100644 index 000000000..d98958805 --- /dev/null +++ b/common/internal/unknowns.cc @@ -0,0 +1,264 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/internal/unknowns.h" + +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/log/absl_check.h" +#include "absl/strings/string_view.h" +#include "base/attribute.h" +#include "base/attribute_matcher.h" +#include "google/protobuf/arena.h" + +namespace cel::common_internal { + +namespace { + +template +[[nodiscard]] +size_t PreflightSetUnion(T* const* first1, T* const* last1, T* const* first2, + T* const* last2) { + size_t preflight = 0; + for (; first1 != last1; ++preflight) { + if (first2 == last2) { + return preflight + static_cast(last1 - first1); + } + if (*first2 < *first1) { + ++first2; + } else { + if (!(*first1 < *first2)) { + ++first2; + } + ++first1; + } + } + return preflight + static_cast(last2 - first2); +} + +template +[[nodiscard]] +T** SetUnion(T* const* first1, T* const* last1, T* const* first2, + T* const* last2, T** out) { + for (; first1 != last1; ++out) { + if (first2 == last2) { + return std::copy(first1, last1, out); + } + if (*first2 < *first1) { + *out = *first2++; + } else { + *out = *first1; + if (!(*first1 < *first2)) { + ++first2; + } + ++first1; + } + } + return std::copy(first2, last2, out); +} + +} // namespace + +absl::string_view ArenaString(google::protobuf::Arena* absl_nonnull arena, + absl::string_view string) { + if (string.empty()) { + return ""; + } + char* str = reinterpret_cast(arena->AllocateAligned(string.size())); + std::memcpy(str, string.data(), string.size()); + return absl::string_view(str, string.size()); +} + +UnknownAttributeNode* absl_nonnull UnknownTree::Convert( + const AttributeMatcherNode* absl_nonnull matcher) { + ABSL_DCHECK(!matcher->key().IsWildcard()); + const size_t scratch_size = matcher->depth() + 1; + if (scratch_size > scratch_.size()) { + scratch_.resize(scratch_size); + } + const AttributeMatcherNode** scratch_data = + reinterpret_cast(scratch_.data()); + const AttributeMatcherNode* node = matcher; + for (size_t i = scratch_size; i > 0; --i) { + scratch_data[i - 1] = node; + node = node->parent(); + } + UnknownAttributeNode* unknown = Root()->Attributes()->Step( + this, *scratch_data[0]->key().ToQualifierView()); + for (size_t i = 1; i < scratch_size; ++i) { + unknown = unknown->Step(this, *scratch_data[i]->key().ToQualifierView()); + } + return unknown; +} + +std::string UnknownTree::ToString( + const AttributeMatcherNode* absl_nonnull matcher, + const AttributeQualifierView& key) { + const bool is_wildcard = matcher->key().IsWildcard(); + ABSL_DCHECK(is_wildcard || matcher->key() == key); + if (is_wildcard) { + std::string result = ToString(matcher->parent()); + key.AppendJoinString(&result); + return result; + } + return ToString(matcher); +} + +std::string UnknownTree::ToString( + const AttributeMatcherNode* absl_nonnull matcher) { + const size_t scratch_size = matcher->depth() + 1; + if (scratch_size > scratch_.size()) { + scratch_.resize(scratch_size); + } + const AttributeMatcherNode** scratch_data = + reinterpret_cast(scratch_.data()); + const AttributeMatcherNode* node = matcher; + for (size_t i = scratch_size; i > 0; --i) { + scratch_data[i - 1] = node; + node = node->parent(); + } + std::string result = scratch_data[0]->key().GetString(); + for (size_t i = 1; i < scratch_size; ++i) { + scratch_data[i]->key().AppendJoinString(&result); + } + return result; +} + +std::string UnknownTree::ToString( + const UnknownAttributeNode* absl_nonnull unknown) { + const size_t scratch_size = unknown->depth(); + if (scratch_size > scratch_.size()) { + scratch_.resize(scratch_size); + } + const UnknownAttributeNode** scratch_data = + reinterpret_cast(scratch_.data()); + const UnknownAttributeNode* node = unknown; + for (size_t i = scratch_size; i > 0; --i) { + scratch_data[i - 1] = node; + node = node->parent(); + } + std::string result = std::string(scratch_data[0]->key().GetString()); + for (size_t i = 1; i < scratch_size; ++i) { + scratch_data[i]->key().AppendJoinString(&result); + } + return result; +} + +void UnknownTree::GraftImpl(const UnknownSet& in, UnknownSet& out) { + ABSL_DCHECK_NE(in.root, RootIfPresent()); + // This should only happen in rare circumstances where somebody passed an + // UnknownValue from one expression evaluation into another. + UnknownAttributeSet* unknown_attribute_set = nullptr; + UnknownFunctionSet* unknown_function_set = nullptr; + if (in.attributes != nullptr && in.attributes->size > 0) { + unknown_attribute_set = + UnknownAttributeSet::Allocate(in.attributes->size, GetArena()); + unknown_attribute_set->max_depth = in.attributes->max_depth; + for (uint32_t i = 0; i < in.attributes->size; ++i) { + unknown_attribute_set->nodes[i] = GraftImpl(in.attributes->nodes[i]); + } + std::sort(unknown_attribute_set->nodes, + unknown_attribute_set->nodes + unknown_attribute_set->size); + } + if (in.functions != nullptr && in.functions->size > 0) { + unknown_function_set = + UnknownFunctionSet::Allocate(in.functions->size, GetArena()); + for (uint32_t i = 0; i < in.functions->size; ++i) { + unknown_function_set->nodes[i] = GraftImpl(in.functions->nodes[i]); + } + std::sort(unknown_function_set->nodes, + unknown_function_set->nodes + unknown_function_set->size); + } + out.root = Root(); + out.attributes = unknown_attribute_set; + out.functions = unknown_function_set; +} + +UnknownAttributeNode* UnknownTree::GraftImpl(const UnknownAttributeNode* in) { + const size_t scratch_size = in->depth(); + if (scratch_size > scratch_.size()) { + scratch_.resize(scratch_size); + } + const UnknownAttributeNode** scratch_data = + reinterpret_cast(scratch_.data()); + for (size_t i = scratch_size; i > 0; --i) { + scratch_data[i - 1] = in; + in = in->parent(); + } + UnknownAttributeNode* out = Root()->Attributes(); + for (size_t i = 0; i < scratch_size; ++i) { + out = out->Step(this, scratch_data[i]->key()); + } + return out; +} + +const UnknownFunctionSet* absl_nonnull MergeUnknownFunctionSets( + const UnknownFunctionSet* absl_nonnull lhs, + const UnknownFunctionSet* absl_nonnull rhs, + google::protobuf::Arena* absl_nonnull arena) { + UnknownFunctionNode* const* const lhs_begin = lhs->nodes; + const size_t lhs_size = lhs->size; + UnknownFunctionNode* const* const lhs_end = lhs_begin + lhs_size; + UnknownFunctionNode* const* const rhs_begin = rhs->nodes; + const size_t rhs_size = rhs->size; + UnknownFunctionNode* const* const rhs_end = rhs_begin + rhs_size; + const size_t size = + (PreflightSetUnion)(lhs_begin, lhs_end, rhs_begin, rhs_end); + if (lhs_size >= rhs_size && lhs_size == size) { + return lhs; + } + if (rhs_size > lhs_size && rhs_size == size) { + return rhs; + } + UnknownFunctionSet* ufs = UnknownFunctionSet::Allocate(size, arena); + UnknownFunctionNode** out_begin = ufs->nodes; + UnknownFunctionNode** out_end = out_begin + size; + out_begin = (SetUnion)(lhs_begin, lhs_end, rhs_begin, rhs_end, out_begin); + ABSL_DCHECK_EQ(out_begin, out_end); + return ufs; +} + +const UnknownAttributeSet* absl_nonnull MergeUnknownAttributeSets( + const UnknownAttributeSet* absl_nonnull lhs, + const UnknownAttributeSet* absl_nonnull rhs, + google::protobuf::Arena* absl_nonnull arena) { + UnknownAttributeNode* const* const lhs_begin = lhs->nodes; + const size_t lhs_size = lhs->size; + UnknownAttributeNode* const* const lhs_end = lhs_begin + lhs_size; + UnknownAttributeNode* const* const rhs_begin = rhs->nodes; + const size_t rhs_size = rhs->size; + UnknownAttributeNode* const* const rhs_end = rhs_begin + rhs_size; + const size_t size = + (PreflightSetUnion)(lhs_begin, lhs_end, rhs_begin, rhs_end); + if (lhs_size >= rhs_size && lhs_size == size) { + return lhs; + } + if (rhs_size > lhs_size && rhs_size == size) { + return rhs; + } + UnknownAttributeSet* uas = UnknownAttributeSet::Allocate(size, arena); + uas->max_depth = std::max(lhs->max_depth, rhs->max_depth); + UnknownAttributeNode** out_begin = uas->nodes; + UnknownAttributeNode** out_end = out_begin + size; + out_begin = (SetUnion)(lhs_begin, lhs_end, rhs_begin, rhs_end, out_begin); + ABSL_DCHECK_EQ(out_begin, out_end); + return uas; +} + +} // namespace cel::common_internal diff --git a/common/internal/unknowns.h b/common/internal/unknowns.h new file mode 100644 index 000000000..b785befb1 --- /dev/null +++ b/common/internal/unknowns.h @@ -0,0 +1,493 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_UNKNOWNS_H_ +#define THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_UNKNOWNS_H_ + +#include +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/base/nullability.h" +#include "absl/base/optimization.h" +#include "absl/container/flat_hash_set.h" +#include "absl/container/inlined_vector.h" +#include "absl/functional/overload.h" +#include "absl/log/absl_check.h" +#include "absl/strings/string_view.h" +#include "base/attribute.h" +#include "base/attribute_matcher.h" +#include "internal/arena_tree.h" +#include "google/protobuf/arena.h" + +namespace cel::common_internal { + +class UnknownAttributeNode; +class UnknownTree; +class UnknownRoot; +struct UnknownFunctionSet; +struct UnknownAttributeSet; + +// Node in an arena-based red black tree. Leaf nodes are matches to unknown +// attributes. The root node is owned by UnknownRoot. Nodes which are direct +// children of the root node are variables. All nodes below them are qualifiers. +class UnknownAttributeNode + : public internal::ArenaTreeNodeCrtp { + private: + struct Compare { + using is_transparent = void; + + int operator()(const AttributeQualifierView& lhs, + const AttributeQualifierView& rhs) const { + if (lhs < rhs) { + return -1; + } + if (lhs == rhs) { + return 0; + } + return 1; + } + + int operator()(const AttributeQualifierView& lhs, + const UnknownAttributeNode& rhs) const { + return (*this)(lhs, rhs.key()); + } + + int operator()(const UnknownAttributeNode& lhs, + const AttributeQualifierView& rhs) const { + return (*this)(lhs.key(), rhs); + } + + int operator()(const UnknownAttributeNode& lhs, + const UnknownAttributeNode& rhs) const { + return (*this)(lhs.key(), rhs.key()); + } + }; + + public: + explicit UnknownAttributeNode(UnknownAttributeNode* absl_nullable parent, + const AttributeQualifierView& key) + : ArenaTreeNodeCrtp(), + parent_(parent), + key_(key), + depth_(parent != nullptr ? parent->depth() + 1 : 0) {} + + [[nodiscard]] + const AttributeQualifierView& key() const { + return key_; + } + + [[nodiscard]] + UnknownAttributeNode* parent() const { + return parent_; + } + + [[nodiscard]] + size_t depth() const { + return depth_; + } + + [[nodiscard]] + UnknownAttributeNode* Step(UnknownTree* absl_nonnull tree, + const AttributeQualifierView& key); + + [[nodiscard]] + UnknownAttributeNode* Find(const AttributeQualifierView& key) const { + return internal::ArenaTreeFind(children_, key, Compare{}); + } + + private: + // Pointer to our parent node. If this is null, then this node is the root + // node and its key is invalid. If this is a pointer to the root node then + // this node is a variable and the key is guaranteed to be a string. + UnknownAttributeNode* const absl_nullability_unknown parent_ = nullptr; + const AttributeQualifierView key_; + // How many nodes between us and the root node. + const size_t depth_; + // Root of the red black tree for children. + UnknownAttributeNode* absl_nullable children_ = nullptr; +}; + +// Node in an arena-based red black tree. Leaf nodes are matches to unknown +// functions. The root node is owned by UnknownRoot. +class UnknownFunctionNode + : public internal::ArenaTreeNodeCrtp { + public: + struct Compare { + using is_transparent = void; + + int operator()(absl::string_view lhs, absl::string_view rhs) const { + return lhs.compare(rhs); + } + + int operator()(absl::string_view lhs, + const UnknownFunctionNode& rhs) const { + return (*this)(lhs, rhs.key()); + } + + int operator()(const UnknownFunctionNode& lhs, + absl::string_view rhs) const { + return (*this)(lhs.key(), rhs); + } + + int operator()(const UnknownFunctionNode& lhs, + const UnknownFunctionNode& rhs) const { + return (*this)(lhs.key(), rhs.key()); + } + }; + + explicit UnknownFunctionNode(absl::string_view key) + : ArenaTreeNodeCrtp(), key_(key) {} + + [[nodiscard]] + absl::string_view key() const { + return key_; + } + + private: + const absl::string_view key_; +}; + +class UnknownTree; + +[[nodiscard]] +absl::string_view ArenaString(google::protobuf::Arena* absl_nonnull arena, + absl::string_view string); + +// Holds the the red black tree for functions and the root attribute node which +// holds a red black tree for variables. The root is lazily created on the arena +// by UnknownTree as needed. +class UnknownRoot { + public: + explicit UnknownRoot(google::protobuf::Arena* absl_nonnull arena) + : arena_(arena), attributes_(nullptr, AttributeQualifierView()) {} + + [[nodiscard]] + google::protobuf::Arena* absl_nonnull GetArena() const { + return arena_; + } + + [[nodiscard]] + UnknownAttributeNode* absl_nonnull Attributes() { + return &attributes_; + } + + [[nodiscard]] + const UnknownAttributeNode* absl_nonnull Attributes() const { + return &attributes_; + } + + [[nodiscard]] + UnknownFunctionNode* absl_nonnull SetFunction(UnknownTree* absl_nonnull tree, + absl::string_view name); + + [[nodiscard]] + UnknownFunctionNode* absl_nullable FindFunction(absl::string_view name) const; + + private: + google::protobuf::Arena* const absl_nonnull arena_; + UnknownAttributeNode attributes_; + UnknownFunctionNode* functions_ = nullptr; +}; + +// Represents a set of unknown function nodes. +struct UnknownFunctionSet { + size_t size; + // Nodes are sorted in ascending order by pointer. There are no duplicates. + UnknownFunctionNode* absl_nonnull nodes[]; + + [[nodiscard]] + static UnknownFunctionSet* Allocate(size_t size, + google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK_GT(size, 0); + UnknownFunctionSet* ptr = reinterpret_cast( + arena->AllocateAligned(offsetof(UnknownFunctionSet, nodes) + + (size * sizeof(UnknownFunctionNode*)))); + ptr->size = size; + return ptr; + } +}; + +// Represents a set of unknown attribute nodes, which themselves represent a +// partial or full match to an unknown attribute. +struct UnknownAttributeSet { + uint32_t size; + uint32_t max_depth; + // Nodes are sorted in ascending order by pointer. There are no duplicates. + UnknownAttributeNode* absl_nonnull nodes[]; + + [[nodiscard]] + static UnknownAttributeSet* Allocate(size_t size, + google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK_GT(size, 0); + ABSL_DCHECK_LE(size, std::numeric_limits::max()); + UnknownAttributeSet* ptr = reinterpret_cast( + arena->AllocateAligned(offsetof(UnknownAttributeSet, nodes) + + (size * sizeof(UnknownAttributeNode*)))); + ptr->size = static_cast(size); + return ptr; + } +}; + +// Underlying representation used by UnknownValue. +struct UnknownSet { + const UnknownRoot* root = nullptr; + const UnknownAttributeSet* attributes = nullptr; + const UnknownFunctionSet* functions = nullptr; +}; + +// Used by the runtime to build unknown attribute and unknown function trees. +class UnknownTree { + public: + UnknownTree() : arena_(nullptr) {} + + UnknownTree(const UnknownTree&) = delete; + UnknownTree& operator=(const UnknownTree&) = delete; + + explicit UnknownTree(google::protobuf::Arena* absl_nonnull arena) : arena_(arena) { + ABSL_DCHECK(arena != nullptr); + } + + void Reset(google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK(arena != nullptr); + arena_ = arena; + root_ = nullptr; + strings_.clear(); + scratch_.clear(); + } + + [[nodiscard]] + google::protobuf::Arena* absl_nonnull GetArena() const { + ABSL_DCHECK(arena_ != nullptr); + return arena_; + } + + [[nodiscard]] + UnknownRoot* absl_nonnull Root() { + UnknownRoot* root = RootIfPresent(); + if (ABSL_PREDICT_FALSE(root == nullptr)) { + root_ = root = google::protobuf::Arena::Create(GetArena(), GetArena()); + } + return root; + } + + [[nodiscard]] + UnknownRoot* absl_nullable RootIfPresent() const { + return root_; + } + + // Called when an unknown attribute match is encountered to materialize the + // match in the unknown attribute tree. + [[nodiscard]] + UnknownAttributeNode* absl_nonnull Convert( + const AttributeMatcherNode* absl_nonnull matcher, + const AttributeQualifierView& key) { + const bool is_wildcard = matcher->key().IsWildcard(); + ABSL_DCHECK(is_wildcard || matcher->key() == key); + if (is_wildcard) { + return Convert(matcher->parent())->Step(this, key); + } + return Convert(matcher); + } + [[nodiscard]] + UnknownAttributeNode* absl_nonnull Convert( + const AttributeMatcherNode* absl_nonnull matcher); + [[nodiscard]] + UnknownAttributeNode* absl_nonnull Convert( + const AttributeMatch& match, const AttributeQualifierView& key) { + return Convert(common_internal::GetAttributeMatcherNode(match), key); + } + [[nodiscard]] + UnknownAttributeNode* absl_nonnull Convert(const AttributeMatch& match) { + return Convert(common_internal::GetAttributeMatcherNode(match)); + } + + // Converts an unknown attribute node to its corresponding attribute pattern + // string, as if the node was converted to AttributePattern and + // AttributePattern::ToString was called. + [[nodiscard]] + std::string ToString(const AttributeMatcherNode* absl_nonnull matcher, + const AttributeQualifierView& key); + [[nodiscard]] + std::string ToString(const AttributeMatch& match, + const AttributeQualifierView& key) { + return ToString(common_internal::GetAttributeMatcherNode(match), key); + } + + // Converts an unknown attribute node to its corresponding attribute string, + // as if the node was converted to Attribute and Attribute::ToString was + // called. + [[nodiscard]] + std::string ToString(const UnknownAttributeNode* absl_nonnull unknown); + + // In the rare event that somebody manages to pass an unknown set from one + // expression into another, we need to be able to graft those nodes into our + // tree. The unknown attribute set and unknown function set sort nodes by + // address for efficiency. This function does that grafting when it is + // required. + [[nodiscard]] + ABSL_ATTRIBUTE_ALWAYS_INLINE UnknownSet Graft(const UnknownSet& in) { + // Fast path, should be taken almost always. + UnknownSet out; + if (ABSL_PREDICT_TRUE(in.root == nullptr || in.root == RootIfPresent())) { + ABSL_DCHECK(in.root != nullptr || in.attributes == nullptr); + ABSL_DCHECK(in.root != nullptr || in.functions == nullptr); + out = in; + } else { + // Slow path, should be taken extremely rarely. + GraftImpl(in, out); + } + return out; + } + + // In the rare event that somebody manages to pass an unknown set from one + // expression into another, we need to be able to graft those nodes into our + // tree. The unknown attribute set and unknown function set sort nodes by + // address for efficiency. This function does that grafting when it is + // required. + ABSL_ATTRIBUTE_ALWAYS_INLINE void GraftAt(UnknownSet& in) { + if (ABSL_PREDICT_TRUE(in.root == nullptr || in.root == RootIfPresent())) { + // Fast path, should be taken almost always. + ABSL_DCHECK(in.root != nullptr || in.attributes == nullptr); + ABSL_DCHECK(in.root != nullptr || in.functions == nullptr); + } else { + // Slow path, should be taken extremely rarely. + GraftImpl(in, in); + } + } + + private: + friend class UnknownAttributeNode; + friend class UnknownRoot; + + ABSL_ATTRIBUTE_NOINLINE void GraftImpl(const UnknownSet& in, UnknownSet& out); + + [[nodiscard]] + UnknownAttributeNode* GraftImpl(const UnknownAttributeNode* in); + + [[nodiscard]] + UnknownFunctionNode* GraftImpl(const UnknownFunctionNode* in) { + return Root()->SetFunction(this, in->key()); + } + + [[nodiscard]] + std::string ToString(const AttributeMatcherNode* absl_nonnull matcher); + + [[nodiscard]] + AttributeQualifierView InternKey(const AttributeQualifierView& key) { + return std::visit( + absl::Overload( + [](std::monostate) -> AttributeQualifierView { + return AttributeQualifierView(); + }, + [](bool value) -> AttributeQualifierView { + return AttributeQualifierView::OfBool(value); + }, + [](int64_t value) -> AttributeQualifierView { + return AttributeQualifierView::OfInt(value); + }, + [](uint64_t value) -> AttributeQualifierView { + return AttributeQualifierView::OfUint(value); + }, + [this](absl::string_view value) -> AttributeQualifierView { + return AttributeQualifierView::OfString(InternStringKey(value)); + }), + common_internal::AsVariant(key)); + } + + [[nodiscard]] + absl::string_view InternStringKey(absl::string_view key) { + if (key.empty()) { + return ""; + } + return *strings_.lazy_emplace(key, [this, key](const auto& constructor) { + constructor((ArenaString)(GetArena(), key)); + }); + } + + google::protobuf::Arena* absl_nullability_unknown arena_; + // Created as needed. + UnknownRoot* absl_nullable root_ = nullptr; + // These strings are actually null terminated. + absl::flat_hash_set strings_; + // Temporary storage for materializing AttributeMatcherNode* trail to + // UnknownAttributeNode* or for converting either type to a string. + absl::InlinedVector scratch_; +}; + +inline UnknownAttributeNode* UnknownAttributeNode::Step( + UnknownTree* absl_nonnull tree, const AttributeQualifierView& key) { + auto [node, emplaced] = internal::ArenaTreeLazyEmplace( + tree->GetArena(), &children_, key, Compare{}, + [this, tree, &key](const auto& constructor) -> void { + constructor(this, tree->InternKey(key)); + }); + return node; +} + +inline UnknownFunctionNode* absl_nonnull UnknownRoot::SetFunction( + UnknownTree* absl_nonnull tree, absl::string_view name) { + auto [node, emplaced] = internal::ArenaTreeLazyEmplace( + tree->GetArena(), &functions_, name, UnknownFunctionNode::Compare{}, + [tree, &name](const auto& constructor) -> void { + constructor((ArenaString)(tree->GetArena(), name)); + }); + return node; +} + +inline UnknownFunctionNode* absl_nullable UnknownRoot::FindFunction( + absl::string_view name) const { + return internal::ArenaTreeFind(functions_, name, + UnknownFunctionNode::Compare{}); +} + +[[nodiscard]] +inline const UnknownFunctionSet* absl_nonnull CreateUnknownFunctionSet( + UnknownFunctionNode* absl_nonnull node, google::protobuf::Arena* absl_nonnull arena) { + UnknownFunctionSet* ufs = UnknownFunctionSet::Allocate(1, arena); + ufs->nodes[0] = node; + return ufs; +} + +// Fast two-way unknown function set merger without temporary storage. +[[nodiscard]] +const UnknownFunctionSet* absl_nonnull MergeUnknownFunctionSets( + const UnknownFunctionSet* absl_nonnull lhs, + const UnknownFunctionSet* absl_nonnull rhs, + google::protobuf::Arena* absl_nonnull arena); + +[[nodiscard]] +inline const UnknownAttributeSet* absl_nonnull CreateUnknownAttributeSet( + UnknownAttributeNode* absl_nonnull node, + google::protobuf::Arena* absl_nonnull arena) { + UnknownAttributeSet* uas = UnknownAttributeSet::Allocate(1, arena); + uas->max_depth = static_cast(node->depth()); + uas->nodes[0] = node; + return uas; +} + +// Fast two-way unknown attribute set merger without temporary storage. +[[nodiscard]] +const UnknownAttributeSet* absl_nonnull MergeUnknownAttributeSets( + const UnknownAttributeSet* absl_nonnull lhs, + const UnknownAttributeSet* absl_nonnull rhs, + google::protobuf::Arena* absl_nonnull arena); + +} // namespace cel::common_internal + +#endif // THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_UNKNOWNS_H_ diff --git a/common/internal/unknowns_test.cc b/common/internal/unknowns_test.cc new file mode 100644 index 000000000..5e10364a7 --- /dev/null +++ b/common/internal/unknowns_test.cc @@ -0,0 +1,274 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/internal/unknowns.h" + +#include + +#include "absl/status/status_matchers.h" +#include "base/attribute.h" +#include "base/attribute_matcher.h" +#include "internal/testing.h" +#include "google/protobuf/arena.h" + +namespace cel::common_internal { +namespace { + +using ::absl_testing::IsOk; +using ::testing::IsNull; +using ::testing::NotNull; + +TEST(UnknownTree, MatchToString) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + IsOk()); + google::protobuf::Arena arena; + UnknownTree tree(&arena); + EXPECT_EQ(tree.ToString( + matcher.MatchVariable("foo") + .MatchQualifier(AttributeQualifierView::OfString("bar")) + .MatchQualifier(AttributeQualifierView::OfString("baz")), + AttributeQualifierView::OfString("baz")), + "foo.bar.baz"); + EXPECT_EQ(tree.ToString(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("bar")), + AttributeQualifierView::OfString("bar")), + "foo.bar"); + EXPECT_EQ(tree.ToString(matcher.MatchVariable("foo"), + AttributeQualifierView::OfString("foo")), + "foo"); +} + +TEST(UnknownTree, UnknownToString) { + google::protobuf::Arena arena; + UnknownTree tree(&arena); + UnknownAttributeNode* node_foo = tree.Root()->Attributes()->Step( + &tree, AttributeQualifierView::OfString("foo")); + UnknownAttributeNode* node_bar = + node_foo->Step(&tree, AttributeQualifierView::OfString("bar")); + UnknownAttributeNode* node_baz = + node_bar->Step(&tree, AttributeQualifierView::OfString("baz")); + EXPECT_EQ(tree.ToString(node_baz), "foo.bar.baz"); + EXPECT_EQ(tree.ToString(node_bar), "foo.bar"); + EXPECT_EQ(tree.ToString(node_foo), "foo"); +} + +TEST(UnknownTree, UnknownAttributeConvert) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + IsOk()); + google::protobuf::Arena arena; + UnknownTree tree(&arena); + UnknownAttributeNode* node_baz = + tree.Convert(matcher.MatchVariable("foo") + .MatchQualifier(AttributeQualifierView::OfString("bar")) + .MatchQualifier(AttributeQualifierView::OfString("baz")), + AttributeQualifierView::OfString("baz")); + UnknownAttributeNode* node_bar = + tree.Convert(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("bar")), + AttributeQualifierView::OfString("bar")); + UnknownAttributeNode* node_foo = tree.Convert( + matcher.MatchVariable("foo"), AttributeQualifierView::OfString("foo")); + EXPECT_EQ(node_baz->parent(), node_bar); + EXPECT_EQ(node_bar->parent(), node_foo); + EXPECT_EQ(tree.ToString(node_baz), "foo.bar.baz"); + EXPECT_EQ(tree.ToString(node_bar), "foo.bar"); + EXPECT_EQ(tree.ToString(node_foo), "foo"); + EXPECT_EQ( + tree.Root()->Attributes()->Find(AttributeQualifierView::OfString("foo")), + node_foo); +} + +TEST(UnknownTree, UnknownFunction) { + google::protobuf::Arena arena; + UnknownTree tree(&arena); + + EXPECT_THAT(tree.Root()->FindFunction("foo"), IsNull()); + UnknownFunctionNode* node_foo = tree.Root()->SetFunction(&tree, "foo"); + EXPECT_THAT(node_foo, NotNull()); + EXPECT_EQ(tree.Root()->FindFunction("foo"), node_foo); + + EXPECT_THAT(tree.Root()->FindFunction("bar"), IsNull()); + UnknownFunctionNode* node_bar = tree.Root()->SetFunction(&tree, "bar"); + EXPECT_THAT(node_bar, NotNull()); + EXPECT_NE(node_bar, node_foo); + EXPECT_EQ(tree.Root()->FindFunction("bar"), node_bar); + + EXPECT_THAT(tree.Root()->FindFunction("baz"), IsNull()); + UnknownFunctionNode* node_baz = tree.Root()->SetFunction(&tree, "baz"); + EXPECT_THAT(node_baz, NotNull()); + EXPECT_NE(node_baz, node_foo); + EXPECT_EQ(tree.Root()->FindFunction("baz"), node_baz); +} + +void TestUnknownAttributeSets(const UnknownAttributeSet* lhs, + const UnknownAttributeSet* rhs) { + EXPECT_EQ(lhs->size, rhs->size); + EXPECT_EQ(lhs->max_depth, rhs->max_depth); + EXPECT_TRUE(std::is_sorted(lhs->nodes, lhs->nodes + lhs->size)); + EXPECT_TRUE(std::is_sorted(rhs->nodes, rhs->nodes + rhs->size)); + EXPECT_TRUE(std::equal(lhs->nodes, lhs->nodes + lhs->size, rhs->nodes, + rhs->nodes + rhs->size)); +} + +TEST(UnknownTree, UnknownAttributeSet) { + google::protobuf::Arena arena; + UnknownTree tree(&arena); + UnknownAttributeNode* node_foo = tree.Root()->Attributes()->Step( + &tree, AttributeQualifierView::OfString("foo")); + UnknownAttributeNode* node_bar = tree.Root()->Attributes()->Step( + &tree, AttributeQualifierView::OfString("bar")); + UnknownAttributeNode* node_baz = tree.Root()->Attributes()->Step( + &tree, AttributeQualifierView::OfString("baz")); + + const UnknownAttributeSet* set_foo = + CreateUnknownAttributeSet(node_foo, tree.GetArena()); + EXPECT_EQ(set_foo->size, 1); + EXPECT_EQ(set_foo->max_depth, 1); + EXPECT_EQ(set_foo->nodes[0], node_foo); + TestUnknownAttributeSets(CreateUnknownAttributeSet(node_foo, tree.GetArena()), + set_foo); + + const UnknownAttributeSet* set_bar = + CreateUnknownAttributeSet(node_bar, tree.GetArena()); + EXPECT_EQ(set_bar->size, 1); + EXPECT_EQ(set_bar->max_depth, 1); + EXPECT_EQ(set_bar->nodes[0], node_bar); + TestUnknownAttributeSets(CreateUnknownAttributeSet(node_bar, tree.GetArena()), + set_bar); + + const UnknownAttributeSet* set_baz = + CreateUnknownAttributeSet(node_baz, tree.GetArena()); + EXPECT_EQ(set_baz->size, 1); + EXPECT_EQ(set_baz->max_depth, 1); + EXPECT_EQ(set_baz->nodes[0], node_baz); + TestUnknownAttributeSets(CreateUnknownAttributeSet(node_baz, tree.GetArena()), + set_baz); + + const UnknownAttributeSet* set_foo_bar = + MergeUnknownAttributeSets(set_foo, set_bar, &arena); + EXPECT_EQ(set_foo_bar->size, 2); + EXPECT_EQ(set_foo_bar->max_depth, 1); + EXPECT_TRUE(std::is_sorted(set_foo_bar->nodes, + set_foo_bar->nodes + set_foo_bar->size)); + TestUnknownAttributeSets( + MergeUnknownAttributeSets(set_bar, set_foo, tree.GetArena()), + set_foo_bar); + + const UnknownAttributeSet* set_foo_bar_baz = + MergeUnknownAttributeSets(set_foo_bar, set_baz, &arena); + EXPECT_EQ(set_foo_bar_baz->size, 3); + EXPECT_EQ(set_foo_bar_baz->max_depth, 1); + EXPECT_TRUE(std::is_sorted(set_foo_bar_baz->nodes, + set_foo_bar_baz->nodes + set_foo_bar_baz->size)); + TestUnknownAttributeSets( + MergeUnknownAttributeSets(set_baz, set_foo_bar, tree.GetArena()), + set_foo_bar_baz); + + EXPECT_EQ(MergeUnknownAttributeSets(set_foo_bar_baz, set_foo, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownAttributeSets(set_foo_bar_baz, set_bar, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownAttributeSets(set_foo_bar_baz, set_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownAttributeSets(set_foo_bar_baz, set_foo_bar, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownAttributeSets(set_foo, set_foo_bar_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownAttributeSets(set_bar, set_foo_bar_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownAttributeSets(set_baz, set_foo_bar_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownAttributeSets(set_foo_bar, set_foo_bar_baz, &arena), + set_foo_bar_baz); +} + +void TestUnknownFunctionSets(const UnknownFunctionSet* lhs, + const UnknownFunctionSet* rhs) { + EXPECT_EQ(lhs->size, rhs->size); + EXPECT_TRUE(std::is_sorted(lhs->nodes, lhs->nodes + lhs->size)); + EXPECT_TRUE(std::is_sorted(rhs->nodes, rhs->nodes + rhs->size)); + EXPECT_TRUE(std::equal(lhs->nodes, lhs->nodes + lhs->size, rhs->nodes, + rhs->nodes + rhs->size)); +} + +TEST(UnknownTree, UnknownFunctionSet) { + google::protobuf::Arena arena; + UnknownTree tree(&arena); + UnknownFunctionNode* node_foo = tree.Root()->SetFunction(&tree, "foo"); + UnknownFunctionNode* node_bar = tree.Root()->SetFunction(&tree, "bar"); + UnknownFunctionNode* node_baz = tree.Root()->SetFunction(&tree, "baz"); + + const UnknownFunctionSet* set_foo = + CreateUnknownFunctionSet(node_foo, tree.GetArena()); + EXPECT_EQ(set_foo->size, 1); + EXPECT_EQ(set_foo->nodes[0], node_foo); + TestUnknownFunctionSets(CreateUnknownFunctionSet(node_foo, tree.GetArena()), + set_foo); + + const UnknownFunctionSet* set_bar = + CreateUnknownFunctionSet(node_bar, tree.GetArena()); + EXPECT_EQ(set_bar->size, 1); + EXPECT_EQ(set_bar->nodes[0], node_bar); + TestUnknownFunctionSets(CreateUnknownFunctionSet(node_bar, tree.GetArena()), + set_bar); + + const UnknownFunctionSet* set_baz = + CreateUnknownFunctionSet(node_baz, tree.GetArena()); + EXPECT_EQ(set_baz->size, 1); + EXPECT_EQ(set_baz->nodes[0], node_baz); + TestUnknownFunctionSets(CreateUnknownFunctionSet(node_baz, tree.GetArena()), + set_baz); + + const UnknownFunctionSet* set_foo_bar = + MergeUnknownFunctionSets(set_foo, set_bar, &arena); + EXPECT_EQ(set_foo_bar->size, 2); + EXPECT_TRUE(std::is_sorted(set_foo_bar->nodes, + set_foo_bar->nodes + set_foo_bar->size)); + TestUnknownFunctionSets(MergeUnknownFunctionSets(set_bar, set_foo, &arena), + set_foo_bar); + + const UnknownFunctionSet* set_foo_bar_baz = + MergeUnknownFunctionSets(set_foo_bar, set_baz, &arena); + EXPECT_EQ(set_foo_bar_baz->size, 3); + EXPECT_TRUE(std::is_sorted(set_foo_bar_baz->nodes, + set_foo_bar_baz->nodes + set_foo_bar_baz->size)); + TestUnknownFunctionSets( + MergeUnknownFunctionSets(set_baz, set_foo_bar, &arena), set_foo_bar_baz); + + EXPECT_EQ(MergeUnknownFunctionSets(set_foo_bar_baz, set_foo, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownFunctionSets(set_foo_bar_baz, set_bar, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownFunctionSets(set_foo_bar_baz, set_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownFunctionSets(set_foo_bar_baz, set_foo_bar, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownFunctionSets(set_foo, set_foo_bar_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownFunctionSets(set_bar, set_foo_bar_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownFunctionSets(set_baz, set_foo_bar_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownFunctionSets(set_foo_bar, set_foo_bar_baz, &arena), + set_foo_bar_baz); +} + +} // namespace +} // namespace cel::common_internal diff --git a/common/internal/value_conversion.cc b/common/internal/value_conversion.cc index 57cf2224b..ef4dda231 100644 --- a/common/internal/value_conversion.cc +++ b/common/internal/value_conversion.cc @@ -224,9 +224,9 @@ absl::StatusOr FromExprValue( case ExprValueKind::kDoubleValue: return cel::DoubleValue(value.double_value()); case ExprValueKind::kStringValue: - return cel::StringValue(value.string_value()); + return cel::StringValue::From(value.string_value(), arena); case ExprValueKind::kBytesValue: - return cel::BytesValue(value.bytes_value()); + return cel::BytesValue::From(value.bytes_value(), arena); case ExprValueKind::kNullValue: return cel::NullValue(); case ExprValueKind::kObjectValue: diff --git a/common/legacy_value.cc b/common/legacy_value.cc index 7fbf16732..0ff096ca5 100644 --- a/common/legacy_value.cc +++ b/common/legacy_value.cc @@ -36,28 +36,26 @@ #include "absl/types/span.h" #include "absl/types/variant.h" #include "base/attribute.h" -#include "common/casting.h" #include "common/kind.h" #include "common/memory.h" #include "common/type.h" #include "common/unknown.h" #include "common/value.h" #include "common/value_kind.h" +#include "common/values/legacy_list_value.h" +#include "common/values/legacy_map_value.h" #include "common/values/list_value_builder.h" #include "common/values/map_value_builder.h" #include "common/values/values.h" #include "eval/internal/cel_value_equal.h" #include "eval/public/cel_value.h" -#include "eval/public/containers/field_backed_list_impl.h" -#include "eval/public/containers/field_backed_map_impl.h" #include "eval/public/message_wrapper.h" -#include "eval/public/structs/cel_proto_wrap_util.h" -#include "eval/public/structs/legacy_type_adapter.h" +#include "eval/public/structs/cel_proto_wrap_value_to_message.h" #include "eval/public/structs/legacy_type_info_apis.h" #include "eval/public/structs/proto_message_type_adapter.h" +#include "eval/public/structs/trivial_legacy_type_info_internal.h" #include "internal/json.h" #include "internal/status_macros.h" -#include "internal/well_known_types.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -71,14 +69,13 @@ namespace cel { namespace { -using google::api::expr::runtime::CelList; -using google::api::expr::runtime::CelMap; -using google::api::expr::runtime::CelValue; -using google::api::expr::runtime::FieldBackedListImpl; -using google::api::expr::runtime::FieldBackedMapImpl; -using google::api::expr::runtime::GetGenericProtoTypeInfoInstance; -using google::api::expr::runtime::LegacyTypeInfoApis; -using google::api::expr::runtime::MessageWrapper; +using ::cel::interop_internal::TrivialTypeInfo; +using ::google::api::expr::runtime::CelList; +using ::google::api::expr::runtime::CelMap; +using ::google::api::expr::runtime::CelValue; +using ::google::api::expr::runtime::GetGenericProtoTypeInfoInstance; +using ::google::api::expr::runtime::LegacyTypeInfoApis; +using ::google::api::expr::runtime::MessageWrapper; using ::google::api::expr::runtime::internal::MaybeWrapValueToMessage; absl::Status InvalidMapKeyTypeError(ValueKind kind) { @@ -262,6 +259,11 @@ CelValue LegacyTrivialStructValue(google::protobuf::Arena* absl_nonnull arena, } if (auto parsed_message_value = value.AsParsedMessage(); parsed_message_value) { + if (interop_internal::IsUnsafeParsedMessageValue(*parsed_message_value)) { + return CelValue::CreateMessageWrapper( + AsMessageWrapper(cel::to_address(*parsed_message_value), + &GetGenericProtoTypeInfoInstance())); + } auto maybe_cloned = parsed_message_value->Clone(arena); return CelValue::CreateMessageWrapper(MessageWrapper( cel::to_address(maybe_cloned), &GetGenericProtoTypeInfoInstance())); @@ -280,19 +282,17 @@ CelValue LegacyTrivialListValue(google::protobuf::Arena* absl_nonnull arena, } if (auto parsed_repeated_field_value = value.AsParsedRepeatedField(); parsed_repeated_field_value) { - auto maybe_cloned = parsed_repeated_field_value->Clone(arena); - return CelValue::CreateList(google::protobuf::Arena::Create( - arena, &maybe_cloned.message(), maybe_cloned.field(), arena)); + auto wrapped = common_internal::WrapLegacyParsedRepeatedField( + *parsed_repeated_field_value, arena); + return CelValue::CreateList( + common_internal::AsLegacyListValue(wrapped)->cel_list()); } if (auto parsed_json_list_value = value.AsParsedJsonList(); parsed_json_list_value) { - auto maybe_cloned = parsed_json_list_value->Clone(arena); - return CelValue::CreateList(google::protobuf::Arena::Create( - arena, cel::to_address(maybe_cloned), - well_known_types::GetListValueReflectionOrDie( - maybe_cloned->GetDescriptor()) - .GetValuesDescriptor(), - arena)); + auto wrapped = common_internal::WrapLegacyParsedJsonList( + *parsed_json_list_value, arena); + return CelValue::CreateList( + common_internal::AsLegacyListValue(wrapped)->cel_list()); } if (auto custom_list_value = value.AsCustomList(); custom_list_value) { auto status_or_compat_list = common_internal::MakeCompatListValue( @@ -318,19 +318,17 @@ CelValue LegacyTrivialMapValue(google::protobuf::Arena* absl_nonnull arena, } if (auto parsed_map_field_value = value.AsParsedMapField(); parsed_map_field_value) { - auto maybe_cloned = parsed_map_field_value->Clone(arena); - return CelValue::CreateMap(google::protobuf::Arena::Create( - arena, &maybe_cloned.message(), maybe_cloned.field(), arena)); + auto wrapped = common_internal::WrapLegacyParsedMapField( + *parsed_map_field_value, arena); + return CelValue::CreateMap( + common_internal::AsLegacyMapValue(wrapped)->cel_map()); } if (auto parsed_json_map_value = value.AsParsedJsonMap(); parsed_json_map_value) { - auto maybe_cloned = parsed_json_map_value->Clone(arena); - return CelValue::CreateMap(google::protobuf::Arena::Create( - arena, cel::to_address(maybe_cloned), - well_known_types::GetStructReflectionOrDie( - maybe_cloned->GetDescriptor()) - .GetFieldsDescriptor(), - arena)); + auto wrapped = + common_internal::WrapLegacyParsedJsonMap(*parsed_json_map_value, arena); + return CelValue::CreateMap( + common_internal::AsLegacyMapValue(wrapped)->cel_map()); } if (auto custom_map_value = value.AsCustomMap(); custom_map_value) { auto status_or_compat_map = common_internal::MakeCompatMapValue( @@ -348,6 +346,25 @@ CelValue LegacyTrivialMapValue(google::protobuf::Arena* absl_nonnull arena, value.GetRuntimeType().DebugString())))); } +LegacyStructValue ParsedMessageToLegacyStructValue( + const ParsedMessageValue& parsed_message) { + return LegacyStructValue(cel::to_address(parsed_message), + &GetGenericProtoTypeInfoInstance()); +} + +LegacyStructValue MakeLegacyStructValue( + const google::protobuf::Message* absl_nonnull message, + const LegacyTypeInfoApis* legacy_type_info) { + // Guard against edge cases where a custom implementation of Message + // misbehaves. + // Modern value handles this with DCHECKs on value creation, legacy value + // would allow it and just report an ErrorValue on accesses. + if (message->GetReflection() == nullptr || legacy_type_info == nullptr) { + legacy_type_info = TrivialTypeInfo::GetInstance(); + } + return LegacyStructValue(message, legacy_type_info); +} + } // namespace google::api::expr::runtime::CelValue UnsafeLegacyValue( @@ -390,10 +407,6 @@ google::api::expr::runtime::CelValue UnsafeLegacyValue( } } -} // namespace common_internal - -namespace common_internal { - std::string LegacyListValue::DebugString() const { return CelValue::CreateList(impl_).DebugString(); } @@ -518,7 +531,8 @@ absl::Status LegacyListValue::Get( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const { if (ABSL_PREDICT_FALSE(index < 0 || index >= impl_->size())) { - *result = ErrorValue(absl::InvalidArgumentError("index out of bounds")); + *result = ErrorValue::From( + absl::InvalidArgumentError("index out of bounds"), arena); return absl::OkStatus(); } CEL_RETURN_IF_ERROR( @@ -700,13 +714,13 @@ absl::Status LegacyMapValue::Get( case ValueKind::kString: break; default: - *result = ErrorValue(InvalidMapKeyTypeError(key.kind())); + *result = ErrorValue::From(InvalidMapKeyTypeError(key.kind()), arena); return absl::OkStatus(); } CEL_ASSIGN_OR_RETURN(auto cel_key, LegacyValue(arena, key)); auto cel_value = impl_->Get(arena, cel_key); if (!cel_value.has_value()) { - *result = NoSuchKeyError(key.DebugString()); + *result = NoSuchKeyError(key.DebugString(), arena); return absl::OkStatus(); } CEL_RETURN_IF_ERROR(ModernValue(arena, *cel_value, *result)); @@ -733,7 +747,7 @@ absl::StatusOr LegacyMapValue::Find( case ValueKind::kString: break; default: - *result = ErrorValue(InvalidMapKeyTypeError(key.kind())); + *result = ErrorValue::From(InvalidMapKeyTypeError(key.kind()), arena); } CEL_ASSIGN_OR_RETURN(auto cel_key, LegacyValue(arena, key)); auto cel_value = impl_->Get(arena, cel_key); @@ -765,13 +779,13 @@ absl::Status LegacyMapValue::Has( case ValueKind::kString: break; default: - *result = ErrorValue(InvalidMapKeyTypeError(key.kind())); + *result = ErrorValue::From(InvalidMapKeyTypeError(key.kind()), arena); return absl::OkStatus(); } CEL_ASSIGN_OR_RETURN(auto cel_key, LegacyValue(arena, key)); absl::StatusOr has = impl_->Has(cel_key); if (!has.ok()) { - *result = ErrorValue(std::move(has).status()); + *result = ErrorValue::From(std::move(has).status(), arena); return absl::OkStatus(); } @@ -833,10 +847,8 @@ absl::Status LegacyStructValue::SerializeTo( ABSL_DCHECK(message_factory != nullptr); ABSL_DCHECK(output != nullptr); - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); if (ABSL_PREDICT_TRUE( - message_wrapper.message_ptr()->SerializePartialToZeroCopyStream( - output))) { + message_ptr_->SerializePartialToZeroCopyStream(output))) { return absl::OkStatus(); } return absl::UnknownError("failed to serialize protocol buffer message"); @@ -846,34 +858,30 @@ absl::Status LegacyStructValue::ConvertToJson( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Message* absl_nonnull json) const { + ABSL_DCHECK(message_ptr_ != nullptr); ABSL_DCHECK(descriptor_pool != nullptr); ABSL_DCHECK(message_factory != nullptr); ABSL_DCHECK(json != nullptr); ABSL_DCHECK_EQ(json->GetDescriptor()->well_known_type(), google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE); - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - - return internal::MessageToJson( - *google::protobuf::DownCastMessage(message_wrapper.message_ptr()), - descriptor_pool, message_factory, json); + return internal::MessageToJson(*message_ptr_, descriptor_pool, + message_factory, json); } absl::Status LegacyStructValue::ConvertToJsonObject( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Message* absl_nonnull json) const { + ABSL_DCHECK(message_ptr_ != nullptr); ABSL_DCHECK(descriptor_pool != nullptr); ABSL_DCHECK(message_factory != nullptr); ABSL_DCHECK(json != nullptr); ABSL_DCHECK_EQ(json->GetDescriptor()->well_known_type(), google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT); - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - - return internal::MessageToJson( - *google::protobuf::DownCastMessage(message_wrapper.message_ptr()), - descriptor_pool, message_factory, json); + return internal::MessageToJson(*message_ptr_, descriptor_pool, + message_factory, json); } absl::Status LegacyStructValue::Equal( @@ -881,39 +889,36 @@ absl::Status LegacyStructValue::Equal( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const { - if (auto legacy_struct_value = common_internal::AsLegacyStructValue(other); - legacy_struct_value.has_value()) { - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - const auto* access_apis = - message_wrapper.legacy_type_info()->GetAccessApis(message_wrapper); - if (ABSL_PREDICT_FALSE(access_apis == nullptr)) { - return absl::UnimplementedError( - absl::StrCat("legacy access APIs missing for ", GetTypeName())); - } - auto other_message_wrapper = - AsMessageWrapper(legacy_struct_value->message_ptr(), - legacy_struct_value->legacy_type_info()); - *result = BoolValue{ - access_apis->IsEqualTo(message_wrapper, other_message_wrapper)}; - return absl::OkStatus(); + ABSL_DCHECK(message_ptr_ != nullptr); + if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { + return absl::UnimplementedError( + absl::StrCat("legacy access APIs missing for ", GetTypeName())); } - if (auto struct_value = other.AsStruct(); struct_value.has_value()) { - return common_internal::StructValueEqual( - common_internal::LegacyStructValue(message_ptr_, legacy_type_info_), - *struct_value, descriptor_pool, message_factory, arena, result); + auto modern_value = UnsafeParsedMessageValue(message_ptr_); + + // Unwrap the rhs if it's a legacy struct so the normal implementation + // doesn't hit the fallback abstract struct comparison. + if (auto other_legacy = common_internal::AsLegacyStructValue(other); + other_legacy) { + if (other_legacy->legacy_type_info_ == TrivialTypeInfo::GetInstance()) { + return absl::UnimplementedError(absl::StrCat( + "legacy access APIs missing for ", other_legacy->GetTypeName())); + } + auto other_message = UnsafeParsedMessageValue(other_legacy->message_ptr_); + return modern_value.Equal(other_message, descriptor_pool, message_factory, + arena, result); } - *result = FalseValue(); - return absl::OkStatus(); + + return modern_value.Equal(other, descriptor_pool, message_factory, arena, + result); } bool LegacyStructValue::IsZeroValue() const { - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - const auto* access_apis = - message_wrapper.legacy_type_info()->GetAccessApis(message_wrapper); - if (ABSL_PREDICT_FALSE(access_apis == nullptr)) { + ABSL_DCHECK(message_ptr_ != nullptr); + if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { return false; } - return access_apis->ListFields(message_wrapper).empty(); + return UnsafeParsedMessageValue(message_ptr_).IsZeroValue(); } absl::Status LegacyStructValue::GetFieldByName( @@ -921,19 +926,26 @@ absl::Status LegacyStructValue::GetFieldByName( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const { - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - const auto* access_apis = - message_wrapper.legacy_type_info()->GetAccessApis(message_wrapper); - if (ABSL_PREDICT_FALSE(access_apis == nullptr)) { - *result = NoSuchFieldError(name); + if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { + *result = NoSuchFieldError(name, arena); return absl::OkStatus(); } - CEL_ASSIGN_OR_RETURN( - auto cel_value, - access_apis->GetField(name, message_wrapper, unboxing_options, - MemoryManagerRef::Pooling(arena))); - CEL_RETURN_IF_ERROR(ModernValue(arena, cel_value, *result)); - return absl::OkStatus(); + + ParsedMessageValue parsed_message = UnsafeParsedMessageValue(message_ptr_); + const auto* descriptor = parsed_message.GetDescriptor(); + const auto* field = descriptor->FindFieldByName(name); + if (field == nullptr) { + field = descriptor->file()->pool()->FindExtensionByPrintableName(descriptor, + name); + if (field == nullptr) { + *result = NoSuchFieldError(name, arena); + return absl::OkStatus(); + } + } + + return interop_internal::WrapLegacyMessageField( + message_ptr_, field, unboxing_options, descriptor_pool, message_factory, + arena, result); } absl::Status LegacyStructValue::GetFieldByNumber( @@ -947,18 +959,19 @@ absl::Status LegacyStructValue::GetFieldByNumber( absl::StatusOr LegacyStructValue::HasFieldByName( absl::string_view name) const { - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - const auto* access_apis = - message_wrapper.legacy_type_info()->GetAccessApis(message_wrapper); - if (ABSL_PREDICT_FALSE(access_apis == nullptr)) { - return NoSuchFieldError(name).NativeValue(); + ABSL_DCHECK(message_ptr_ != nullptr); + if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { + return common_internal::MakeNoSuchFieldError(name); } - return access_apis->HasField(name, message_wrapper); + return UnsafeParsedMessageValue(message_ptr_).HasFieldByName(name); } absl::StatusOr LegacyStructValue::HasFieldByNumber(int64_t number) const { - return absl::UnimplementedError( - "access to fields by numbers is not available for legacy structs"); + ABSL_DCHECK(message_ptr_ != nullptr); + if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { + return common_internal::MakeNoSuchFieldError(absl::StrCat(number)); + } + return UnsafeParsedMessageValue(message_ptr_).HasFieldByNumber(number); } absl::Status LegacyStructValue::ForEachField( @@ -966,28 +979,13 @@ absl::Status LegacyStructValue::ForEachField( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) const { - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - const auto* access_apis = - message_wrapper.legacy_type_info()->GetAccessApis(message_wrapper); - if (ABSL_PREDICT_FALSE(access_apis == nullptr)) { + ABSL_DCHECK(message_ptr_ != nullptr); + if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { return absl::UnimplementedError( absl::StrCat("legacy access APIs missing for ", GetTypeName())); } - auto field_names = access_apis->ListFields(message_wrapper); - Value value; - for (const auto& field_name : field_names) { - CEL_ASSIGN_OR_RETURN( - auto cel_value, - access_apis->GetField(field_name, message_wrapper, - ProtoWrapperTypeOptions::kUnsetNull, - MemoryManagerRef::Pooling(arena))); - CEL_RETURN_IF_ERROR(ModernValue(arena, cel_value, value)); - CEL_ASSIGN_OR_RETURN(auto ok, callback(field_name, value)); - if (!ok) { - break; - } - } - return absl::OkStatus(); + return UnsafeParsedMessageValue(message_ptr_) + .ForEachField(callback, descriptor_pool, message_factory, arena); } absl::Status LegacyStructValue::Qualify( @@ -999,10 +997,7 @@ absl::Status LegacyStructValue::Qualify( if (ABSL_PREDICT_FALSE(qualifiers.empty())) { return absl::InvalidArgumentError("invalid select qualifier path."); } - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - const auto* access_apis = - message_wrapper.legacy_type_info()->GetAccessApis(message_wrapper); - if (ABSL_PREDICT_FALSE(access_apis == nullptr)) { + if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { absl::string_view field_name = absl::visit( absl::Overload( [](const FieldSpecifier& field) -> absl::string_view { @@ -1012,16 +1007,17 @@ absl::Status LegacyStructValue::Qualify( return field.GetStringKey().value_or(""); }), qualifiers.front()); - *result = NoSuchFieldError(field_name); + *result = NoSuchFieldError(field_name, arena); *count = -1; return absl::OkStatus(); } - CEL_ASSIGN_OR_RETURN( - auto legacy_result, - access_apis->Qualify(qualifiers, message_wrapper, presence_test, - MemoryManager::Pooling(arena))); - CEL_RETURN_IF_ERROR(ModernValue(arena, legacy_result.value, *result)); - *count = legacy_result.qualifier_count; + + ParsedMessageValue parsed_message = UnsafeParsedMessageValue(message_ptr_); + CEL_RETURN_IF_ERROR(parsed_message.Qualify(qualifiers, presence_test, + descriptor_pool, message_factory, + arena, result, count)); + + interop_internal::WrapLegacyFieldAccessResult(arena, result); return absl::OkStatus(); } @@ -1047,16 +1043,14 @@ absl::Status ModernValue(google::protobuf::Arena* arena, result = DoubleValue{legacy_value.DoubleOrDie()}; return absl::OkStatus(); case CelValue::Type::kString: - result = StringValue(Borrower::Arena(arena), - legacy_value.StringOrDie().value()); + result = StringValue::Wrap(legacy_value.StringOrDie().value(), nullptr); return absl::OkStatus(); case CelValue::Type::kBytes: - result = - BytesValue(Borrower::Arena(arena), legacy_value.BytesOrDie().value()); + result = BytesValue::Wrap(legacy_value.BytesOrDie().value(), nullptr); return absl::OkStatus(); case CelValue::Type::kMessage: { auto message_wrapper = legacy_value.MessageWrapperOrDie(); - result = common_internal::LegacyStructValue( + result = common_internal::MakeLegacyStructValue( google::protobuf::DownCastMessage( message_wrapper.message_ptr()), message_wrapper.legacy_type_info()); @@ -1069,15 +1063,14 @@ absl::Status ModernValue(google::protobuf::Arena* arena, result = UnsafeTimestampValue(legacy_value.TimestampOrDie()); return absl::OkStatus(); case CelValue::Type::kList: - result = - ListValue(common_internal::LegacyListValue(legacy_value.ListOrDie())); + result = common_internal::FromLegacyList(legacy_value.ListOrDie()); return absl::OkStatus(); case CelValue::Type::kMap: - result = - MapValue(common_internal::LegacyMapValue(legacy_value.MapOrDie())); + result = common_internal::FromLegacyMap(legacy_value.MapOrDie()); return absl::OkStatus(); case CelValue::Type::kUnknownSet: - result = UnknownValue{*legacy_value.UnknownSetOrDie()}; + result = common_internal::MakeUnknownValue( + *legacy_value.UnknownSetOrDie(), arena); return absl::OkStatus(); case CelValue::Type::kCelType: { auto type_name = legacy_value.CelTypeOrDie().value(); @@ -1088,7 +1081,7 @@ absl::Status ModernValue(google::protobuf::Arena* arena, return absl::OkStatus(); } case CelValue::Type::kError: - result = ErrorValue{*legacy_value.ErrorOrDie()}; + result = ErrorValue::From(*legacy_value.ErrorOrDie(), arena); return absl::OkStatus(); case CelValue::Type::kAny: return absl::InternalError(absl::StrCat( @@ -1107,15 +1100,15 @@ absl::StatusOr LegacyValue( case ValueKind::kNull: return CelValue::CreateNull(); case ValueKind::kBool: - return CelValue::CreateBool(Cast(modern_value).NativeValue()); + return CelValue::CreateBool(modern_value.Get().NativeValue()); case ValueKind::kInt: - return CelValue::CreateInt64(Cast(modern_value).NativeValue()); + return CelValue::CreateInt64(modern_value.Get().NativeValue()); case ValueKind::kUint: return CelValue::CreateUint64( - Cast(modern_value).NativeValue()); + modern_value.Get().NativeValue()); case ValueKind::kDouble: return CelValue::CreateDouble( - Cast(modern_value).NativeValue()); + modern_value.Get().NativeValue()); case ValueKind::kString: return CelValue::CreateStringView(common_internal::LegacyStringValue( modern_value.GetString(), /*stable=*/false, arena)); @@ -1136,14 +1129,14 @@ absl::StatusOr LegacyValue( return common_internal::LegacyTrivialMapValue(arena, modern_value); case ValueKind::kUnknown: return CelValue::CreateUnknownSet(google::protobuf::Arena::Create( - arena, Cast(modern_value).NativeValue())); + arena, common_internal::ToUnknown(modern_value.Get()))); case ValueKind::kType: return CelValue::CreateCelType( CelValue::CelTypeHolder(google::protobuf::Arena::Create( - arena, Cast(modern_value).NativeValue().name()))); + arena, modern_value.Get().NativeValue().name()))); case ValueKind::kError: return CelValue::CreateError(google::protobuf::Arena::Create( - arena, Cast(modern_value).NativeValue())); + arena, modern_value.Get().NativeValue())); default: return absl::InvalidArgumentError( absl::StrCat("google::api::expr::runtime::CelValue does not support ", @@ -1167,14 +1160,12 @@ absl::StatusOr FromLegacyValue(google::protobuf::Arena* arena, case CelValue::Type::kDouble: return DoubleValue(legacy_value.DoubleOrDie()); case CelValue::Type::kString: - return StringValue(Borrower::Arena(arena), - legacy_value.StringOrDie().value()); + return StringValue::Wrap(legacy_value.StringOrDie().value(), nullptr); case CelValue::Type::kBytes: - return BytesValue(Borrower::Arena(arena), - legacy_value.BytesOrDie().value()); + return BytesValue::Wrap(legacy_value.BytesOrDie().value(), nullptr); case CelValue::Type::kMessage: { auto message_wrapper = legacy_value.MessageWrapperOrDie(); - return common_internal::LegacyStructValue( + return common_internal::MakeLegacyStructValue( google::protobuf::DownCastMessage( message_wrapper.message_ptr()), message_wrapper.legacy_type_info()); @@ -1184,17 +1175,17 @@ absl::StatusOr FromLegacyValue(google::protobuf::Arena* arena, case CelValue::Type::kTimestamp: return UnsafeTimestampValue(legacy_value.TimestampOrDie()); case CelValue::Type::kList: - return ListValue( - common_internal::LegacyListValue(legacy_value.ListOrDie())); + return common_internal::FromLegacyList(legacy_value.ListOrDie()); case CelValue::Type::kMap: - return MapValue(common_internal::LegacyMapValue(legacy_value.MapOrDie())); + return common_internal::FromLegacyMap(legacy_value.MapOrDie()); case CelValue::Type::kUnknownSet: - return UnknownValue{*legacy_value.UnknownSetOrDie()}; + return common_internal::MakeUnknownValue(*legacy_value.UnknownSetOrDie(), + arena); case CelValue::Type::kCelType: return CreateTypeValueFromView(arena, legacy_value.CelTypeOrDie().value()); case CelValue::Type::kError: - return ErrorValue(*legacy_value.ErrorOrDie()); + return ErrorValue::From(*legacy_value.ErrorOrDie(), arena); case CelValue::Type::kAny: return absl::InternalError(absl::StrCat( "illegal attempt to convert special CelValue type ", @@ -1213,13 +1204,13 @@ absl::StatusOr ToLegacyValue( case ValueKind::kNull: return CelValue::CreateNull(); case ValueKind::kBool: - return CelValue::CreateBool(Cast(value).NativeValue()); + return CelValue::CreateBool(value.Get().NativeValue()); case ValueKind::kInt: - return CelValue::CreateInt64(Cast(value).NativeValue()); + return CelValue::CreateInt64(value.Get().NativeValue()); case ValueKind::kUint: - return CelValue::CreateUint64(Cast(value).NativeValue()); + return CelValue::CreateUint64(value.Get().NativeValue()); case ValueKind::kDouble: - return CelValue::CreateDouble(Cast(value).NativeValue()); + return CelValue::CreateDouble(value.Get().NativeValue()); case ValueKind::kString: return CelValue::CreateStringView(common_internal::LegacyStringValue( value.GetString(), /*stable=*/false, arena)); @@ -1230,24 +1221,24 @@ absl::StatusOr ToLegacyValue( return common_internal::LegacyTrivialStructValue(arena, value); case ValueKind::kDuration: return CelValue::CreateUncheckedDuration( - Cast(value).NativeValue()); + value.Get().NativeValue()); case ValueKind::kTimestamp: return CelValue::CreateTimestamp( - Cast(value).NativeValue()); + value.Get().NativeValue()); case ValueKind::kList: return common_internal::LegacyTrivialListValue(arena, value); case ValueKind::kMap: return common_internal::LegacyTrivialMapValue(arena, value); case ValueKind::kUnknown: return CelValue::CreateUnknownSet(google::protobuf::Arena::Create( - arena, Cast(value).NativeValue())); + arena, common_internal::ToUnknown(value.Get()))); case ValueKind::kType: return CelValue::CreateCelType( CelValue::CelTypeHolder(google::protobuf::Arena::Create( - arena, Cast(value).NativeValue().name()))); + arena, value.Get().NativeValue().name()))); case ValueKind::kError: return CelValue::CreateError(google::protobuf::Arena::Create( - arena, Cast(value).NativeValue())); + arena, value.Get().NativeValue())); default: return absl::InvalidArgumentError( absl::StrCat("google::api::expr::runtime::CelValue does not support ", @@ -1283,11 +1274,63 @@ google::api::expr::runtime::CelValue ModernValueToLegacyValueOrDie( return std::move(*status_or_value); } +void WrapLegacyFieldAccessResult(google::protobuf::Arena* absl_nonnull arena, + Value* absl_nonnull result) { + if (result->IsParsedMessage()) { + *result = common_internal::ParsedMessageToLegacyStructValue( + result->GetParsedMessage()); + } else if (result->IsParsedRepeatedField()) { + *result = + WrapLegacyParsedRepeatedField(result->GetParsedRepeatedField(), arena); + } else if (result->IsParsedJsonList()) { + *result = WrapLegacyParsedJsonList(result->GetParsedJsonList(), arena); + } else if (result->IsParsedMapField()) { + *result = WrapLegacyParsedMapField(result->GetParsedMapField(), arena); + } else if (result->IsParsedJsonMap()) { + *result = WrapLegacyParsedJsonMap(result->GetParsedJsonMap(), arena); + } +} + TypeValue CreateTypeValueFromView(google::protobuf::Arena* arena, absl::string_view input) { return TypeValue(common_internal::LegacyRuntimeType(input)); } +const google::protobuf::Message* absl_nullable GetLegacyMessage(const Value& value) { + if (!common_internal::IsLegacyStructValue(value)) { + return nullptr; + } + + auto legacy = common_internal::GetLegacyStructValue(value); + const auto* legacy_type_info = legacy.legacy_type_info(); + if (legacy_type_info == nullptr || + legacy_type_info == TrivialTypeInfo::GetInstance()) { + return nullptr; + } + // This should not be possible using the normal public APIs, but possible + // if someone used the MessageWrapper class directly. + if (IsWellKnownMessageType(legacy.message_ptr()->GetDescriptor())) { + return nullptr; + } + return legacy.message_ptr(); +} + +absl::Status WrapLegacyMessageField( + const google::protobuf::Message* absl_nonnull message, + const google::protobuf::FieldDescriptor* absl_nonnull field_descriptor, + ProtoWrapperTypeOptions unboxing_option, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* arena, + Value* absl_nonnull out) { + ParsedMessageValue parsed_message = UnsafeParsedMessageValue(message); + CEL_RETURN_IF_ERROR(parsed_message.GetField(field_descriptor, unboxing_option, + descriptor_pool, message_factory, + arena, out)); + WrapLegacyFieldAccessResult(arena, out); + + return absl::OkStatus(); +} + } // namespace interop_internal } // namespace cel diff --git a/common/legacy_value.h b/common/legacy_value.h index 7e703cea1..8fea22ec0 100644 --- a/common/legacy_value.h +++ b/common/legacy_value.h @@ -27,7 +27,9 @@ #include "common/value.h" #include "eval/public/cel_value.h" #include "internal/status_macros.h" +#include "runtime/runtime_options.h" #include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" namespace cel { @@ -57,8 +59,38 @@ google::api::expr::runtime::CelValue UnsafeLegacyValue( } // namespace cel +namespace proto2 { +class MessageFactory; +} // namespace proto2 + namespace cel::interop_internal { +inline bool IsUnsafeParsedMessageValue(const cel::ParsedMessageValue& value) { + return value.is_unsafe(); +} + +// Returns the underlying `google::protobuf::Message` of a `cel::Value` if it is a legacy +// message with the default type info, or `nullptr` otherwise. +const google::protobuf::Message* absl_nullable GetLegacyMessage(const Value& value); + +// Helper for wrapping a field accesses for the legacy runtime. +// +// Adapts the output to avoid further allocations when converting to a legacy +// value when possible. +void WrapLegacyFieldAccessResult(google::protobuf::Arena* absl_nonnull arena, + Value* absl_nonnull result); + +// Access a field on a legacy message value, writing the result to `out`. +// Prefers wrapping legacy values instead of using the modern value +// representation. +absl::Status WrapLegacyMessageField( + const google::protobuf::Message* absl_nonnull message, + const google::protobuf::FieldDescriptor* absl_nonnull field_descriptor, + ProtoWrapperTypeOptions unboxing_option, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* arena, + Value* absl_nonnull out); + absl::StatusOr FromLegacyValue( google::protobuf::Arena* arena, const google::api::expr::runtime::CelValue& legacy_value, @@ -89,14 +121,6 @@ inline MapValue CreateLegacyMapValue( return common_internal::LegacyMapValue(value); } -inline Value CreateDurationValue(absl::Duration value, bool unchecked = false) { - return DurationValue{value}; -} - -inline TimestampValue CreateTimestampValue(absl::Time value) { - return TimestampValue{value}; -} - Value LegacyValueToModernValueOrDie( google::protobuf::Arena* arena, const google::api::expr::runtime::CelValue& value, bool unchecked = false); diff --git a/common/memory.cc b/common/memory.cc index c00c12ed8..46373ba5b 100644 --- a/common/memory.cc +++ b/common/memory.cc @@ -19,10 +19,8 @@ #include #include -#include "absl/base/no_destructor.h" #include "absl/log/absl_check.h" #include "absl/numeric/bits.h" -#include "google/protobuf/arena.h" namespace cel { @@ -73,11 +71,4 @@ bool ReferenceCountingMemoryManager::Deallocate(void* ptr, size_t size, return true; } -MemoryManager MemoryManager::Unmanaged() { - // A static singleton arena, using `absl::NoDestructor` to avoid warnings - // related static variables without trivial destructors. - static absl::NoDestructor arena; - return MemoryManager::Pooling(&*arena); -} - } // namespace cel diff --git a/common/memory.h b/common/memory.h index b19f54f94..684e708f8 100644 --- a/common/memory.h +++ b/common/memory.h @@ -16,25 +16,16 @@ #define THIRD_PARTY_CEL_CPP_COMMON_MEMORY_H_ #include -#include #include #include #include #include #include "absl/base/attributes.h" -#include "absl/base/macros.h" #include "absl/base/nullability.h" -#include "absl/base/optimization.h" #include "absl/log/absl_check.h" #include "absl/numeric/bits.h" #include "common/allocator.h" -#include "common/arena.h" -#include "common/data.h" -#include "common/internal/metadata.h" -#include "common/internal/reference_count.h" -#include "common/reference_count.h" -#include "internal/exceptions.h" #include "internal/to_address.h" // IWYU pragma: keep #include "google/protobuf/arena.h" @@ -57,1252 +48,11 @@ enum class MemoryManagement { std::ostream& operator<<(std::ostream& out, MemoryManagement memory_management); -class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Owner; -class Borrower; -template -class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Unique; -template -class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Owned; -template -class Borrowed; -template -struct Ownable; -template -struct Borrowable; - class MemoryManager; -class ReferenceCountingMemoryManager; -class PoolingMemoryManager; - -namespace common_internal { -template -inline constexpr bool kNotMessageLiteAndNotData = - std::conjunction_v>, - std::negation>>; -template -inline constexpr bool kIsPointerConvertible = std::is_convertible_v; -template -inline constexpr bool kNotSameAndIsPointerConvertible = - std::conjunction_v>, - std::bool_constant>>; - -// Clears the contents of `owner`, and returns the reference count if in use. -const ReferenceCount* absl_nullable OwnerRelease(Owner owner) noexcept; -const ReferenceCount* absl_nullable BorrowerRelease(Borrower borrower) noexcept; -template -Owned WrapEternal(const T* value); - -// Pointer tag used by `cel::Unique` to indicate that the destructor needs to be -// registered with the arena, but it has not been done yet. Must be done when -// releasing. -inline constexpr uintptr_t kUniqueArenaUnownedBit = uintptr_t{1} << 0; -inline constexpr uintptr_t kUniqueArenaBits = kUniqueArenaUnownedBit; -inline constexpr uintptr_t kUniqueArenaPointerMask = ~kUniqueArenaBits; -} // namespace common_internal - -template -Owned AllocateShared(Allocator<> allocator, Args&&... args); - -template -Owned WrapShared(T* object, Allocator<> allocator); - -// `Owner` represents a reference to some co-owned data, of which this owner is -// one of the co-owners. When using reference counting, `Owner` performs -// increment/decrement where appropriate similar to `std::shared_ptr`. -// `Borrower` is similar to `Owner`, except that it is always trivially -// copyable/destructible. In that sense, `Borrower` is similar to -// `std::reference_wrapper`. -class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Owner final { - private: - static constexpr uintptr_t kNone = common_internal::kMetadataOwnerNone; - static constexpr uintptr_t kReferenceCountBit = - common_internal::kMetadataOwnerReferenceCountBit; - static constexpr uintptr_t kArenaBit = - common_internal::kMetadataOwnerArenaBit; - static constexpr uintptr_t kBits = common_internal::kMetadataOwnerBits; - static constexpr uintptr_t kPointerMask = - common_internal::kMetadataOwnerPointerMask; - - public: - static Owner None() noexcept { return Owner(); } - - static Owner Allocator(Allocator<> allocator) noexcept { - auto* arena = allocator.arena(); - return arena != nullptr ? Arena(arena) : None(); - } - - static Owner Arena(google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept { - ABSL_DCHECK(arena != nullptr); - return Owner(reinterpret_cast(arena) | kArenaBit); - } - - static Owner Arena(std::nullptr_t) = delete; - - static Owner ReferenceCount(const ReferenceCount* absl_nonnull reference_count - ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept { - ABSL_DCHECK(reference_count != nullptr); - common_internal::StrongRef(*reference_count); - return Owner(reinterpret_cast(reference_count) | - kReferenceCountBit); - } - - static Owner ReferenceCount(std::nullptr_t) = delete; - - Owner() = default; - - Owner(const Owner& other) noexcept : Owner(CopyFrom(other.ptr_)) {} - - Owner(Owner&& other) noexcept : Owner(MoveFrom(other.ptr_)) {} - - template - // NOLINTNEXTLINE(google-explicit-constructor) - Owner(const Owned& owned) noexcept; - - template - // NOLINTNEXTLINE(google-explicit-constructor) - Owner(Owned&& owned) noexcept; - - explicit Owner(Borrower borrower) noexcept; - - template - explicit Owner(Borrowed borrowed) noexcept; - - ~Owner() { Destroy(ptr_); } - - Owner& operator=(const Owner& other) noexcept { - if (ptr_ != other.ptr_) { - Destroy(ptr_); - ptr_ = CopyFrom(other.ptr_); - } - return *this; - } - - Owner& operator=(Owner&& other) noexcept { - if (ABSL_PREDICT_TRUE(this != &other)) { - Destroy(ptr_); - ptr_ = MoveFrom(other.ptr_); - } - return *this; - } - - template - // NOLINTNEXTLINE(google-explicit-constructor) - Owner& operator=(const Owned& owned) noexcept; - - template - // NOLINTNEXTLINE(google-explicit-constructor) - Owner& operator=(Owned&& owned) noexcept; - - explicit operator bool() const noexcept { return !IsNone(ptr_); } - - google::protobuf::Arena* absl_nullable arena() const noexcept { - return (ptr_ & Owner::kBits) == Owner::kArenaBit - ? reinterpret_cast(ptr_ & Owner::kPointerMask) - : nullptr; - } - - void reset() noexcept { - Destroy(ptr_); - ptr_ = 0; - } - - // Tests whether two owners have ownership over the same data, that is they - // are co-owners. - friend bool operator==(const Owner& lhs, const Owner& rhs) noexcept { - // A reference count and arena can never occupy the same memory address, so - // we can compare for equality without masking off the bits. - return lhs.ptr_ == rhs.ptr_; - } - - private: - template - friend class Unique; - friend class Borrower; - template - friend Owned AllocateShared(cel::Allocator<> allocator, Args&&... args); - template - friend Owned WrapShared(T* object, cel::Allocator<> allocator); - template - friend struct Ownable; - friend const common_internal::ReferenceCount* absl_nullable - common_internal::OwnerRelease(Owner owner) noexcept; - friend const common_internal::ReferenceCount* absl_nullable - common_internal::BorrowerRelease(Borrower borrower) noexcept; - friend struct ArenaTraits; - - constexpr explicit Owner(uintptr_t ptr) noexcept : ptr_(ptr) {} - - static constexpr bool IsNone(uintptr_t ptr) noexcept { return ptr == kNone; } - - static constexpr bool IsArena(uintptr_t ptr) noexcept { - return (ptr & kArenaBit) != kNone; - } - - static constexpr bool IsReferenceCount(uintptr_t ptr) noexcept { - return (ptr & kReferenceCountBit) != kNone; - } - - ABSL_ATTRIBUTE_RETURNS_NONNULL - static google::protobuf::Arena* absl_nonnull AsArena(uintptr_t ptr) noexcept { - ABSL_ASSERT(IsArena(ptr)); - return reinterpret_cast(ptr & kPointerMask); - } - - ABSL_ATTRIBUTE_RETURNS_NONNULL - static const common_internal::ReferenceCount* absl_nonnull AsReferenceCount( - uintptr_t ptr) noexcept { - ABSL_ASSERT(IsReferenceCount(ptr)); - return reinterpret_cast( - ptr & kPointerMask); - } - - static uintptr_t CopyFrom(uintptr_t other) noexcept { return Own(other); } - - static uintptr_t MoveFrom(uintptr_t& other) noexcept { - return std::exchange(other, kNone); - } - - static void Destroy(uintptr_t ptr) noexcept { Unown(ptr); } - - static uintptr_t Own(uintptr_t ptr) noexcept { - if (IsReferenceCount(ptr)) { - const auto* refcount = Owner::AsReferenceCount(ptr); - ABSL_ASSUME(refcount != nullptr); - common_internal::StrongRef(refcount); - } - return ptr; - } - - static void Unown(uintptr_t ptr) noexcept { - if (IsReferenceCount(ptr)) { - const auto* reference_count = AsReferenceCount(ptr); - ABSL_ASSUME(reference_count != nullptr); - common_internal::StrongUnref(reference_count); - } - } - - uintptr_t ptr_ = kNone; -}; - -inline bool operator!=(const Owner& lhs, const Owner& rhs) noexcept { - return !operator==(lhs, rhs); -} - -namespace common_internal { - -inline const ReferenceCount* absl_nullable OwnerRelease(Owner owner) noexcept { - uintptr_t ptr = std::exchange(owner.ptr_, kMetadataOwnerNone); - if (Owner::IsReferenceCount(ptr)) { - return Owner::AsReferenceCount(ptr); - } - return nullptr; -} - -} // namespace common_internal - -template <> -struct ArenaTraits { - static bool trivially_destructible(const Owner& owner) { - return !Owner::IsReferenceCount(owner.ptr_); - } -}; - -// `Borrower` represents a reference to some borrowed data, where the data has -// at least one owner. When using reference counting, `Borrower` does not -// participate in incrementing/decrementing the reference count. Thus `Borrower` -// will not keep the underlying data alive. -class Borrower final { - public: - static Borrower None() noexcept { return Borrower(); } - - static Borrower Allocator(Allocator<> allocator) noexcept { - auto* arena = allocator.arena(); - return arena != nullptr ? Arena(arena) : None(); - } - - static Borrower Arena(google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept { - ABSL_DCHECK(arena != nullptr); - return Borrower(reinterpret_cast(arena) | Owner::kArenaBit); - } - - static Borrower Arena(std::nullptr_t) = delete; - - static Borrower ReferenceCount( - const ReferenceCount* absl_nonnull reference_count - ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept { - ABSL_DCHECK(reference_count != nullptr); - return Borrower(reinterpret_cast(reference_count) | - Owner::kReferenceCountBit); - } - - static Borrower ReferenceCount(std::nullptr_t) = delete; - - Borrower() = default; - Borrower(const Borrower&) = default; - Borrower(Borrower&&) = default; - Borrower& operator=(const Borrower&) = default; - Borrower& operator=(Borrower&&) = default; - - template - // NOLINTNEXTLINE(google-explicit-constructor) - Borrower(const Owned& owned ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept; - - template - // NOLINTNEXTLINE(google-explicit-constructor) - Borrower(Borrowed borrowed) noexcept; - - // NOLINTNEXTLINE(google-explicit-constructor) - Borrower(const Owner& owner ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept - : ptr_(owner.ptr_) {} - - // NOLINTNEXTLINE(google-explicit-constructor) - Borrower& operator=( - const Owner& owner ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept { - ptr_ = owner.ptr_; - return *this; - } - - Borrower& operator=(Owner&&) = delete; - - template - Borrower& operator=( - const Owned& owned ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept; - - template - Borrower& operator=(Owned&&) = delete; - - template - // NOLINTNEXTLINE(google-explicit-constructor) - Borrower& operator=(Borrowed borrowed) noexcept; - - explicit operator bool() const noexcept { return !Owner::IsNone(ptr_); } - - google::protobuf::Arena* absl_nullable arena() const noexcept { - return (ptr_ & Owner::kBits) == Owner::kArenaBit - ? reinterpret_cast(ptr_ & Owner::kPointerMask) - : nullptr; - } - - void reset() noexcept { ptr_ = 0; } - - // Tests whether two borrowers are borrowing the same data. - friend bool operator==(Borrower lhs, Borrower rhs) noexcept { - // A reference count and arena can never occupy the same memory address, so - // we can compare for equality without masking off the bits. - return lhs.ptr_ == rhs.ptr_; - } - - private: - friend class Owner; - template - friend struct Borrowable; - friend const common_internal::ReferenceCount* absl_nullable - common_internal::BorrowerRelease(Borrower borrower) noexcept; - - constexpr explicit Borrower(uintptr_t ptr) noexcept : ptr_(ptr) {} - - uintptr_t ptr_ = Owner::kNone; -}; - -inline bool operator!=(Borrower lhs, Borrower rhs) noexcept { - return !operator==(lhs, rhs); -} - -inline bool operator==(Borrower lhs, const Owner& rhs) noexcept { - return operator==(lhs, Borrower(rhs)); -} - -inline bool operator==(const Owner& lhs, Borrower rhs) noexcept { - return operator==(Borrower(lhs), rhs); -} - -inline bool operator!=(Borrower lhs, const Owner& rhs) noexcept { - return !operator==(lhs, rhs); -} - -inline bool operator!=(const Owner& lhs, Borrower rhs) noexcept { - return !operator==(lhs, rhs); -} - -inline Owner::Owner(Borrower borrower) noexcept - : ptr_(Owner::Own(borrower.ptr_)) {} - -namespace common_internal { - -inline const ReferenceCount* absl_nullable BorrowerRelease( - Borrower borrower) noexcept { - uintptr_t ptr = borrower.ptr_; - if (Owner::IsReferenceCount(ptr)) { - return Owner::AsReferenceCount(ptr); - } - return nullptr; -} - -} // namespace common_internal - -template -Unique AllocateUnique(Allocator<> allocator, Args&&... args); - -// Wrap an already created `T` in `Unique`. Requires that `T` is not const, -// otherwise `GetArena()` may return slightly unexpected results depending on if -// it is the default value. -template -std::enable_if_t, Unique> WrapUnique(T* object); - -template -Unique WrapUnique(T* object, Allocator<> allocator); - -// `Unique` points to an object which was allocated using `Allocator<>` or -// `Allocator`. It has ownership over the object, and will perform any -// destruction and deallocation required. `Unique` must not outlive the -// underlying arena, if any. Unlike `Owned` and `Borrowed`, `Unique` supports -// arena incompatible objects. It is very similar to `std::unique_ptr` when -// using a custom deleter. -// -// IMPLEMENTATION NOTES: -// When utilizing arenas, we optionally perform a risky optimization via -// `AllocateUnique`. We do not use `Arena::Create`, instead we directly allocate -// the bytes and construct it in place ourselves. This avoids registering the -// destructor when required. Instead we register the destructor ourselves, if -// required, during `Unique::release`. This allows us to avoid deferring -// destruction of the object until the arena is destroyed, avoiding the cost -// involved in doing so. -template -class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Unique final { - public: - using element_type = T; - - static_assert(!std::is_array_v, "T must not be an array"); - static_assert(!std::is_reference_v, "T must not be a reference"); - static_assert(!std::is_volatile_v, "T must not be volatile qualified"); - - Unique() = default; - Unique(const Unique&) = delete; - Unique& operator=(const Unique&) = delete; - - explicit Unique(T* ptr) noexcept - : Unique(ptr, common_internal::GetArena(ptr)) {} - - // NOLINTNEXTLINE(google-explicit-constructor) - Unique(std::nullptr_t) noexcept : Unique() {} - - Unique(Unique&& other) noexcept : Unique(other.ptr_, other.arena_) { - other.ptr_ = nullptr; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Unique(Unique&& other) noexcept : Unique(other.ptr_, other.arena_) { - other.ptr_ = nullptr; - } - - ~Unique() { Delete(); } - - Unique& operator=(Unique&& other) noexcept { - if (ABSL_PREDICT_TRUE(this != &other)) { - Delete(); - ptr_ = other.ptr_; - arena_ = other.arena_; - other.ptr_ = nullptr; - } - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Unique& operator=(U* other) noexcept { - reset(other); - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Unique& operator=(Unique&& other) noexcept { - Delete(); - ptr_ = other.ptr_; - arena_ = other.arena_; - other.ptr_ = nullptr; - return *this; - } - - // NOLINTNEXTLINE(google-explicit-constructor) - Unique& operator=(std::nullptr_t) noexcept { - reset(); - return *this; - } - - T& operator*() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_DCHECK(static_cast(*this)); - return *get(); - } - - T* absl_nonnull operator->() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_DCHECK(static_cast(*this)); - return get(); - } - - // Relinquishes ownership of `T*`, returning it. If `T` was allocated and - // constructed using an arena, no further action is required. If `T` was - // allocated and constructed without an arena, the caller must eventually call - // `delete`. - ABSL_MUST_USE_RESULT T* release() noexcept { - PreRelease(); - return std::exchange(ptr_, nullptr); - } - - void reset() noexcept { reset(nullptr); } - - void reset(T* ptr) noexcept { - Delete(); - ptr_ = ptr; - arena_ = reinterpret_cast(common_internal::GetArena(ptr)); - } - - void reset(std::nullptr_t) noexcept { - Delete(); - ptr_ = nullptr; - arena_ = 0; - } - - explicit operator bool() const noexcept { return get() != nullptr; } - - google::protobuf::Arena* absl_nullable arena() const noexcept { - return reinterpret_cast( - arena_ & common_internal::kUniqueArenaPointerMask); - } - - friend void swap(Unique& lhs, Unique& rhs) noexcept { - using std::swap; - swap(lhs.ptr_, rhs.ptr_); - swap(lhs.arena_, rhs.arena_); - } - - private: - template - friend class Unique; - template - friend class Owned; - template - friend Unique AllocateUnique(Allocator<> allocator, Args&&... args); - template - friend Unique WrapUnique(U* object, Allocator<> allocator); - friend class ReferenceCountingMemoryManager; - friend class PoolingMemoryManager; - friend struct std::pointer_traits>; - friend struct ArenaTraits>; - - Unique(T* ptr, uintptr_t arena) noexcept : ptr_(ptr), arena_(arena) {} - - Unique(T* ptr, google::protobuf::Arena* arena, bool unowned = false) noexcept - : Unique(ptr, - reinterpret_cast(arena) | - (unowned ? common_internal::kUniqueArenaUnownedBit : 0)) { - ABSL_ASSERT(!unowned || (unowned && arena != nullptr)); - } - - Unique(google::protobuf::Arena* arena, T* ptr, bool unowned = false) noexcept - : Unique(ptr, arena, unowned) {} - - T* get() const noexcept { return ptr_; } - - void Delete() const noexcept { - if (static_cast(*this)) { - if (arena_ != 0) { - if ((arena_ & common_internal::kUniqueArenaBits) == - common_internal::kUniqueArenaUnownedBit) { - // We never registered the destructor, call it if necessary. - if constexpr (!std::is_trivially_destructible_v && - !google::protobuf::Arena::is_destructor_skippable::value) { - std::destroy_at(ptr_); - } - } - } else { - delete ptr_; - } - } - } - - void PreRelease() noexcept { - if constexpr (!std::is_trivially_destructible_v && - !google::protobuf::Arena::is_destructor_skippable::value) { - if (static_cast(*this) && - (arena_ & common_internal::kUniqueArenaBits) == - common_internal::kUniqueArenaUnownedBit) { - // We never registered the destructor, call it if necessary. - arena()->OwnDestructor(const_cast*>(ptr_)); - arena_ &= common_internal::kUniqueArenaPointerMask; - } - } - } - - void Release(T** ptr, Owner* owner) noexcept { - if (ptr_ == nullptr) { - *ptr = nullptr; - return; - } - PreRelease(); - *ptr = std::exchange(ptr_, nullptr); - if (arena_ == 0) { - owner->ptr_ = reinterpret_cast( - common_internal::MakeDeletingReferenceCount(*ptr)) | - common_internal::kMetadataOwnerReferenceCountBit; - } else { - owner->ptr_ = reinterpret_cast(arena()) | - common_internal::kMetadataOwnerArenaBit; - } - } - - T* ptr_ = nullptr; - // Potentially tagged pointer to `google::protobuf::Arena`. The tag is used to determine - // whether we still need to register the destructor with the `google::protobuf::Arena`. - uintptr_t arena_ = 0; -}; - -template -Unique(T*) -> Unique; - -template -Unique AllocateUnique(Allocator<> allocator, Args&&... args) { - using U = std::remove_cv_t; - static_assert(!std::is_reference_v, "T must not be a reference"); - static_assert(!std::is_array_v, "T must not be an array"); - - U* object; - google::protobuf::Arena* absl_nullable arena = allocator.arena(); - bool unowned; - if constexpr (google::protobuf::Arena::is_arena_constructable::value) { - object = google::protobuf::Arena::Create(arena, std::forward(args)...); - // For arena-compatible proto types, let the Arena::Create handle - // registering the destructor call. - // Otherwise, Unique retains a pointer to the owning arena so it may - // conditionally register T::~T depending on usage. - unowned = false; - } else { - void* p = allocator.allocate_bytes(sizeof(U), alignof(U)); - CEL_INTERNAL_TRY { - if constexpr (ArenaTraits<>::constructible()) { - object = ::new (p) U(arena, std::forward(args)...); - } else { - object = ::new (p) U(std::forward(args)...); - } - } - CEL_INTERNAL_CATCH_ANY { - allocator.deallocate_bytes(p, sizeof(U), alignof(U)); - CEL_INTERNAL_RETHROW; - } - unowned = - arena != nullptr && !ArenaTraits<>::trivially_destructible(*object); - } - return Unique(object, arena, unowned); -} - -template -std::enable_if_t, Unique> WrapUnique(T* object) { - return Unique(object); -} - -template -Unique WrapUnique(T* object, Allocator<> allocator) { - return Unique(object, allocator.arena()); -} - -template -inline bool operator==(const Unique& lhs, std::nullptr_t) { - return !static_cast(lhs); -} - -template -inline bool operator==(std::nullptr_t, const Unique& rhs) { - return !static_cast(rhs); -} - -template -inline bool operator!=(const Unique& lhs, std::nullptr_t) { - return static_cast(lhs); -} - -template -inline bool operator!=(std::nullptr_t, const Unique& rhs) { - return static_cast(rhs); -} - -} // namespace cel - -namespace std { - -template -struct pointer_traits> { - using pointer = cel::Unique; - using element_type = typename cel::Unique::element_type; - using difference_type = ptrdiff_t; - - template - using rebind = cel::Unique; - - static element_type* to_address(const pointer& p) noexcept { return p.ptr_; } -}; - -} // namespace std - -namespace cel { - -template -struct ArenaTraits> { - static bool trivially_destructible(const Unique& unique) { - return unique.arena_ != 0 && - (unique.arena_ & common_internal::kUniqueArenaBits) == 0; - } -}; - -// `Owned` points to an object which was allocated using `Allocator<>` or -// `Allocator`. It has co-ownership over the object. `T` must meet the named -// requirement `ArenaConstructable`. -template -class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Owned final { - public: - using element_type = T; - - static_assert(!std::is_array_v, "T must not be an array"); - static_assert(!std::is_reference_v, "T must not be a reference"); - static_assert(!std::is_volatile_v, "T must not be volatile qualified"); - static_assert(!std::is_void_v, "T must not be void"); - - Owned() = default; - Owned(const Owned&) = default; - Owned& operator=(const Owned&) = default; - - Owned(Owned&& other) noexcept - : Owned(std::exchange(other.value_, nullptr), std::move(other.owner_)) {} - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Owned(const Owned& other) noexcept : Owned(other.value_, other.owner_) {} - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Owned(Owned&& other) noexcept - : Owned(std::exchange(other.value_, nullptr), std::move(other.owner_)) {} - - template >> - explicit Owned(Borrowed other) noexcept; - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Owned(Unique&& other) : Owned() { - other.Release(&value_, &owner_); - } - - Owned(Owner owner, T* value ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept - : Owned(value, std::move(owner)) {} - - // NOLINTNEXTLINE(google-explicit-constructor) - Owned(std::nullptr_t) noexcept : Owned() {} - - Owned& operator=(Owned&& other) noexcept { - if (ABSL_PREDICT_TRUE(this != &other)) { - value_ = std::exchange(other.value_, nullptr); - owner_ = std::move(other.owner_); - } - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Owned& operator=(const Owned& other) noexcept { - value_ = other.value_; - owner_ = other.owner_; - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Owned& operator=(Owned&& other) noexcept { - value_ = std::exchange(other.value_, nullptr); - owner_ = std::move(other.owner_); - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Owned& operator=(Borrowed other) noexcept; - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Owned& operator=(Unique&& other) { - owner_.reset(); - other.Release(&value_, &owner_); - return *this; - } - - // NOLINTNEXTLINE(google-explicit-constructor) - Owned& operator=(std::nullptr_t) noexcept { - reset(); - return *this; - } - - T& operator*() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_DCHECK(static_cast(*this)); - return *get(); - } - - T* absl_nonnull operator->() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_DCHECK(static_cast(*this)); - return get(); - } - - void reset() noexcept { - value_ = nullptr; - owner_.reset(); - } - - google::protobuf::Arena* absl_nullable arena() const noexcept { return owner_.arena(); } - - explicit operator bool() const noexcept { return get() != nullptr; } - - friend void swap(Owned& lhs, Owned& rhs) noexcept { - using std::swap; - swap(lhs.value_, rhs.value_); - swap(lhs.owner_, rhs.owner_); - } - - private: - friend class Owner; - friend class Borrower; - template - friend class Owned; - template - friend class Borrowed; - template - friend struct Ownable; - template - friend Owned AllocateShared(Allocator<> allocator, Args&&... args); - template - friend Owned WrapShared(U* object, Allocator<> allocator); - template - friend Owned common_internal::WrapEternal(const U* value); - friend struct std::pointer_traits>; - friend struct ArenaTraits>; - - Owned(T* value, Owner owner) noexcept - : value_(value), owner_(std::move(owner)) {} - - T* get() const noexcept { return value_; } - - T* value_ = nullptr; - Owner owner_; -}; - -template -Owned(T*) -> Owned; -template -Owned(Unique) -> Owned; -template -Owned(Owner, T*) -> Owned; -template -Owned(Borrowed) -> Owned; - -} // namespace cel - -namespace std { - -template -struct pointer_traits> { - using pointer = cel::Owned; - using element_type = typename cel::Owned::element_type; - using difference_type = ptrdiff_t; - - template - using rebind = cel::Owned; - - static element_type* to_address(const pointer& p) noexcept { - return p.value_; - } -}; - -} // namespace std - -namespace cel { - -template -struct ArenaTraits> { - static bool trivially_destructible(const Owned& owned) { - return ArenaTraits<>::trivially_destructible(owned.owner_); - } -}; - -template -Owner::Owner(const Owned& owned) noexcept : Owner(owned.owner_) {} - -template -Owner::Owner(Owned&& owned) noexcept : Owner(std::move(owned.owner_)) { - owned.value_ = nullptr; -} - -template -Owner& Owner::operator=(const Owned& owned) noexcept { - *this = owned.owner_; - return *this; -} - -template -Owner& Owner::operator=(Owned&& owned) noexcept { - *this = std::move(owned.owner_); - owned.value_ = nullptr; - return *this; -} - -template -bool operator==(const Owned& lhs, std::nullptr_t) noexcept { - return !static_cast(lhs); -} - -template -bool operator==(std::nullptr_t, const Owned& rhs) noexcept { - return rhs == nullptr; -} - -template -bool operator!=(const Owned& lhs, std::nullptr_t) noexcept { - return !operator==(lhs, nullptr); -} - -template -bool operator!=(std::nullptr_t, const Owned& rhs) noexcept { - return !operator==(nullptr, rhs); -} - -template -Owned AllocateShared(Allocator<> allocator, Args&&... args) { - using U = std::remove_cv_t; - static_assert(!std::is_reference_v, "T must not be a reference"); - static_assert(!std::is_array_v, "T must not be an array"); - - U* object; - Owner owner; - if (google::protobuf::Arena* absl_nullable arena = allocator.arena(); - arena != nullptr) { - object = ArenaAllocator(arena).template new_object( - std::forward(args)...); - owner.ptr_ = reinterpret_cast(arena) | - common_internal::kMetadataOwnerArenaBit; - } else { - const common_internal::ReferenceCount* refcount; - std::tie(object, refcount) = common_internal::MakeEmplacedReferenceCount( - std::forward(args)...); - owner.ptr_ = reinterpret_cast(refcount) | - common_internal::kMetadataOwnerReferenceCountBit; - } - return Owned(object, std::move(owner)); -} - -template -Owned WrapShared(T* object, Allocator<> allocator) { - Owner owner; - if (object == nullptr) { - } else if (allocator.arena() != nullptr) { - owner.ptr_ = reinterpret_cast( - static_cast(allocator.arena())) | - common_internal::kMetadataOwnerArenaBit; - } else { - owner.ptr_ = reinterpret_cast( - common_internal::MakeDeletingReferenceCount(object)) | - common_internal::kMetadataOwnerReferenceCountBit; - } - return Owned(object, std::move(owner)); -} - -template -std::enable_if_t, Owned> WrapShared(T* object) { - return WrapShared(object, object->GetArena()); -} - -namespace common_internal { - -template -Owned WrapEternal(const T* value) { - return Owned(value, Owner::None()); -} - -} // namespace common_internal - -// `Borrowed` points to an object which was allocated using `Allocator<>` or -// `Allocator`. It has no ownership over the object, and is only valid so -// long as one or more owners of the object exist. `T` must meet the named -// requirement `ArenaConstructable`. -template -class Borrowed final { - public: - using element_type = T; - - static_assert(!std::is_array_v, "T must not be an array"); - static_assert(!std::is_reference_v, "T must not be a reference"); - static_assert(!std::is_volatile_v, "T must not be volatile qualified"); - static_assert(!std::is_void_v, "T must not be void"); - - Borrowed() = default; - Borrowed(const Borrowed&) = default; - Borrowed(Borrowed&&) = default; - Borrowed& operator=(const Borrowed&) = default; - Borrowed& operator=(Borrowed&&) = default; - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed(const Borrowed& other) noexcept - : Borrowed(other.value_, other.borrower_) {} - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed(Borrowed&& other) noexcept - : Borrowed(other.value_, other.borrower_) {} - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed(const Owned& other ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept - : Borrowed(other.value_, other.owner_) {} - - Borrowed(Borrower borrower, T* ptr) noexcept : Borrowed(ptr, borrower) {} - - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed(std::nullptr_t) noexcept : Borrowed() {} - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed& operator=(const Borrowed& other) noexcept { - value_ = other.value_; - borrower_ = other.borrower_; - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed& operator=(Borrowed&& other) noexcept { - value_ = other.value_; - borrower_ = other.borrower_; - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed& operator=( - const Owned& other ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept { - value_ = other.value_; - borrower_ = other.borrower_; - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed& operator=(Owned&&) = delete; - - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed& operator=(std::nullptr_t) noexcept { - reset(); - return *this; - } - - T& operator*() const noexcept { - ABSL_DCHECK(static_cast(*this)); - return *get(); - } - - T* absl_nonnull operator->() const noexcept { - ABSL_DCHECK(static_cast(*this)); - return get(); - } - - void reset() noexcept { - value_ = nullptr; - borrower_.reset(); - } - - google::protobuf::Arena* absl_nullable arena() const noexcept { - return borrower_.arena(); - } - - explicit operator bool() const noexcept { return get() != nullptr; } - - private: - friend class Owner; - friend class Borrower; - template - friend class Owned; - template - friend class Borrowed; - template - friend struct Borrowable; - friend struct std::pointer_traits>; - - constexpr Borrowed(T* value, Borrower borrower) noexcept - : value_(value), borrower_(borrower) {} - - T* get() const noexcept { return value_; } - - T* value_ = nullptr; - Borrower borrower_; -}; - -template -Borrowed(T*) -> Borrowed; -template -Borrowed(Borrower, T*) -> Borrowed; -template -Borrowed(Owned) -> Borrowed; - -} // namespace cel - -namespace std { - -template -struct pointer_traits> { - using pointer = cel::Borrowed; - using element_type = typename cel::Borrowed::element_type; - using difference_type = ptrdiff_t; - - template - using rebind = cel::Borrowed; - - static element_type* to_address(pointer p) noexcept { return p.value_; } -}; - -} // namespace std - -namespace cel { - -template -Owner::Owner(Borrowed borrowed) noexcept : Owner(borrowed.borrower_) {} - -template -Borrower::Borrower(const Owned& owned ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept - : Borrower(owned.owner_) {} - -template -Borrower::Borrower(Borrowed borrowed) noexcept - : Borrower(borrowed.borrower_) {} - -template -Borrower& Borrower::operator=( - const Owned& owned ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept { - *this = owned.owner_; - return *this; -} - -template -Borrower& Borrower::operator=(Borrowed borrowed) noexcept { - *this = borrowed.borrower_; - return *this; -} - -template -bool operator==(Borrowed lhs, std::nullptr_t) noexcept { - return !static_cast(lhs); -} - -template -bool operator==(std::nullptr_t, Borrowed rhs) noexcept { - return rhs == nullptr; -} - -template -bool operator!=(Borrowed lhs, std::nullptr_t) noexcept { - return !operator==(lhs, nullptr); -} - -template -bool operator!=(std::nullptr_t, Borrowed rhs) noexcept { - return !operator==(nullptr, rhs); -} - -template -template -Owned::Owned(Borrowed other) noexcept - : Owned(other.value_, Owner(other.borrower_)) {} - -template -template -Owned& Owned::operator=(Borrowed other) noexcept { - value_ = other.value_; - owner_ = Owner(other.borrower_); - return *this; -} - -// `Ownable` is a mixin for enabling the ability to get `Owned` that refer to -// this. -template -struct Ownable { - protected: - Owned Own() const noexcept { - static_assert(std::is_base_of_v, "T must be derived from Data"); - const T* const that = static_cast(this); - return Owned( - Owner(Owner::Own(static_cast(that)->owner_)), that); - } - - Owned Own() noexcept { - static_assert(std::is_base_of_v, "T must be derived from Data"); - T* const that = static_cast(this); - return Owned(Owner(Owner::Own(static_cast(that)->owner_)), that); - } - - ABSL_DEPRECATED("Use Own") - Owned shared_from_this() const noexcept { return Own(); } - - ABSL_DEPRECATED("Use Own") - Owned shared_from_this() noexcept { return Own(); } -}; - -// `Borrowable` is a mixin for enabling the ability to get `Borrowed` that -// refer to this. -template -struct Borrowable { - protected: - Borrowed Borrow() const noexcept { - static_assert(std::is_base_of_v, "T must be derived from Data"); - const T* const that = static_cast(this); - return Borrowed(Borrower(static_cast(that)->owner_), - that); - } - - Borrowed Borrow() noexcept { - static_assert(std::is_base_of_v, "T must be derived from Data"); - T* const that = static_cast(this); - return Borrowed(Borrower(static_cast(that)->owner_), that); - } -}; // `ReferenceCountingMemoryManager` is a `MemoryManager` which employs automatic // memory management through reference counting. -class ReferenceCountingMemoryManager final { +class ABSL_DEPRECATED("Do not use") ReferenceCountingMemoryManager final { public: ReferenceCountingMemoryManager(const ReferenceCountingMemoryManager&) = delete; @@ -1324,7 +74,7 @@ class ReferenceCountingMemoryManager final { // `PoolingMemoryManager` is a `MemoryManager` which employs automatic // memory management through memory pooling. -class PoolingMemoryManager final { +class ABSL_DEPRECATED("Do not use") PoolingMemoryManager final { public: PoolingMemoryManager(const PoolingMemoryManager&) = delete; PoolingMemoryManager(PoolingMemoryManager&&) = delete; @@ -1400,17 +150,6 @@ class PoolingMemoryManager final { // resources. class MemoryManager final { public: - // Returns a `MemoryManager` which utilizes an arena but never frees its - // memory. It is effectively a memory leak and should only be used for limited - // use cases, such as initializing singletons which live for the life of the - // program. - static MemoryManager Unmanaged(); - - // Returns a `MemoryManager` which utilizes reference counting. - ABSL_MUST_USE_RESULT static MemoryManager ReferenceCounting() { - return MemoryManager(nullptr); - } - // Returns a `MemoryManager` which utilizes an arena. ABSL_MUST_USE_RESULT static MemoryManager Pooling( google::protobuf::Arena* absl_nonnull arena) { diff --git a/common/memory_test.cc b/common/memory_test.cc deleted file mode 100644 index 7f3e7a82a..000000000 --- a/common/memory_test.cc +++ /dev/null @@ -1,466 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This header contains primitives for reference counting, roughly equivalent to -// the primitives used to implement `std::shared_ptr`. These primitives should -// not be used directly in most cases, instead `cel::ManagedMemory` should be -// used instead. - -#include "common/memory.h" - -#include - -#include "google/protobuf/struct.pb.h" -#include "absl/base/nullability.h" -#include "common/allocator.h" -#include "common/data.h" -#include "common/internal/reference_count.h" -#include "internal/testing.h" -#include "google/protobuf/arena.h" - -#ifdef ABSL_HAVE_EXCEPTIONS -#include -#endif - -namespace cel { -namespace { - -using ::testing::IsFalse; -using ::testing::IsNull; -using ::testing::IsTrue; -using ::testing::TestParamInfo; -using ::testing::TestWithParam; - -TEST(Owner, None) { - EXPECT_THAT(Owner::None(), IsFalse()); - EXPECT_THAT(Owner::None().arena(), IsNull()); -} - -TEST(Owner, Allocator) { - google::protobuf::Arena arena; - EXPECT_THAT(Owner::Allocator(NewDeleteAllocator<>{}), IsFalse()); - EXPECT_THAT(Owner::Allocator(ArenaAllocator<>{&arena}), IsTrue()); -} - -TEST(Owner, Arena) { - google::protobuf::Arena arena; - EXPECT_THAT(Owner::Arena(&arena), IsTrue()); - EXPECT_EQ(Owner::Arena(&arena).arena(), &arena); -} - -TEST(Owner, ReferenceCount) { - auto* refcount = new common_internal::ReferenceCounted(); - EXPECT_THAT(Owner::ReferenceCount(refcount), IsTrue()); - EXPECT_THAT(Owner::ReferenceCount(refcount).arena(), IsNull()); - common_internal::StrongUnref(refcount); -} - -TEST(Owner, Equality) { - google::protobuf::Arena arena1; - google::protobuf::Arena arena2; - EXPECT_EQ(Owner::None(), Owner::None()); - EXPECT_EQ(Owner::Allocator(NewDeleteAllocator<>{}), Owner::None()); - EXPECT_EQ(Owner::Arena(&arena1), Owner::Arena(&arena1)); - EXPECT_NE(Owner::Arena(&arena1), Owner::None()); - EXPECT_NE(Owner::None(), Owner::Arena(&arena1)); - EXPECT_NE(Owner::Arena(&arena1), Owner::Arena(&arena2)); - EXPECT_EQ(Owner::Allocator(ArenaAllocator<>{&arena1}), Owner::Arena(&arena1)); -} - -TEST(Borrower, None) { - EXPECT_THAT(Borrower::None(), IsFalse()); - EXPECT_THAT(Borrower::None().arena(), IsNull()); -} - -TEST(Borrower, Allocator) { - google::protobuf::Arena arena; - EXPECT_THAT(Borrower::Allocator(NewDeleteAllocator<>{}), IsFalse()); - EXPECT_THAT(Borrower::Allocator(ArenaAllocator<>{&arena}), IsTrue()); -} - -TEST(Borrower, Arena) { - google::protobuf::Arena arena; - EXPECT_THAT(Borrower::Arena(&arena), IsTrue()); - EXPECT_EQ(Borrower::Arena(&arena).arena(), &arena); -} - -TEST(Borrower, ReferenceCount) { - auto* refcount = new common_internal::ReferenceCounted(); - EXPECT_THAT(Borrower::ReferenceCount(refcount), IsTrue()); - EXPECT_THAT(Borrower::ReferenceCount(refcount).arena(), IsNull()); - common_internal::StrongUnref(refcount); -} - -TEST(Borrower, Equality) { - google::protobuf::Arena arena1; - google::protobuf::Arena arena2; - EXPECT_EQ(Borrower::None(), Borrower::None()); - EXPECT_EQ(Borrower::Allocator(NewDeleteAllocator<>{}), Borrower::None()); - EXPECT_EQ(Borrower::Arena(&arena1), Borrower::Arena(&arena1)); - EXPECT_NE(Borrower::Arena(&arena1), Borrower::None()); - EXPECT_NE(Borrower::None(), Borrower::Arena(&arena1)); - EXPECT_NE(Borrower::Arena(&arena1), Borrower::Arena(&arena2)); - EXPECT_EQ(Borrower::Allocator(ArenaAllocator<>{&arena1}), - Borrower::Arena(&arena1)); -} - -TEST(OwnerBorrower, CopyConstruct) { - auto* refcount = new common_internal::ReferenceCounted(); - Owner owner1 = Owner::ReferenceCount(refcount); - common_internal::StrongUnref(refcount); - Owner owner2(owner1); - Borrower borrower(owner1); - EXPECT_EQ(owner1, owner2); - EXPECT_EQ(owner1, borrower); - EXPECT_EQ(borrower, owner1); -} - -TEST(OwnerBorrower, MoveConstruct) { - auto* refcount = new common_internal::ReferenceCounted(); - Owner owner1 = Owner::ReferenceCount(refcount); - common_internal::StrongUnref(refcount); - Owner owner2(std::move(owner1)); - Borrower borrower(owner2); - EXPECT_EQ(owner2, borrower); - EXPECT_EQ(borrower, owner2); -} - -TEST(OwnerBorrower, CopyAssign) { - auto* refcount = new common_internal::ReferenceCounted(); - Owner owner1 = Owner::ReferenceCount(refcount); - common_internal::StrongUnref(refcount); - Owner owner2; - owner2 = owner1; - Borrower borrower(owner1); - EXPECT_EQ(owner1, owner2); - EXPECT_EQ(owner1, borrower); - EXPECT_EQ(borrower, owner1); -} - -TEST(OwnerBorrower, MoveAssign) { - auto* refcount = new common_internal::ReferenceCounted(); - Owner owner1 = Owner::ReferenceCount(refcount); - common_internal::StrongUnref(refcount); - Owner owner2; - owner2 = std::move(owner1); - Borrower borrower(owner2); - EXPECT_EQ(owner2, borrower); - EXPECT_EQ(borrower, owner2); -} - -TEST(Unique, ToAddress) { - Unique unique; - EXPECT_EQ(cel::to_address(unique), nullptr); - unique = AllocateUnique(NewDeleteAllocator<>{}); - EXPECT_EQ(cel::to_address(unique), unique.operator->()); -} - -class OwnedTest : public TestWithParam { - public: - Allocator<> GetAllocator() { - switch (GetParam()) { - case AllocatorKind::kArena: - return ArenaAllocator<>{&arena_}; - case AllocatorKind::kNewDelete: - return NewDeleteAllocator<>{}; - } - } - - private: - google::protobuf::Arena arena_; -}; - -TEST_P(OwnedTest, Default) { - Owned owned; - EXPECT_FALSE(owned); - EXPECT_EQ(cel::to_address(owned), nullptr); - EXPECT_FALSE(owned != nullptr); - EXPECT_FALSE(nullptr != owned); -} - -class TestData final : public Data { - public: - using InternalArenaConstructable_ = void; - using DestructorSkippable_ = void; - - TestData() noexcept : Data() {} - - explicit TestData(google::protobuf::Arena* absl_nullable arena) noexcept - : Data(arena) {} -}; - -TEST_P(OwnedTest, AllocateSharedData) { - auto owned = AllocateShared(GetAllocator()); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - EXPECT_EQ(Owner(owned).arena(), GetAllocator().arena()); - EXPECT_EQ(Borrower(owned).arena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, AllocateSharedMessageLite) { - auto owned = AllocateShared(GetAllocator()); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - EXPECT_EQ(Owner(owned).arena(), GetAllocator().arena()); - EXPECT_EQ(Borrower(owned).arena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, WrapSharedData) { - auto owned = - WrapShared(google::protobuf::Arena::Create(GetAllocator().arena())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - EXPECT_EQ(Owner(owned).arena(), GetAllocator().arena()); - EXPECT_EQ(Borrower(owned).arena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, WrapSharedMessageLite) { - auto owned = WrapShared( - google::protobuf::Arena::Create(GetAllocator().arena())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - EXPECT_EQ(Owner(owned).arena(), GetAllocator().arena()); - EXPECT_EQ(Borrower(owned).arena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, SharedFromUniqueData) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - EXPECT_EQ(Owner(owned).arena(), GetAllocator().arena()); - EXPECT_EQ(Borrower(owned).arena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, SharedFromUniqueMessageLite) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - EXPECT_EQ(Owner(owned).arena(), GetAllocator().arena()); - EXPECT_EQ(Borrower(owned).arena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, CopyConstruct) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned copied_owned(owned); - EXPECT_EQ(copied_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, MoveConstruct) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned moved_owned(std::move(owned)); - EXPECT_EQ(moved_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, CopyConstructOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned copied_owned(owned); - EXPECT_EQ(copied_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, MoveConstructOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned moved_owned(std::move(owned)); - EXPECT_EQ(moved_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, ConstructBorrowed) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned borrowed_owned(Borrowed{owned}); - EXPECT_EQ(borrowed_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, ConstructOwner) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned owner_owned(Owner(owned), cel::to_address(owned)); - EXPECT_EQ(owner_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, ConstructNullPtr) { - Owned owned(nullptr); - EXPECT_EQ(owned, nullptr); -} - -TEST_P(OwnedTest, CopyAssign) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned copied_owned; - copied_owned = owned; - EXPECT_EQ(copied_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, MoveAssign) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned moved_owned; - moved_owned = std::move(owned); - EXPECT_EQ(moved_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, CopyAssignOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned copied_owned; - copied_owned = owned; - EXPECT_EQ(copied_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, MoveAssignOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned moved_owned; - moved_owned = std::move(owned); - EXPECT_EQ(moved_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, AssignBorrowed) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned borrowed_owned; - borrowed_owned = Borrowed{owned}; - EXPECT_EQ(borrowed_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, AssignUnique) { - Owned owned; - owned = AllocateUnique(GetAllocator()); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, AssignNullPtr) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - EXPECT_TRUE(owned); - owned = nullptr; - EXPECT_FALSE(owned); -} - -INSTANTIATE_TEST_SUITE_P(OwnedTest, OwnedTest, - ::testing::Values(AllocatorKind::kArena, - AllocatorKind::kNewDelete)); - -class BorrowedTest : public TestWithParam { - public: - Allocator<> GetAllocator() { - switch (GetParam()) { - case AllocatorKind::kArena: - return ArenaAllocator<>{&arena_}; - case AllocatorKind::kNewDelete: - return NewDeleteAllocator<>{}; - } - } - - private: - google::protobuf::Arena arena_; -}; - -TEST_P(BorrowedTest, Default) { - Borrowed borrowed; - EXPECT_FALSE(borrowed); - EXPECT_EQ(cel::to_address(borrowed), nullptr); - EXPECT_FALSE(borrowed != nullptr); - EXPECT_FALSE(nullptr != borrowed); -} - -TEST_P(BorrowedTest, CopyConstruct) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed copied_borrowed(borrowed); - EXPECT_EQ(copied_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, MoveConstruct) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed moved_borrowed(std::move(borrowed)); - EXPECT_EQ(moved_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, CopyConstructOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed copied_borrowed(borrowed); - EXPECT_EQ(copied_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, MoveConstructOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed moved_borrowed(std::move(borrowed)); - EXPECT_EQ(moved_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, ConstructNullPtr) { - Borrowed borrowed(nullptr); - EXPECT_FALSE(borrowed); -} - -TEST_P(BorrowedTest, CopyAssign) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed copied_borrowed; - copied_borrowed = borrowed; - EXPECT_EQ(copied_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, MoveAssign) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed moved_borrowed; - moved_borrowed = std::move(borrowed); - EXPECT_EQ(moved_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, CopyAssignOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed copied_borrowed; - copied_borrowed = borrowed; - EXPECT_EQ(copied_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, MoveAssignOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed moved_borrowed; - moved_borrowed = std::move(borrowed); - EXPECT_EQ(moved_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, AssignOwned) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Borrowed borrowed = owned; - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, AssignNullPtr) { - Borrowed borrowed; - borrowed = nullptr; - EXPECT_FALSE(borrowed); -} - -INSTANTIATE_TEST_SUITE_P(BorrowedTest, BorrowedTest, - ::testing::Values(AllocatorKind::kArena, - AllocatorKind::kNewDelete)); - -} // namespace -} // namespace cel diff --git a/common/memory_testing.h b/common/memory_testing.h index 37244dd8f..7c06de298 100644 --- a/common/memory_testing.h +++ b/common/memory_testing.h @@ -39,8 +39,6 @@ class ThreadCompatibleMemoryTest MemoryManagerRef memory_manager() { switch (memory_management()) { case MemoryManagement::kReferenceCounting: - return MemoryManager::ReferenceCounting(); - break; case MemoryManagement::kPooling: if (!arena_) { arena_.emplace(); diff --git a/common/optional_ref.h b/common/optional_ref.h index c7ba580fc..f7f21b7a1 100644 --- a/common/optional_ref.h +++ b/common/optional_ref.h @@ -79,6 +79,7 @@ class optional_ref final { constexpr bool has_value() const { return value_ != nullptr; } + ABSL_DEPRECATE_AND_INLINE() constexpr explicit operator bool() const { return has_value(); } constexpr T& value() const { diff --git a/common/signature.cc b/common/signature.cc new file mode 100644 index 000000000..3f4ea8d29 --- /dev/null +++ b/common/signature.cc @@ -0,0 +1,640 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/signature.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/types/optional.h" +#include "common/ast.h" +#include "common/type.h" +#include "common/type_spec_resolver.h" +#include "internal/status_macros.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" + +namespace cel { + +// Signature generator helper functions. +namespace { + +void AppendEscaped(std::string* result, std::string_view str, bool escape_dot) { + for (char c : str) { + switch (c) { + case '\\': + case '(': + case ')': + case '<': + case '>': + case '"': + case ',': + case '~': + result->push_back('\\'); + break; + case '.': + if (escape_dot) { + result->push_back('\\'); + } + break; + } + result->push_back(c); + } +} + +absl::Status AppendTypeDesc(std::string* result, const TypeSpec& type_spec); + +absl::Status AppendTypeSpecList(std::string* result, + const std::vector& params) { + if (!params.empty()) { + result->push_back('<'); + for (size_t i = 0; i < params.size(); ++i) { + CEL_RETURN_IF_ERROR(AppendTypeDesc(result, params[i])); + if (i < params.size() - 1) { + result->push_back(','); + } + } + result->push_back('>'); + } + return absl::OkStatus(); +} + +absl::Status AppendTypeDesc(std::string* result, const TypeSpec& type_spec) { + if (type_spec.has_null()) { + absl::StrAppend(result, "null"); + } else if (type_spec.has_dyn()) { + absl::StrAppend(result, "dyn"); + } else if (type_spec.has_primitive()) { + switch (type_spec.primitive()) { + case PrimitiveType::kBool: + absl::StrAppend(result, "bool"); + break; + case PrimitiveType::kInt64: + absl::StrAppend(result, "int"); + break; + case PrimitiveType::kUint64: + absl::StrAppend(result, "uint"); + break; + case PrimitiveType::kDouble: + absl::StrAppend(result, "double"); + break; + case PrimitiveType::kString: + absl::StrAppend(result, "string"); + break; + case PrimitiveType::kBytes: + absl::StrAppend(result, "bytes"); + break; + default: + return absl::InvalidArgumentError("Unsupported primitive type"); + } + } else if (type_spec.has_well_known()) { + switch (type_spec.well_known()) { + case WellKnownTypeSpec::kAny: + absl::StrAppend(result, "any"); + break; + case WellKnownTypeSpec::kTimestamp: + absl::StrAppend(result, "timestamp"); + break; + case WellKnownTypeSpec::kDuration: + absl::StrAppend(result, "duration"); + break; + default: + return absl::InvalidArgumentError("Unsupported well-known type"); + } + } else if (type_spec.has_wrapper()) { + switch (type_spec.wrapper()) { + case PrimitiveType::kBool: + absl::StrAppend(result, "bool_wrapper"); + break; + case PrimitiveType::kInt64: + absl::StrAppend(result, "int_wrapper"); + break; + case PrimitiveType::kUint64: + absl::StrAppend(result, "uint_wrapper"); + break; + case PrimitiveType::kDouble: + absl::StrAppend(result, "double_wrapper"); + break; + case PrimitiveType::kString: + absl::StrAppend(result, "string_wrapper"); + break; + case PrimitiveType::kBytes: + absl::StrAppend(result, "bytes_wrapper"); + break; + default: + return absl::InvalidArgumentError("Unsupported wrapper type"); + } + } else if (type_spec.has_list_type()) { + absl::StrAppend(result, "list<"); + if (type_spec.list_type().elem_type().is_specified()) { + CEL_RETURN_IF_ERROR( + AppendTypeDesc(result, type_spec.list_type().elem_type())); + } else { + absl::StrAppend(result, "dyn"); + } + result->push_back('>'); + } else if (type_spec.has_map_type()) { + absl::StrAppend(result, "map<"); + if (type_spec.map_type().key_type().is_specified()) { + CEL_RETURN_IF_ERROR( + AppendTypeDesc(result, type_spec.map_type().key_type())); + } else { + absl::StrAppend(result, "dyn"); + } + result->push_back(','); + if (type_spec.map_type().value_type().is_specified()) { + CEL_RETURN_IF_ERROR( + AppendTypeDesc(result, type_spec.map_type().value_type())); + } else { + absl::StrAppend(result, "dyn"); + } + result->push_back('>'); + } else if (type_spec.has_function()) { + absl::StrAppend(result, "function<"); + if (type_spec.function().result_type().is_specified()) { + CEL_RETURN_IF_ERROR( + AppendTypeDesc(result, type_spec.function().result_type())); + } else { + absl::StrAppend(result, "dyn"); + } + for (const auto& arg : type_spec.function().arg_types()) { + result->push_back(','); + CEL_RETURN_IF_ERROR(AppendTypeDesc(result, arg)); + } + result->push_back('>'); + } else if (type_spec.has_type()) { + absl::StrAppend(result, "type"); + result->push_back('<'); + CEL_RETURN_IF_ERROR(AppendTypeDesc(result, type_spec.type())); + result->push_back('>'); + } else if (type_spec.has_type_param()) { + absl::StrAppend(result, "~"); + AppendEscaped(result, type_spec.type_param().type(), /*escape_dot=*/true); + } else if (type_spec.has_abstract_type()) { + AppendEscaped(result, type_spec.abstract_type().name(), + /*escape_dot=*/false); + CEL_RETURN_IF_ERROR(AppendTypeSpecList( + result, type_spec.abstract_type().parameter_types())); + } else if (type_spec.has_message_type()) { + AppendEscaped(result, type_spec.message_type().type(), + /*escape_dot=*/false); + } else { + return absl::InvalidArgumentError(absl::StrCat( + "Unsupported type in signature: ", FormatTypeSpec(type_spec))); + } + return absl::OkStatus(); +} +} // namespace + +absl::StatusOr MakeTypeSignature(const Type& type) { + std::string result; + CEL_ASSIGN_OR_RETURN(TypeSpec type_spec, ConvertTypeToTypeSpec(type)); + CEL_RETURN_IF_ERROR(AppendTypeDesc(&result, type_spec)); + return result; +} + +absl::StatusOr MakeTypeSpecSignature(const TypeSpec& type_spec) { + std::string result; + CEL_RETURN_IF_ERROR(AppendTypeDesc(&result, type_spec)); + return result; +} + +absl::StatusOr MakeOverloadSignature( + std::string_view function_name, const std::vector& args, + bool is_member) { + std::vector arg_type_specs; + arg_type_specs.reserve(args.size()); + for (const auto& arg : args) { + CEL_ASSIGN_OR_RETURN(TypeSpec type_spec, ConvertTypeToTypeSpec(arg)); + arg_type_specs.push_back(type_spec); + } + return MakeOverloadSignature(function_name, arg_type_specs, is_member); +} + +absl::StatusOr MakeOverloadSignature( + std::string_view function_name, const std::vector& args, + bool is_member) { + std::string result; + if (is_member) { + if (!args.empty()) { + CEL_RETURN_IF_ERROR(AppendTypeDesc(&result, args[0])); + } else { + return absl::InvalidArgumentError("Member function with no receiver"); + } + result.push_back('.'); + } + AppendEscaped(&result, function_name, /*escape_dot=*/true); + result.push_back('('); + for (size_t i = is_member ? 1 : 0; i < args.size(); ++i) { + CEL_RETURN_IF_ERROR(AppendTypeDesc(&result, args[i])); + if (i < args.size() - 1) { + result.push_back(','); + } + } + result.push_back(')'); + + return result; +} + +// Signature parser helper functions. +namespace { + +std::string StripUnescapedWhitespace(std::string_view str) { + std::string result; + result.reserve(str.size()); + bool escaped = false; + for (char c : str) { + if (escaped) { + result.push_back(c); + escaped = false; + continue; + } + if (c == '\\') { + result.push_back(c); + escaped = true; + continue; + } + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + continue; + } + result.push_back(c); + } + return result; +} + +absl::optional ParseBuiltinOrWrapper(std::string_view name_str) { + if (name_str == "null") return TypeSpec(NullTypeSpec()); + if (name_str == "bool") return TypeSpec(PrimitiveType::kBool); + if (name_str == "int") return TypeSpec(PrimitiveType::kInt64); + if (name_str == "uint") return TypeSpec(PrimitiveType::kUint64); + if (name_str == "double") return TypeSpec(PrimitiveType::kDouble); + if (name_str == "string") return TypeSpec(PrimitiveType::kString); + if (name_str == "bytes") return TypeSpec(PrimitiveType::kBytes); + if (name_str == "any" || name_str == "google.protobuf.Any") + return TypeSpec(WellKnownTypeSpec::kAny); + if (name_str == "timestamp" || name_str == "google.protobuf.Timestamp") + return TypeSpec(WellKnownTypeSpec::kTimestamp); + if (name_str == "duration" || name_str == "google.protobuf.Duration") + return TypeSpec(WellKnownTypeSpec::kDuration); + if (name_str == "dyn" || name_str == "google.protobuf.Value") + return TypeSpec(DynTypeSpec()); + + // Handle standard Protobuf well-known wrapper types to preserve + // backward compatibility for users migrating YAML configuration files. + if (name_str == "bool_wrapper" || name_str == "google.protobuf.BoolValue") + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBool)); + if (name_str == "int_wrapper" || name_str == "google.protobuf.Int64Value" || + name_str == "google.protobuf.Int32Value") + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kInt64)); + if (name_str == "uint_wrapper" || name_str == "google.protobuf.UInt64Value" || + name_str == "google.protobuf.UInt32Value") + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kUint64)); + if (name_str == "double_wrapper" || + name_str == "google.protobuf.DoubleValue" || + name_str == "google.protobuf.FloatValue") + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kDouble)); + if (name_str == "string_wrapper" || name_str == "google.protobuf.StringValue") + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kString)); + if (name_str == "bytes_wrapper" || name_str == "google.protobuf.BytesValue") + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBytes)); + + if (name_str == "google.protobuf.ListValue") { + return TypeSpec(ListTypeSpec(std::make_unique(DynTypeSpec()))); + } + if (name_str == "google.protobuf.Struct") { + return TypeSpec( + MapTypeSpec(std::make_unique(PrimitiveType::kString), + std::make_unique(DynTypeSpec()))); + } + + return std::nullopt; +} + +std::string Unescape(std::string_view str) { + size_t first_escape = str.find('\\'); + if (first_escape == std::string_view::npos) { + return std::string(str); + } + std::string result; + result.reserve(str.size()); + result.append(str.substr(0, first_escape)); + bool escaped = false; + for (size_t i = first_escape; i < str.size(); ++i) { + char c = str[i]; + if (escaped) { + result.push_back(c); + escaped = false; + } else if (c == '\\') { + escaped = true; + } else { + result.push_back(c); + } + } + if (escaped) { + result.push_back('\\'); + } + return result; +} + +class SignatureScanner { + public: + explicit SignatureScanner(std::string_view input, + std::string_view error_prefix = "Invalid signature") + : input_(input), error_prefix_(error_prefix) {} + + absl::StatusOr FindTopLevelChar(char target, bool find_last = false) { + size_t found_idx = std::string_view::npos; + int nesting = 0; + bool escaped = false; + // Scanning str for delimiter boundaries while ensuring + // brackets are balanced and escape backslashes are bypassed. + for (size_t i = 0; i < input_.size(); ++i) { + char c = input_[i]; + if (escaped) { + escaped = false; + continue; + } + if (c == '\\') { + escaped = true; + continue; + } + if (c == target && nesting == 0) { + if (find_last || found_idx == std::string_view::npos) { + found_idx = i; + } + } + if (c == '<') { + nesting++; + } else if (c == '>') { + nesting--; + if (nesting < 0) { + return absl::InvalidArgumentError( + absl::StrCat(error_prefix_, ": mismatched brackets")); + } + } + } + if (nesting != 0) { + return absl::InvalidArgumentError( + absl::StrCat(error_prefix_, ": mismatched brackets")); + } + return found_idx; + } + + absl::StatusOr> SplitTopLevel(char delimiter) { + std::vector result; + int nesting = 0; + bool escaped = false; + size_t start = 0; + // Scanning str for delimiter while ensuring brackets are balanced and + // escape backslashes are bypassed. + for (size_t i = 0; i < input_.size(); ++i) { + char c = input_[i]; + if (escaped) { + escaped = false; + continue; + } + if (c == '\\') { + escaped = true; + continue; + } + if (c == delimiter && nesting == 0) { + result.push_back(input_.substr(start, i - start)); + start = i + 1; + } + if (c == '<') { + nesting++; + } else if (c == '>') { + nesting--; + if (nesting < 0) { + return absl::InvalidArgumentError( + absl::StrCat(error_prefix_, ": mismatched brackets")); + } + } + } + if (nesting != 0) { + return absl::InvalidArgumentError( + absl::StrCat(error_prefix_, ": mismatched brackets")); + } + result.push_back(input_.substr(start)); + return result; + } + + private: + std::string_view input_; + std::string_view error_prefix_; +}; + +absl::StatusOr> SplitTypeList( + std::string_view params) { + return SignatureScanner(params, "Invalid type signature").SplitTopLevel(','); +} + +absl::StatusOr ParseTypeSignature(std::string_view signature) { + if (signature.empty()) { + return absl::InvalidArgumentError("Empty type signature"); + } + + if (signature[0] == '~') { + std::string_view param_name = signature.substr(1); + if (param_name.empty()) { + return absl::InvalidArgumentError( + "Invalid type signature: invalid type parameter name"); + } + CEL_ASSIGN_OR_RETURN(size_t less_idx, + SignatureScanner(param_name) + .FindTopLevelChar('<', /*find_last=*/false)); + CEL_ASSIGN_OR_RETURN(size_t comma_idx, + SignatureScanner(param_name) + .FindTopLevelChar(',', /*find_last=*/false)); + if (less_idx != std::string_view::npos || + comma_idx != std::string_view::npos) { + return absl::InvalidArgumentError( + "Invalid type signature: invalid type parameter name"); + } + return TypeSpec(ParamTypeSpec(Unescape(param_name))); + } + + CEL_ASSIGN_OR_RETURN(size_t less_idx, + SignatureScanner(signature, "Invalid type signature") + .FindTopLevelChar('<', /*find_last=*/false)); + + std::string name_str; + std::vector params; + + if (less_idx != std::string_view::npos) { + // If the signature contains a '<', it must also contain a matching '>'. + if (signature.back() != '>') { + return absl::InvalidArgumentError( + "Invalid type signature: missing closing >"); + } + name_str = Unescape(signature.substr(0, less_idx)); + std::string_view params_str = + signature.substr(less_idx + 1, signature.size() - less_idx - 2); + CEL_ASSIGN_OR_RETURN(auto param_list, SplitTypeList(params_str)); + for (std::string_view param_str : param_list) { + CEL_ASSIGN_OR_RETURN(auto param, ParseTypeSignature(param_str)); + params.push_back(std::move(param)); + } + } else { + name_str = Unescape(signature); + } + + auto read_param_or_dyn = [¶ms](size_t index) { + auto spec = std::make_unique(DynTypeSpec()); + if (params.size() > index) { + *spec = std::move(params[index]); + } + return spec; + }; + + if (!params.empty()) { + if (ParseBuiltinOrWrapper(name_str).has_value()) { + return absl::InvalidArgumentError( + absl::StrCat("Invalid type signature: ", name_str, + " cannot have type parameters")); + } + } else { + if (auto builtin = ParseBuiltinOrWrapper(name_str); builtin.has_value()) { + return *builtin; + } + } + + if (name_str == "type") { + if (params.size() > 1) { + return absl::InvalidArgumentError( + "Invalid type signature: type expects at most 1 parameter"); + } + return TypeSpec(read_param_or_dyn(0)); + } + + if (name_str == "list") { + if (params.size() > 1) { + return absl::InvalidArgumentError( + "Invalid type signature: list expects at most 1 parameter"); + } + return TypeSpec(ListTypeSpec(read_param_or_dyn(0))); + } + + if (name_str == "map") { + if (!params.empty() && params.size() != 2) { + return absl::InvalidArgumentError( + "Invalid type signature: map expects 0 or 2 parameters"); + } + auto key = read_param_or_dyn(0); + auto value = read_param_or_dyn(1); + return TypeSpec(MapTypeSpec(std::move(key), std::move(value))); + } + + if (name_str == "function") { + auto result_type = read_param_or_dyn(0); + std::vector arg_types; + for (size_t i = 1; i < params.size(); ++i) { + arg_types.push_back(std::move(params[i])); + } + return TypeSpec( + FunctionTypeSpec(std::move(result_type), std::move(arg_types))); + } + + if (name_str.empty() || absl::StrContains(name_str, "..")) { + return absl::InvalidArgumentError( + "Invalid type signature: invalid identifier"); + } + + return TypeSpec(AbstractType(name_str, std::move(params))); +} + +} // namespace + +absl::StatusOr ParseFunctionSignature( + std::string_view signature) { + std::string stripped_sig = StripUnescapedWhitespace(signature); + if (stripped_sig.empty()) { + return absl::InvalidArgumentError("Empty function signature"); + } + + CEL_ASSIGN_OR_RETURN( + size_t paren_idx, + SignatureScanner(stripped_sig, "Invalid function signature") + .FindTopLevelChar('(', /*find_last=*/false)); + + if (paren_idx == std::string_view::npos || stripped_sig.back() != ')') { + return absl::InvalidArgumentError("Invalid function signature"); + } + + std::string_view prefix = std::string_view(stripped_sig).substr(0, paren_idx); + std::string_view args_str = + std::string_view(stripped_sig) + .substr(paren_idx + 1, stripped_sig.size() - paren_idx - 2); + + std::vector arg_types; + ParsedFunctionOverload out; + + CEL_ASSIGN_OR_RETURN(size_t dot_idx, + SignatureScanner(prefix, "Invalid function signature") + .FindTopLevelChar('.', /*find_last=*/true)); + + if (dot_idx != std::string_view::npos) { + out.is_member = true; + std::string_view receiver_str = prefix.substr(0, dot_idx); + std::string_view func_str = prefix.substr(dot_idx + 1); + + CEL_ASSIGN_OR_RETURN(auto receiver_param, ParseTypeSignature(receiver_str)); + arg_types.push_back(std::move(receiver_param)); + out.function_name = Unescape(func_str); + } else { + out.is_member = false; + out.function_name = Unescape(prefix); + } + + if (out.function_name.empty()) { + return absl::InvalidArgumentError( + "Invalid function signature: empty function name"); + } + + if (!args_str.empty()) { + CEL_ASSIGN_OR_RETURN(auto arg_list, SplitTypeList(args_str)); + for (std::string_view arg_str : arg_list) { + CEL_ASSIGN_OR_RETURN(auto arg_param, ParseTypeSignature(arg_str)); + arg_types.push_back(std::move(arg_param)); + } + } + + auto result_type = std::make_unique(DynTypeSpec()); + out.signature_type = + TypeSpec(FunctionTypeSpec(std::move(result_type), std::move(arg_types))); + + return out; +} + +absl::StatusOr ParseTypeSpec(std::string_view signature) { + std::string stripped_sig = StripUnescapedWhitespace(signature); + return ParseTypeSignature(stripped_sig); +} + +absl::StatusOr ParseType(std::string_view signature, google::protobuf::Arena* arena, + const google::protobuf::DescriptorPool& pool) { + CEL_ASSIGN_OR_RETURN(auto type_spec, ParseTypeSpec(signature)); + return cel::ConvertTypeSpecToType(type_spec, pool, arena); +} + +} // namespace cel diff --git a/common/signature.h b/common/signature.h new file mode 100644 index 000000000..777f03439 --- /dev/null +++ b/common/signature.h @@ -0,0 +1,101 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_COMMON_SIGNATURE_H_ +#define THIRD_PARTY_CEL_CPP_COMMON_SIGNATURE_H_ + +#include +#include +#include + +#include "absl/status/statusor.h" +#include "common/ast.h" +#include "common/type.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" + +namespace cel { + +// Generates a signature for a `cel::Type`, which is a string representation of +// the type. +// +// Examples: +// +// - `int` +// - `list` +// - `list>` +absl::StatusOr MakeTypeSignature(const Type& type); + +// Generates a signature for a `cel::TypeSpec`, which is a string +// representation of the type. +// +// Examples: +// +// - `int` +// - `list` +// - `list>` +absl::StatusOr MakeTypeSpecSignature(const TypeSpec& type_spec); + +// Generates a signature for a function overload based on the function name +// and the types of the arguments. If `is_member` is true, the first argument +// type is used as the receiver and is prepended to the function name, followed +// by a dollar sign. +// +// Examples: +// +// - `foo()` +// - `foo(int)` +// - `bar.foo(int)` +// - `foo(int,string)` +// - `foo(list,list)` +// - `bar.foo(list,list>)` +// +// If the function name contains a period, it is escaped with a backslash, e.g. +// `foo.bar` becomes `foo\.bar`. This allows to disambiguate between a member +// function and qualified target type name. +// +absl::StatusOr MakeOverloadSignature( + std::string_view function_name, const std::vector& args, + bool is_member); + +// Generates a signature for a function overload based on the function name +// and the type specs of the arguments. See above for more details. +absl::StatusOr MakeOverloadSignature( + std::string_view function_name, const std::vector& args, + bool is_member); + +// Parses a string type signature directly into a `cel::TypeSpec`. +absl::StatusOr ParseTypeSpec(std::string_view signature); + +// Parses a string type signature directly into a `cel::Type`. +absl::StatusOr ParseType(std::string_view signature, google::protobuf::Arena* arena, + const google::protobuf::DescriptorPool& pool); + +// A parsed function overload signature with the function name, flag for member +// function, and the function signature type. +struct ParsedFunctionOverload { + std::string function_name; + bool is_member = false; + // The function signature type, configured as a `FunctionTypeSpec`. + TypeSpec signature_type; +}; + +// Parses a string function overload signature directly into a +// `cel::TypeSpec` configured as a `FunctionTypeSpec`. +absl::StatusOr ParseFunctionSignature( + std::string_view signature); + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_COMMON_SIGNATURE_H_ diff --git a/common/signature_test.cc b/common/signature_test.cc new file mode 100644 index 000000000..e157d5be0 --- /dev/null +++ b/common/signature_test.cc @@ -0,0 +1,785 @@ +#include "common/signature.h" +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "absl/base/no_destructor.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "common/ast.h" +#include "common/type.h" +#include "common/type_kind.h" +#include "common/type_spec_resolver.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "google/protobuf/arena.h" + +namespace cel { +namespace { + +using ::absl_testing::IsOkAndHolds; +using ::absl_testing::StatusIs; +using ::cel::internal::GetTestingDescriptorPool; +using ::testing::HasSubstr; +using ::testing::ValuesIn; + +google::protobuf::Arena* GetTestArena() { + static absl::NoDestructor arena; + return &*arena; +} + +void VerifyParsedMatchesType(const TypeSpec& parsed, const TypeSpec& expected) { + EXPECT_EQ(parsed, expected); +} +void VerifyTypesEqual(const Type& lhs, const Type& rhs) { + EXPECT_EQ(lhs.kind(), rhs.kind()); + if (lhs.kind() != rhs.kind()) return; + + if (lhs.kind() == TypeKind::kOpaque || lhs.kind() == TypeKind::kStruct || + lhs.kind() == TypeKind::kTypeParam) { + EXPECT_EQ(lhs.name(), rhs.name()); + } + + const auto& lhs_params = lhs.GetParameters(); + const auto& rhs_params = rhs.GetParameters(); + EXPECT_EQ(lhs_params.size(), rhs_params.size()); + if (lhs_params.size() == rhs_params.size()) { + for (size_t i = 0; i < lhs_params.size(); ++i) { + VerifyTypesEqual(lhs_params[i], rhs_params[i]); + } + } +} + +struct TypeSignatureTestCase { + TypeSpec type; + std::string expected_signature; + std::string expected_error; +}; + +using TypeSignatureTest = testing::TestWithParam; + +TEST_P(TypeSignatureTest, TypeSignature) { + const auto& param = GetParam(); + + absl::StatusOr signature = MakeTypeSpecSignature(param.type); + if (!param.expected_error.empty()) { + EXPECT_THAT(signature, StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr(param.expected_error))); + } else { + EXPECT_THAT(signature, IsOkAndHolds(param.expected_signature)); + + absl::StatusOr type = ConvertTypeSpecToType( + param.type, *GetTestingDescriptorPool(), GetTestArena()); + ASSERT_THAT(type, ::absl_testing::IsOk()); + EXPECT_THAT(MakeTypeSignature(*type), + IsOkAndHolds(param.expected_signature)); + } +} + +std::vector GetTypeSignatureTestCases() { + return { + { + .type = TypeSpec(NullTypeSpec{}), + .expected_signature = "null", + }, + { + .type = TypeSpec(PrimitiveType::kBool), + .expected_signature = "bool", + }, + { + .type = TypeSpec(PrimitiveType::kInt64), + .expected_signature = "int", + }, + { + .type = TypeSpec(PrimitiveType::kUint64), + .expected_signature = "uint", + }, + { + .type = TypeSpec(PrimitiveType::kDouble), + .expected_signature = "double", + }, + { + .type = TypeSpec(PrimitiveType::kString), + .expected_signature = "string", + }, + { + .type = TypeSpec(PrimitiveType::kBytes), + .expected_signature = "bytes", + }, + { + .type = TypeSpec( + MessageTypeSpec("cel.expr.conformance.proto3.TestAllTypes")), + .expected_signature = "cel.expr.conformance.proto3.TestAllTypes", + }, + { + .type = TypeSpec( + AbstractType("cel.expr.conformance.proto3.TestAllTypes", {})), + .expected_signature = "cel.expr.conformance.proto3.TestAllTypes", + }, + { + .type = TypeSpec(WellKnownTypeSpec::kDuration), + .expected_signature = "duration", + }, + { + .type = TypeSpec(WellKnownTypeSpec::kTimestamp), + .expected_signature = "timestamp", + }, + { + .type = TypeSpec( + ListTypeSpec(std::make_unique(PrimitiveType::kString))), + .expected_signature = "list", + }, + { + .type = TypeSpec( + ListTypeSpec(std::make_unique(ParamTypeSpec("A")))), + .expected_signature = "list<~A>", + }, + { + .type = TypeSpec( + ListTypeSpec(std::make_unique(ParamTypeSpec("A(ParamTypeSpec(R"(a,b..(d)\e)")))), + .expected_signature = R"(list<~a\,b\.\\.\(d\)\\e>)", + }, + { + .type = TypeSpec( + MapTypeSpec(std::make_unique(PrimitiveType::kInt64), + std::make_unique(DynTypeSpec()))), + .expected_signature = "map", + }, + { + .type = TypeSpec( + MapTypeSpec(std::make_unique(ParamTypeSpec("B")), + std::make_unique(ParamTypeSpec("C")))), + .expected_signature = "map<~B,~C>", + }, + { + .type = TypeSpec(MapTypeSpec( + std::make_unique(PrimitiveType::kInt64), nullptr)), + .expected_signature = "map", + }, + { + .type = TypeSpec(MapTypeSpec(nullptr, nullptr)), + .expected_signature = "map", + }, + { + .type = TypeSpec(std::make_unique(PrimitiveType::kInt64)), + .expected_signature = "type", + }, + { + .type = TypeSpec(WellKnownTypeSpec::kAny), + .expected_signature = "any", + }, + { + .type = TypeSpec(DynTypeSpec{}), + .expected_signature = "dyn", + }, + { + .type = TypeSpec(AbstractType( + "bar", {TypeSpec(FunctionTypeSpec( + std::make_unique(ParamTypeSpec("D")), + {TypeSpec(PrimitiveType::kString), + TypeSpec(PrimitiveType::kBool)}))})), + .expected_signature = "bar>", + }, + { + .type = + TypeSpec(AbstractType("bar", {TypeSpec(PrimitiveType::kInt64), + TypeSpec(PrimitiveType::kString)})), + .expected_signature = "bar", + }, + { + .type = TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBool)), + .expected_signature = "bool_wrapper", + }, + { + .type = TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kInt64)), + .expected_signature = "int_wrapper", + }, + { + .type = TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kUint64)), + .expected_signature = "uint_wrapper", + }, + { + .type = TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kDouble)), + .expected_signature = "double_wrapper", + }, + { + .type = TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kString)), + .expected_signature = "string_wrapper", + }, + { + .type = TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBytes)), + .expected_signature = "bytes_wrapper", + }, + { + .type = TypeSpec( + FunctionTypeSpec(nullptr, {TypeSpec(PrimitiveType::kInt64)})), + .expected_signature = "function", + }, + { + .type = TypeSpec(FunctionTypeSpec( + std::make_unique(PrimitiveType::kInt64), {})), + .expected_signature = "function", + }, + { + .type = TypeSpec(FunctionTypeSpec(nullptr, {})), + .expected_signature = "function", + }, + }; +} + +INSTANTIATE_TEST_SUITE_P(TypeSignatureTest, TypeSignatureTest, + ValuesIn(GetTypeSignatureTestCases())); + +TEST(TypeSignatureTest, UnsupportedTypes) { + EXPECT_THAT(MakeTypeSignature(UnknownType{}), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unsupported Type kind: *unknown*"))); + + EXPECT_THAT(MakeTypeSignature(ErrorType{}), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unsupported type in signature: *error*"))); + + EXPECT_THAT(MakeTypeSpecSignature(TypeSpec(static_cast(999))), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unsupported primitive type"))); + + EXPECT_THAT( + MakeTypeSpecSignature(TypeSpec(static_cast(999))), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unsupported well-known type"))); + + EXPECT_THAT(MakeTypeSpecSignature(TypeSpec( + PrimitiveTypeWrapper(static_cast(999)))), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unsupported wrapper type"))); +} + +TEST_P(TypeSignatureTest, ParseTypeCheck) { + const auto& param = GetParam(); + if (!param.expected_signature.empty() && param.expected_error.empty()) { + auto parsed = ParseType(param.expected_signature, GetTestArena(), + *GetTestingDescriptorPool()); + ASSERT_THAT(parsed, ::absl_testing::IsOk()); + ASSERT_OK_AND_ASSIGN( + auto expected_type, + ConvertTypeSpecToType(param.type, *GetTestingDescriptorPool(), + GetTestArena())); + VerifyTypesEqual(*parsed, expected_type); + } +} + +struct OverloadSignatureTestCase { + std::string function_name = "hello"; + std::vector args; + bool is_member = false; + std::string expected_signature; + std::string expected_error; +}; + +using OverloadSignatureTest = testing::TestWithParam; + +TEST_P(OverloadSignatureTest, OverloadSignature) { + const auto& param = GetParam(); + + absl::StatusOr signature = + MakeOverloadSignature(param.function_name, param.args, param.is_member); + if (!param.expected_error.empty()) { + EXPECT_THAT(signature, StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr(param.expected_error))); + } else { + EXPECT_THAT(signature, IsOkAndHolds(param.expected_signature)); + } +} + +std::vector GetOverloadSignatureTestCases() { + return { + { + .args = {TypeSpec(PrimitiveType::kString)}, + .expected_signature = "hello(string)", + }, + { + .args = {TypeSpec(PrimitiveType::kInt64), + TypeSpec(PrimitiveType::kUint64)}, + .expected_signature = "hello(int,uint)", + }, + { + .args = {TypeSpec(ListTypeSpec( + std::make_unique(PrimitiveType::kString)))}, + .expected_signature = "hello(list)", + }, + { + .args = {TypeSpec( + ListTypeSpec(std::make_unique(ParamTypeSpec("A"))))}, + .expected_signature = "hello(list<~A>)", + }, + { + .args = {TypeSpec( + MapTypeSpec(std::make_unique(PrimitiveType::kInt64), + std::make_unique(DynTypeSpec())))}, + .expected_signature = "hello(map)", + }, + { + .args = {TypeSpec( + MapTypeSpec(std::make_unique(ParamTypeSpec("B")), + std::make_unique(ParamTypeSpec("C"))))}, + .expected_signature = "hello(map<~B,~C>)", + }, + + { + .args = {TypeSpec(AbstractType( + "bar", + {TypeSpec(FunctionTypeSpec( + std::make_unique(ParamTypeSpec("D")), {}))}))}, + .expected_signature = "hello(bar>)", + }, + { + .args = {TypeSpec(WellKnownTypeSpec::kAny)}, + .expected_signature = "hello(any)", + }, + { + .args = {TypeSpec(WellKnownTypeSpec::kDuration)}, + .expected_signature = "hello(duration)", + }, + { + .args = {TypeSpec(WellKnownTypeSpec::kTimestamp)}, + .expected_signature = "hello(timestamp)", + }, + { + .args = {TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBool))}, + .expected_signature = "hello(bool_wrapper)", + }, + { + .args = {TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kInt64))}, + .expected_signature = "hello(int_wrapper)", + }, + { + .args = {TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kUint64))}, + .expected_signature = "hello(uint_wrapper)", + }, + { + .args = {TypeSpec( + AbstractType("cel.expr.conformance.proto3.TestAllTypes", {}))}, + .expected_signature = + "hello(cel.expr.conformance.proto3.TestAllTypes)", + }, + { + .args = {TypeSpec(PrimitiveType::kString)}, + .is_member = true, + .expected_signature = "string.hello()", + }, + { + .args = {TypeSpec(PrimitiveType::kString), + TypeSpec(ListTypeSpec( + std::make_unique(PrimitiveType::kBool)))}, + .is_member = true, + .expected_signature = "string.hello(list)", + }, + { + .args = {TypeSpec(PrimitiveType::kString), + TypeSpec(PrimitiveType::kBool), TypeSpec(DynTypeSpec())}, + .is_member = true, + .expected_signature = "string.hello(bool,dyn)", + }, + { + .function_name = "hello", + .args = {TypeSpec( + AbstractType("bar", {TypeSpec(ParamTypeSpec("dummy.type"))}))}, + .is_member = true, + .expected_signature = R"(bar<~dummy\.type>.hello())", + }, + { + .function_name = "inspect", + .args = {TypeSpec( + std::make_unique(PrimitiveType::kString))}, + .expected_signature = "inspect(type)", + }, + { + .function_name = R"(h.(e),l\o)", + .args = {TypeSpec(PrimitiveType::kString), + TypeSpec(ListTypeSpec(std::make_unique( + ParamTypeSpec(R"(a,b..(d)\e)"))))}, + .is_member = true, + .expected_signature = + R"(string.h\.\(e\)\,l\\\o(list<~a\,b\.\\.\(d\)\\e>))", + }, + }; +} + +TEST(OverloadSignatureTest, MemberFunctionNoReceiverError) { + auto signature = + MakeOverloadSignature("hello", std::vector{}, true); + EXPECT_THAT(signature, + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Member function with no receiver"))); +} + +INSTANTIATE_TEST_SUITE_P(OverloadIdTest, OverloadSignatureTest, + ValuesIn(GetOverloadSignatureTestCases())); + +TEST_P(OverloadSignatureTest, ExhaustiveFunctionParseCheck) { + const auto& param = GetParam(); + if (!param.expected_signature.empty()) { + auto parsed = ParseFunctionSignature(param.expected_signature); + ASSERT_THAT(parsed, ::absl_testing::IsOk()); + EXPECT_EQ(parsed->function_name, param.function_name); + EXPECT_EQ(parsed->is_member, param.is_member); + EXPECT_TRUE(parsed->signature_type.has_function()); + const auto& func = parsed->signature_type.function(); + for (size_t i = 0; i < param.args.size(); ++i) { + VerifyParsedMatchesType(func.arg_types()[i], param.args[i]); + } + } +} + +TEST(ParseSignatureTest, ProtoParsing) { + ASSERT_OK_AND_ASSIGN( + auto t1, ParseType("int", GetTestArena(), *GetTestingDescriptorPool())); + EXPECT_TRUE(t1.IsInt()); + + ASSERT_OK_AND_ASSIGN(auto t2, ParseType("list<~A>", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(t2.IsList()); + + ASSERT_OK_AND_ASSIGN(auto t3, ParseType(R"(~abc\)", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(t3.IsTypeParam()); + EXPECT_EQ(t3.GetTypeParam().name(), R"(abc\)"); + + ASSERT_OK_AND_ASSIGN(auto w1, + ParseType("google.protobuf.BoolValue", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w1.IsBoolWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w2, + ParseType("google.protobuf.Int64Value", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w2.IsIntWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w3, + ParseType("google.protobuf.Int32Value", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w3.IsIntWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w4, + ParseType("google.protobuf.UInt64Value", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w4.IsUintWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w5, + ParseType("google.protobuf.UInt32Value", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w5.IsUintWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w6, + ParseType("google.protobuf.DoubleValue", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w6.IsDoubleWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w7, + ParseType("google.protobuf.FloatValue", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w7.IsDoubleWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w8, + ParseType("google.protobuf.StringValue", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w8.IsStringWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w9, + ParseType("google.protobuf.BytesValue", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w9.IsBytesWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w10, ParseType("string_wrapper", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w10.IsStringWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w11, ParseType("bytes_wrapper", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w11.IsBytesWrapper()); + + ASSERT_OK_AND_ASSIGN(auto gp_any, + ParseType("google.protobuf.Any", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(gp_any.IsAny()); + + ASSERT_OK_AND_ASSIGN(auto gp_timestamp, + ParseType("google.protobuf.Timestamp", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(gp_timestamp.IsTimestamp()); + + ASSERT_OK_AND_ASSIGN(auto gp_duration, + ParseType("google.protobuf.Duration", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(gp_duration.IsDuration()); + + ASSERT_OK_AND_ASSIGN(auto gp_value, + ParseType("google.protobuf.Value", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(gp_value.IsDyn()); + + ASSERT_OK_AND_ASSIGN(auto gp_list_value, + ParseType("google.protobuf.ListValue", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(gp_list_value.IsList()); + + ASSERT_OK_AND_ASSIGN(auto gp_struct, + ParseType("google.protobuf.Struct", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(gp_struct.IsMap()); + + // Legal whitespace handling tests + ASSERT_OK_AND_ASSIGN(auto ws_type1, + ParseType("map < int , string > ", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(ws_type1.IsMap()); + + ASSERT_OK_AND_ASSIGN(auto ws_type2, + ParseType("map\t<\nint\r,\tstring\n>\r", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(ws_type2.IsMap()); +} + +TEST(ParseSignatureTest, FunctionParsing) { + ASSERT_OK_AND_ASSIGN(auto f1, ParseFunctionSignature("hello(string)")); + EXPECT_TRUE(f1.signature_type.has_function()); + EXPECT_EQ(f1.signature_type.function().arg_types().size(), 1); + + // Legal whitespace handling tests + ASSERT_OK_AND_ASSIGN(auto ws_func1, + ParseFunctionSignature(" hello ( string ) ")); + EXPECT_TRUE(ws_func1.signature_type.has_function()); + EXPECT_EQ(ws_func1.signature_type.function().arg_types().size(), 1); + + ASSERT_OK_AND_ASSIGN(auto ws_func2, + ParseFunctionSignature("\thello\n(\rstring\t)\n\r")); + EXPECT_TRUE(ws_func2.signature_type.has_function()); + EXPECT_EQ(ws_func2.signature_type.function().arg_types().size(), 1); + + ASSERT_OK_AND_ASSIGN(auto f2, ParseFunctionSignature("a.b.c()")); + EXPECT_TRUE(f2.is_member); + EXPECT_EQ(f2.function_name, "c"); +} + +TEST(ParseSignatureTest, ParsingErrors) { + // Mismatched template brackets and parentheses. + EXPECT_THAT( + ParseType("list>", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + EXPECT_THAT( + ParseType("list", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + EXPECT_THAT(ParseType("list><", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + EXPECT_THAT(ParseFunctionSignature("hello(list>)"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + EXPECT_THAT(ParseFunctionSignature("hello(list)"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + EXPECT_THAT(ParseFunctionSignature("foo"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid function signature"))); + EXPECT_THAT( + ParseType("list b < c>", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + + // Parameter count validations for list, map and type types. + EXPECT_THAT(ParseType("list", GetTestArena(), + *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("list expects at most 1 parameter"))); + EXPECT_THAT( + ParseType("map", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("map expects 0 or 2 parameters"))); + EXPECT_THAT(ParseType("map", GetTestArena(), + *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("map expects 0 or 2 parameters"))); + EXPECT_THAT(ParseType("type", GetTestArena(), + *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("type expects at most 1 parameter"))); + + // Invalid parameter name validations. + EXPECT_THAT(ParseType("~", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("invalid type parameter name"))); + EXPECT_THAT(ParseType("~A", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("invalid type parameter name"))); + + // Enforcing valid function and identifier names. + EXPECT_THAT(ParseFunctionSignature("()"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("empty function name"))); + EXPECT_THAT(ParseFunctionSignature("string.()"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("empty function name"))); + + // Missing closing operators and boundary checks. + EXPECT_THAT( + ParseType("listfoo", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("missing closing >"))); + + EXPECT_THAT(ParseFunctionSignature("hello>(string)"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + EXPECT_THAT( + ParseType("list<", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + + EXPECT_THAT(ParseType("map", GetTestArena(), + *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + + EXPECT_THAT(ParseType("map int, string>", GetTestArena(), + *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + + EXPECT_THAT(ParseType("list", GetTestArena(), + *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid type signature"))); + + EXPECT_THAT(ParseFunctionSignature("a..b.c()"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid type signature"))); + EXPECT_THAT( + ParseType("list", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Empty type signature"))); + + EXPECT_THAT( + ParseType("~list", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid type signature"))); + + // Checks that builtin types cannot have type parameters. + EXPECT_THAT( + ParseType("int", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("cannot have type parameters"))); +} + +TEST(ParseSignatureTest, MessageTypeWithParamsError) { + EXPECT_THAT(ParseType("cel.expr.conformance.proto3.TestAllTypes", + GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("cannot have type parameters"))); +} + +TEST(ParseSignatureTest, MissingClosingParenthesisError) { + EXPECT_THAT(ParseFunctionSignature("hello(string"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid function signature"))); + EXPECT_THAT(ParseFunctionSignature("hello)"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid function signature"))); +} + +TEST(ParseSignatureTest, NestedDotsNonMember) { + auto f1 = ParseFunctionSignature( + "my_opaque()"); + ASSERT_THAT(f1, ::absl_testing::IsOk()); + EXPECT_FALSE(f1->is_member); + EXPECT_EQ(f1->function_name, + "my_opaque"); +} + +TEST(ParseSignatureTest, OverlyComplexSignatures) { + auto t1 = ParseType("map>,map>>", + GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t1, ::absl_testing::IsOk()); + EXPECT_TRUE(t1->IsMap()); + + auto t2 = ParseType(R"(~abc\\)", GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t2, ::absl_testing::IsOk()); + EXPECT_TRUE(t2->IsTypeParam()); + EXPECT_EQ(t2->GetTypeParam().name(), R"(abc\)"); + + auto t3 = + ParseType(R"(~abc\\\\)", GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t3, ::absl_testing::IsOk()); + EXPECT_TRUE(t3->IsTypeParam()); + EXPECT_EQ(t3->GetTypeParam().name(), R"(abc\\)"); + + auto f1 = ParseFunctionSignature( + "bar>,map>.func(string)"); + ASSERT_THAT(f1, ::absl_testing::IsOk()); + EXPECT_TRUE(f1->is_member); + EXPECT_EQ(f1->function_name, "func"); + EXPECT_TRUE(f1->signature_type.has_function()); + EXPECT_EQ(f1->signature_type.function().arg_types().size(), 2); +} + +TEST(ParseSignatureTest, EmptyOrWhitespaceErrors) { + EXPECT_THAT(ParseType("", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Empty type signature"))); + EXPECT_THAT(ParseFunctionSignature(""), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Empty function signature"))); + EXPECT_THAT(ParseType("list>", GetTestArena(), + *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Empty type signature"))); +} + +TEST(OverloadSignatureTest, ArgumentTypeVector) { + std::vector args; + args.push_back(Type(IntType())); + args.push_back(Type(StringType())); + args.push_back(Type(ListType(GetTestArena(), IntType()))); + args.push_back( + Type(MessageType(GetTestingDescriptorPool()->FindMessageTypeByName( + "cel.expr.conformance.proto3.TestAllTypes")))); + args.push_back(Type(OpaqueType(GetTestArena(), "Foo", {TypeParamType("T")}))); + ASSERT_OK_AND_ASSIGN(auto sig, MakeOverloadSignature("foo", args, false)); + EXPECT_EQ(sig, + "foo(int,string,list,cel.expr.conformance.proto3.TestAllTypes," + "Foo<~T>)"); +} + +} // namespace +} // namespace cel diff --git a/common/source.cc b/common/source.cc index 8c32ad6ba..0d770b228 100644 --- a/common/source.cc +++ b/common/source.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -162,10 +163,17 @@ namespace common_internal { class SourceImpl : public Source { public: + struct OffsetEntry { + int32_t byte_offset; + SourcePosition codepoint; + }; + SourceImpl(std::string description, - absl::InlinedVector line_offsets) + absl::InlinedVector line_offsets, + std::vector byte_offsets = {}) : description_(std::move(description)), - line_offsets_(std::move(line_offsets)) {} + line_offsets_(std::move(line_offsets)), + byte_offsets_(std::move(byte_offsets)) {} absl::string_view description() const final { return description_; } @@ -173,9 +181,50 @@ class SourceImpl : public Source { return absl::MakeConstSpan(line_offsets_); } + absl::optional GetPositionFromByteOffset( + SourcePosition byte_offset) const final { + // handle out of bounds and all ascii. + if (byte_offset < 0) { + return absl::nullopt; + } + if (byte_offsets_.empty()) { + if (byte_offset > content().size()) { + return absl::nullopt; + } + return byte_offset; + } + if (byte_offset > byte_offsets_.back().byte_offset) { + return absl::nullopt; + } + + // binary search for the next alignment shift after the given byte offset. + auto it = std::upper_bound( + byte_offsets_.begin(), byte_offsets_.end(), byte_offset, + [](SourcePosition val, const OffsetEntry& entry) { + return val < entry.byte_offset; + }); + if (it == byte_offsets_.begin()) { + return byte_offset; + } + const OffsetEntry& prev = *(it - 1); + if (it == byte_offsets_.end()) { + // matched the EOF entry. + return prev.codepoint; + } + + // the total shift at the next multi-byte character. + int32_t next_delta = it->byte_offset - it->codepoint; + + // The offset is either in the previous multi-byte character or in the + // run of ascii characters after it. if negative, then in, otherwise + // shifted down. + return std::max(prev.codepoint, byte_offset - next_delta); + } + private: const std::string description_; const absl::InlinedVector line_offsets_; + const std::vector byte_offsets_; }; namespace { @@ -200,8 +249,9 @@ class Latin1Source final : public SourceImpl { public: Latin1Source(std::string description, absl::InlinedVector line_offsets, - std::vector text) - : SourceImpl(std::move(description), std::move(line_offsets)), + std::vector byte_offsets, std::vector text) + : SourceImpl(std::move(description), std::move(line_offsets), + std::move(byte_offsets)), text_(std::move(text)) {} ContentView content() const override { @@ -216,8 +266,10 @@ class BasicPlaneSource final : public SourceImpl { public: BasicPlaneSource(std::string description, absl::InlinedVector line_offsets, + std::vector byte_offsets, std::vector text) - : SourceImpl(std::move(description), std::move(line_offsets)), + : SourceImpl(std::move(description), std::move(line_offsets), + std::move(byte_offsets)), text_(std::move(text)) {} ContentView content() const override { @@ -232,8 +284,10 @@ class SupplementalPlaneSource final : public SourceImpl { public: SupplementalPlaneSource(std::string description, absl::InlinedVector line_offsets, + std::vector byte_offsets, std::vector text) - : SourceImpl(std::move(description), std::move(line_offsets)), + : SourceImpl(std::move(description), std::move(line_offsets), + std::move(byte_offsets)), text_(std::move(text)) {} ContentView content() const override { @@ -306,12 +360,18 @@ struct SourceTextTraits { template absl::StatusOr NewSourceImpl(std::string description, const T& text, - const size_t text_size) { + const size_t text_size, + const size_t max_codepoints) { if (ABSL_PREDICT_FALSE( text_size > static_cast(std::numeric_limits::max()))) { return absl::InvalidArgumentError("expression larger than 2GiB limit"); } + if ((text_size >> 2) > max_codepoints) { + // If byte size is 4 times the codepoint limit, then definitely exceeded. + return absl::InvalidArgumentError(absl::StrCat( + "expression is larger than codepoint limit ", max_codepoints)); + } using Traits = SourceTextTraits; size_t index = 0; typename Traits::iterator_type it = Traits::Begin(text); @@ -322,7 +382,12 @@ absl::StatusOr NewSourceImpl(std::string description, const T& text, std::vector data16; std::vector data32; absl::InlinedVector line_offsets; + std::vector byte_offsets; while (index < text_size) { + if (offset >= max_codepoints) { + return absl::InvalidArgumentError(absl::StrCat( + "expression is larger than codepoint limit ", max_codepoints)); + } std::tie(code_point, code_units) = cel::internal::Utf8Decode(it); if (ABSL_PREDICT_FALSE(code_point == cel::internal::kUnicodeReplacementCharacter && @@ -339,6 +404,7 @@ absl::StatusOr NewSourceImpl(std::string description, const T& text, ++offset; continue; } + byte_offsets.push_back({static_cast(index), offset}); if (code_point <= 0xff) { data8.reserve(text_size); Traits::AppendTo(data8, text, index); @@ -374,6 +440,10 @@ absl::StatusOr NewSourceImpl(std::string description, const T& text, std::move(description), std::move(line_offsets), Traits::ToVector(text)); latin1: while (index < text_size) { + if (offset >= max_codepoints) { + return absl::InvalidArgumentError(absl::StrCat( + "expression is larger than codepoint limit ", max_codepoints)); + } std::tie(code_point, code_units) = internal::Utf8Decode(it); if (ABSL_PREDICT_FALSE(code_point == internal::kUnicodeReplacementCharacter && @@ -384,6 +454,9 @@ absl::StatusOr NewSourceImpl(std::string description, const T& text, if (code_point == '\n') { line_offsets.push_back(offset + 1); } + if (code_units > 1) { + byte_offsets.push_back({static_cast(index), offset}); + } if (code_point <= 0xff) { data8.push_back(static_cast(code_point)); Traits::Advance(it, code_units); @@ -415,10 +488,16 @@ absl::StatusOr NewSourceImpl(std::string description, const T& text, goto supplemental; } line_offsets.push_back(offset + 1); + byte_offsets.push_back({static_cast(text_size), offset}); return std::make_unique( - std::move(description), std::move(line_offsets), std::move(data8)); + std::move(description), std::move(line_offsets), std::move(byte_offsets), + std::move(data8)); basic: while (index < text_size) { + if (offset >= max_codepoints) { + return absl::InvalidArgumentError(absl::StrCat( + "expression is larger than codepoint limit ", max_codepoints)); + } std::tie(code_point, code_units) = internal::Utf8Decode(it); if (ABSL_PREDICT_FALSE(code_point == internal::kUnicodeReplacementCharacter && @@ -429,6 +508,9 @@ absl::StatusOr NewSourceImpl(std::string description, const T& text, if (code_point == '\n') { line_offsets.push_back(offset + 1); } + if (code_units > 1) { + byte_offsets.push_back({static_cast(index), offset}); + } if (code_point <= 0xffff) { data16.push_back(static_cast(code_point)); Traits::Advance(it, code_units); @@ -448,10 +530,16 @@ absl::StatusOr NewSourceImpl(std::string description, const T& text, goto supplemental; } line_offsets.push_back(offset + 1); + byte_offsets.push_back({static_cast(text_size), offset}); return std::make_unique( - std::move(description), std::move(line_offsets), std::move(data16)); + std::move(description), std::move(line_offsets), std::move(byte_offsets), + std::move(data16)); supplemental: while (index < text_size) { + if (offset >= max_codepoints) { + return absl::InvalidArgumentError(absl::StrCat( + "expression is larger than codepoint limit ", max_codepoints)); + } std::tie(code_point, code_units) = internal::Utf8Decode(it); if (ABSL_PREDICT_FALSE(code_point == internal::kUnicodeReplacementCharacter && @@ -462,14 +550,19 @@ absl::StatusOr NewSourceImpl(std::string description, const T& text, if (code_point == '\n') { line_offsets.push_back(offset + 1); } + if (code_units > 1) { + byte_offsets.push_back({static_cast(index), offset}); + } data32.push_back(code_point); Traits::Advance(it, code_units); index += code_units; ++offset; } line_offsets.push_back(offset + 1); + byte_offsets.push_back({static_cast(text_size), offset}); return std::make_unique( - std::move(description), std::move(line_offsets), std::move(data32)); + std::move(description), std::move(line_offsets), std::move(byte_offsets), + std::move(data32)); } } // namespace @@ -483,26 +576,26 @@ absl::optional Source::GetLocation( return SourceLocation{line_and_offset->first, position - line_and_offset->second}; } - return absl::nullopt; + return std::nullopt; } absl::optional Source::GetPosition( const SourceLocation& location) const { if (ABSL_PREDICT_FALSE(location.line < 1 || location.column < 0)) { - return absl::nullopt; + return std::nullopt; } if (auto position = FindLinePosition(location.line); ABSL_PREDICT_TRUE(position.has_value())) { return *position + location.column; } - return absl::nullopt; + return std::nullopt; } absl::optional Source::Snippet(int32_t line) const { auto content = this->content(); auto start = FindLinePosition(line); if (ABSL_PREDICT_FALSE(!start.has_value() || content.empty())) { - return absl::nullopt; + return std::nullopt; } auto end = FindLinePosition(line + 1); if (end.has_value()) { @@ -554,7 +647,7 @@ std::string Source::DisplayErrorLocation(SourceLocation location) const { absl::optional Source::FindLinePosition(int32_t line) const { if (ABSL_PREDICT_FALSE(line < 1)) { - return absl::nullopt; + return std::nullopt; } if (line == 1) { return SourcePosition{0}; @@ -563,13 +656,13 @@ absl::optional Source::FindLinePosition(int32_t line) const { if (ABSL_PREDICT_TRUE(line <= static_cast(line_offsets.size()))) { return line_offsets[static_cast(line - 2)]; } - return absl::nullopt; + return std::nullopt; } absl::optional> Source::FindLine( SourcePosition position) const { if (ABSL_PREDICT_FALSE(position < 0)) { - return absl::nullopt; + return std::nullopt; } int32_t line = 1; const auto line_offsets = this->line_offsets(); @@ -585,16 +678,72 @@ absl::optional> Source::FindLine( return std::make_pair(line, line_offsets[static_cast(line) - 2]); } +SourceSubrange::SourceSubrange(const Source& source, SourceRange range) + : source_(source), range_(range) { + SourcePosition size = source_.content().size(); + ABSL_DCHECK(range_.begin >= 0); + ABSL_DCHECK(range_.begin <= size); + ABSL_DCHECK(range_.end >= range_.begin); + ABSL_DCHECK(range_.end <= size); + if (range_.begin < 0) { + range_.begin = 0; + } + if (range_.begin > size) { + range_.begin = size; + } + if (range_.end < range_.begin) { + range_.end = range_.begin; + } + if (range_.end > size) { + range_.end = size; + } + for (const auto& line_offset : source_.line_offsets()) { + if (line_offset > range_.begin && line_offset <= range_.end) { + line_offsets_.push_back(line_offset - range_.begin); + } + } + line_offsets_.push_back(range_.end - range_.begin + 1); +} + +SourceContentView SourceSubrange::content() const { + auto parent_content = source_.content(); + if (parent_content.empty() || range_.begin >= range_.end) { + return EmptyContentView(); + } + return absl::visit( + [this](auto view) { + return SourceContentView( + view.subspan(static_cast(range_.begin), + static_cast(range_.end - range_.begin))); + }, + parent_content.view_); +} + +absl::Span SourceSubrange::line_offsets() const { + return absl::MakeConstSpan(line_offsets_); +} + +static size_t ClampLimit(int value) { + if (value < 0) { + return std::numeric_limits::max(); + } + return static_cast(value); +} + absl::StatusOr NewSource(absl::string_view content, - std::string description) { + std::string description, + const SourceOptions& options) { return common_internal::NewSourceImpl(std::move(description), content, - content.size()); + content.size(), + ClampLimit(options.max_codepoint_size)); } absl::StatusOr NewSource(const absl::Cord& content, - std::string description) { + std::string description, + const SourceOptions& options) { return common_internal::NewSourceImpl(std::move(description), content, - content.size()); + content.size(), + ClampLimit(options.max_codepoint_size)); } } // namespace cel diff --git a/common/source.h b/common/source.h index 6453363a8..5f517e0fa 100644 --- a/common/source.h +++ b/common/source.h @@ -22,6 +22,7 @@ #include "absl/base/attributes.h" #include "absl/base/nullability.h" +#include "absl/container/inlined_vector.h" #include "absl/status/statusor.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" @@ -36,6 +37,7 @@ class SourceImpl; } // namespace common_internal class Source; +class SourceSubrange; // SourcePosition represents an offset in source text. using SourcePosition = int32_t; @@ -94,6 +96,7 @@ class SourceContentView final { private: friend class Source; + friend class SourceSubrange; constexpr SourceContentView() = default; @@ -141,6 +144,20 @@ class Source { absl::optional GetPosition( const SourceLocation& location) const; + // Maps a 0-based byte offset in the original UTF-8 source text to a + // `SourcePosition` (0-based code point offset). Returns an empty + // `absl::optional` when `byte_offset` is invalid or the information required + // to perform the mapping is not present. + // + // If `byte_offset` falls within a multi-byte UTF-8 sequence, returns the + // code point position of that sequence. + // + // The offset for the size in bytes is mapped to the size in code points. + virtual absl::optional GetPositionFromByteOffset( + SourcePosition byte_offset) const { + return absl::nullopt; + } + absl::optional Snippet(int32_t line) const; // Formats an annotated snippet highlighting an error at location, e.g. @@ -178,6 +195,7 @@ class Source { private: friend class common_internal::SourceImpl; + friend class SourceSubrange; Source() = default; @@ -187,13 +205,72 @@ class Source { SourcePosition position) const; }; +// `SourceSubrange` is a view of a subrange fo an underlying `Source` object. +// Intended to be used when the CEL expression is embedded in a larger text +// representation (such as a celpolicy). +// +// The parent `Source` must outlive this object. +// +// Note: `GetPositionFromByteOffset` is not supported by `SourceSubrange`. +class SourceSubrange final : public Source { + public: + SourceSubrange(const Source& source ABSL_ATTRIBUTE_LIFETIME_BOUND, + SourceRange range); + + absl::string_view description() const ABSL_ATTRIBUTE_LIFETIME_BOUND override { + return source_.description(); + } + + // Returns a view of the underlying expression text. + ContentView content() const ABSL_ATTRIBUTE_LIFETIME_BOUND override; + + // Returns a `absl::Span` of `SourcePosition` which represent the positions + // where new lines occur. + absl::Span line_offsets() const + ABSL_ATTRIBUTE_LIFETIME_BOUND override; + + private: + const Source& source_; + SourceRange range_; + absl::InlinedVector line_offsets_; +}; + using SourcePtr = std::unique_ptr; -absl::StatusOr NewSource( - absl::string_view content, std::string description = ""); +struct SourceOptions { + // The maximum number of code points allowed in the source. + // A negative value indicates no limit (though still limited by + // int32_t max value). + int max_codepoint_size = 100'000; +}; -absl::StatusOr NewSource( - const absl::Cord& content, std::string description = ""); +absl::StatusOr NewSource(absl::string_view content, + std::string description, + const SourceOptions& options); + +absl::StatusOr NewSource(const absl::Cord& content, + std::string description, + const SourceOptions& options); + +inline absl::StatusOr NewSource( + absl::string_view content, std::string description) { + return NewSource(content, std::move(description), SourceOptions{}); +} + +inline absl::StatusOr NewSource( + const absl::Cord& content, std::string description) { + return NewSource(content, std::move(description), SourceOptions{}); +} + +inline absl::StatusOr NewSource( + absl::string_view content) { + return NewSource(content, "", SourceOptions{}); +} + +inline absl::StatusOr NewSource( + const absl::Cord& content) { + return NewSource(content, "", SourceOptions{}); +} } // namespace cel diff --git a/common/source_test.cc b/common/source_test.cc index 2a3b78893..167d88481 100644 --- a/common/source_test.cc +++ b/common/source_test.cc @@ -14,8 +14,12 @@ #include "common/source.h" +#include +#include + +#include "absl/status/status.h" #include "absl/strings/cord.h" -#include "absl/types/optional.h" +#include "absl/strings/string_view.h" #include "internal/testing.h" namespace cel { @@ -81,37 +85,37 @@ TEST(StringSource, PositionAndLocation) { Optional(Eq(SourceLocation{int32_t{1}, int32_t{2}}))); EXPECT_THAT(source->GetLocation(*end), Optional(Eq(SourceLocation{int32_t{3}, int32_t{2}}))); - EXPECT_THAT(source->GetLocation(-1), Eq(absl::nullopt)); + EXPECT_THAT(source->GetLocation(-1), Eq(std::nullopt)); EXPECT_THAT(source->content().ToString(*start, *end), Eq("d &&\n\t b.c.arg(10) &&\n\t ")); EXPECT_THAT(source->GetPosition(SourceLocation{int32_t{0}, int32_t{0}}), - Eq(absl::nullopt)); + Eq(std::nullopt)); EXPECT_THAT(source->GetPosition(SourceLocation{int32_t{1}, int32_t{-1}}), - Eq(absl::nullopt)); + Eq(std::nullopt)); EXPECT_THAT(source->GetPosition(SourceLocation{int32_t{4}, int32_t{0}}), - Eq(absl::nullopt)); + Eq(std::nullopt)); } TEST(StringSource, SnippetSingle) { ASSERT_OK_AND_ASSIGN(auto source, NewSource("hello, world", "one-line-test")); EXPECT_THAT(source->Snippet(1), Optional(Eq("hello, world"))); - EXPECT_THAT(source->Snippet(2), Eq(absl::nullopt)); + EXPECT_THAT(source->Snippet(2), Eq(std::nullopt)); } TEST(StringSource, SnippetMulti) { ASSERT_OK_AND_ASSIGN(auto source, NewSource("hello\nworld\nmy\nbub\n", "four-line-test")); - EXPECT_THAT(source->Snippet(0), Eq(absl::nullopt)); + EXPECT_THAT(source->Snippet(0), Eq(std::nullopt)); EXPECT_THAT(source->Snippet(1), Optional(Eq("hello"))); EXPECT_THAT(source->Snippet(2), Optional(Eq("world"))); EXPECT_THAT(source->Snippet(3), Optional(Eq("my"))); EXPECT_THAT(source->Snippet(4), Optional(Eq("bub"))); EXPECT_THAT(source->Snippet(5), Optional(Eq(""))); - EXPECT_THAT(source->Snippet(6), Eq(absl::nullopt)); + EXPECT_THAT(source->Snippet(6), Eq(std::nullopt)); } TEST(CordSource, Description) { @@ -150,17 +154,17 @@ TEST(CordSource, PositionAndLocation) { Optional(Eq(SourceLocation{int32_t{1}, int32_t{2}}))); EXPECT_THAT(source->GetLocation(*end), Optional(Eq(SourceLocation{int32_t{3}, int32_t{2}}))); - EXPECT_THAT(source->GetLocation(-1), Eq(absl::nullopt)); + EXPECT_THAT(source->GetLocation(-1), Eq(std::nullopt)); EXPECT_THAT(source->content().ToString(*start, *end), Eq("d &&\n\t b.c.arg(10) &&\n\t ")); EXPECT_THAT(source->GetPosition(SourceLocation{int32_t{0}, int32_t{0}}), - Eq(absl::nullopt)); + Eq(std::nullopt)); EXPECT_THAT(source->GetPosition(SourceLocation{int32_t{1}, int32_t{-1}}), - Eq(absl::nullopt)); + Eq(std::nullopt)); EXPECT_THAT(source->GetPosition(SourceLocation{int32_t{4}, int32_t{0}}), - Eq(absl::nullopt)); + Eq(std::nullopt)); } TEST(CordSource, SnippetSingle) { @@ -168,7 +172,7 @@ TEST(CordSource, SnippetSingle) { NewSource(absl::Cord("hello, world"), "one-line-test")); EXPECT_THAT(source->Snippet(1), Optional(Eq("hello, world"))); - EXPECT_THAT(source->Snippet(2), Eq(absl::nullopt)); + EXPECT_THAT(source->Snippet(2), Eq(std::nullopt)); } TEST(CordSource, SnippetMulti) { @@ -176,13 +180,13 @@ TEST(CordSource, SnippetMulti) { auto source, NewSource(absl::Cord("hello\nworld\nmy\nbub\n"), "four-line-test")); - EXPECT_THAT(source->Snippet(0), Eq(absl::nullopt)); + EXPECT_THAT(source->Snippet(0), Eq(std::nullopt)); EXPECT_THAT(source->Snippet(1), Optional(Eq("hello"))); EXPECT_THAT(source->Snippet(2), Optional(Eq("world"))); EXPECT_THAT(source->Snippet(3), Optional(Eq("my"))); EXPECT_THAT(source->Snippet(4), Optional(Eq("bub"))); EXPECT_THAT(source->Snippet(5), Optional(Eq(""))); - EXPECT_THAT(source->Snippet(6), Eq(absl::nullopt)); + EXPECT_THAT(source->Snippet(6), Eq(std::nullopt)); } TEST(Source, DisplayErrorLocationBasic) { @@ -223,5 +227,197 @@ TEST(Source, DisplayErrorLocationFullWidth) { "\n | .．＾"); } +TEST(SourceSubrange, Description) { + ASSERT_OK_AND_ASSIGN(auto source, NewSource("hello world", "subrange-test")); + SourceSubrange subrange(*source, SourceRange{0, 5}); + EXPECT_THAT(subrange.description(), Eq("subrange-test")); +} + +TEST(SourceSubrange, Content) { + ASSERT_OK_AND_ASSIGN(auto source, NewSource("hello world", "subrange-test")); + SourceSubrange subrange(*source, SourceRange{6, 11}); + EXPECT_THAT(subrange.content().ToString(), Eq("world")); +} + +TEST(SourceSubrange, ContentEmpty) { + ASSERT_OK_AND_ASSIGN(auto source, NewSource("hello world", "subrange-test")); + SourceSubrange subrange(*source, SourceRange{5, 5}); + EXPECT_THAT(subrange.content().ToString(), Eq("")); +} + +TEST(SourceSubrange, LineOffsetsNoNewlines) { + ASSERT_OK_AND_ASSIGN(auto source, + NewSource("hello\nworld\n", "subrange-test")); + SourceSubrange subrange(*source, SourceRange{0, 5}); + EXPECT_THAT(subrange.line_offsets(), ElementsAre(6)); +} + +TEST(SourceSubrange, LineOffsetsWithNewlines) { + ASSERT_OK_AND_ASSIGN(auto source, + NewSource("hello\nworld\ncel", "subrange-test")); + SourceSubrange subrange(*source, SourceRange{0, 11}); + EXPECT_THAT(subrange.line_offsets(), ElementsAre(6, 12)); +} + +TEST(SourceSubrange, LineOffsetsMiddleSubrange) { + ASSERT_OK_AND_ASSIGN(auto source, + NewSource("hello\nworld\ncel\ncpp", "subrange-test")); + SourceSubrange subrange(*source, SourceRange{6, 15}); + EXPECT_THAT(subrange.line_offsets(), ElementsAre(6, 10)); +} + +TEST(StringSource, CodepointLimitExceeded) { + SourceOptions options; + options.max_codepoint_size = 5; + + EXPECT_THAT( + NewSource("123456", "test", options), + ::absl_testing::StatusIs( + absl::StatusCode::kInvalidArgument, + ::testing::HasSubstr("expression is larger than codepoint limit 5"))); + + ASSERT_OK_AND_ASSIGN(auto source, NewSource("12345", "test", options)); + EXPECT_THAT(source->content().ToString(), ::testing::Eq("12345")); +} + +TEST(StringSource, CodepointLimitMultibyteUtf8) { + SourceOptions options; + options.max_codepoint_size = 5; + + // "Ｈｅｌｌｏ" consists of 5 full-width Unicode characters (15 bytes in + // UTF-8). + ASSERT_OK_AND_ASSIGN(auto source, NewSource("Ｈｅｌｌｏ", "test", options)); + EXPECT_THAT(source->content().ToString(), ::testing::Eq("Ｈｅｌｌｏ")); + + options.max_codepoint_size = 4; + EXPECT_THAT( + NewSource("Ｈｅｌｌｏ", "test", options), + ::absl_testing::StatusIs( + absl::StatusCode::kInvalidArgument, + ::testing::HasSubstr("expression is larger than codepoint limit 4"))); +} + +TEST(CordSource, CodepointLimitExceeded) { + SourceOptions options; + options.max_codepoint_size = 5; + + EXPECT_THAT( + NewSource(absl::Cord("123456"), "test", options), + ::absl_testing::StatusIs( + absl::StatusCode::kInvalidArgument, + ::testing::HasSubstr("expression is larger than codepoint limit 5"))); + + ASSERT_OK_AND_ASSIGN(SourcePtr source, + NewSource(absl::Cord("12345"), "test", options)); + EXPECT_THAT(source->content().ToString(), ::testing::Eq("12345")); +} + +TEST(Source, GetPositionFromByteOffsetAscii) { + ASSERT_OK_AND_ASSIGN(SourcePtr empty_source, NewSource("")); + EXPECT_THAT(empty_source->GetPositionFromByteOffset(-1), Eq(std::nullopt)); + EXPECT_THAT(empty_source->GetPositionFromByteOffset(0), Optional(Eq(0))); + EXPECT_THAT(empty_source->GetPositionFromByteOffset(1), Eq(std::nullopt)); + + ASSERT_OK_AND_ASSIGN(SourcePtr source, NewSource("hello")); + EXPECT_THAT(source->GetPositionFromByteOffset(-1), Eq(std::nullopt)); + for (SourcePosition i = 0; i <= 5; ++i) { + EXPECT_THAT(source->GetPositionFromByteOffset(i), Optional(Eq(i))); + } + EXPECT_THAT(source->GetPositionFromByteOffset(6), Eq(std::nullopt)); +} + +TEST(Source, GetPositionFromByteOffsetLatin1) { + // "a£b": 'a' (1 byte: 0), '£' (2 bytes: 1..2), 'b' (1 byte: 3). Total 4 + // bytes, 3 codepoints. + ASSERT_OK_AND_ASSIGN(SourcePtr source, NewSource("a£b")); + EXPECT_THAT(source->GetPositionFromByteOffset(-1), Eq(std::nullopt)); + EXPECT_THAT(source->GetPositionFromByteOffset(0), Optional(Eq(0))); + EXPECT_THAT(source->GetPositionFromByteOffset(1), Optional(Eq(1))); + EXPECT_THAT(source->GetPositionFromByteOffset(2), Optional(Eq(1))); + EXPECT_THAT(source->GetPositionFromByteOffset(3), Optional(Eq(2))); + EXPECT_THAT(source->GetPositionFromByteOffset(4), Optional(Eq(3))); + EXPECT_THAT(source->GetPositionFromByteOffset(5), Eq(std::nullopt)); +} + +TEST(Source, GetPositionFromByteOffsetBasicPlane) { + // "a€b": 'a' (1 byte: 0), '€' (3 bytes: 1..3), 'b' (1 byte: 4). Total 5 + // bytes, 3 codepoints. + ASSERT_OK_AND_ASSIGN(SourcePtr source, NewSource("a€b")); + EXPECT_THAT(source->GetPositionFromByteOffset(-1), Eq(std::nullopt)); + EXPECT_THAT(source->GetPositionFromByteOffset(0), Optional(Eq(0))); + EXPECT_THAT(source->GetPositionFromByteOffset(1), Optional(Eq(1))); + EXPECT_THAT(source->GetPositionFromByteOffset(2), Optional(Eq(1))); + EXPECT_THAT(source->GetPositionFromByteOffset(3), Optional(Eq(1))); + EXPECT_THAT(source->GetPositionFromByteOffset(4), Optional(Eq(2))); + EXPECT_THAT(source->GetPositionFromByteOffset(5), Optional(Eq(3))); + EXPECT_THAT(source->GetPositionFromByteOffset(6), Eq(std::nullopt)); +} + +TEST(Source, GetPositionFromByteOffsetSupplementalPlaneAndMixed) { + // "a£€𐍈b": + // cp 0: 'a' (byte 0, 1 byte) + // cp 1: '£' (bytes 1..2, 2 bytes) + // cp 2: '€' (bytes 3..5, 3 bytes) + // cp 3: '𐍈' (bytes 6..9, 4 bytes) + // cp 4: 'b' (byte 10, 1 byte) + // Total: 11 bytes, 5 codepoints. + constexpr absl::string_view kText = "a£€𐍈b"; + ASSERT_OK_AND_ASSIGN(SourcePtr source, NewSource(kText)); + ASSERT_OK_AND_ASSIGN(SourcePtr cord_source, NewSource(absl::Cord(kText))); + + for (const Source* s : {source.get(), cord_source.get()}) { + EXPECT_THAT(s->GetPositionFromByteOffset(-1), Eq(std::nullopt)); + EXPECT_THAT(s->GetPositionFromByteOffset(0), Optional(Eq(0))); + EXPECT_THAT(s->GetPositionFromByteOffset(1), Optional(Eq(1))); + EXPECT_THAT(s->GetPositionFromByteOffset(2), Optional(Eq(1))); + EXPECT_THAT(s->GetPositionFromByteOffset(3), Optional(Eq(2))); + EXPECT_THAT(s->GetPositionFromByteOffset(4), Optional(Eq(2))); + EXPECT_THAT(s->GetPositionFromByteOffset(5), Optional(Eq(2))); + EXPECT_THAT(s->GetPositionFromByteOffset(6), Optional(Eq(3))); + EXPECT_THAT(s->GetPositionFromByteOffset(7), Optional(Eq(3))); + EXPECT_THAT(s->GetPositionFromByteOffset(8), Optional(Eq(3))); + EXPECT_THAT(s->GetPositionFromByteOffset(9), Optional(Eq(3))); + EXPECT_THAT(s->GetPositionFromByteOffset(10), Optional(Eq(4))); + EXPECT_THAT(s->GetPositionFromByteOffset(11), Optional(Eq(5))); + EXPECT_THAT(s->GetPositionFromByteOffset(12), Eq(std::nullopt)); + } +} + +TEST(Source, GetPositionFromByteOffsetStartsAndEndsWithMultibyte) { + // "£a€b𐍈": + // cp 0: '£' (bytes 0..1, 2 bytes) + // cp 1: 'a' (byte 2, 1 byte) + // cp 2: '€' (bytes 3..5, 3 bytes) + // cp 3: 'b' (byte 6, 1 byte) + // cp 4: '𐍈' (bytes 7..10, 4 bytes) + // Total: 11 bytes, 5 codepoints. + constexpr absl::string_view kText = "£a€b𐍈"; + ASSERT_OK_AND_ASSIGN(SourcePtr source, NewSource(kText)); + ASSERT_OK_AND_ASSIGN(SourcePtr cord_source, NewSource(absl::Cord(kText))); + + for (const Source* s : {source.get(), cord_source.get()}) { + EXPECT_THAT(s->GetPositionFromByteOffset(-1), Eq(std::nullopt)); + EXPECT_THAT(s->GetPositionFromByteOffset(0), Optional(Eq(0))); + EXPECT_THAT(s->GetPositionFromByteOffset(1), Optional(Eq(0))); + EXPECT_THAT(s->GetPositionFromByteOffset(2), Optional(Eq(1))); + EXPECT_THAT(s->GetPositionFromByteOffset(3), Optional(Eq(2))); + EXPECT_THAT(s->GetPositionFromByteOffset(4), Optional(Eq(2))); + EXPECT_THAT(s->GetPositionFromByteOffset(5), Optional(Eq(2))); + EXPECT_THAT(s->GetPositionFromByteOffset(6), Optional(Eq(3))); + EXPECT_THAT(s->GetPositionFromByteOffset(7), Optional(Eq(4))); + EXPECT_THAT(s->GetPositionFromByteOffset(8), Optional(Eq(4))); + EXPECT_THAT(s->GetPositionFromByteOffset(9), Optional(Eq(4))); + EXPECT_THAT(s->GetPositionFromByteOffset(10), Optional(Eq(4))); + EXPECT_THAT(s->GetPositionFromByteOffset(11), Optional(Eq(5))); + EXPECT_THAT(s->GetPositionFromByteOffset(12), Eq(std::nullopt)); + } +} + +TEST(SourceSubrange, GetPositionFromByteOffsetReturnsNullopt) { + ASSERT_OK_AND_ASSIGN(SourcePtr source, NewSource("hello world")); + SourceSubrange subrange(*source, SourceRange{0, 5}); + EXPECT_THAT(subrange.GetPositionFromByteOffset(0), Eq(std::nullopt)); +} + } // namespace } // namespace cel diff --git a/common/standard_definitions.h b/common/standard_definitions.h index eea185f6b..b35e67af2 100644 --- a/common/standard_definitions.h +++ b/common/standard_definitions.h @@ -210,7 +210,7 @@ struct StandardOverloadIds { static constexpr absl::string_view kGreaterEqualsUintInt = "greater_equals_uint64_int64"; static constexpr absl::string_view kGreaterEqualsUintDouble = - "greater_equals_uint_double"; + "greater_equals_uint64_double"; static constexpr absl::string_view kLessEqualsBool = "less_equals_bool"; static constexpr absl::string_view kLessEqualsString = "less_equals_string"; static constexpr absl::string_view kLessEqualsBytes = "less_equals_bytes"; diff --git a/common/type.cc b/common/type.cc index ce8c7a89a..5396bc57f 100644 --- a/common/type.cc +++ b/common/type.cc @@ -31,6 +31,8 @@ #include "common/types/types.h" #include "google/protobuf/descriptor.h" +#undef GetMessage + namespace cel { using ::google::protobuf::Descriptor; @@ -97,7 +99,7 @@ static constexpr std::array kTypeToKindArray = { TypeKind::kUnknown}; static_assert(kTypeToKindArray.size() == - absl::variant_size(), + std::variant_size(), "Kind indexer must match variant declaration for cel::Type."); } // namespace @@ -158,7 +160,7 @@ absl::optional GetOrNullopt(const common_internal::TypeVariant& variant) { if (const auto* alt = absl::get_if(&variant); alt != nullptr) { return *alt; } - return absl::nullopt; + return std::nullopt; } } // namespace @@ -243,7 +245,7 @@ absl::optional Type::AsOptional() const { if (auto maybe_opaque = AsOpaque(); maybe_opaque.has_value()) { return maybe_opaque->AsOptional(); } - return absl::nullopt; + return std::nullopt; } absl::optional Type::AsString() const { @@ -263,7 +265,7 @@ absl::optional Type::AsStruct() const { if (const auto* alt = absl::get_if(&variant_); alt != nullptr) { return *alt; } - return absl::nullopt; + return std::nullopt; } absl::optional Type::AsTimestamp() const { @@ -603,7 +605,12 @@ absl::optional StructTypeField::AsMessage() const { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; +} + +MessageTypeField StructTypeField::GetMessage() const { + ABSL_DCHECK(IsMessage()); + return absl::get(variant_); } StructTypeField::operator MessageTypeField() const { @@ -640,8 +647,6 @@ constexpr absl::string_view kUInt64TypeName = "uint"; constexpr absl::string_view kDoubleTypeName = "double"; constexpr absl::string_view kStringTypeName = "string"; constexpr absl::string_view kBytesTypeName = "bytes"; -constexpr absl::string_view kDurationTypeName = "google.protobuf.Duration"; -constexpr absl::string_view kTimestampTypeName = "google.protobuf.Timestamp"; constexpr absl::string_view kListTypeName = "list"; constexpr absl::string_view kMapTypeName = "map"; constexpr absl::string_view kCelTypeTypeName = "type"; @@ -670,12 +675,6 @@ Type LegacyRuntimeType(absl::string_view name) { if (name == kBytesTypeName) { return BytesType{}; } - if (name == kDurationTypeName) { - return DurationType{}; - } - if (name == kTimestampTypeName) { - return TimestampType{}; - } if (name == kListTypeName) { return ListType{}; } @@ -685,6 +684,53 @@ Type LegacyRuntimeType(absl::string_view name) { if (name == kCelTypeTypeName) { return TypeType{}; } + if (cel::IsWellKnownMessageType(name)) { + if (name == "google.protobuf.Any") { + return AnyType(); + } + if (name == "google.protobuf.BoolValue") { + return BoolWrapperType(); + } + if (name == "google.protobuf.BytesValue") { + return BytesWrapperType(); + } + if (name == "google.protobuf.DoubleValue") { + return DoubleWrapperType(); + } + if (name == "google.protobuf.Duration") { + return DurationType(); + } + if (name == "google.protobuf.FloatValue") { + return DoubleWrapperType(); + } + if (name == "google.protobuf.Int32Value") { + return IntWrapperType(); + } + if (name == "google.protobuf.Int64Value") { + return IntWrapperType(); + } + if (name == "google.protobuf.ListValue") { + return ListType(); + } + if (name == "google.protobuf.StringValue") { + return StringWrapperType(); + } + if (name == "google.protobuf.Struct") { + return JsonMapType(); + } + if (name == "google.protobuf.Timestamp") { + return TimestampType(); + } + if (name == "google.protobuf.UInt32Value") { + return UintWrapperType(); + } + if (name == "google.protobuf.UInt64Value") { + return UintWrapperType(); + } + if (name == "google.protobuf.Value") { + return DynType(); + } + } return common_internal::MakeBasicStructType(name); } diff --git a/common/type.h b/common/type.h index c8851dd4e..a8d116355 100644 --- a/common/type.h +++ b/common/type.h @@ -69,6 +69,10 @@ #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" +// GetMessage in windows API headers might be defined as a macro. +#pragma push_macro("GetMessage") +#undef GetMessage + namespace cel { class Type; @@ -1140,6 +1144,7 @@ class StructTypeField final { } absl::optional AsMessage() const; + MessageTypeField GetMessage() const; explicit operator MessageTypeField() const; @@ -1259,6 +1264,19 @@ inline H AbslHashValue(H state, const MapType& type) { size_t{2}); } +inline bool operator==(const TypeType& lhs, const TypeType& rhs) { + return absl::c_equal(lhs.GetParameters(), rhs.GetParameters()); +} + +template +inline H AbslHashValue(H state, const TypeType& type) { + auto parameters = type.GetParameters(); + for (const auto& parameter : parameters) { + state = H::combine(std::move(state), parameter); + } + return H::combine(std::move(state), parameters.size()); +} + inline bool operator==(const OpaqueType& lhs, const OpaqueType& rhs) { return lhs.name() == rhs.name() && absl::c_equal(lhs.GetParameters(), rhs.GetParameters()); @@ -1299,4 +1317,6 @@ Type LegacyRuntimeType(absl::string_view name); } // namespace cel +#pragma pop_macro("GetMessage") + #endif // THIRD_PARTY_CEL_CPP_COMMON_TYPE_H_ diff --git a/common/type_introspector.cc b/common/type_introspector.cc index 26f53685e..3846ab58b 100644 --- a/common/type_introspector.cc +++ b/common/type_introspector.cc @@ -104,7 +104,7 @@ struct WellKnownType { auto it = std::lower_bound(fields_by_name.begin(), fields_by_name.end(), name, FieldNameComparer{}); if (it == fields_by_name.end() || it->name() != name) { - return absl::nullopt; + return std::nullopt; } return *it; } @@ -114,7 +114,7 @@ struct WellKnownType { auto it = std::lower_bound(fields_by_number.begin(), fields_by_number.end(), number, FieldNumberComparer{}); if (it == fields_by_number.end() || it->number() != number) { - return absl::nullopt; + return std::nullopt; } return *it; } @@ -216,25 +216,25 @@ const WellKnownTypesMap& GetWellKnownTypesMap() { absl::StatusOr> TypeIntrospector::FindTypeImpl( absl::string_view) const { - return absl::nullopt; + return std::nullopt; } absl::StatusOr> TypeIntrospector::FindEnumConstantImpl(absl::string_view, absl::string_view) const { - return absl::nullopt; + return std::nullopt; } absl::StatusOr> TypeIntrospector::FindStructTypeFieldByNameImpl(absl::string_view, absl::string_view) const { - return absl::nullopt; + return std::nullopt; } absl::StatusOr< absl::optional>> TypeIntrospector::ListFieldsForStructTypeImpl(absl::string_view) const { - return absl::nullopt; + return std::nullopt; } absl::optional FindWellKnownType(absl::string_view name) { @@ -242,7 +242,7 @@ absl::optional FindWellKnownType(absl::string_view name) { if (auto it = well_known_types.find(name); it != well_known_types.end()) { return it->second.type; } - return absl::nullopt; + return std::nullopt; } absl::optional FindWellKnownTypeEnumConstant( @@ -251,7 +251,7 @@ absl::optional FindWellKnownTypeEnumConstant( return TypeIntrospector::EnumConstant{ IntType{}, "google.protobuf.NullValue", "NULL_VALUE", 0}; } - return absl::nullopt; + return std::nullopt; } absl::optional FindWellKnownTypeFieldByName( @@ -260,7 +260,7 @@ absl::optional FindWellKnownTypeFieldByName( if (auto it = well_known_types.find(type); it != well_known_types.end()) { return it->second.FieldByName(name); } - return absl::nullopt; + return std::nullopt; } absl::optional> @@ -268,7 +268,7 @@ ListFieldsForWellKnownType(absl::string_view type) { const auto& well_known_types = GetWellKnownTypesMap(); auto it = well_known_types.find(type); if (it == well_known_types.end()) { - return absl::nullopt; + return std::nullopt; } // The fields are not normally gettable. return {}; diff --git a/common/type_kind.h b/common/type_kind.h index 34df8e385..01dcce3bb 100644 --- a/common/type_kind.h +++ b/common/type_kind.h @@ -77,7 +77,7 @@ constexpr Kind TypeKindToKind(TypeKind kind) { return static_cast(static_cast>(kind)); } -constexpr bool KindIsTypeKind(Kind kind ABSL_ATTRIBUTE_UNUSED) { +constexpr bool KindIsTypeKind(Kind kind [[maybe_unused]]) { // Currently all Kind are valid TypeKind. return true; } diff --git a/common/type_proto.cc b/common/type_proto.cc index 66c16689d..b6b66f73a 100644 --- a/common/type_proto.cc +++ b/common/type_proto.cc @@ -71,7 +71,7 @@ absl::optional MaybeWellKnownType(absl::string_view type_name) { return it->second; } - return absl::nullopt; + return std::nullopt; } absl::Status TypeToProtoInternal(const cel::Type& type, diff --git a/common/type_reflector_test.cc b/common/type_reflector_test.cc index f2ff2c322..52cd176ab 100644 --- a/common/type_reflector_test.cc +++ b/common/type_reflector_test.cc @@ -20,7 +20,6 @@ #include "absl/status/status_matchers.h" #include "absl/time/time.h" #include "absl/types/optional.h" -#include "common/casting.h" #include "common/value.h" #include "common/value_testing.h" #include "common/values/list_value.h" @@ -32,6 +31,7 @@ namespace cel { namespace { +using ::absl_testing::IsOk; using ::absl_testing::IsOkAndHolds; using ::absl_testing::StatusIs; using ::cel::test::ErrorValueIs; @@ -178,8 +178,8 @@ TEST_F(TypeReflectorTest, NewMapValueBuilderCoverage_DynamicDynamic) { EXPECT_OK(builder->Put(IntValue(1), IntValue(4))); EXPECT_OK(builder->Put(UintValue(0), IntValue(5))); EXPECT_OK(builder->Put(UintValue(1), IntValue(6))); - EXPECT_OK(builder->Put(StringValue("a"), IntValue(7))); - EXPECT_OK(builder->Put(StringValue("b"), IntValue(8))); + EXPECT_THAT(builder->Put(StringValue::WrapUnsafe("a"), IntValue(7)), IsOk()); + EXPECT_THAT(builder->Put(StringValue::WrapUnsafe("b"), IntValue(8)), IsOk()); EXPECT_EQ(builder->Size(), 8); EXPECT_FALSE(builder->IsEmpty()); auto value = std::move(*builder).Build(); @@ -210,7 +210,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_BoolValue) { internal::GetTestingMessageFactory(), "google.protobuf.BoolValue"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", BoolValue(true)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -218,7 +218,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_BoolValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, BoolValue(true)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -226,8 +226,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_BoolValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), true); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), true); } TEST_F(TypeReflectorTest, NewValueBuilder_Int32Value) { @@ -236,7 +236,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Int32Value) { internal::GetTestingMessageFactory(), "google.protobuf.Int32Value"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -248,7 +248,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Int32Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kOutOfRange))))); EXPECT_THAT(builder->SetFieldByNumber(1, IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -260,8 +260,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Int32Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kOutOfRange))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), 1); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), 1); } TEST_F(TypeReflectorTest, NewValueBuilder_Int64Value) { @@ -270,7 +270,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Int64Value) { internal::GetTestingMessageFactory(), "google.protobuf.Int64Value"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -278,7 +278,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Int64Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -286,8 +286,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Int64Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), 1); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), 1); } TEST_F(TypeReflectorTest, NewValueBuilder_UInt32Value) { @@ -296,7 +296,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_UInt32Value) { internal::GetTestingMessageFactory(), "google.protobuf.UInt32Value"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", UintValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", UintValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -308,7 +308,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_UInt32Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kOutOfRange))))); EXPECT_THAT(builder->SetFieldByNumber(1, UintValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, UintValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -320,8 +320,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_UInt32Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kOutOfRange))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), 1); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), 1); } TEST_F(TypeReflectorTest, NewValueBuilder_UInt64Value) { @@ -330,7 +330,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_UInt64Value) { internal::GetTestingMessageFactory(), "google.protobuf.UInt64Value"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", UintValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", UintValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -338,7 +338,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_UInt64Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, UintValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, UintValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -346,8 +346,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_UInt64Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), 1); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), 1); } TEST_F(TypeReflectorTest, NewValueBuilder_FloatValue) { @@ -356,7 +356,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_FloatValue) { internal::GetTestingMessageFactory(), "google.protobuf.FloatValue"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", DoubleValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", DoubleValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -364,7 +364,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_FloatValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, DoubleValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, DoubleValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -372,8 +372,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_FloatValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), 1); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), 1); } TEST_F(TypeReflectorTest, NewValueBuilder_DoubleValue) { @@ -382,7 +382,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_DoubleValue) { internal::GetTestingMessageFactory(), "google.protobuf.DoubleValue"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", DoubleValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", DoubleValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -390,7 +390,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_DoubleValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, DoubleValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, DoubleValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -398,8 +398,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_DoubleValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), 1); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), 1); } TEST_F(TypeReflectorTest, NewValueBuilder_StringValue) { @@ -407,25 +407,26 @@ TEST_F(TypeReflectorTest, NewValueBuilder_StringValue) { arena(), internal::GetTestingDescriptorPool(), internal::GetTestingMessageFactory(), "google.protobuf.StringValue"); ASSERT_THAT(builder, NotNull()); - EXPECT_THAT(builder->SetFieldByName("value", StringValue("foo")), - IsOkAndHolds(Eq(absl::nullopt))); - EXPECT_THAT(builder->SetFieldByName("does_not_exist", StringValue("foo")), - IsOkAndHolds(Optional( - ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); + EXPECT_THAT(builder->SetFieldByName("value", StringValue::WrapUnsafe("foo")), + IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT( + builder->SetFieldByName("does_not_exist", StringValue::WrapUnsafe("foo")), + IsOkAndHolds( + Optional(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); EXPECT_THAT(builder->SetFieldByName("value", BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); - EXPECT_THAT(builder->SetFieldByNumber(1, StringValue("foo")), - IsOkAndHolds(Eq(absl::nullopt))); - EXPECT_THAT(builder->SetFieldByNumber(2, StringValue("foo")), + EXPECT_THAT(builder->SetFieldByNumber(1, StringValue::WrapUnsafe("foo")), + IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT(builder->SetFieldByNumber(2, StringValue::WrapUnsafe("foo")), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); EXPECT_THAT(builder->SetFieldByNumber(1, BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeString(), "foo"); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeString(), "foo"); } TEST_F(TypeReflectorTest, NewValueBuilder_BytesValue) { @@ -433,25 +434,26 @@ TEST_F(TypeReflectorTest, NewValueBuilder_BytesValue) { arena(), internal::GetTestingDescriptorPool(), internal::GetTestingMessageFactory(), "google.protobuf.BytesValue"); ASSERT_THAT(builder, NotNull()); - EXPECT_THAT(builder->SetFieldByName("value", BytesValue("foo")), - IsOkAndHolds(Eq(absl::nullopt))); - EXPECT_THAT(builder->SetFieldByName("does_not_exist", BytesValue("foo")), - IsOkAndHolds(Optional( - ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); + EXPECT_THAT(builder->SetFieldByName("value", BytesValue::WrapUnsafe("foo")), + IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT( + builder->SetFieldByName("does_not_exist", BytesValue::WrapUnsafe("foo")), + IsOkAndHolds( + Optional(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); EXPECT_THAT(builder->SetFieldByName("value", BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); - EXPECT_THAT(builder->SetFieldByNumber(1, BytesValue("foo")), - IsOkAndHolds(Eq(absl::nullopt))); - EXPECT_THAT(builder->SetFieldByNumber(2, BytesValue("foo")), + EXPECT_THAT(builder->SetFieldByNumber(1, BytesValue::WrapUnsafe("foo")), + IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT(builder->SetFieldByNumber(2, BytesValue::WrapUnsafe("foo")), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); EXPECT_THAT(builder->SetFieldByNumber(1, BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeString(), "foo"); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeString(), "foo"); } TEST_F(TypeReflectorTest, NewValueBuilder_Duration) { @@ -460,7 +462,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Duration) { internal::GetTestingMessageFactory(), "google.protobuf.Duration"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("seconds", IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -468,7 +470,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Duration) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByName("nanos", IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName( "nanos", IntValue(std::numeric_limits::max())), IsOkAndHolds(Optional( @@ -477,7 +479,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Duration) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(3, IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -485,7 +487,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Duration) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(2, IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber( 2, IntValue(std::numeric_limits::max())), IsOkAndHolds(Optional( @@ -494,8 +496,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Duration) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), absl::Seconds(1) + absl::Nanoseconds(1)); } @@ -505,7 +507,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Timestamp) { internal::GetTestingMessageFactory(), "google.protobuf.Timestamp"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("seconds", IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -513,7 +515,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Timestamp) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByName("nanos", IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName( "nanos", IntValue(std::numeric_limits::max())), IsOkAndHolds(Optional( @@ -522,7 +524,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Timestamp) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(3, IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -530,7 +532,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Timestamp) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(2, IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber( 2, IntValue(std::numeric_limits::max())), IsOkAndHolds(Optional( @@ -539,8 +541,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Timestamp) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), absl::UnixEpoch() + absl::Seconds(1) + absl::Nanoseconds(1)); } @@ -549,10 +551,11 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Any) { arena(), internal::GetTestingDescriptorPool(), internal::GetTestingMessageFactory(), "google.protobuf.Any"); ASSERT_THAT(builder, NotNull()); - EXPECT_THAT(builder->SetFieldByName( - "type_url", - StringValue("type.googleapis.com/google.protobuf.BoolValue")), - IsOkAndHolds(Eq(absl::nullopt))); + EXPECT_THAT( + builder->SetFieldByName( + "type_url", StringValue::WrapUnsafe( + "type.googleapis.com/google.protobuf.BoolValue")), + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -560,14 +563,14 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Any) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByName("value", BytesValue()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("value", BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); - EXPECT_THAT( - builder->SetFieldByNumber( - 1, StringValue("type.googleapis.com/google.protobuf.BoolValue")), - IsOkAndHolds(Eq(absl::nullopt))); + EXPECT_THAT(builder->SetFieldByNumber( + 1, StringValue::WrapUnsafe( + "type.googleapis.com/google.protobuf.BoolValue")), + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(3, IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -575,13 +578,13 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Any) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(2, BytesValue()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), false); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), false); } } // namespace diff --git a/common/type_spec_resolver.cc b/common/type_spec_resolver.cc new file mode 100644 index 000000000..c3b4bc171 --- /dev/null +++ b/common/type_spec_resolver.cc @@ -0,0 +1,326 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/type_spec_resolver.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "common/ast.h" +#include "common/descriptor_pool_type_introspector.h" +#include "common/type.h" +#include "common/type_introspector.h" +#include "common/type_kind.h" +#include "internal/status_macros.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" + +#undef GetMessage + +namespace cel { +namespace { + +absl::StatusOr> ResolveNamedType( + absl::string_view name, const TypeIntrospector& type_introspector) { + absl::optional type = FindWellKnownType(name); + if (type.has_value()) { + return type; + } + return type_introspector.FindType(name); +} + +bool TypeAcceptsParameters(const Type& type) { + return type.IsOpaque() || type.IsType() || type.IsList() || type.IsMap(); +} +} // namespace + +absl::StatusOr ConvertTypeSpecToType( + const TypeSpec& type_spec, const TypeIntrospector& type_introspector, + google::protobuf::Arena* arena) { + if (type_spec.has_null()) return Type(NullType{}); + if (type_spec.has_dyn()) return Type(DynType{}); + + if (type_spec.has_primitive()) { + switch (type_spec.primitive()) { + case PrimitiveType::kBool: + return Type(BoolType{}); + case PrimitiveType::kInt64: + return Type(IntType{}); + case PrimitiveType::kUint64: + return Type(UintType{}); + case PrimitiveType::kDouble: + return Type(DoubleType{}); + case PrimitiveType::kString: + return Type(StringType{}); + case PrimitiveType::kBytes: + return Type(BytesType{}); + default: + return absl::InvalidArgumentError("Unsupported primitive type"); + } + } + + if (type_spec.has_well_known()) { + switch (type_spec.well_known()) { + case WellKnownTypeSpec::kAny: + return Type(AnyType{}); + case WellKnownTypeSpec::kTimestamp: + return Type(TimestampType{}); + case WellKnownTypeSpec::kDuration: + return Type(DurationType{}); + default: + return absl::InvalidArgumentError("Unsupported well-known type"); + } + } + + if (type_spec.has_wrapper()) { + switch (type_spec.wrapper()) { + case PrimitiveType::kBool: + return Type(BoolWrapperType{}); + case PrimitiveType::kInt64: + return Type(IntWrapperType{}); + case PrimitiveType::kUint64: + return Type(UintWrapperType{}); + case PrimitiveType::kDouble: + return Type(DoubleWrapperType{}); + case PrimitiveType::kString: + return Type(StringWrapperType{}); + case PrimitiveType::kBytes: + return Type(BytesWrapperType{}); + default: + return absl::InvalidArgumentError("Unsupported wrapper type"); + } + } + + if (type_spec.has_list_type()) { + Type elem_type; + if (type_spec.list_type().elem_type().is_specified()) { + CEL_ASSIGN_OR_RETURN( + elem_type, ConvertTypeSpecToType(type_spec.list_type().elem_type(), + type_introspector, arena)); + } + return Type(ListType(arena, elem_type)); + } + + if (type_spec.has_map_type()) { + Type key_type; + if (type_spec.map_type().key_type().is_specified()) { + CEL_ASSIGN_OR_RETURN( + key_type, ConvertTypeSpecToType(type_spec.map_type().key_type(), + type_introspector, arena)); + } + + Type value_type; + if (type_spec.map_type().value_type().is_specified()) { + CEL_ASSIGN_OR_RETURN( + value_type, ConvertTypeSpecToType(type_spec.map_type().value_type(), + type_introspector, arena)); + } + return Type(MapType(arena, key_type, value_type)); + } + + if (type_spec.has_function()) { + const auto& func_spec = type_spec.function(); + Type result_type; + if (func_spec.result_type().is_specified()) { + CEL_ASSIGN_OR_RETURN(result_type, + ConvertTypeSpecToType(func_spec.result_type(), + type_introspector, arena)); + } + std::vector arg_types; + arg_types.reserve(func_spec.arg_types().size()); + for (const auto& arg_spec : func_spec.arg_types()) { + CEL_ASSIGN_OR_RETURN( + auto arg_type, + ConvertTypeSpecToType(arg_spec, type_introspector, arena)); + arg_types.push_back(std::move(arg_type)); + } + return Type(FunctionType(arena, result_type, arg_types)); + } + + if (type_spec.has_type_param()) { + const std::string& name = type_spec.type_param().type(); + auto* allocated_name = google::protobuf::Arena::Create(arena, name); + return Type(TypeParamType(absl::string_view(*allocated_name))); + } + + if (type_spec.has_message_type()) { + const std::string& name = type_spec.message_type().type(); + CEL_ASSIGN_OR_RETURN(absl::optional type, + ResolveNamedType(name, type_introspector)); + if (!type.has_value()) { + return absl::InvalidArgumentError(absl::StrCat( + "Message type '", name, "' not found in descriptor pool")); + } + return *type; + } + + if (type_spec.has_abstract_type()) { + const std::string& name = type_spec.abstract_type().name(); + + CEL_ASSIGN_OR_RETURN(absl::optional type, + ResolveNamedType(name, type_introspector)); + if (type.has_value()) { + if (!TypeAcceptsParameters(*type) && + !type_spec.abstract_type().parameter_types().empty()) { + return absl::InvalidArgumentError( + absl::StrCat("Type '", name, "' cannot have type parameters")); + } + return *type; + } + + // Otherwise fallback to OpaqueType + std::vector params; + params.reserve(type_spec.abstract_type().parameter_types().size()); + for (const auto& param_spec : type_spec.abstract_type().parameter_types()) { + CEL_ASSIGN_OR_RETURN( + auto param, + ConvertTypeSpecToType(param_spec, type_introspector, arena)); + params.push_back(std::move(param)); + } + auto* allocated_name = google::protobuf::Arena::Create(arena, name); + return Type(OpaqueType(arena, absl::string_view(*allocated_name), params)); + } + + if (type_spec.has_type()) { + CEL_ASSIGN_OR_RETURN( + auto contained_type, + ConvertTypeSpecToType(type_spec.type(), type_introspector, arena)); + return Type(TypeType(arena, contained_type)); + } + + if (type_spec.has_error()) { + return Type(ErrorType{}); + } + + return absl::InvalidArgumentError("Unknown TypeSpec kind"); +} + +absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, + const google::protobuf::DescriptorPool& pool, + google::protobuf::Arena* arena) { + // In this configuration we will only ever reference objects backed by the + // given DescriptorPool or global constants so it is safe for `introspector` + // to go out of scope, but this is not sound for arbitrary TypeIntrospectors. + DescriptorPoolTypeIntrospector introspector(&pool); + return ConvertTypeSpecToType(type_spec, introspector, arena); +} + +absl::StatusOr ConvertTypeToTypeSpec(const Type& type) { + switch (type.kind()) { + case TypeKind::kNull: + return TypeSpec(NullTypeSpec{}); + case TypeKind::kDyn: + return TypeSpec(DynTypeSpec{}); + case TypeKind::kBool: + return TypeSpec(PrimitiveType::kBool); + case TypeKind::kInt: + return TypeSpec(PrimitiveType::kInt64); + case TypeKind::kUint: + return TypeSpec(PrimitiveType::kUint64); + case TypeKind::kDouble: + return TypeSpec(PrimitiveType::kDouble); + case TypeKind::kString: + return TypeSpec(PrimitiveType::kString); + case TypeKind::kBytes: + return TypeSpec(PrimitiveType::kBytes); + case TypeKind::kAny: + return TypeSpec(WellKnownTypeSpec::kAny); + case TypeKind::kTimestamp: + return TypeSpec(WellKnownTypeSpec::kTimestamp); + case TypeKind::kDuration: + return TypeSpec(WellKnownTypeSpec::kDuration); + case TypeKind::kBoolWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBool)); + case TypeKind::kIntWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kInt64)); + case TypeKind::kUintWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kUint64)); + case TypeKind::kDoubleWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kDouble)); + case TypeKind::kStringWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kString)); + case TypeKind::kBytesWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBytes)); + case TypeKind::kList: { + CEL_ASSIGN_OR_RETURN(auto elem_type, + ConvertTypeToTypeSpec(type.GetList().element())); + return TypeSpec( + ListTypeSpec(std::make_unique(std::move(elem_type)))); + } + case TypeKind::kMap: { + CEL_ASSIGN_OR_RETURN(auto key_type, + ConvertTypeToTypeSpec(type.GetMap().key())); + CEL_ASSIGN_OR_RETURN(auto value_type, + ConvertTypeToTypeSpec(type.GetMap().value())); + return TypeSpec( + MapTypeSpec(std::make_unique(std::move(key_type)), + std::make_unique(std::move(value_type)))); + } + case TypeKind::kFunction: { + auto func_type = type.GetFunction(); + CEL_ASSIGN_OR_RETURN(auto result_type, + ConvertTypeToTypeSpec(func_type.result())); + std::vector arg_types; + arg_types.reserve(func_type.args().size()); + for (const auto& arg : func_type.args()) { + CEL_ASSIGN_OR_RETURN(auto arg_type, ConvertTypeToTypeSpec(arg)); + arg_types.push_back(std::move(arg_type)); + } + return TypeSpec( + FunctionTypeSpec(std::make_unique(std::move(result_type)), + std::move(arg_types))); + } + case TypeKind::kTypeParam: + return TypeSpec(ParamTypeSpec(std::string(type.GetTypeParam().name()))); + case TypeKind::kStruct: { + if (type.IsMessage()) { + return TypeSpec(MessageTypeSpec(std::string(type.GetMessage().name()))); + } + return absl::InvalidArgumentError("Unsupported struct type"); + } + case TypeKind::kOpaque: { + auto opaque_type = type.GetOpaque(); + std::vector params; + params.reserve(opaque_type.GetParameters().size()); + for (const auto& param : opaque_type.GetParameters()) { + CEL_ASSIGN_OR_RETURN(auto param_type, ConvertTypeToTypeSpec(param)); + params.push_back(std::move(param_type)); + } + return TypeSpec( + AbstractType(std::string(opaque_type.name()), std::move(params))); + } + case TypeKind::kType: { + CEL_ASSIGN_OR_RETURN(auto nested_type, + ConvertTypeToTypeSpec(type.GetType().GetType())); + return TypeSpec(std::make_unique(std::move(nested_type))); + } + case TypeKind::kError: + return TypeSpec(ErrorTypeSpec::kValue); + case TypeKind::kEnum: + return TypeSpec( + AbstractType(std::string(type.GetEnum().name()), /*params=*/{})); + default: + return absl::InvalidArgumentError(absl::StrCat( + "Unsupported Type kind: ", TypeKindToString(type.kind()))); + } +} + +} // namespace cel diff --git a/common/type_spec_resolver.h b/common/type_spec_resolver.h new file mode 100644 index 000000000..17fbf27de --- /dev/null +++ b/common/type_spec_resolver.h @@ -0,0 +1,46 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_COMMON_TYPE_SPEC_RESOLVER_H_ +#define THIRD_PARTY_CEL_CPP_COMMON_TYPE_SPEC_RESOLVER_H_ + +#include "absl/status/statusor.h" +#include "common/ast.h" +#include "common/type.h" +#include "common/type_introspector.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" + +namespace cel { + +// Resolves a `cel::TypeSpec` to a `cel::Type`. +// +// TypeSpec only specifies a type while Type provides support for inspecting +// properties of the type when used in CEL. Returns a status with code +// `InvalidArgument` if the input cannot be resolved to a type. +absl::StatusOr ConvertTypeSpecToType( + const TypeSpec& type_spec, const cel::TypeIntrospector& type_introspector, + google::protobuf::Arena* arena); + +// Convenience overload for type resolution using only a descriptor pool. +absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, + const google::protobuf::DescriptorPool& pool, + google::protobuf::Arena* arena); + +// Resolves a `cel::Type` to a `cel::TypeSpec`. +absl::StatusOr ConvertTypeToTypeSpec(const Type& type); + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_COMMON_TYPE_SPEC_RESOLVER_H_ diff --git a/common/type_spec_resolver_test.cc b/common/type_spec_resolver_test.cc new file mode 100644 index 000000000..056ba8f67 --- /dev/null +++ b/common/type_spec_resolver_test.cc @@ -0,0 +1,342 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/type_spec_resolver.h" + +#include +#include +#include +#include +#include + +#include "absl/base/no_destructor.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "common/ast.h" +#include "common/type.h" +#include "common/type_introspector.h" +#include "common/type_kind.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "google/protobuf/arena.h" + +namespace cel { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::IsOkAndHolds; +using ::absl_testing::StatusIs; +using ::cel::internal::GetTestingDescriptorPool; +using ::testing::HasSubstr; +using ::testing::TestWithParam; +using ::testing::Values; + +google::protobuf::Arena* GetTestArena() { + static absl::NoDestructor arena; + return &*arena; +} + +TEST(TypeSpecResolverTest, NullTypeSpec) { + TypeSpec spec(NullTypeSpec{}); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsNull()); +} + +TEST(TypeSpecResolverTest, DynTypeSpec) { + TypeSpec spec(DynTypeSpec{}); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsDyn()); +} + +using ConversionTest = testing::TestWithParam>; + +TEST_P(ConversionTest, TestTypeSpecConversion) { + ASSERT_OK_AND_ASSIGN( + auto t, + ConvertTypeSpecToType(std::get<0>(GetParam()), + *GetTestingDescriptorPool(), GetTestArena())); + EXPECT_EQ(t.kind(), std::get<1>(GetParam())); + EXPECT_THAT(ConvertTypeToTypeSpec(t), IsOkAndHolds(std::get<0>(GetParam()))); +} + +INSTANTIATE_TEST_SUITE_P( + TypeSpecResolverTest, ConversionTest, + testing::Values( + std::make_tuple(TypeSpec(PrimitiveType::kBool), TypeKind::kBool), + std::make_tuple(TypeSpec(PrimitiveType::kInt64), TypeKind::kInt), + std::make_tuple(TypeSpec(PrimitiveType::kUint64), TypeKind::kUint), + std::make_tuple(TypeSpec(PrimitiveType::kDouble), TypeKind::kDouble), + std::make_tuple(TypeSpec(PrimitiveType::kString), TypeKind::kString), + std::make_tuple(TypeSpec(PrimitiveType::kBytes), TypeKind::kBytes), + std::make_tuple(TypeSpec(WellKnownTypeSpec::kAny), TypeKind::kAny), + std::make_tuple(TypeSpec(WellKnownTypeSpec::kTimestamp), + TypeKind::kTimestamp), + std::make_tuple(TypeSpec(WellKnownTypeSpec::kDuration), + TypeKind::kDuration), + std::make_tuple(TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBool)), + TypeKind::kBoolWrapper), + std::make_tuple(TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kInt64)), + TypeKind::kIntWrapper), + std::make_tuple(TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kUint64)), + TypeKind::kUintWrapper), + std::make_tuple(TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kDouble)), + TypeKind::kDoubleWrapper), + std::make_tuple(TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kString)), + TypeKind::kStringWrapper), + std::make_tuple(TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBytes)), + TypeKind::kBytesWrapper))); + +TEST(TypeSpecResolverTest, ListTypeConversion) { + auto elem = std::make_unique(PrimitiveType::kInt64); + TypeSpec spec(ListTypeSpec(std::move(elem))); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsList()); + EXPECT_TRUE(t->GetList().element().IsInt()); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); +} + +TEST(TypeSpecResolverTest, MapTypeConversion) { + auto key = std::make_unique(PrimitiveType::kString); + auto val = std::make_unique(PrimitiveType::kBytes); + TypeSpec spec(MapTypeSpec(std::move(key), std::move(val))); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsMap()); + EXPECT_TRUE(t->GetMap().key().IsString()); + EXPECT_TRUE(t->GetMap().value().IsBytes()); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); +} + +TEST(TypeSpecResolverTest, FunctionTypeConversion) { + auto result = std::make_unique(PrimitiveType::kBool); + std::vector args; + args.push_back(TypeSpec(PrimitiveType::kString)); + TypeSpec spec(FunctionTypeSpec(std::move(result), std::move(args))); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsFunction()); + EXPECT_EQ(t->GetFunction().args().size(), 1); + EXPECT_TRUE(t->GetFunction().result().IsBool()); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); +} + +TEST(TypeSpecResolverTest, TypeParamConversion) { + TypeSpec spec(ParamTypeSpec("T")); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsTypeParam()); + EXPECT_EQ(t->GetTypeParam().name(), "T"); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); +} + +TEST(TypeSpecResolverTest, MessageTypeConversion) { + TypeSpec spec( + AbstractType("cel.expr.conformance.proto3.TestAllTypes", /*params=*/{})); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsMessage()); + EXPECT_EQ(t->name(), "cel.expr.conformance.proto3.TestAllTypes"); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ( + spec2, + TypeSpec(MessageTypeSpec("cel.expr.conformance.proto3.TestAllTypes"))); +} + +TEST(TypeSpecResolverTest, MessageTypeWithParamsError) { + std::vector params; + params.push_back(TypeSpec(PrimitiveType::kInt64)); + TypeSpec spec(AbstractType("cel.expr.conformance.proto3.TestAllTypes", + std::move(params))); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + EXPECT_THAT(t, StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("cannot have type parameters"))); +} + +TEST(TypeSpecResolverTest, UnresolvedAbstractTypeFallbackToOpaque) { + std::vector params; + params.push_back(TypeSpec(PrimitiveType::kInt64)); + TypeSpec spec(AbstractType("my.custom.OpaqueType", std::move(params))); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsOpaque()); + EXPECT_EQ(t->name(), "my.custom.OpaqueType"); + EXPECT_EQ(t->GetParameters().size(), 1); + EXPECT_TRUE(t->GetParameters()[0].IsInt()); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); +} + +TEST(TypeSpecResolverTest, OptionalType) { + std::vector params; + params.push_back(TypeSpec(PrimitiveType::kInt64)); + TypeSpec spec(AbstractType("optional_type", std::move(params))); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsOpaque()); + EXPECT_EQ(t->name(), "optional_type"); + EXPECT_EQ(t->GetParameters().size(), 1); + EXPECT_TRUE(t->GetParameters()[0].IsInt()); + EXPECT_TRUE(t->IsOptional()); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); +} + +TEST(TypeSpecResolverTest, TypeTypeConversion) { + auto nested = std::make_unique(PrimitiveType::kInt64); + TypeSpec spec(std::move(nested)); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsType()); + EXPECT_TRUE(t->GetType().GetType().IsInt()); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); +} + +TEST(TypeSpecResolverTest, ErrorTypeConversion) { + TypeSpec spec(ErrorTypeSpec::kValue); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsError()); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); +} + +TEST(TypeSpecResolverTest, MessageTypeSpecConversion) { + TypeSpec spec(MessageTypeSpec("cel.expr.conformance.proto3.TestAllTypes")); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsMessage()); + EXPECT_EQ(t->name(), "cel.expr.conformance.proto3.TestAllTypes"); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); +} + +TEST(TypeSpecResolverTest, MessageTypeSpecNotFoundError) { + TypeSpec spec(MessageTypeSpec("cel.expr.conformance.proto3.NonExistentType")); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + EXPECT_THAT(t, StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("not found in descriptor pool"))); +} + +TEST(TypeSpecResolverTest, EnumTypeConversion) { + TypeSpec spec(AbstractType( + "cel.expr.conformance.proto3.TestAllTypes.NestedEnum", /*params=*/{})); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsEnum()); + EXPECT_EQ(t->name(), "cel.expr.conformance.proto3.TestAllTypes.NestedEnum"); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); +} + +TEST(TypeSpecResolverTest, EnumTypeWithParamsError) { + std::vector params; + params.push_back(TypeSpec(PrimitiveType::kInt64)); + TypeSpec spec( + AbstractType("cel.expr.conformance.proto3.TestAllTypes.NestedEnum", + std::move(params))); + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + EXPECT_THAT(t, StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("cannot have type parameters"))); +} + +TEST(TypeSpecResolverTest, WellKnownMessageTypeFiltering) { + TypeSpec message_spec(MessageTypeSpec("google.protobuf.Timestamp")); + ASSERT_OK_AND_ASSIGN( + auto message_type, + ConvertTypeSpecToType(message_spec, *GetTestingDescriptorPool(), + GetTestArena())); + EXPECT_TRUE(message_type.IsTimestamp()); + + TypeSpec abstract_spec( + AbstractType("google.protobuf.Duration", /*params=*/{})); + ASSERT_OK_AND_ASSIGN( + auto abstract_type, + ConvertTypeSpecToType(abstract_spec, *GetTestingDescriptorPool(), + GetTestArena())); + EXPECT_TRUE(abstract_type.IsDuration()); + + std::vector params; + params.push_back(TypeSpec(PrimitiveType::kInt64)); + TypeSpec abstract_with_params( + AbstractType("google.protobuf.Timestamp", std::move(params))); + EXPECT_THAT( + ConvertTypeSpecToType(abstract_with_params, *GetTestingDescriptorPool(), + GetTestArena()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("cannot have type parameters"))); +} + +TEST(TypeSpecResolverTest, WellKnownTypeIntrospector) { + WellKnownTypeIntrospector introspector; + + TypeSpec message_spec(MessageTypeSpec("google.protobuf.Timestamp")); + ASSERT_OK_AND_ASSIGN( + auto message_type, + ConvertTypeSpecToType(message_spec, introspector, GetTestArena())); + EXPECT_TRUE(message_type.IsTimestamp()); + + TypeSpec abstract_spec( + AbstractType("google.protobuf.Duration", /*params=*/{})); + ASSERT_OK_AND_ASSIGN( + auto abstract_type, + ConvertTypeSpecToType(abstract_spec, introspector, GetTestArena())); + EXPECT_TRUE(abstract_type.IsDuration()); + + TypeSpec primitive_spec(PrimitiveType::kInt64); + ASSERT_OK_AND_ASSIGN( + auto primitive_type, + ConvertTypeSpecToType(primitive_spec, introspector, GetTestArena())); + EXPECT_TRUE(primitive_type.IsInt()); + + TypeSpec custom_spec(MessageTypeSpec("custom.UnknownMessage")); + EXPECT_THAT( + ConvertTypeSpecToType(custom_spec, introspector, GetTestArena()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Message type 'custom.UnknownMessage' not found"))); +} + +TEST(TypeSpecResolverTest, UnknownTypeSpecKindError) { + TypeSpec spec; + auto t = + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); + EXPECT_THAT(t, StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unknown TypeSpec kind"))); +} + +} // namespace +} // namespace cel diff --git a/common/type_test.cc b/common/type_test.cc index 2cebf27ba..d6a613c3c 100644 --- a/common/type_test.cc +++ b/common/type_test.cc @@ -638,5 +638,39 @@ TEST(Type, Wrap) { EXPECT_EQ(Type(AnyType()).Wrap(), AnyType()); } +TEST(Type, LegacyRuntimeType) { + EXPECT_EQ(common_internal::LegacyRuntimeType("bool"), BoolType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.Any"), + AnyType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.BoolValue"), + BoolWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.BytesValue"), + BytesWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.DoubleValue"), + DoubleWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.Duration"), + DurationType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.FloatValue"), + DoubleWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.Int32Value"), + IntWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.Int64Value"), + IntWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.ListValue"), + ListType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.StringValue"), + StringWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.Struct"), + JsonMapType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.Timestamp"), + TimestampType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.UInt32Value"), + UintWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.UInt64Value"), + UintWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.Value"), + DynType()); +} + } // namespace } // namespace cel diff --git a/common/typeinfo.cc b/common/typeinfo.cc index 86bae1934..b07275712 100644 --- a/common/typeinfo.cc +++ b/common/typeinfo.cc @@ -57,18 +57,13 @@ std::string TypeInfo::DebugString() const { } return std::string(demangled.get()); #else - size_t length = 0; int status = 0; std::unique_ptr demangled( - abi::__cxa_demangle(rep_->name(), nullptr, &length, &status)); + abi::__cxa_demangle(rep_->name(), nullptr, nullptr, &status)); if (status != 0 || demangled == nullptr) { return std::string(rep_->name()); } - while (length != 0 && demangled.get()[length - 1] == '\0') { - // length includes the null terminator, remove it. - --length; - } - return std::string(demangled.get(), length); + return std::string(demangled.get()); #endif #else return absl::StrCat("0x", absl::Hex(absl::bit_cast(rep_))); diff --git a/common/types/message_type.h b/common/types/message_type.h index 782af87aa..eea811971 100644 --- a/common/types/message_type.h +++ b/common/types/message_type.h @@ -87,6 +87,11 @@ class MessageType final { return descriptor_; } + const google::protobuf::Descriptor* absl_nonnull descriptor() const { + ABSL_DCHECK(*this); + return descriptor_; + } + explicit operator bool() const { return descriptor_ != nullptr; } private: @@ -166,8 +171,12 @@ class MessageTypeField final { return *descriptor_; } - const google::protobuf::FieldDescriptor* absl_nonnull operator->() const - ABSL_ATTRIBUTE_LIFETIME_BOUND { + const google::protobuf::FieldDescriptor* absl_nonnull operator->() const { + ABSL_DCHECK(*this); + return descriptor_; + } + + const google::protobuf::FieldDescriptor* absl_nonnull descriptor() const { ABSL_DCHECK(*this); return descriptor_; } diff --git a/common/types/opaque_type.cc b/common/types/opaque_type.cc index 002319d1d..9c58e8289 100644 --- a/common/types/opaque_type.cc +++ b/common/types/opaque_type.cc @@ -98,7 +98,7 @@ absl::optional OpaqueType::AsOptional() const { if (IsOptional()) { return OptionalType(absl::in_place, *this); } - return absl::nullopt; + return std::nullopt; } OptionalType OpaqueType::GetOptional() const { diff --git a/common/types/struct_type.cc b/common/types/struct_type.cc index 4540cec9c..14e9fc4e2 100644 --- a/common/types/struct_type.cc +++ b/common/types/struct_type.cc @@ -22,12 +22,14 @@ #include "common/type.h" #include "common/types/types.h" +#undef GetMessage + namespace cel { absl::string_view StructType::name() const { ABSL_DCHECK(*this); return absl::visit( - absl::Overload([](absl::monostate) { return absl::string_view(); }, + absl::Overload([](std::monostate) { return absl::string_view(); }, [](const common_internal::BasicStructType& alt) { return alt.name(); }, @@ -39,7 +41,7 @@ TypeParameters StructType::GetParameters() const { ABSL_DCHECK(*this); return absl::visit( absl::Overload( - [](absl::monostate) { return TypeParameters(); }, + [](std::monostate) { return TypeParameters(); }, [](const common_internal::BasicStructType& alt) { return alt.GetParameters(); }, @@ -49,7 +51,7 @@ TypeParameters StructType::GetParameters() const { std::string StructType::DebugString() const { return absl::visit( - absl::Overload([](absl::monostate) { return std::string(); }, + absl::Overload([](std::monostate) { return std::string(); }, [](common_internal::BasicStructType alt) { return alt.DebugString(); }, @@ -61,7 +63,7 @@ absl::optional StructType::AsMessage() const { if (const auto* alt = absl::get_if(&variant_); alt != nullptr) { return *alt; } - return absl::nullopt; + return std::nullopt; } MessageType StructType::GetMessage() const { @@ -72,7 +74,7 @@ MessageType StructType::GetMessage() const { common_internal::TypeVariant StructType::ToTypeVariant() const { return absl::visit( absl::Overload( - [](absl::monostate) { return common_internal::TypeVariant(); }, + [](std::monostate) { return common_internal::TypeVariant(); }, [](common_internal::BasicStructType alt) { return static_cast(alt) ? common_internal::TypeVariant(alt) : common_internal::TypeVariant(); diff --git a/common/types/struct_type.h b/common/types/struct_type.h index 6e20ea007..f947f88c3 100644 --- a/common/types/struct_type.h +++ b/common/types/struct_type.h @@ -33,6 +33,9 @@ #include "common/types/message_type.h" #include "common/types/types.h" +#pragma push_macro("GetMessage") +#undef GetMessage + namespace cel { class Type; @@ -155,4 +158,6 @@ inline std::ostream& operator<<(std::ostream& out, const StructType& type) { } // namespace cel +#pragma pop_macro("GetMessage") + #endif // THIRD_PARTY_CEL_CPP_COMMON_TYPES_STRUCT_TYPE_H_ diff --git a/common/types/type_type.h b/common/types/type_type.h index 652f99008..2de78ca1a 100644 --- a/common/types/type_type.h +++ b/common/types/type_type.h @@ -68,20 +68,16 @@ class TypeType final { const common_internal::TypeTypeData* absl_nullable data_ = nullptr; }; -inline constexpr bool operator==(const TypeType&, const TypeType&) { - return true; -} +// Two `TypeType` are equal if their type parameters are equal. Note that the +// parameterless `type` is not equal to `type(dyn)`. +bool operator==(const TypeType& lhs, const TypeType& rhs); -inline constexpr bool operator!=(const TypeType& lhs, const TypeType& rhs) { +inline bool operator!=(const TypeType& lhs, const TypeType& rhs) { return !operator==(lhs, rhs); } template -H AbslHashValue(H state, const TypeType&) { - // TypeType is really a singleton and all instances are equal. Nothing to - // hash. - return std::move(state); -} +H AbslHashValue(H state, const TypeType& type); inline std::ostream& operator<<(std::ostream& out, const TypeType& type) { return out << type.DebugString(); diff --git a/common/types/type_type_test.cc b/common/types/type_type_test.cc index 978027f98..2c62f1732 100644 --- a/common/types/type_type_test.cc +++ b/common/types/type_type_test.cc @@ -18,6 +18,7 @@ #include "absl/hash/hash.h" #include "internal/testing.h" +#include "google/protobuf/arena.h" namespace cel { namespace { @@ -56,5 +57,25 @@ TEST(TypeType, Equal) { EXPECT_EQ(Type(TypeType()), Type(TypeType())); } +TEST(TypeType, EqualComparesParameter) { + google::protobuf::Arena arena; + EXPECT_EQ(TypeType(&arena, IntType()), TypeType(&arena, IntType())); + EXPECT_NE(TypeType(&arena, IntType()), TypeType(&arena, StringType())); + EXPECT_NE(TypeType(&arena, IntType()), TypeType()); + EXPECT_NE(TypeType(&arena, DynType()), TypeType()); + EXPECT_EQ(Type(TypeType(&arena, ListType(&arena, IntType()))), + Type(TypeType(&arena, ListType(&arena, IntType())))); + EXPECT_NE(Type(ListType(&arena, TypeType(&arena, IntType()))), + Type(ListType(&arena, TypeType(&arena, UintType())))); +} + +TEST(TypeType, HashIncludesParameter) { + google::protobuf::Arena arena; + EXPECT_EQ(absl::HashOf(TypeType(&arena, IntType())), + absl::HashOf(TypeType(&arena, IntType()))); + EXPECT_NE(absl::HashOf(TypeType(&arena, IntType())), + absl::HashOf(TypeType(&arena, StringType()))); +} + } // namespace } // namespace cel diff --git a/common/value.cc b/common/value.cc index 535ddead8..bc8a3d7a0 100644 --- a/common/value.cc +++ b/common/value.cc @@ -17,13 +17,16 @@ #include #include #include +#include #include #include #include #include +#include #include "google/protobuf/struct.pb.h" #include "absl/base/attributes.h" +#include "absl/base/no_destructor.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/functional/overload.h" @@ -37,10 +40,9 @@ #include "absl/strings/string_view.h" #include "absl/time/time.h" #include "absl/types/optional.h" +#include "absl/types/source_location.h" #include "absl/types/variant.h" #include "common/allocator.h" -#include "common/memory.h" -#include "common/optional_ref.h" #include "common/type.h" #include "common/value_kind.h" #include "common/values/list_value_builder.h" @@ -50,6 +52,7 @@ #include "internal/number.h" #include "internal/protobuf_runtime_version.h" #include "internal/status_macros.h" +#include "internal/utf8.h" #include "internal/well_known_types.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -115,7 +118,7 @@ Type Value::GetRuntimeType() const { namespace { template -struct IsMonostate : std::is_same, absl::monostate> {}; +struct IsMonostate : std::is_same, std::monostate> {}; } // namespace @@ -171,7 +174,7 @@ absl::Status Value::ConvertToJsonArray( google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE); return variant_.Visit(absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError("use of invalid Value"); }, [descriptor_pool, message_factory, json]( @@ -195,9 +198,8 @@ absl::Status Value::ConvertToJsonArray( json); }, [](const auto& alternative) -> absl::Status { - return TypeConversionError(alternative.GetTypeName(), - "google.protobuf.ListValue") - .NativeValue(); + return common_internal::MakeTypeConversionError( + alternative.GetTypeName(), "google.protobuf.ListValue"); })); } @@ -212,7 +214,7 @@ absl::Status Value::ConvertToJsonObject( google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT); return variant_.Visit(absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError("use of invalid Value"); }, [descriptor_pool, message_factory, json]( @@ -252,9 +254,8 @@ absl::Status Value::ConvertToJsonObject( json); }, [](const auto& alternative) -> absl::Status { - return TypeConversionError(alternative.GetTypeName(), - "google.protobuf.Struct") - .NativeValue(); + return common_internal::MakeTypeConversionError( + alternative.GetTypeName(), "google.protobuf.Struct"); })); } @@ -320,12 +321,20 @@ Value NonNullEnumValue(const google::protobuf::EnumValueDescriptor* absl_nonnull } Value NonNullEnumValue(const google::protobuf::EnumDescriptor* absl_nonnull type, - int32_t number) { + int32_t number, google::protobuf::Arena* absl_nullable arena) { ABSL_DCHECK(type != nullptr); if (type->is_closed()) { if (ABSL_PREDICT_FALSE(type->FindValueByNumber(number) == nullptr)) { - return ErrorValue(absl::InvalidArgumentError(absl::StrCat( - "closed enum has no such value: ", type->full_name(), ".", number))); + if (arena == nullptr) { + static const absl::NoDestructor error( + absl::InvalidArgumentError("closed enum has no such value", + absl::SourceLocation())); + return ErrorValue::WrapUnsafe(&*error); + } + return ErrorValue::From(absl::InvalidArgumentError(absl::StrCat( + "closed enum has no such value: ", + type->full_name(), ".", number)), + arena); } } return IntValue(number); @@ -349,7 +358,18 @@ Value Value::Enum(const google::protobuf::EnumDescriptor* absl_nonnull type, ABSL_DCHECK_EQ(number, 0); return NullValue(); } - return NonNullEnumValue(type, number); + return NonNullEnumValue(type, number, nullptr); +} + +Value Value::Enum(const google::protobuf::EnumDescriptor* absl_nonnull type, + int32_t number, google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK(type != nullptr); + ABSL_DCHECK(arena != nullptr); + if (type->full_name() == "google.protobuf.NullValue") { + ABSL_DCHECK_EQ(number, 0); + return NullValue(); + } + return NonNullEnumValue(type, number, arena); } namespace common_internal { @@ -420,10 +440,14 @@ void StringMapFieldKeyAccessor(const google::protobuf::MapKey& key, ABSL_DCHECK(result != nullptr); #if CEL_INTERNAL_PROTOBUF_OSS_VERSION_PREREQ(5, 30, 0) - *result = StringValue(Borrower::Arena(MessageArenaOr(message, arena)), - key.GetStringValue()); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + *result = StringValue::Wrap(key.GetStringValue(), message_arena); + } else { + *result = StringValue::From(key.GetStringValue(), arena); + } #else - *result = StringValue(arena, key.GetStringValue()); + *result = StringValue::From(key.GetStringValue(), arena); #endif } @@ -602,9 +626,9 @@ void StringMapFieldValueAccessor( ABSL_DCHECK_EQ(field->type(), google::protobuf::FieldDescriptor::TYPE_STRING); if (message->GetArena() == nullptr) { - *result = StringValue(arena, value.GetStringValue()); + *result = StringValue::From(value.GetStringValue(), arena); } else { - *result = StringValue(Borrower::Arena(arena), value.GetStringValue()); + *result = StringValue::Wrap(value.GetStringValue(), message->GetArena()); } } @@ -645,9 +669,9 @@ void BytesMapFieldValueAccessor( ABSL_DCHECK_EQ(field->type(), google::protobuf::FieldDescriptor::TYPE_BYTES); if (message->GetArena() == nullptr) { - *result = BytesValue(arena, value.GetStringValue()); + *result = BytesValue::From(value.GetStringValue(), arena); } else { - *result = BytesValue(Borrower::Arena(arena), value.GetStringValue()); + *result = BytesValue::Wrap(value.GetStringValue(), message->GetArena()); } } @@ -667,7 +691,7 @@ void EnumMapFieldValueAccessor( ABSL_DCHECK(!field->is_repeated()); ABSL_DCHECK_EQ(field->cpp_type(), google::protobuf::FieldDescriptor::CPPTYPE_ENUM); - *result = NonNullEnumValue(field->enum_type(), value.GetEnumValue()); + *result = NonNullEnumValue(field->enum_type(), value.GetEnumValue(), arena); } void NullMapFieldValueAccessor( @@ -938,16 +962,18 @@ void StringRepeatedFieldAccessor( [&](absl::string_view string) { if (string.data() == scratch.data() && string.size() == scratch.size()) { - *result = StringValue(arena, std::move(scratch)); + *result = StringValue::From(std::move(scratch), arena); } else { if (message->GetArena() == nullptr) { - *result = StringValue(arena, string); + *result = StringValue::From(string, arena); } else { - *result = StringValue(Borrower::Arena(arena), string); + *result = StringValue::Wrap(string, message->GetArena()); } } }, - [&](absl::Cord&& cord) { *result = StringValue(std::move(cord)); }), + [&](absl::Cord&& cord) { + *result = StringValue::From(std::move(cord), arena); + }), well_known_types::AsVariant(well_known_types::GetRepeatedStringField( *message, field, index, scratch))); } @@ -1005,16 +1031,18 @@ void BytesRepeatedFieldAccessor( [&](absl::string_view string) { if (string.data() == scratch.data() && string.size() == scratch.size()) { - *result = BytesValue(arena, std::move(scratch)); + *result = BytesValue::From(std::move(scratch), arena); } else { if (message->GetArena() == nullptr) { - *result = BytesValue(arena, string); + *result = BytesValue::From(string, arena); } else { - *result = BytesValue(Borrower::Arena(arena), string); + *result = BytesValue::Wrap(string, message->GetArena()); } } }, - [&](absl::Cord&& cord) { *result = BytesValue(std::move(cord)); }), + [&](absl::Cord&& cord) { + *result = BytesValue::From(std::move(cord), arena); + }), well_known_types::AsVariant(well_known_types::GetRepeatedBytesField( *message, field, index, scratch))); } @@ -1042,7 +1070,7 @@ void EnumRepeatedFieldAccessor( *result = NonNullEnumValue( field->enum_type(), - reflection->GetRepeatedEnumValue(*message, field, index)); + reflection->GetRepeatedEnumValue(*message, field, index), arena); } void NullRepeatedFieldAccessor( @@ -1144,10 +1172,13 @@ Value VistWellKnownTypeValue(float value) { return DoubleValue(value); } Value VistWellKnownTypeValue(double value) { return DoubleValue(value); } Value VistWellKnownTypeValue(absl::Duration value) { - return DurationValue(value); + // Tolerate out-of-range values. + return UnsafeDurationValue(value); } -Value VistWellKnownTypeValue(absl::Time value) { return TimestampValue(value); } +Value VistWellKnownTypeValue(absl::Time value) { + return UnsafeTimestampValue(value); +} struct OwningWellKnownTypesValueVisitor { google::protobuf::Arena* absl_nullable arena; @@ -1161,15 +1192,16 @@ struct OwningWellKnownTypesValueVisitor { } if (scratch->data() == string.data() && scratch->size() == string.size()) { - return BytesValue(arena, std::move(*scratch)); + return BytesValue::From(std::move(*scratch), + arena); } - return BytesValue(arena, string); + return BytesValue::From(string, arena); }, [&](absl::Cord&& cord) -> BytesValue { if (cord.empty()) { return BytesValue(); } - return BytesValue(arena, cord); + return BytesValue::From(cord, arena); }), well_known_types::AsVariant(std::move(value))); } @@ -1182,15 +1214,16 @@ struct OwningWellKnownTypesValueVisitor { } if (scratch->data() == string.data() && scratch->size() == string.size()) { - return StringValue(arena, std::move(*scratch)); + return StringValue::From(std::move(*scratch), + arena); } - return StringValue(arena, string); + return StringValue::From(string, arena); }, [&](absl::Cord&& cord) -> StringValue { if (cord.empty()) { return StringValue(); } - return StringValue(arena, cord); + return StringValue::From(cord, arena); }), well_known_types::AsVariant(std::move(value))); } @@ -1209,7 +1242,8 @@ struct OwningWellKnownTypesValueVisitor { cloned->CopyFrom(*value); return ParsedJsonListValue(cloned, arena); } - return ParsedJsonListValue(value.release(), arena); + return ParsedJsonListValue(value.try_as_arena_ptr()->get(), + arena); }), well_known_types::AsVariant(std::move(value))); } @@ -1223,23 +1257,23 @@ struct OwningWellKnownTypesValueVisitor { return ParsedJsonMapValue(cloned, arena); }, [&](well_known_types::StructPtr value) -> MapValue { - if (value.arena() != arena) { + if (value.GetOwningArena() != arena) { auto* cloned = value->New(arena); cloned->CopyFrom(*value); return ParsedJsonMapValue(cloned, arena); } - return ParsedJsonMapValue(value.release(), arena); + return ParsedJsonMapValue(value.try_as_arena_ptr()->get(), arena); }), well_known_types::AsVariant(std::move(value))); } - Value operator()(Unique value) const { + Value operator()(google::protobuf::Arena::UniquePtr value) const { if (value->GetArena() != arena) { auto* cloned = value->New(arena); cloned->CopyFrom(*value); return ParsedMessageValue(cloned, arena); } - return ParsedMessageValue(value.release(), arena); + return ParsedMessageValue(value.try_as_arena_ptr()->get(), arena); } template @@ -1259,14 +1293,17 @@ struct BorrowingWellKnownTypesValueVisitor { [&](absl::string_view string) -> BytesValue { if (string.data() == scratch->data() && string.size() == scratch->size()) { - return BytesValue(arena, std::move(*scratch)); + return BytesValue::From(std::move(*scratch), arena); } else { - return BytesValue( - Borrower::Arena(MessageArenaOr(message, arena)), string); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return BytesValue::Wrap(string, message_arena); + } + return BytesValue::From(string, arena); } }, [&](absl::Cord&& cord) -> BytesValue { - return BytesValue(std::move(cord)); + return BytesValue::From(std::move(cord), arena); }), well_known_types::AsVariant(std::move(value))); } @@ -1277,14 +1314,17 @@ struct BorrowingWellKnownTypesValueVisitor { [&](absl::string_view string) -> StringValue { if (string.data() == scratch->data() && string.size() == scratch->size()) { - return StringValue(arena, std::move(*scratch)); + return StringValue::From(std::move(*scratch), arena); } else { - return StringValue( - Borrower::Arena(MessageArenaOr(message, arena)), string); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return StringValue::Wrap(string, message_arena); + } + return StringValue::From(string, arena); } }, [&](absl::Cord&& cord) -> StringValue { - return StringValue(std::move(cord)); + return StringValue::From(std::move(cord), arena); }), well_known_types::AsVariant(std::move(value))); } @@ -1303,7 +1343,8 @@ struct BorrowingWellKnownTypesValueVisitor { cloned->CopyFrom(*value); return ParsedJsonListValue(cloned, arena); } - return ParsedJsonListValue(value.release(), arena); + return ParsedJsonListValue(value.try_as_arena_ptr()->get(), + arena); }), well_known_types::AsVariant(std::move(value))); } @@ -1321,18 +1362,18 @@ struct BorrowingWellKnownTypesValueVisitor { cloned->CopyFrom(*value); return ParsedJsonMapValue(cloned, arena); } - return ParsedJsonMapValue(value.release(), arena); + return ParsedJsonMapValue(value.try_as_arena_ptr()->get(), arena); }), well_known_types::AsVariant(std::move(value))); } - Value operator()(Unique&& value) const { + Value operator()(google::protobuf::Arena::UniquePtr&& value) const { if (value->GetArena() != arena) { auto* cloned = value->New(arena); cloned->CopyFrom(*value); return ParsedMessageValue(cloned, arena); } - return ParsedMessageValue(value.release(), arena); + return ParsedMessageValue(value.try_as_arena_ptr()->get(), arena); } template @@ -1358,12 +1399,12 @@ Value Value::FromMessage( auto status_or_adapted = well_known_types::AdaptFromMessage( arena, message, descriptor_pool, message_factory, scratch); if (ABSL_PREDICT_FALSE(!status_or_adapted.ok())) { - return ErrorValue(std::move(status_or_adapted).status()); + return ErrorValue::From(std::move(status_or_adapted).status(), arena); } return absl::visit( absl::Overload(OwningWellKnownTypesValueVisitor{ /* .arena = */ arena, /* .scratch = */ &scratch}, - [&](absl::monostate) -> Value { + [&](std::monostate) -> Value { auto* cloned = message.New(arena); cloned->CopyFrom(message); return ParsedMessageValue(cloned, arena); @@ -1386,12 +1427,12 @@ Value Value::FromMessage( auto status_or_adapted = well_known_types::AdaptFromMessage( arena, message, descriptor_pool, message_factory, scratch); if (ABSL_PREDICT_FALSE(!status_or_adapted.ok())) { - return ErrorValue(std::move(status_or_adapted).status()); + return ErrorValue::From(std::move(status_or_adapted).status(), arena); } return absl::visit( absl::Overload(OwningWellKnownTypesValueVisitor{ /* .arena = */ arena, /* .scratch = */ &scratch}, - [&](absl::monostate) -> Value { + [&](std::monostate) -> Value { auto* cloned = message.New(arena); cloned->GetReflection()->Swap(cloned, &message); return ParsedMessageValue(cloned, arena); @@ -1416,13 +1457,13 @@ Value Value::WrapMessage( well_known_types::AdaptFromMessage(arena, *message, descriptor_pool, message_factory, scratch); if (ABSL_PREDICT_FALSE(!adapted_value.ok())) { - return ErrorValue(std::move(adapted_value).status()); + return ErrorValue::From(std::move(adapted_value).status(), arena); } return absl::visit( absl::Overload(BorrowingWellKnownTypesValueVisitor{ /* .message = */ message, /* .arena = */ arena, /* .scratch = */ &scratch}, - [&](absl::monostate) -> Value { + [&](std::monostate) -> Value { if (message->GetArena() != arena) { auto* cloned = message->New(arena); cloned->CopyFrom(*message); @@ -1450,13 +1491,13 @@ Value Value::WrapMessageUnsafe( well_known_types::AdaptFromMessage(arena, *message, descriptor_pool, message_factory, scratch); if (ABSL_PREDICT_FALSE(!adapted_value.ok())) { - return ErrorValue(std::move(adapted_value).status()); + return ErrorValue::From(std::move(adapted_value).status(), arena); } return absl::visit( absl::Overload(BorrowingWellKnownTypesValueVisitor{ /* .message = */ message, /* .arena = */ arena, /* .scratch = */ &scratch}, - [&](absl::monostate) -> Value { + [&](std::monostate) -> Value { if (message->GetArena() != arena) { return UnsafeParsedMessageValue(message); } @@ -1511,10 +1552,15 @@ Value WrapFieldImpl( ABSL_DCHECK(!IsWellKnownMessageType(message->GetDescriptor())); const auto* reflection = message->GetReflection(); + if (ABSL_PREDICT_FALSE(reflection == nullptr)) { + // This only happens for special implementations of Message that + // should not normally be used with CEL. + return ErrorValue::From(absl::InvalidArgumentError(absl::StrCat( + "failed to get reflection for message type: ", + message->GetDescriptor()->full_name())), + arena); + } if (field->is_map()) { - if (reflection->FieldSize(*message, field) == 0) { - return MapValue(); - } if constexpr (Unsafe::value) { return UnsafeParsedMapFieldValue(message, field); } else { @@ -1523,9 +1569,6 @@ Value WrapFieldImpl( } } if (field->is_repeated()) { - if (reflection->FieldSize(*message, field) == 0) { - return ListValue(); - } if constexpr (Unsafe::value) { return UnsafeParsedRepeatedFieldValue(message, field); } else { @@ -1557,17 +1600,20 @@ Value WrapFieldImpl( [&](absl::string_view string) -> StringValue { if (string.data() == scratch.data() && string.size() == scratch.size()) { - return StringValue(arena, std::move(scratch)); + return StringValue::From(std::move(scratch), arena); } if constexpr (Unsafe::value) { return StringValue::WrapUnsafe(string); } else { - return StringValue( - Borrower::Arena(MessageArenaOr(message, arena)), string); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return StringValue::Wrap(string, message_arena); + } + return StringValue::From(string, arena); } }, [&](absl::Cord&& cord) -> StringValue { - return StringValue(std::move(cord)); + return StringValue::From(std::move(cord), arena); }), well_known_types::AsVariant( well_known_types::GetStringField(*message, field, scratch))); @@ -1582,11 +1628,12 @@ Value WrapFieldImpl( } if constexpr (Unsafe::value) { return Value::WrapMessageUnsafe( - &reflection->GetMessage(*message, field), descriptor_pool, - message_factory, arena); + &reflection->GetMessage(*message, field, message_factory), + descriptor_pool, message_factory, arena); } else { - return Value::WrapMessage(&reflection->GetMessage(*message, field), - descriptor_pool, message_factory, arena); + return Value::WrapMessage( + &reflection->GetMessage(*message, field, message_factory), + descriptor_pool, message_factory, arena); } case google::protobuf::FieldDescriptor::TYPE_BYTES: { std::string scratch; @@ -1595,17 +1642,20 @@ Value WrapFieldImpl( [&](absl::string_view string) -> BytesValue { if (string.data() == scratch.data() && string.size() == scratch.size()) { - return BytesValue(arena, std::move(scratch)); + return BytesValue::From(std::move(scratch), arena); } if constexpr (Unsafe::value) { return BytesValue::WrapUnsafe(string); } else { - return BytesValue( - Borrower::Arena(MessageArenaOr(message, arena)), string); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return BytesValue::Wrap(string, message_arena); + } + return BytesValue::From(string, arena); } }, [&](absl::Cord&& cord) -> BytesValue { - return BytesValue(std::move(cord)); + return BytesValue::From(std::move(cord), arena); }), well_known_types::AsVariant( well_known_types::GetBytesField(*message, field, scratch))); @@ -1624,9 +1674,11 @@ Value WrapFieldImpl( case google::protobuf::FieldDescriptor::TYPE_SINT64: return IntValue(reflection->GetInt64(*message, field)); default: - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("unexpected protocol buffer message field type: ", - field->type_name()))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrCat("unexpected protocol buffer message field type: ", + field->type_name())), + arena); } } @@ -1651,10 +1703,19 @@ Value WrapRepeatedFieldImpl( ABSL_DCHECK(arena != nullptr); const auto* reflection = message->GetReflection(); + if (ABSL_PREDICT_FALSE(reflection == nullptr)) { + // This only happens for special implementations of Message that + // should not normally be used with CEL. + return ErrorValue::From(absl::InvalidArgumentError(absl::StrCat( + "failed to get reflection for message type: ", + message->GetDescriptor()->full_name())), + arena); + } const int size = reflection->FieldSize(*message, field); if (ABSL_PREDICT_FALSE(index < 0 || index >= size)) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("index out of bounds: ", index))); + return ErrorValue::From(absl::InvalidArgumentError( + absl::StrCat("index out of bounds: ", index)), + arena); } switch (field->type()) { case google::protobuf::FieldDescriptor::TYPE_DOUBLE: @@ -1686,17 +1747,20 @@ Value WrapRepeatedFieldImpl( [&](absl::string_view string) -> StringValue { if (string.data() == scratch.data() && string.size() == scratch.size()) { - return StringValue(arena, std::move(scratch)); + return StringValue::From(std::move(scratch), arena); } if constexpr (Unsafe::value) { return StringValue::WrapUnsafe(string); } else { - return StringValue( - Borrower::Arena(MessageArenaOr(message, arena)), string); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return StringValue::Wrap(string, message_arena); + } + return StringValue::From(string, arena); } }, [&](absl::Cord&& cord) -> StringValue { - return StringValue(std::move(cord)); + return StringValue::From(std::move(cord), arena); }), well_known_types::AsVariant(well_known_types::GetRepeatedStringField( reflection, *message, field, index, scratch))); @@ -1720,17 +1784,20 @@ Value WrapRepeatedFieldImpl( [&](absl::string_view string) -> BytesValue { if (string.data() == scratch.data() && string.size() == scratch.size()) { - return BytesValue(arena, std::move(scratch)); + return BytesValue::From(std::move(scratch), arena); } if constexpr (Unsafe::value) { return BytesValue::WrapUnsafe(string); } else { - return BytesValue( - Borrower::Arena(MessageArenaOr(message, arena)), string); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return BytesValue::Wrap(string, message_arena); + } + return BytesValue::From(string, arena); } }, [&](absl::Cord&& cord) -> BytesValue { - return BytesValue(std::move(cord)); + return BytesValue::From(std::move(cord), arena); }), well_known_types::AsVariant(well_known_types::GetRepeatedBytesField( reflection, *message, field, index, scratch))); @@ -1743,8 +1810,10 @@ Value WrapRepeatedFieldImpl( return Value::Enum(field->enum_type(), reflection->GetRepeatedEnumValue( *message, field, index)); default: - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("unexpected message field type: ", field->type_name()))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrCat( + "unexpected message field type: ", field->type_name())), + arena); } } @@ -1796,8 +1865,7 @@ Value WrapMapFieldValueImpl( if constexpr (Unsafe::value) { return StringValue::WrapUnsafe(value.GetStringValue()); } else { - return StringValue(Borrower::Arena(MessageArenaOr(message, arena)), - value.GetStringValue()); + return StringValue::From(value.GetStringValue(), arena); } case google::protobuf::FieldDescriptor::TYPE_GROUP: ABSL_FALLTHROUGH_INTENDED; @@ -1813,8 +1881,7 @@ Value WrapMapFieldValueImpl( if constexpr (Unsafe::value) { return BytesValue::WrapUnsafe(value.GetStringValue()); } else { - return BytesValue(Borrower::Arena(MessageArenaOr(message, arena)), - value.GetStringValue()); + return BytesValue::From(value.GetStringValue(), arena); } case google::protobuf::FieldDescriptor::TYPE_FIXED32: ABSL_FALLTHROUGH_INTENDED; @@ -1823,8 +1890,10 @@ Value WrapMapFieldValueImpl( case google::protobuf::FieldDescriptor::TYPE_ENUM: return Value::Enum(field->enum_type(), value.GetEnumValue()); default: - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("unexpected message field type: ", field->type_name()))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrCat( + "unexpected message field type: ", field->type_name())), + arena); } } @@ -1899,11 +1968,12 @@ StringValue Value::WrapMapFieldKeyString( ABSL_DCHECK_EQ(key.type(), google::protobuf::FieldDescriptor::CPPTYPE_STRING); #if CEL_INTERNAL_PROTOBUF_OSS_VERSION_PREREQ(5, 30, 0) - return StringValue(Borrower::Arena(MessageArenaOr(message, arena)), - key.GetStringValue()); -#else - return StringValue(arena, key.GetStringValue()); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return StringValue::Wrap(key.GetStringValue(), message_arena); + } #endif + return StringValue::From(key.GetStringValue(), arena); } Value Value::WrapMapFieldValue( @@ -1936,60 +2006,6 @@ Value Value::WrapMapFieldValueUnsafe( message_factory, arena); } -optional_ref Value::AsBytes() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsBytes() && { - if (auto* alternative = variant_.As(); alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; -} - -absl::optional Value::AsDouble() const { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsDuration() const { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -optional_ref Value::AsError() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsError() && { - if (auto* alternative = variant_.As(); alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; -} - -absl::optional Value::AsInt() const { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - absl::optional Value::AsList() const& { if (const auto* alternative = variant_.As(); alternative != nullptr) { @@ -2007,7 +2023,7 @@ absl::optional Value::AsList() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsList() && { @@ -2027,7 +2043,7 @@ absl::optional Value::AsList() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsMap() const& { @@ -2047,7 +2063,7 @@ absl::optional Value::AsMap() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsMap() && { @@ -2067,206 +2083,7 @@ absl::optional Value::AsMap() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; -} - -absl::optional Value::AsMessage() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsMessage() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; -} - -absl::optional Value::AsNull() const { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -optional_ref Value::AsOpaque() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsOpaque() && { - if (auto* alternative = variant_.As(); alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; -} - -optional_ref Value::AsOptional() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr && alternative->IsOptional()) { - return static_cast(*alternative); - } - return absl::nullopt; -} - -absl::optional Value::AsOptional() && { - if (auto* alternative = variant_.As(); - alternative != nullptr && alternative->IsOptional()) { - return static_cast(*alternative); - } - return absl::nullopt; -} - -optional_ref Value::AsParsedJsonList() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsParsedJsonList() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; -} - -optional_ref Value::AsParsedJsonMap() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsParsedJsonMap() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; -} - -optional_ref Value::AsCustomList() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsCustomList() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; -} - -optional_ref Value::AsCustomMap() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsCustomMap() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; -} - -optional_ref Value::AsParsedMapField() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsParsedMapField() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; -} - -optional_ref Value::AsParsedMessage() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsParsedMessage() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; -} - -optional_ref Value::AsParsedRepeatedField() - const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsParsedRepeatedField() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; -} - -optional_ref Value::AsCustomStruct() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsCustomStruct() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; -} - -optional_ref Value::AsString() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsString() && { - if (auto* alternative = variant_.As(); alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsStruct() const& { @@ -2283,7 +2100,7 @@ absl::optional Value::AsStruct() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsStruct() && { @@ -2299,88 +2116,7 @@ absl::optional Value::AsStruct() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; -} - -absl::optional Value::AsTimestamp() const { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -optional_ref Value::AsType() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsType() && { - if (auto* alternative = variant_.As(); alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; -} - -absl::optional Value::AsUint() const { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -optional_ref Value::AsUnknown() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return absl::nullopt; -} - -absl::optional Value::AsUnknown() && { - if (auto* alternative = variant_.As(); alternative != nullptr) { - return std::move(*alternative); - } - return absl::nullopt; -} - -const BytesValue& Value::GetBytes() const& { - ABSL_DCHECK(IsBytes()) << *this; - return variant_.Get(); -} - -BytesValue Value::GetBytes() && { - ABSL_DCHECK(IsBytes()) << *this; - return std::move(variant_).Get(); -} - -DoubleValue Value::GetDouble() const { - ABSL_DCHECK(IsDouble()) << *this; - return variant_.Get(); -} - -DurationValue Value::GetDuration() const { - ABSL_DCHECK(IsDuration()) << *this; - return variant_.Get(); -} - -const ErrorValue& Value::GetError() const& { - ABSL_DCHECK(IsError()) << *this; - return variant_.Get(); -} - -ErrorValue Value::GetError() && { - ABSL_DCHECK(IsError()) << *this; - return std::move(variant_).Get(); -} - -IntValue Value::GetInt() const { - ABSL_DCHECK(IsInt()) << *this; - return variant_.Get(); + return std::nullopt; } #ifdef ABSL_HAVE_EXCEPTIONS @@ -2474,131 +2210,6 @@ MapValue Value::GetMap() && { CEL_VALUE_THROW_BAD_VARIANT_ACCESS(); } -MessageValue Value::GetMessage() const& { - ABSL_DCHECK(IsMessage()) << *this; - return variant_.Get(); -} - -MessageValue Value::GetMessage() && { - ABSL_DCHECK(IsMessage()) << *this; - return std::move(variant_).Get(); -} - -NullValue Value::GetNull() const { - ABSL_DCHECK(IsNull()) << *this; - return variant_.Get(); -} - -const OpaqueValue& Value::GetOpaque() const& { - ABSL_DCHECK(IsOpaque()) << *this; - return variant_.Get(); -} - -OpaqueValue Value::GetOpaque() && { - ABSL_DCHECK(IsOpaque()) << *this; - return std::move(variant_).Get(); -} - -const OptionalValue& Value::GetOptional() const& { - ABSL_DCHECK(IsOptional()) << *this; - return static_cast(variant_.Get()); -} - -OptionalValue Value::GetOptional() && { - ABSL_DCHECK(IsOptional()) << *this; - return static_cast(std::move(variant_).Get()); -} - -const ParsedJsonListValue& Value::GetParsedJsonList() const& { - ABSL_DCHECK(IsParsedJsonList()) << *this; - return variant_.Get(); -} - -ParsedJsonListValue Value::GetParsedJsonList() && { - ABSL_DCHECK(IsParsedJsonList()) << *this; - return std::move(variant_).Get(); -} - -const ParsedJsonMapValue& Value::GetParsedJsonMap() const& { - ABSL_DCHECK(IsParsedJsonMap()) << *this; - return variant_.Get(); -} - -ParsedJsonMapValue Value::GetParsedJsonMap() && { - ABSL_DCHECK(IsParsedJsonMap()) << *this; - return std::move(variant_).Get(); -} - -const CustomListValue& Value::GetCustomList() const& { - ABSL_DCHECK(IsCustomList()) << *this; - return variant_.Get(); -} - -CustomListValue Value::GetCustomList() && { - ABSL_DCHECK(IsCustomList()) << *this; - return std::move(variant_).Get(); -} - -const CustomMapValue& Value::GetCustomMap() const& { - ABSL_DCHECK(IsCustomMap()) << *this; - return variant_.Get(); -} - -CustomMapValue Value::GetCustomMap() && { - ABSL_DCHECK(IsCustomMap()) << *this; - return std::move(variant_).Get(); -} - -const ParsedMapFieldValue& Value::GetParsedMapField() const& { - ABSL_DCHECK(IsParsedMapField()) << *this; - return variant_.Get(); -} - -ParsedMapFieldValue Value::GetParsedMapField() && { - ABSL_DCHECK(IsParsedMapField()) << *this; - return std::move(variant_).Get(); -} - -const ParsedMessageValue& Value::GetParsedMessage() const& { - ABSL_DCHECK(IsParsedMessage()) << *this; - return variant_.Get(); -} - -ParsedMessageValue Value::GetParsedMessage() && { - ABSL_DCHECK(IsParsedMessage()) << *this; - return std::move(variant_).Get(); -} - -const ParsedRepeatedFieldValue& Value::GetParsedRepeatedField() const& { - ABSL_DCHECK(IsParsedRepeatedField()) << *this; - return variant_.Get(); -} - -ParsedRepeatedFieldValue Value::GetParsedRepeatedField() && { - ABSL_DCHECK(IsParsedRepeatedField()) << *this; - return std::move(variant_).Get(); -} - -const CustomStructValue& Value::GetCustomStruct() const& { - ABSL_DCHECK(IsCustomStruct()) << *this; - return variant_.Get(); -} - -CustomStructValue Value::GetCustomStruct() && { - ABSL_DCHECK(IsCustomStruct()) << *this; - return std::move(variant_).Get(); -} - -const StringValue& Value::GetString() const& { - ABSL_DCHECK(IsString()) << *this; - return variant_.Get(); -} - -StringValue Value::GetString() && { - ABSL_DCHECK(IsString()) << *this; - return std::move(variant_).Get(); -} - StructValue Value::GetStruct() const& { ABSL_DCHECK(IsStruct()) << *this; if (const auto* alternative = @@ -2634,36 +2245,6 @@ StructValue Value::GetStruct() && { CEL_VALUE_THROW_BAD_VARIANT_ACCESS(); } -TimestampValue Value::GetTimestamp() const { - ABSL_DCHECK(IsTimestamp()) << *this; - return variant_.Get(); -} - -const TypeValue& Value::GetType() const& { - ABSL_DCHECK(IsType()) << *this; - return variant_.Get(); -} - -TypeValue Value::GetType() && { - ABSL_DCHECK(IsType()) << *this; - return std::move(variant_).Get(); -} - -UintValue Value::GetUint() const { - ABSL_DCHECK(IsUint()) << *this; - return variant_.Get(); -} - -const UnknownValue& Value::GetUnknown() const& { - ABSL_DCHECK(IsUnknown()) << *this; - return variant_.Get(); -} - -UnknownValue Value::GetUnknown() && { - ABSL_DCHECK(IsUnknown()) << *this; - return std::move(variant_).Get(); -} - namespace { class EmptyValueIterator final : public ValueIterator { @@ -2787,4 +2368,14 @@ absl::StatusOr ValueIterator::Next1( return false; } +StringValue::StringValue(const BytesValue& other) : StringValue(other.value_) { + ABSL_DCHECK(value_.Visit(absl::Overload( + [](absl::string_view string) -> bool { + return internal::Utf8IsValid(string); + }, + [](const absl::Cord& cord) -> bool { + return internal::Utf8IsValid(cord); + }))); +} + } // namespace cel diff --git a/common/value.h b/common/value.h index 34b4714a7..fc7992c49 100644 --- a/common/value.h +++ b/common/value.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -102,8 +103,11 @@ class Value final : private common_internal::ValueMixin { // enums, returns `cel::IntValue`. For closed enums, returns `cel::ErrorValue` // if the value is not present in the enum otherwise returns `cel::IntValue`. static Value Enum(const google::protobuf::EnumValueDescriptor* absl_nonnull value); + ABSL_DEPRECATED("Use overload which takes an arena pointer") static Value Enum(const google::protobuf::EnumDescriptor* absl_nonnull type, int32_t number); + static Value Enum(const google::protobuf::EnumDescriptor* absl_nonnull type, + int32_t number, google::protobuf::Arena* absl_nonnull arena); // SFINAE overload for generated protobuf enums which are not well-known. // Always returns `cel::IntValue`. @@ -294,103 +298,54 @@ class Value final : private common_internal::ValueMixin { Value() = default; Value(const Value&) = default; Value& operator=(const Value&) = default; - Value(Value&& other) = default; - Value& operator=(Value&&) = default; // NOLINTNEXTLINE(google-explicit-constructor) Value(const ListValue& value) : variant_(value.ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(ListValue&& value) : variant_(std::move(value).ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const ListValue& value) { variant_ = value.ToValueVariant(); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(ListValue&& value) { - variant_ = std::move(value).ToValueVariant(); - return *this; - } - // NOLINTNEXTLINE(google-explicit-constructor) Value(const MapValue& value) : variant_(value.ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(MapValue&& value) : variant_(std::move(value).ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const MapValue& value) { variant_ = value.ToValueVariant(); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(MapValue&& value) { - variant_ = std::move(value).ToValueVariant(); - return *this; - } - // NOLINTNEXTLINE(google-explicit-constructor) Value(const StructValue& value) : variant_(value.ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(StructValue&& value) : variant_(std::move(value).ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const StructValue& value) { variant_ = value.ToValueVariant(); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(StructValue&& value) { - variant_ = std::move(value).ToValueVariant(); - return *this; - } - // NOLINTNEXTLINE(google-explicit-constructor) Value(const MessageValue& value) : variant_(value.ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(MessageValue&& value) : variant_(std::move(value).ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const MessageValue& value) { variant_ = value.ToValueVariant(); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(MessageValue&& value) { - variant_ = std::move(value).ToValueVariant(); - return *this; - } - // NOLINTNEXTLINE(google-explicit-constructor) Value(const OptionalValue& value) : variant_(absl::in_place_type, static_cast(value)) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(OptionalValue&& value) - : variant_(absl::in_place_type, - static_cast(value)) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const OptionalValue& value) { variant_.Assign(static_cast(value)); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(OptionalValue&& value) { - variant_.Assign(static_cast(value)); - return *this; - } - template >>> @@ -1065,9 +1020,7 @@ class Value final : private common_internal::ValueMixin { // Performs a checked cast from a value to an unknown value, // returning a non-empty optional with either a value or reference to the // unknown value. Otherwise an empty optional is returned. - optional_ref AsUnknown() & ABSL_ATTRIBUTE_LIFETIME_BOUND { - return std::as_const(*this).AsUnknown(); - } + optional_ref AsUnknown() & ABSL_ATTRIBUTE_LIFETIME_BOUND; optional_ref AsUnknown() const& ABSL_ATTRIBUTE_LIFETIME_BOUND; absl::optional AsUnknown() &&; @@ -1711,7 +1664,7 @@ class Value final : private common_internal::ValueMixin { // `AsUnknown()`. template std::enable_if_t, - optional_ref> + optional_ref> As() & ABSL_ATTRIBUTE_LIFETIME_BOUND { return AsUnknown(); } @@ -2609,7 +2562,7 @@ static_assert(std::is_nothrow_swappable_v); inline common_internal::ImplicitlyConvertibleStatus ErrorValueAssign::operator()(absl::Status status) const { - *value_ = ErrorValue(std::move(status)); + *value_ = ErrorValue::From(std::move(status), arena_); return common_internal::ImplicitlyConvertibleStatus(); } @@ -2802,7 +2755,7 @@ absl::StatusOr> StructValueMixin::Qualify( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) const { - ABSL_DCHECK_GT(qualifiers.size(), 0); + ABSL_DCHECK(!qualifiers.empty()); ABSL_DCHECK(descriptor_pool != nullptr); ABSL_DCHECK(message_factory != nullptr); ABSL_DCHECK(arena != nullptr); @@ -2940,6 +2893,508 @@ absl::StatusOr RepeatedFieldAccessorFor( } // namespace common_internal +inline BytesValue::BytesValue(const StringValue& other) + : BytesValue(other.value_) {} + +inline optional_ref Value::AsBytes() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsBytes() && { + if (auto* alternative = variant_.As(); alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline absl::optional Value::AsDouble() const { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsDuration() const { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline optional_ref Value::AsError() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsError() && { + if (auto* alternative = variant_.As(); alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline absl::optional Value::AsInt() const { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsMessage() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsMessage() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline absl::optional Value::AsNull() const { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline optional_ref Value::AsOpaque() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsOpaque() && { + if (auto* alternative = variant_.As(); alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsOptional() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr && alternative->IsOptional()) { + return static_cast(*alternative); + } + return std::nullopt; +} + +inline absl::optional Value::AsOptional() && { + if (auto* alternative = variant_.As(); + alternative != nullptr && alternative->IsOptional()) { + return static_cast(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsParsedJsonList() + const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsParsedJsonList() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsParsedJsonMap() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsParsedJsonMap() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsCustomList() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsCustomList() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsCustomMap() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsCustomMap() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsParsedMapField() + const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsParsedMapField() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsParsedMessage() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsParsedMessage() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref +Value::AsParsedRepeatedField() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional +Value::AsParsedRepeatedField() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsCustomStruct() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsCustomStruct() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsString() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsString() && { + if (auto* alternative = variant_.As(); alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline absl::optional Value::AsTimestamp() const { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline optional_ref Value::AsType() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsType() && { + if (auto* alternative = variant_.As(); alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline absl::optional Value::AsUint() const { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline optional_ref Value::AsUnknown() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline optional_ref Value::AsUnknown() & { + if (auto* alternative = variant_.As(); alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsUnknown() && { + if (auto* alternative = variant_.As(); alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline const BytesValue& Value::GetBytes() const& { + ABSL_DCHECK(IsBytes()) << *this; + return variant_.Get(); +} + +inline BytesValue Value::GetBytes() && { + ABSL_DCHECK(IsBytes()) << *this; + return std::move(variant_).Get(); +} + +inline DoubleValue Value::GetDouble() const { + ABSL_DCHECK(IsDouble()) << *this; + return variant_.Get(); +} + +inline DurationValue Value::GetDuration() const { + ABSL_DCHECK(IsDuration()) << *this; + return variant_.Get(); +} + +inline const ErrorValue& Value::GetError() const& { + ABSL_DCHECK(IsError()) << *this; + return variant_.Get(); +} + +inline ErrorValue Value::GetError() && { + ABSL_DCHECK(IsError()) << *this; + return std::move(variant_).Get(); +} + +inline IntValue Value::GetInt() const { + ABSL_DCHECK(IsInt()) << *this; + return variant_.Get(); +} + +inline MessageValue Value::GetMessage() const& { + ABSL_DCHECK(IsMessage()) << *this; + return variant_.Get(); +} + +inline MessageValue Value::GetMessage() && { + ABSL_DCHECK(IsMessage()) << *this; + return std::move(variant_).Get(); +} + +inline NullValue Value::GetNull() const { + ABSL_DCHECK(IsNull()) << *this; + return variant_.Get(); +} + +inline const OpaqueValue& Value::GetOpaque() const& { + ABSL_DCHECK(IsOpaque()) << *this; + return variant_.Get(); +} + +inline OpaqueValue Value::GetOpaque() && { + ABSL_DCHECK(IsOpaque()) << *this; + return std::move(variant_).Get(); +} + +inline const OptionalValue& Value::GetOptional() const& { + ABSL_DCHECK(IsOptional()) << *this; + return static_cast(variant_.Get()); +} + +inline OptionalValue Value::GetOptional() && { + ABSL_DCHECK(IsOptional()) << *this; + return static_cast(std::move(variant_).Get()); +} + +inline const ParsedJsonListValue& Value::GetParsedJsonList() const& { + ABSL_DCHECK(IsParsedJsonList()) << *this; + return variant_.Get(); +} + +inline ParsedJsonListValue Value::GetParsedJsonList() && { + ABSL_DCHECK(IsParsedJsonList()) << *this; + return std::move(variant_).Get(); +} + +inline const ParsedJsonMapValue& Value::GetParsedJsonMap() const& { + ABSL_DCHECK(IsParsedJsonMap()) << *this; + return variant_.Get(); +} + +inline ParsedJsonMapValue Value::GetParsedJsonMap() && { + ABSL_DCHECK(IsParsedJsonMap()) << *this; + return std::move(variant_).Get(); +} + +inline const CustomListValue& Value::GetCustomList() const& { + ABSL_DCHECK(IsCustomList()) << *this; + return variant_.Get(); +} + +inline CustomListValue Value::GetCustomList() && { + ABSL_DCHECK(IsCustomList()) << *this; + return std::move(variant_).Get(); +} + +inline const CustomMapValue& Value::GetCustomMap() const& { + ABSL_DCHECK(IsCustomMap()) << *this; + return variant_.Get(); +} + +inline CustomMapValue Value::GetCustomMap() && { + ABSL_DCHECK(IsCustomMap()) << *this; + return std::move(variant_).Get(); +} + +inline const ParsedMapFieldValue& Value::GetParsedMapField() const& { + ABSL_DCHECK(IsParsedMapField()) << *this; + return variant_.Get(); +} + +inline ParsedMapFieldValue Value::GetParsedMapField() && { + ABSL_DCHECK(IsParsedMapField()) << *this; + return std::move(variant_).Get(); +} + +inline const ParsedMessageValue& Value::GetParsedMessage() const& { + ABSL_DCHECK(IsParsedMessage()) << *this; + return variant_.Get(); +} + +inline ParsedMessageValue Value::GetParsedMessage() && { + ABSL_DCHECK(IsParsedMessage()) << *this; + return std::move(variant_).Get(); +} + +inline const ParsedRepeatedFieldValue& Value::GetParsedRepeatedField() const& { + ABSL_DCHECK(IsParsedRepeatedField()) << *this; + return variant_.Get(); +} + +inline ParsedRepeatedFieldValue Value::GetParsedRepeatedField() && { + ABSL_DCHECK(IsParsedRepeatedField()) << *this; + return std::move(variant_).Get(); +} + +inline const CustomStructValue& Value::GetCustomStruct() const& { + ABSL_DCHECK(IsCustomStruct()) << *this; + return variant_.Get(); +} + +inline CustomStructValue Value::GetCustomStruct() && { + ABSL_DCHECK(IsCustomStruct()) << *this; + return std::move(variant_).Get(); +} + +inline const StringValue& Value::GetString() const& { + ABSL_DCHECK(IsString()) << *this; + return variant_.Get(); +} + +inline StringValue Value::GetString() && { + ABSL_DCHECK(IsString()) << *this; + return std::move(variant_).Get(); +} + +inline TimestampValue Value::GetTimestamp() const { + ABSL_DCHECK(IsTimestamp()) << *this; + return variant_.Get(); +} + +inline const TypeValue& Value::GetType() const& { + ABSL_DCHECK(IsType()) << *this; + return variant_.Get(); +} + +inline TypeValue Value::GetType() && { + ABSL_DCHECK(IsType()) << *this; + return std::move(variant_).Get(); +} + +inline UintValue Value::GetUint() const { + ABSL_DCHECK(IsUint()) << *this; + return variant_.Get(); +} + +inline const UnknownValue& Value::GetUnknown() const& { + ABSL_DCHECK(IsUnknown()) << *this; + return variant_.Get(); +} + +inline UnknownValue Value::GetUnknown() && { + ABSL_DCHECK(IsUnknown()) << *this; + return std::move(variant_).Get(); +} + } // namespace cel #pragma pop_macro("GetMessage") diff --git a/common/value_test.cc b/common/value_test.cc index fb346423b..56cd5252b 100644 --- a/common/value_test.cc +++ b/common/value_test.cc @@ -28,9 +28,11 @@ #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" #include "internal/testing_message_factory.h" +#include "cel/expr/conformance/proto2/test_all_types.pb.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/dynamic_message.h" #include "google/protobuf/generated_enum_reflection.h" namespace cel { @@ -190,7 +192,7 @@ TEST(Value, As) { google::protobuf::Arena arena; EXPECT_THAT(Value(BoolValue()).As(), Optional(An())); - EXPECT_THAT(Value(BoolValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(BoolValue()).As(), Eq(std::nullopt)); { Value value(BytesValue{}); @@ -207,11 +209,11 @@ TEST(Value, As) { EXPECT_THAT(Value(DoubleValue()).As(), Optional(An())); - EXPECT_THAT(Value(DoubleValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(DoubleValue()).As(), Eq(std::nullopt)); EXPECT_THAT(Value(DurationValue()).As(), Optional(An())); - EXPECT_THAT(Value(DurationValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(DurationValue()).As(), Eq(std::nullopt)); { Value value(ErrorValue{}); @@ -224,11 +226,11 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(ErrorValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(ErrorValue()).As(), Eq(std::nullopt)); } EXPECT_THAT(Value(IntValue()).As(), Optional(An())); - EXPECT_THAT(Value(IntValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(IntValue()).As(), Eq(std::nullopt)); { Value value(ListValue{}); @@ -241,7 +243,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(ListValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(ListValue()).As(), Eq(std::nullopt)); } { @@ -255,7 +257,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(ListValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(ListValue()).As(), Eq(std::nullopt)); } { @@ -282,7 +284,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(ListValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(ListValue()).As(), Eq(std::nullopt)); } { @@ -346,7 +348,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(MapValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(MapValue()).As(), Eq(std::nullopt)); } { @@ -360,7 +362,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(MapValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(MapValue()).As(), Eq(std::nullopt)); } { @@ -387,7 +389,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(MapValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(MapValue()).As(), Eq(std::nullopt)); } { @@ -460,11 +462,11 @@ TEST(Value, As) { GetTestingMessageFactory()), &arena}) .As(), - Eq(absl::nullopt)); + Eq(std::nullopt)); } EXPECT_THAT(Value(NullValue()).As(), Optional(An())); - EXPECT_THAT(Value(NullValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(NullValue()).As(), Eq(std::nullopt)); { Value value(OptionalValue{}); @@ -478,7 +480,7 @@ TEST(Value, As) { EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); EXPECT_THAT(Value(OpaqueValue(OptionalValue())).As(), - Eq(absl::nullopt)); + Eq(std::nullopt)); } { @@ -492,7 +494,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(OptionalValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(OptionalValue()).As(), Eq(std::nullopt)); } { @@ -536,7 +538,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(StringValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(StringValue()).As(), Eq(std::nullopt)); } { @@ -558,7 +560,7 @@ TEST(Value, As) { EXPECT_THAT(Value(TimestampValue()).As(), Optional(An())); - EXPECT_THAT(Value(TimestampValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(TimestampValue()).As(), Eq(std::nullopt)); { Value value(TypeValue(StringType{})); @@ -572,11 +574,11 @@ TEST(Value, As) { EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); EXPECT_THAT(Value(TypeValue(StringType())).As(), - Eq(absl::nullopt)); + Eq(std::nullopt)); } EXPECT_THAT(Value(UintValue()).As(), Optional(An())); - EXPECT_THAT(Value(UintValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(UintValue()).As(), Eq(std::nullopt)); { Value value(UnknownValue{}); @@ -589,7 +591,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(UnknownValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(UnknownValue()).As(), Eq(std::nullopt)); } } @@ -973,6 +975,43 @@ TEST(Value, NumericHeterogeneousEquality) { EXPECT_NE(DoubleValue(1), UintValue(2)); } +TEST(Value, WrapDynamicExtensionMessageField) { + google::protobuf::DescriptorPool pool(google::protobuf::DescriptorPool::generated_pool()); + google::protobuf::FileDescriptorProto file_proto; + file_proto.set_name("dynamic_extension.proto"); + file_proto.set_package("cel.expr.conformance.proto2"); + file_proto.add_dependency( + cel::expr::conformance::proto2::TestAllTypes::descriptor() + ->file() + ->name()); + + auto* msg_type = file_proto.add_message_type(); + msg_type->set_name("DynamicExtensionMessage"); + + auto* ext = file_proto.add_extension(); + ext->set_name("dynamic_ext"); + ext->set_number(1000); + ext->set_label(google::protobuf::FieldDescriptorProto::LABEL_OPTIONAL); + ext->set_type(google::protobuf::FieldDescriptorProto::TYPE_MESSAGE); + ext->set_type_name(".cel.expr.conformance.proto2.DynamicExtensionMessage"); + ext->set_extendee( + cel::expr::conformance::proto2::TestAllTypes::descriptor()->full_name()); + + const google::protobuf::FileDescriptor* file_desc = pool.BuildFile(file_proto); + ASSERT_NE(file_desc, nullptr); + + const google::protobuf::FieldDescriptor* ext_desc = + pool.FindExtensionByName("cel.expr.conformance.proto2.dynamic_ext"); + ASSERT_NE(ext_desc, nullptr); + + google::protobuf::DynamicMessageFactory dynamic_factory(&pool); + google::protobuf::Arena arena; + cel::expr::conformance::proto2::TestAllTypes test_msg; + Value val = + Value::WrapField(&test_msg, ext_desc, &pool, &dynamic_factory, &arena); + EXPECT_TRUE(val.IsMessage()); +} + using ValueIteratorTest = common_internal::ValueTest<>; TEST_F(ValueIteratorTest, Empty) { @@ -985,13 +1024,13 @@ TEST_F(ValueIteratorTest, Empty) { TEST_F(ValueIteratorTest, Empty1) { auto iterator = NewEmptyValueIterator(); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ValueIteratorTest, Empty2) { auto iterator = NewEmptyValueIterator(); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } } // namespace diff --git a/common/value_testing_test.cc b/common/value_testing_test.cc index d7a7a4c07..6aead7ce0 100644 --- a/common/value_testing_test.cc +++ b/common/value_testing_test.cc @@ -21,10 +21,12 @@ #include "absl/time/time.h" #include "common/value.h" #include "internal/testing.h" +#include "google/protobuf/arena.h" namespace cel::test { namespace { +using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::testing::_; using ::testing::ElementsAre; @@ -124,11 +126,12 @@ TEST(TimestampValueIs, NonMatchMessage) { } TEST(StringValueIs, Match) { - EXPECT_THAT(StringValue("hello!"), StringValueIs("hello!")); + EXPECT_THAT(StringValue::WrapUnsafe("hello!"), StringValueIs("hello!")); } TEST(StringValueIs, NoMatch) { - EXPECT_THAT(StringValue("hello!"), Not(StringValueIs("goodbye!"))); + EXPECT_THAT(StringValue::WrapUnsafe("hello!"), + Not(StringValueIs("goodbye!"))); EXPECT_THAT(IntValue(2), Not(StringValueIs("goodbye!"))); } @@ -139,11 +142,11 @@ TEST(StringValueIs, NonMatchMessage) { } TEST(BytesValueIs, Match) { - EXPECT_THAT(BytesValue("hello!"), BytesValueIs("hello!")); + EXPECT_THAT(BytesValue::WrapUnsafe("hello!"), BytesValueIs("hello!")); } TEST(BytesValueIs, NoMatch) { - EXPECT_THAT(BytesValue("hello!"), Not(BytesValueIs("goodbye!"))); + EXPECT_THAT(BytesValue::WrapUnsafe("hello!"), Not(BytesValueIs("goodbye!"))); EXPECT_THAT(IntValue(2), Not(BytesValueIs("goodbye!"))); } @@ -154,12 +157,14 @@ TEST(BytesValueIs, NonMatchMessage) { } TEST(ErrorValueIs, Match) { - EXPECT_THAT(ErrorValue(absl::InternalError("test")), + google::protobuf::Arena arena; + EXPECT_THAT(ErrorValue::From(absl::InternalError("test"), &arena), ErrorValueIs(StatusIs(absl::StatusCode::kInternal, "test"))); } TEST(ErrorValueIs, NoMatch) { - EXPECT_THAT(ErrorValue(absl::UnknownError("test")), + google::protobuf::Arena arena; + EXPECT_THAT(ErrorValue::From(absl::UnknownError("test"), &arena), Not(ErrorValueIs(StatusIs(absl::StatusCode::kInternal, "test")))); EXPECT_THAT(IntValue(2), Not(ErrorValueIs(_))); } @@ -265,8 +270,10 @@ TEST_F(ValueMatcherTest, MapMatcherBasic) { TEST_F(ValueMatcherTest, MapMatcherMatchesElements) { auto builder = NewMapValueBuilder(arena()); - ASSERT_OK(builder->Put(IntValue(42), StringValue("answer"))); - ASSERT_OK(builder->Put(IntValue(1337), StringValue("leet"))); + ASSERT_THAT(builder->Put(IntValue(42), StringValue::WrapUnsafe("answer")), + IsOk()); + ASSERT_THAT(builder->Put(IntValue(1337), StringValue::WrapUnsafe("leet")), + IsOk()); EXPECT_THAT( std::move(*builder).Build(), MapValueIs(MapValueElements( diff --git a/common/values/bytes_value.cc b/common/values/bytes_value.cc index c9fc32ac2..5c57bd740 100644 --- a/common/values/bytes_value.cc +++ b/common/values/bytes_value.cc @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include "google/protobuf/wrappers.pb.h" @@ -123,72 +122,4 @@ absl::Status BytesValue::Equal( return absl::OkStatus(); } -BytesValue BytesValue::Clone(google::protobuf::Arena* absl_nonnull arena) const { - return BytesValue(value_.Clone(arena)); -} - -size_t BytesValue::Size() const { - return NativeValue( - [](const auto& alternative) -> size_t { return alternative.size(); }); -} - -bool BytesValue::IsEmpty() const { - return NativeValue( - [](const auto& alternative) -> bool { return alternative.empty(); }); -} - -bool BytesValue::Equals(absl::string_view bytes) const { - return NativeValue([bytes](const auto& alternative) -> bool { - return alternative == bytes; - }); -} - -bool BytesValue::Equals(const absl::Cord& bytes) const { - return NativeValue([&bytes](const auto& alternative) -> bool { - return alternative == bytes; - }); -} - -bool BytesValue::Equals(const BytesValue& bytes) const { - return bytes.NativeValue( - [this](const auto& alternative) -> bool { return Equals(alternative); }); -} - -namespace { - -int CompareImpl(absl::string_view lhs, absl::string_view rhs) { - return lhs.compare(rhs); -} - -int CompareImpl(absl::string_view lhs, const absl::Cord& rhs) { - return -rhs.Compare(lhs); -} - -int CompareImpl(const absl::Cord& lhs, absl::string_view rhs) { - return lhs.Compare(rhs); -} - -int CompareImpl(const absl::Cord& lhs, const absl::Cord& rhs) { - return lhs.Compare(rhs); -} - -} // namespace - -int BytesValue::Compare(absl::string_view bytes) const { - return NativeValue([bytes](const auto& alternative) -> int { - return CompareImpl(alternative, bytes); - }); -} - -int BytesValue::Compare(const absl::Cord& bytes) const { - return NativeValue([&bytes](const auto& alternative) -> int { - return CompareImpl(alternative, bytes); - }); -} - -int BytesValue::Compare(const BytesValue& bytes) const { - return bytes.NativeValue( - [this](const auto& alternative) -> int { return Compare(alternative); }); -} - } // namespace cel diff --git a/common/values/bytes_value.h b/common/values/bytes_value.h index c18381a6a..42be2120e 100644 --- a/common/values/bytes_value.h +++ b/common/values/bytes_value.h @@ -26,15 +26,11 @@ #include "absl/base/attributes.h" #include "absl/base/nullability.h" -#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/allocator.h" -#include "common/arena.h" #include "common/internal/byte_string.h" -#include "common/memory.h" #include "common/type.h" #include "common/value_kind.h" #include "common/values/values.h" @@ -62,75 +58,61 @@ class BytesValue final : private common_internal::ValueMixin { static BytesValue From(const char* absl_nullable value, google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return BytesValue(common_internal::ByteString::From(value, arena)); + } static BytesValue From(absl::string_view value, google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); - static BytesValue From(const absl::Cord& value); + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return BytesValue(common_internal::ByteString::From(value, arena)); + } + static BytesValue From(const absl::Cord& value, + google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return BytesValue(common_internal::ByteString::From(value, arena)); + } static BytesValue From(std::string&& value, google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return BytesValue( + common_internal::ByteString::From(std::move(value), arena)); + } static BytesValue Wrap(absl::string_view value, google::protobuf::Arena* absl_nullable arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); - static BytesValue Wrap(absl::string_view value) = delete; - static BytesValue Wrap(const absl::Cord& value); - static BytesValue Wrap(std::string&& value) = delete; - static BytesValue Wrap(std::string&& value, - google::protobuf::Arena* absl_nullable arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) = delete; + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return BytesValue(common_internal::ByteString::Wrap(value, arena)); + } + static BytesValue Wrap( + const absl::Cord* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND, + google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return BytesValue(common_internal::ByteString::Wrap(value, arena)); + } + static BytesValue Wrap(std::nullptr_t, google::protobuf::Arena*) = delete; + static BytesValue Wrap(std::string&& value, google::protobuf::Arena*) = delete; // Returns a BytesValue that aliases the provided string. Caller must ensure // the provided string outlives the use of the returned BytesValue. - static BytesValue WrapUnsafe(absl::string_view value); + static BytesValue WrapUnsafe(absl::string_view value) { + return BytesValue(common_internal::ByteString::WrapUnsafe(value)); + } + static BytesValue WrapUnsafe(const absl::Cord* absl_nonnull value) { + return BytesValue(common_internal::ByteString::WrapUnsafe(value)); + } + static BytesValue WrapUnsafe(std::nullptr_t) = delete; static BytesValue Concat(const BytesValue& lhs, const BytesValue& rhs, google::protobuf::Arena* absl_nonnull arena ABSL_ATTRIBUTE_LIFETIME_BOUND); - ABSL_DEPRECATED("Use From") - explicit BytesValue(const char* absl_nullable value) : value_(value) {} - - ABSL_DEPRECATED("Use From") - explicit BytesValue(absl::string_view value) : value_(value) {} - - ABSL_DEPRECATED("Use From") - explicit BytesValue(const absl::Cord& value) : value_(value) {} - - ABSL_DEPRECATED("Use From") - explicit BytesValue(std::string&& value) : value_(std::move(value)) {} - - ABSL_DEPRECATED("Use From") - BytesValue(Allocator<> allocator, const char* absl_nullable value) - : value_(allocator, value) {} - - ABSL_DEPRECATED("Use From") - BytesValue(Allocator<> allocator, absl::string_view value) - : value_(allocator, value) {} - - ABSL_DEPRECATED("Use From") - BytesValue(Allocator<> allocator, const absl::Cord& value) - : value_(allocator, value) {} - - ABSL_DEPRECATED("Use From") - BytesValue(Allocator<> allocator, std::string&& value) - : value_(allocator, std::move(value)) {} - - ABSL_DEPRECATED("Use Wrap") - BytesValue(Borrower borrower, absl::string_view value) - : value_(borrower, value) {} - - ABSL_DEPRECATED("Use Wrap") - BytesValue(Borrower borrower, const absl::Cord& value) - : value_(borrower, value) {} - BytesValue() = default; BytesValue(const BytesValue&) = default; BytesValue(BytesValue&&) = default; BytesValue& operator=(const BytesValue&) = default; BytesValue& operator=(BytesValue&&) = default; + explicit BytesValue(const StringValue& other); + constexpr ValueKind kind() const { return kKind; } absl::string_view GetTypeName() const { return BytesType::kName; } @@ -156,11 +138,11 @@ class BytesValue final : private common_internal::ValueMixin { Value* absl_nonnull result) const; using ValueMixin::Equal; - bool IsZeroValue() const { - return NativeValue([](const auto& value) -> bool { return value.empty(); }); - } + bool IsZeroValue() const { return value_.empty(); } - BytesValue Clone(google::protobuf::Arena* absl_nonnull arena) const; + BytesValue Clone(google::protobuf::Arena* absl_nonnull arena) const { + return BytesValue(value_.Clone(arena)); + } ABSL_DEPRECATED("Use ToString()") std::string NativeString() const { return value_.ToString(); } @@ -185,22 +167,26 @@ class BytesValue final : private common_internal::ValueMixin { return value_.Visit(std::forward(visitor)); } - void swap(BytesValue& other) noexcept { + friend void swap(BytesValue& lhs, BytesValue& rhs) noexcept { using std::swap; - swap(value_, other.value_); + swap(lhs.value_, rhs.value_); } - size_t Size() const; + size_t Size() const { return value_.size(); } - bool IsEmpty() const; + bool IsEmpty() const { return value_.empty(); } - bool Equals(absl::string_view bytes) const; - bool Equals(const absl::Cord& bytes) const; - bool Equals(const BytesValue& bytes) const; + bool Equals(absl::string_view bytes) const { return value_.Equals(bytes); } + bool Equals(const absl::Cord& bytes) const { return value_.Equals(bytes); } + bool Equals(const BytesValue& bytes) const { + return value_.Equals(bytes.value_); + } - int Compare(absl::string_view bytes) const; - int Compare(const absl::Cord& bytes) const; - int Compare(const BytesValue& bytes) const; + int Compare(absl::string_view bytes) const { return value_.Compare(bytes); } + int Compare(const absl::Cord& bytes) const { return value_.Compare(bytes); } + int Compare(const BytesValue& bytes) const { + return value_.Compare(bytes.value_); + } absl::optional TryFlat() const ABSL_ATTRIBUTE_LIFETIME_BOUND { @@ -238,12 +224,12 @@ class BytesValue final : private common_internal::ValueMixin { } private: + friend class StringValue; friend class common_internal::ValueMixin; friend class BytesValueInputStream; friend class BytesValueOutputStream; friend absl::string_view common_internal::LegacyBytesValue( const BytesValue& value, bool stable, google::protobuf::Arena* absl_nonnull arena); - friend struct ArenaTraits; explicit BytesValue(common_internal::ByteString value) noexcept : value_(std::move(value)) {} @@ -251,8 +237,6 @@ class BytesValue final : private common_internal::ValueMixin { common_internal::ByteString value_; }; -inline void swap(BytesValue& lhs, BytesValue& rhs) noexcept { lhs.swap(rhs); } - inline std::ostream& operator<<(std::ostream& out, const BytesValue& value) { return out << value.DebugString(); } @@ -273,48 +257,6 @@ inline bool operator!=(absl::string_view lhs, const BytesValue& rhs) { return rhs != lhs; } -inline BytesValue BytesValue::From(const char* absl_nullable value, - google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - return From(absl::NullSafeStringView(value), arena); -} - -inline BytesValue BytesValue::From(absl::string_view value, - google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - ABSL_DCHECK(arena != nullptr); - - return BytesValue(arena, value); -} - -inline BytesValue BytesValue::From(const absl::Cord& value) { - return BytesValue(value); -} - -inline BytesValue BytesValue::From(std::string&& value, - google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - ABSL_DCHECK(arena != nullptr); - - return BytesValue(arena, std::move(value)); -} - -inline BytesValue BytesValue::Wrap(absl::string_view value, - google::protobuf::Arena* absl_nullable arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - ABSL_DCHECK(arena != nullptr); - - return BytesValue(Borrower::Arena(arena), value); -} - -inline BytesValue BytesValue::WrapUnsafe(absl::string_view value) { - return BytesValue(common_internal::ByteString::FromExternal(value)); -} - -inline BytesValue BytesValue::Wrap(const absl::Cord& value) { - return BytesValue(value); -} - namespace common_internal { inline absl::string_view LegacyBytesValue(const BytesValue& value, bool stable, @@ -324,15 +266,6 @@ inline absl::string_view LegacyBytesValue(const BytesValue& value, bool stable, } // namespace common_internal -template <> -struct ArenaTraits { - using constructible = std::true_type; - - static bool trivially_destructible(const BytesValue& value) { - return ArenaTraits<>::trivially_destructible(value.value_); - } -}; - } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_VALUES_BYTES_VALUE_H_ diff --git a/common/values/bytes_value_input_stream.h b/common/values/bytes_value_input_stream.h index c4224f30d..fc2972a05 100644 --- a/common/values/bytes_value_input_stream.h +++ b/common/values/bytes_value_input_stream.h @@ -21,15 +21,12 @@ #include #include #include -#include +#include +#include -#include "absl/base/attributes.h" -#include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" -#include "absl/types/variant.h" -#include "absl/utility/utility.h" #include "common/internal/byte_string.h" #include "common/values/bytes_value.h" #include "google/protobuf/io/zero_copy_stream.h" @@ -39,65 +36,44 @@ namespace cel { class BytesValueInputStream final : public google::protobuf::io::ZeroCopyInputStream { public: - explicit BytesValueInputStream( - const BytesValue* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND) { - Construct(value); - } - - ~BytesValueInputStream() override { AsVariant().~variant(); } + explicit BytesValueInputStream(const BytesValue& value) { Construct(value); } bool Next(const void** data, int* size) override { - return absl::visit( - [&data, &size](auto& alternative) -> bool { - return alternative.Next(data, size); - }, - AsVariant()); + return stream_->Next(data, size); } - void BackUp(int count) override { - absl::visit( - [&count](auto& alternative) -> void { alternative.BackUp(count); }, - AsVariant()); - } + void BackUp(int count) override { stream_->BackUp(count); } - bool Skip(int count) override { - return absl::visit( - [&count](auto& alternative) -> bool { return alternative.Skip(count); }, - AsVariant()); - } + bool Skip(int count) override { return stream_->Skip(count); } - int64_t ByteCount() const override { - return absl::visit( - [](const auto& alternative) -> int64_t { - return alternative.ByteCount(); - }, - AsVariant()); - } + int64_t ByteCount() const override { return stream_->ByteCount(); } bool ReadCord(absl::Cord* cord, int count) override { - return absl::visit( - [&cord, &count](auto& alternative) -> bool { - return alternative.ReadCord(cord, count); - }, - AsVariant()); + return stream_->ReadCord(cord, count); } private: - using Variant = - absl::variant; - - void Construct(const BytesValue* absl_nonnull value) { - ABSL_DCHECK(value != nullptr); - - switch (value->value_.GetKind()) { + using ArrayStream = google::protobuf::io::ArrayInputStream; + struct CordStream { + explicit CordStream(absl::Cord cord) + : cord(std::move(cord)), stream(&this->cord) {} + + absl::Cord cord; + google::protobuf::io::CordInputStream stream; + }; + using Variant = std::variant; + + void Construct(const BytesValue& value) { + switch (value.value_.GetKind()) { case common_internal::ByteStringKind::kSmall: - Construct(value->value_.GetSmall()); + small_ = value.value_.rep_.small; + Construct(absl::string_view(small_.data, small_.size)); break; case common_internal::ByteStringKind::kMedium: - Construct(value->value_.GetMedium()); + Construct(value.value_.GetMedium()); break; case common_internal::ByteStringKind::kLarge: - Construct(&value->value_.GetLarge()); + Construct(value.value_.GetLarge()); break; } } @@ -105,27 +81,17 @@ class BytesValueInputStream final : public google::protobuf::io::ZeroCopyInputSt void Construct(absl::string_view value) { ABSL_DCHECK_LE(value.size(), static_cast(std::numeric_limits::max())); - ::new (static_cast(&impl_[0])) - Variant(absl::in_place_type, value.data(), - static_cast(value.size())); - } - - void Construct(const absl::Cord* absl_nonnull value) { - ::new (static_cast(&impl_[0])) - Variant(absl::in_place_type, value); - } - - void Destruct() { AsVariant().~variant(); } - - Variant& AsVariant() ABSL_ATTRIBUTE_LIFETIME_BOUND { - return *std::launder(reinterpret_cast(&impl_[0])); + stream_ = &variant_.emplace(value.data(), + static_cast(value.size())); } - const Variant& AsVariant() const ABSL_ATTRIBUTE_LIFETIME_BOUND { - return *std::launder(reinterpret_cast(&impl_[0])); + void Construct(absl::Cord value) { + stream_ = &variant_.emplace(std::move(value)).stream; } - alignas(Variant) char impl_[sizeof(Variant)]; + google::protobuf::io::ZeroCopyInputStream* stream_; + common_internal::SmallByteStringRep small_; + Variant variant_; }; } // namespace cel diff --git a/common/values/bytes_value_output_stream.h b/common/values/bytes_value_output_stream.h index 0773e40e7..5ec3b7ea7 100644 --- a/common/values/bytes_value_output_stream.h +++ b/common/values/bytes_value_output_stream.h @@ -19,17 +19,16 @@ #define THIRD_PARTY_CEL_CPP_COMMON_VALUES_BYTES_VALUE_OUTPUT_STREAM_H_ #include -#include #include #include +#include -#include "absl/base/attributes.h" #include "absl/base/nullability.h" +#include "absl/base/optimization.h" #include "absl/functional/overload.h" +#include "absl/log/absl_check.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" -#include "absl/types/variant.h" -#include "absl/utility/utility.h" #include "common/internal/byte_string.h" #include "common/values/bytes_value.h" #include "google/protobuf/arena.h" @@ -40,107 +39,63 @@ namespace cel { class BytesValueOutputStream final : public google::protobuf::io::ZeroCopyOutputStream { public: - explicit BytesValueOutputStream(const BytesValue& value) - : BytesValueOutputStream(value, /*arena=*/nullptr) {} + BytesValueOutputStream() { Construct(); } - BytesValueOutputStream(const BytesValue& value, - google::protobuf::Arena* absl_nullable arena) { - Construct(value, arena); - } + explicit BytesValueOutputStream(const BytesValue& value) { Construct(value); } bool Next(void** data, int* size) override { - return absl::visit(absl::Overload( - [&data, &size](String& string) -> bool { - return string.stream.Next(data, size); - }, - [&data, &size](Cord& cord) -> bool { - return cord.Next(data, size); - }), - AsVariant()); + return stream_->Next(data, size); } - void BackUp(int count) override { - absl::visit( - absl::Overload( - [&count](String& string) -> void { string.stream.BackUp(count); }, - [&count](Cord& cord) -> void { cord.BackUp(count); }), - AsVariant()); - } + void BackUp(int count) override { stream_->BackUp(count); } - int64_t ByteCount() const override { - return absl::visit( - absl::Overload( - [](const String& string) -> int64_t { - return string.stream.ByteCount(); - }, - [](const Cord& cord) -> int64_t { return cord.ByteCount(); }), - AsVariant()); - } + int64_t ByteCount() const override { return stream_->ByteCount(); } bool WriteAliasedRaw(const void* data, int size) override { - return absl::visit(absl::Overload( - [&data, &size](String& string) -> bool { - return string.stream.WriteAliasedRaw(data, size); - }, - [&data, &size](Cord& cord) -> bool { - return cord.WriteAliasedRaw(data, size); - }), - AsVariant()); + return stream_->WriteAliasedRaw(data, size); } - bool AllowsAliasing() const override { - return absl::visit( - absl::Overload( - [](const String& string) -> bool { - return string.stream.AllowsAliasing(); - }, - [](const Cord& cord) -> bool { return cord.AllowsAliasing(); }), - AsVariant()); - } + bool AllowsAliasing() const override { return stream_->AllowsAliasing(); } bool WriteCord(const absl::Cord& out) override { - return absl::visit( - absl::Overload( - [&out](String& string) -> bool { - return string.stream.WriteCord(out); - }, - [&out](Cord& cord) -> bool { return cord.WriteCord(out); }), - AsVariant()); + return stream_->WriteCord(out); } - BytesValue Consume() && { - return absl::visit(absl::Overload( - [](String& string) -> BytesValue { - return BytesValue(string.arena, - std::move(string.target)); - }, - [](Cord& cord) -> BytesValue { - return BytesValue(cord.Consume()); - }), - AsVariant()); + [[nodiscard]] + BytesValue Consume(google::protobuf::Arena* absl_nonnull arena) && { + ABSL_DCHECK(arena != nullptr); + return std::visit( + absl::Overload([](std::monostate) -> BytesValue { ABSL_UNREACHABLE(); }, + [arena](StringStream& stream) -> BytesValue { + return BytesValue::From(std::move(stream.target), + arena); + }, + [arena](CordStream& stream) -> BytesValue { + return BytesValue::From(stream.Consume(), arena); + }), + variant_); } private: - struct String final { - String(absl::string_view target, google::protobuf::Arena* absl_nullable arena) - : target(target), stream(&this->target), arena(arena) {} + struct StringStream final { + explicit StringStream(absl::string_view target) + : target(target), stream(&this->target) {} std::string target; google::protobuf::io::StringOutputStream stream; - google::protobuf::Arena* absl_nullable arena; }; + using CordStream = google::protobuf::io::CordOutputStream; + using Variant = std::variant; - using Cord = google::protobuf::io::CordOutputStream; - - using Variant = absl::variant; + void Construct() { stream_ = &variant_.emplace(); } - void Construct(const BytesValue& value, google::protobuf::Arena* absl_nullable arena) { + void Construct(const BytesValue& value) { switch (value.value_.GetKind()) { case common_internal::ByteStringKind::kSmall: - Construct(value.value_.GetSmall(), arena); + Construct(value.value_.GetSmall()); break; case common_internal::ByteStringKind::kMedium: - Construct(value.value_.GetMedium(), arena); + Construct(value.value_.GetMedium()); break; case common_internal::ByteStringKind::kLarge: Construct(value.value_.GetLarge()); @@ -148,27 +103,16 @@ class BytesValueOutputStream final : public google::protobuf::io::ZeroCopyOutput } } - void Construct(absl::string_view value, google::protobuf::Arena* absl_nullable arena) { - ::new (static_cast(&impl_[0])) - Variant(absl::in_place_type, value, arena); - } - - void Construct(const absl::Cord& value) { - ::new (static_cast(&impl_[0])) - Variant(absl::in_place_type, value); - } - - void Destruct() { AsVariant().~variant(); } - - Variant& AsVariant() ABSL_ATTRIBUTE_LIFETIME_BOUND { - return *std::launder(reinterpret_cast(&impl_[0])); + void Construct(absl::string_view value) { + stream_ = &variant_.emplace(value).stream; } - const Variant& AsVariant() const ABSL_ATTRIBUTE_LIFETIME_BOUND { - return *std::launder(reinterpret_cast(&impl_[0])); + void Construct(absl::Cord value) { + stream_ = &variant_.emplace(std::move(value)); } - alignas(Variant) char impl_[sizeof(Variant)]; + google::protobuf::io::ZeroCopyOutputStream* stream_; + Variant variant_; }; } // namespace cel diff --git a/common/values/bytes_value_test.cc b/common/values/bytes_value_test.cc index 58219e3a4..d2d62b5af 100644 --- a/common/values/bytes_value_test.cc +++ b/common/values/bytes_value_test.cc @@ -20,7 +20,6 @@ #include "absl/status/status_matchers.h" #include "absl/strings/cord.h" #include "absl/strings/cord_test_helpers.h" -#include "absl/types/optional.h" #include "common/native_type.h" #include "common/value.h" #include "common/value_testing.h" @@ -38,131 +37,156 @@ using ::testing::Optional; using BytesValueTest = common_internal::ValueTest<>; TEST_F(BytesValueTest, Kind) { - EXPECT_EQ(BytesValue("foo").kind(), BytesValue::kKind); - EXPECT_EQ(Value(BytesValue(absl::Cord("foo"))).kind(), BytesValue::kKind); + EXPECT_EQ(BytesValue::WrapUnsafe("foo").kind(), BytesValue::kKind); + EXPECT_EQ(Value(BytesValue::From(absl::Cord("foo"), arena())).kind(), + BytesValue::kKind); } TEST_F(BytesValueTest, DebugString) { { std::ostringstream out; - out << BytesValue("foo"); + out << BytesValue::WrapUnsafe("foo"); EXPECT_EQ(out.str(), "b\"foo\""); } { std::ostringstream out; - out << BytesValue(absl::MakeFragmentedCord({"f", "o", "o"})); + out << BytesValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()); EXPECT_EQ(out.str(), "b\"foo\""); } { std::ostringstream out; - out << Value(BytesValue(absl::Cord("foo"))); + out << Value(BytesValue::From(absl::Cord("foo"), arena())); EXPECT_EQ(out.str(), "b\"foo\""); } } TEST_F(BytesValueTest, ConvertToJson) { auto* message = NewArenaValueMessage(); - EXPECT_THAT(BytesValue("foo").ConvertToJson(descriptor_pool(), - message_factory(), message), + EXPECT_THAT(BytesValue::WrapUnsafe("foo").ConvertToJson( + descriptor_pool(), message_factory(), message), IsOk()); EXPECT_THAT(*message, EqualsValueTextProto(R"pb(string_value: "Zm9v")pb")); } TEST_F(BytesValueTest, NativeValue) { std::string scratch; - EXPECT_EQ(BytesValue("foo").NativeString(), "foo"); - EXPECT_EQ(BytesValue("foo").NativeString(scratch), "foo"); - EXPECT_EQ(BytesValue("foo").NativeCord(), "foo"); + EXPECT_EQ(BytesValue::WrapUnsafe("foo").NativeString(), "foo"); + EXPECT_EQ(BytesValue::WrapUnsafe("foo").NativeString(scratch), "foo"); + EXPECT_EQ(BytesValue::WrapUnsafe("foo").NativeCord(), "foo"); } TEST_F(BytesValueTest, TryFlat) { - EXPECT_THAT(BytesValue("foo").TryFlat(), Optional(Eq("foo"))); + EXPECT_THAT(BytesValue::WrapUnsafe("foo").TryFlat(), Optional(Eq("foo"))); EXPECT_THAT( - BytesValue(absl::MakeFragmentedCord({"Hello, World!", "World, Hello!"})) + BytesValue::From( + absl::MakeFragmentedCord({"Hello, World!", "World, Hello!"}), arena()) .TryFlat(), - Eq(absl::nullopt)); + Eq(std::nullopt)); } TEST_F(BytesValueTest, ToString) { - EXPECT_EQ(BytesValue("foo").ToString(), "foo"); - EXPECT_EQ(BytesValue(absl::MakeFragmentedCord({"f", "o", "o"})).ToString(), + EXPECT_EQ(BytesValue::WrapUnsafe("foo").ToString(), "foo"); + EXPECT_EQ(BytesValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .ToString(), "foo"); } TEST_F(BytesValueTest, CopyToString) { std::string out; - BytesValue("foo").CopyToString(&out); + BytesValue::WrapUnsafe("foo").CopyToString(&out); EXPECT_EQ(out, "foo"); - BytesValue(absl::MakeFragmentedCord({"f", "o", "o"})).CopyToString(&out); + BytesValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .CopyToString(&out); EXPECT_EQ(out, "foo"); } TEST_F(BytesValueTest, AppendToString) { std::string out; - BytesValue("foo").AppendToString(&out); + BytesValue::WrapUnsafe("foo").AppendToString(&out); EXPECT_EQ(out, "foo"); - BytesValue(absl::MakeFragmentedCord({"f", "o", "o"})).AppendToString(&out); + BytesValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .AppendToString(&out); EXPECT_EQ(out, "foofoo"); } TEST_F(BytesValueTest, ToCord) { - EXPECT_EQ(BytesValue("foo").ToCord(), "foo"); - EXPECT_EQ(BytesValue(absl::MakeFragmentedCord({"f", "o", "o"})).ToCord(), + EXPECT_EQ(BytesValue::WrapUnsafe("foo").ToCord(), "foo"); + EXPECT_EQ(BytesValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .ToCord(), "foo"); } TEST_F(BytesValueTest, CopyToCord) { absl::Cord out; - BytesValue("foo").CopyToCord(&out); + BytesValue::WrapUnsafe("foo").CopyToCord(&out); EXPECT_EQ(out, "foo"); - BytesValue(absl::MakeFragmentedCord({"f", "o", "o"})).CopyToCord(&out); + BytesValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .CopyToCord(&out); EXPECT_EQ(out, "foo"); } TEST_F(BytesValueTest, AppendToCord) { absl::Cord out; - BytesValue("foo").AppendToCord(&out); + BytesValue::WrapUnsafe("foo").AppendToCord(&out); EXPECT_EQ(out, "foo"); - BytesValue(absl::MakeFragmentedCord({"f", "o", "o"})).AppendToCord(&out); + BytesValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .AppendToCord(&out); EXPECT_EQ(out, "foofoo"); } TEST_F(BytesValueTest, NativeTypeId) { - EXPECT_EQ(NativeTypeId::Of(BytesValue("foo")), - NativeTypeId::For()); - EXPECT_EQ(NativeTypeId::Of(Value(BytesValue(absl::Cord("foo")))), + EXPECT_EQ(NativeTypeId::Of(BytesValue::WrapUnsafe("foo")), NativeTypeId::For()); + EXPECT_EQ( + NativeTypeId::Of(Value(BytesValue::From(absl::Cord("foo"), arena()))), + NativeTypeId::For()); } TEST_F(BytesValueTest, StringViewEquality) { // NOLINTBEGIN(readability/check) - EXPECT_TRUE(BytesValue("foo") == "foo"); - EXPECT_FALSE(BytesValue("foo") == "bar"); + EXPECT_TRUE(BytesValue::WrapUnsafe("foo") == "foo"); + EXPECT_FALSE(BytesValue::WrapUnsafe("foo") == "bar"); - EXPECT_TRUE("foo" == BytesValue("foo")); - EXPECT_FALSE("bar" == BytesValue("foo")); + EXPECT_TRUE("foo" == BytesValue::WrapUnsafe("foo")); + EXPECT_FALSE("bar" == BytesValue::WrapUnsafe("foo")); // NOLINTEND(readability/check) } TEST_F(BytesValueTest, StringViewInequality) { // NOLINTBEGIN(readability/check) - EXPECT_FALSE(BytesValue("foo") != "foo"); - EXPECT_TRUE(BytesValue("foo") != "bar"); + EXPECT_FALSE(BytesValue::WrapUnsafe("foo") != "foo"); + EXPECT_TRUE(BytesValue::WrapUnsafe("foo") != "bar"); - EXPECT_FALSE("foo" != BytesValue("foo")); - EXPECT_TRUE("bar" != BytesValue("foo")); + EXPECT_FALSE("foo" != BytesValue::WrapUnsafe("foo")); + EXPECT_TRUE("bar" != BytesValue::WrapUnsafe("foo")); // NOLINTEND(readability/check) } TEST_F(BytesValueTest, Comparison) { - EXPECT_LT(BytesValue("bar"), BytesValue("foo")); - EXPECT_FALSE(BytesValue("foo") < BytesValue("foo")); - EXPECT_FALSE(BytesValue("foo") < BytesValue("bar")); + EXPECT_LT(BytesValue::WrapUnsafe("bar"), BytesValue::WrapUnsafe("foo")); + EXPECT_FALSE(BytesValue::WrapUnsafe("foo") < BytesValue::WrapUnsafe("foo")); + EXPECT_FALSE(BytesValue::WrapUnsafe("foo") < BytesValue::WrapUnsafe("bar")); +} + +TEST_F(BytesValueTest, SmallStringInputStream) { + BytesValue value = BytesValue::From("foo", arena()); + BytesValueInputStream stream(value); + const void* data; + int size; + absl::Cord cord; + ASSERT_TRUE(stream.Next(&data, &size)); + EXPECT_THAT(data, NotNull()); + EXPECT_EQ(size, 3); + EXPECT_EQ(stream.ByteCount(), 3); + stream.BackUp(size); + ASSERT_TRUE(stream.Skip(3)); + EXPECT_FALSE(stream.ReadCord(&cord, 3)); + EXPECT_FALSE(stream.Next(&data, &size)); } -TEST_F(BytesValueTest, StringInputStream) { - BytesValue value = BytesValue("foo"); - BytesValueInputStream stream(&value); +TEST_F(BytesValueTest, MediumStringInputStream) { + BytesValue value = BytesValue::WrapUnsafe("foo"); + BytesValueInputStream stream(value); const void* data; int size; absl::Cord cord; @@ -177,8 +201,9 @@ TEST_F(BytesValueTest, StringInputStream) { } TEST_F(BytesValueTest, CordInputStream) { - BytesValue value = BytesValue(absl::Cord("foo")); - BytesValueInputStream stream(&value); + absl::Cord value_cord("foo"); + BytesValue value = BytesValue::WrapUnsafe(&value_cord); + BytesValueInputStream stream(value); const void* data; int size; absl::Cord cord; @@ -193,9 +218,9 @@ TEST_F(BytesValueTest, CordInputStream) { } TEST_F(BytesValueTest, ArenaStringOutputStream) { - BytesValue value = BytesValue(""); + BytesValue value = BytesValue(); { - BytesValueOutputStream stream(value, arena()); + BytesValueOutputStream stream(value); EXPECT_THAT(stream.AllowsAliasing(), An()); EXPECT_EQ(stream.ByteCount(), 0); google::protobuf::Value value_proto; @@ -203,17 +228,17 @@ TEST_F(BytesValueTest, ArenaStringOutputStream) { (*struct_proto->mutable_fields())["foo"].set_string_value("bar"); (*struct_proto->mutable_fields())["baz"].set_number_value(3.14159); ASSERT_TRUE(value_proto.SerializePartialToZeroCopyStream(&stream)); - EXPECT_EQ(std::move(stream).Consume(), + EXPECT_EQ(std::move(stream).Consume(arena()), value_proto.SerializePartialAsString()); } { BytesValueOutputStream stream(value); - EXPECT_EQ(std::move(stream).Consume(), ""); + EXPECT_EQ(std::move(stream).Consume(arena()), ""); } } TEST_F(BytesValueTest, StringOutputStream) { - BytesValue value = BytesValue(""); + BytesValue value = BytesValue(); { BytesValueOutputStream stream(value); EXPECT_THAT(stream.AllowsAliasing(), An()); @@ -223,17 +248,18 @@ TEST_F(BytesValueTest, StringOutputStream) { (*struct_proto->mutable_fields())["foo"].set_string_value("bar"); (*struct_proto->mutable_fields())["baz"].set_number_value(3.14159); ASSERT_TRUE(value_proto.SerializePartialToZeroCopyStream(&stream)); - EXPECT_EQ(std::move(stream).Consume(), + EXPECT_EQ(std::move(stream).Consume(arena()), value_proto.SerializePartialAsString()); } { BytesValueOutputStream stream(value); - EXPECT_EQ(std::move(stream).Consume(), ""); + EXPECT_EQ(std::move(stream).Consume(arena()), ""); } } TEST_F(BytesValueTest, CordOutputStream) { - BytesValue value = BytesValue(absl::Cord()); + absl::Cord cord; + BytesValue value = BytesValue::WrapUnsafe(&cord); { BytesValueOutputStream stream(value); EXPECT_THAT(stream.AllowsAliasing(), An()); @@ -243,12 +269,12 @@ TEST_F(BytesValueTest, CordOutputStream) { (*struct_proto->mutable_fields())["foo"].set_string_value("bar"); (*struct_proto->mutable_fields())["baz"].set_number_value(3.14159); ASSERT_TRUE(value_proto.SerializePartialToZeroCopyStream(&stream)); - EXPECT_EQ(std::move(stream).Consume(), + EXPECT_EQ(std::move(stream).Consume(arena()), value_proto.SerializePartialAsString()); } { BytesValueOutputStream stream(value); - EXPECT_EQ(std::move(stream).Consume(), ""); + EXPECT_EQ(std::move(stream).Consume(arena()), ""); } } diff --git a/common/values/custom_list_value.cc b/common/values/custom_list_value.cc index fbba38cfa..5a34b42cf 100644 --- a/common/values/custom_list_value.cc +++ b/common/values/custom_list_value.cc @@ -24,7 +24,6 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "common/casting.h" #include "common/native_type.h" #include "common/value.h" #include "common/values/list_value_builder.h" @@ -54,6 +53,8 @@ class EmptyListValue final : public common_internal::CompatListValue { EmptyListValue() = default; + google::protobuf::Arena* GetOwningArena() const override { return nullptr; } + std::string DebugString() const override { return "[]"; } bool IsEmpty() const override { return true; } @@ -97,9 +98,9 @@ class EmptyListValue final : public common_internal::CompatListValue { private: absl::Status Get(size_t index, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - google::protobuf::Arena* absl_nonnull, + google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const override { - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); return absl::OkStatus(); } }; @@ -334,7 +335,7 @@ absl::Status CustomListValueInterface::Contains( [&](size_t index, const Value& element) -> absl::StatusOr { CEL_RETURN_IF_ERROR(element.Equal(other, descriptor_pool, message_factory, arena, &equal)); - if (auto bool_result = As(equal); + if (auto bool_result = equal.AsBool(); bool_result.has_value() && bool_result->NativeValue()) { outcome = BoolValue(true); return false; @@ -486,10 +487,7 @@ CustomListValue CustomListValue::Clone( CustomListValueInterface::Content content = content_.To(); ABSL_DCHECK(content.interface != nullptr); - if (content.arena != arena) { - return content.interface->Clone(arena); - } - return *this; + return content.interface->Clone(arena); } return dispatcher_->clone(dispatcher_, content_, arena); } @@ -599,7 +597,7 @@ absl::Status CustomListValue::Contains( [&](size_t index, const Value& element) -> absl::StatusOr { CEL_RETURN_IF_ERROR(element.Equal(other, descriptor_pool, message_factory, arena, &equal)); - if (auto bool_result = As(equal); + if (auto bool_result = equal.As(); bool_result.has_value() && bool_result->NativeValue()) { outcome = BoolValue(true); return false; diff --git a/common/values/custom_list_value_test.cc b/common/values/custom_list_value_test.cc index 79c3f2419..eefc9a481 100644 --- a/common/values/custom_list_value_test.cc +++ b/common/values/custom_list_value_test.cc @@ -24,7 +24,6 @@ #include "absl/status/statusor.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "common/memory.h" #include "common/native_type.h" #include "common/value.h" @@ -121,7 +120,7 @@ class CustomListValueInterfaceTest final : public CustomListValueInterface { *result = IntValue(1); return absl::OkStatus(); } - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); return absl::OkStatus(); } @@ -221,7 +220,7 @@ class CustomListValueTest : public common_internal::ValueTest<> { *result = IntValue(1); return absl::OkStatus(); } - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); return absl::OkStatus(); }, .clone = [](const CustomListValueDispatcher* absl_nonnull dispatcher, @@ -442,7 +441,7 @@ TEST_F(CustomListValueTest, Dispatcher_NewIterator1) { EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(IntValueIs(1)))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomListValueTest, Interface_NewIterator1) { @@ -453,7 +452,7 @@ TEST_F(CustomListValueTest, Interface_NewIterator1) { EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(IntValueIs(1)))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomListValueTest, Dispatcher_NewIterator2) { @@ -464,7 +463,7 @@ TEST_F(CustomListValueTest, Dispatcher_NewIterator2) { EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(Pair(IntValueIs(1), IntValueIs(1))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomListValueTest, Interface_NewIterator2) { @@ -475,7 +474,7 @@ TEST_F(CustomListValueTest, Interface_NewIterator2) { EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(Pair(IntValueIs(1), IntValueIs(1))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomListValueTest, Dispatcher_Contains) { diff --git a/common/values/custom_map_value.cc b/common/values/custom_map_value.cc index ae07f7723..ce463f890 100644 --- a/common/values/custom_map_value.cc +++ b/common/values/custom_map_value.cc @@ -14,7 +14,9 @@ #include #include +#include #include +#include #include "absl/base/attributes.h" #include "absl/base/no_destructor.h" @@ -68,6 +70,8 @@ class EmptyMapValue final : public common_internal::CompatMapValue { EmptyMapValue() = default; + google::protobuf::Arena* GetOwningArena() const override { return nullptr; } + std::string DebugString() const override { return "{}"; } bool IsEmpty() const override { return true; } @@ -106,13 +110,13 @@ class EmptyMapValue final : public common_internal::CompatMapValue { } absl::optional operator[](CelValue key) const override { - return absl::nullopt; + return std::nullopt; } using CompatMapValue::Get; absl::optional Get(google::protobuf::Arena* arena, CelValue key) const override { - return absl::nullopt; + return std::nullopt; } absl::StatusOr Has(const CelValue& key) const override { return false; } @@ -424,7 +428,7 @@ absl::Status CustomMapValueInterface::ForEach( CEL_ASSIGN_OR_RETURN( bool found, Find(key, descriptor_pool, message_factory, arena, &value)); if (!found) { - value = ErrorValue(NoSuchKeyError(key)); + value = ErrorValue::From(NoSuchKeyError(key), arena); } CEL_ASSIGN_OR_RETURN(auto ok, callback(key, value)); if (!ok) { @@ -587,10 +591,7 @@ CustomMapValue CustomMapValue::Clone(google::protobuf::Arena* absl_nonnull arena CustomMapValueInterface::Content content = content_.To(); ABSL_DCHECK(content.interface != nullptr); - if (content.arena != arena) { - return content.interface->Clone(arena); - } - return *this; + return content.interface->Clone(arena); } return dispatcher_->clone(dispatcher_, content_, arena); } @@ -637,7 +638,7 @@ absl::Status CustomMapValue::Get( case ValueKind::kUnknown: break; default: - *result = ErrorValue(NoSuchKeyError(key)); + *result = ErrorValue::From(NoSuchKeyError(key), arena); break; } } @@ -669,28 +670,38 @@ absl::StatusOr CustomMapValue::Find( case ValueKind::kString: break; default: - *result = ErrorValue(InvalidMapKeyTypeError(key.kind())); + *result = ErrorValue::From(InvalidMapKeyTypeError(key.kind()), arena); return false; } - bool ok; if (dispatcher_ == nullptr) { CustomMapValueInterface::Content content = content_.To(); ABSL_DCHECK(content.interface != nullptr); - CEL_ASSIGN_OR_RETURN( - ok, content.interface->Find(key, descriptor_pool, message_factory, - arena, result)); - } else { - CEL_ASSIGN_OR_RETURN( - ok, dispatcher_->find(dispatcher_, content_, key, descriptor_pool, - message_factory, arena, result)); - } - if (ok) { + auto status_or_found = content.interface->Find( + key, descriptor_pool, message_factory, arena, result); + if (!status_or_found.ok()) { + *result = ErrorValue::From(std::move(status_or_found).status(), arena); + return false; + } + if (!*status_or_found) { + *result = NullValue(); + return false; + } return true; } - *result = NullValue{}; - return false; + auto status_or_found = + dispatcher_->find(dispatcher_, content_, key, descriptor_pool, + message_factory, arena, result); + if (!status_or_found.ok()) { + *result = ErrorValue::From(std::move(status_or_found).status(), arena); + return false; + } + if (!*status_or_found) { + *result = NullValue(); + return false; + } + return true; } absl::Status CustomMapValue::Has( @@ -718,22 +729,29 @@ absl::Status CustomMapValue::Has( case ValueKind::kString: break; default: - *result = ErrorValue(InvalidMapKeyTypeError(key.kind())); + *result = ErrorValue::From(InvalidMapKeyTypeError(key.kind()), arena); return absl::OkStatus(); } - bool has; if (dispatcher_ == nullptr) { CustomMapValueInterface::Content content = content_.To(); ABSL_DCHECK(content.interface != nullptr); - CEL_ASSIGN_OR_RETURN(has, content.interface->Has(key, descriptor_pool, - message_factory, arena)); - } else { - CEL_ASSIGN_OR_RETURN( - has, dispatcher_->has(dispatcher_, content_, key, descriptor_pool, - message_factory, arena)); + auto status_or_has = + content.interface->Has(key, descriptor_pool, message_factory, arena); + if (!status_or_has.ok()) { + *result = ErrorValue::From(std::move(status_or_has).status(), arena); + return absl::OkStatus(); + } + *result = BoolValue(*status_or_has); + return absl::OkStatus(); + } + auto status_or_has = dispatcher_->has( + dispatcher_, content_, key, descriptor_pool, message_factory, arena); + if (!status_or_has.ok()) { + *result = ErrorValue::From(std::move(status_or_has).status(), arena); + return absl::OkStatus(); } - *result = BoolValue(has); + *result = BoolValue(*status_or_has); return absl::OkStatus(); } @@ -795,7 +813,7 @@ absl::Status CustomMapValue::ForEach( dispatcher_->find(dispatcher_, content_, key, descriptor_pool, message_factory, arena, &value)); if (!found) { - value = ErrorValue(NoSuchKeyError(key)); + value = ErrorValue::From(NoSuchKeyError(key), arena); } CEL_ASSIGN_OR_RETURN(auto ok, callback(key, value)); if (!ok) { diff --git a/common/values/custom_map_value.h b/common/values/custom_map_value.h index ca6e1e025..5d2a1dfb5 100644 --- a/common/values/custom_map_value.h +++ b/common/values/custom_map_value.h @@ -54,6 +54,12 @@ class CustomMapValueInterfaceKeysIterator; class CustomMapValue; using CustomMapValueContent = CustomValueContent; +// Dispatch table for `CustomMapValue`. +// +// See the documentation for `CustomMapValueInterface` for more details on +// composite functions. +// +// See documentation for `UnsafeCustomMapValue` on how to use this class. struct CustomMapValueDispatcher { using GetTypeId = NativeTypeId (*)(const CustomMapValueDispatcher* absl_nonnull dispatcher, @@ -247,12 +253,21 @@ class CustomMapValueInterface { virtual CustomMapValue Clone(google::protobuf::Arena* absl_nonnull arena) const = 0; + // Tests whether the map contains the given key. If it does, the value + // associated with the key is written to `result` and the function returns + // true. Otherwise, the function returns false and `result` is set to + // `NullValue`. + // + // A non-ok status is converted to an ErrorValue (e.g. wrong key type). virtual absl::StatusOr Find( const Value& key, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const = 0; + // Whether the map has the given key. + // + // A non-ok status is converted to an ErrorValue (e.g. wrong key type). virtual absl::StatusOr Has( const Value& key, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, diff --git a/common/values/custom_map_value_test.cc b/common/values/custom_map_value_test.cc index 8c3183cf8..23fc18cf0 100644 --- a/common/values/custom_map_value_test.cc +++ b/common/values/custom_map_value_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include #include @@ -24,7 +25,6 @@ #include "absl/status/statusor.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "common/memory.h" #include "common/native_type.h" #include "common/value.h" @@ -109,8 +109,8 @@ class CustomMapValueInterfaceTest final : public CustomMapValueInterface { ListValue* absl_nonnull result) const override { auto builder = common_internal::NewListValueBuilder(arena); builder->Reserve(2); - CEL_RETURN_IF_ERROR(builder->Add(StringValue("foo"))); - CEL_RETURN_IF_ERROR(builder->Add(StringValue("bar"))); + CEL_RETURN_IF_ERROR(builder->Add(StringValue::WrapUnsafe("foo"))); + CEL_RETURN_IF_ERROR(builder->Add(StringValue::WrapUnsafe("bar"))); *result = std::move(*builder).Build(); return absl::OkStatus(); } @@ -139,6 +139,9 @@ class CustomMapValueInterfaceTest final : public CustomMapValueInterface { *result = IntValue(1); return true; } + if (*string_key == "error") { + return absl::InvalidArgumentError("custom error"); + } } return false; } @@ -155,6 +158,9 @@ class CustomMapValueInterfaceTest final : public CustomMapValueInterface { if (*string_key == "bar") { return true; } + if (*string_key == "error") { + return absl::InvalidArgumentError("custom error"); + } } return false; } @@ -254,6 +260,9 @@ class CustomMapValueTest : public common_internal::ValueTest<> { *result = IntValue(1); return true; } + if (*string_key == "error") { + return absl::InvalidArgumentError("custom error"); + } } return false; }, @@ -269,6 +278,9 @@ class CustomMapValueTest : public common_internal::ValueTest<> { if (*string_key == "bar") { return true; } + if (*string_key == "error") { + return absl::InvalidArgumentError("custom error"); + } } return false; }, @@ -281,8 +293,8 @@ class CustomMapValueTest : public common_internal::ValueTest<> { ListValue* absl_nonnull result) -> absl::Status { auto builder = common_internal::NewListValueBuilder(arena); builder->Reserve(2); - CEL_RETURN_IF_ERROR(builder->Add(StringValue("foo"))); - CEL_RETURN_IF_ERROR(builder->Add(StringValue("bar"))); + CEL_RETURN_IF_ERROR(builder->Add(StringValue::WrapUnsafe("foo"))); + CEL_RETURN_IF_ERROR(builder->Add(StringValue::WrapUnsafe("bar"))); *result = std::move(*builder).Build(); return absl::OkStatus(); }, @@ -444,84 +456,332 @@ TEST_F(CustomMapValueTest, Interface_Size) { TEST_F(CustomMapValueTest, Dispatcher_Get) { CustomMapValue map = MakeDispatcher(); - ASSERT_THAT(map.Get(StringValue("foo"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Get(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - ASSERT_THAT(map.Get(StringValue("bar"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Get(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(IntValueIs(1))); ASSERT_THAT( - map.Get(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), + map.Get(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound)))); } TEST_F(CustomMapValueTest, Interface_Get) { CustomMapValue map = MakeInterface(); - ASSERT_THAT(map.Get(StringValue("foo"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Get(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - ASSERT_THAT(map.Get(StringValue("bar"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Get(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(IntValueIs(1))); ASSERT_THAT( - map.Get(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), + map.Get(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound)))); } TEST_F(CustomMapValueTest, Dispatcher_Find) { CustomMapValue map = MakeDispatcher(); - ASSERT_THAT(map.Find(StringValue("foo"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(true)))); - ASSERT_THAT(map.Find(StringValue("bar"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(Optional(IntValueIs(1)))); - ASSERT_THAT(map.Find(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), - IsOkAndHolds(Eq(absl::nullopt))); + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomMapValueTest, Interface_Find) { CustomMapValue map = MakeInterface(); - ASSERT_THAT(map.Find(StringValue("foo"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(true)))); - ASSERT_THAT(map.Find(StringValue("bar"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(Optional(IntValueIs(1)))); - ASSERT_THAT(map.Find(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), - IsOkAndHolds(Eq(absl::nullopt))); + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); +} + +TEST_F(CustomMapValueTest, Dispatcher_Find_Error) { + CustomMapValue map = MakeDispatcher(); + Value result; + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("error"), descriptor_pool(), + message_factory(), arena(), &result), + IsOkAndHolds(false)); + EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, + "custom error"))); + ASSERT_THAT(map.Get(StringValue::WrapUnsafe("error"), descriptor_pool(), + message_factory(), arena(), &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, + "custom error"))); + EXPECT_THAT(map.Get(StringValue::WrapUnsafe("error"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs( + absl::StatusCode::kInvalidArgument, "custom error")))); + EXPECT_THAT(map.Find(StringValue::WrapUnsafe("error"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); +} + +TEST_F(CustomMapValueTest, Interface_Find_Error) { + CustomMapValue map = MakeInterface(); + Value result; + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("error"), descriptor_pool(), + message_factory(), arena(), &result), + IsOkAndHolds(false)); + EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, + "custom error"))); + ASSERT_THAT(map.Get(StringValue::WrapUnsafe("error"), descriptor_pool(), + message_factory(), arena(), &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, + "custom error"))); + EXPECT_THAT(map.Get(StringValue::WrapUnsafe("error"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs( + absl::StatusCode::kInvalidArgument, "custom error")))); + EXPECT_THAT(map.Find(StringValue::WrapUnsafe("error"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); +} + +TEST_F(CustomMapValueTest, Dispatcher_Find_InvalidKeyType) { + CustomMapValue map = MakeDispatcher(); + Value result; + ASSERT_THAT(map.Find(DoubleValue(1.0), descriptor_pool(), message_factory(), + arena(), &result), + IsOkAndHolds(false)); + EXPECT_THAT(result, + ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))); + ASSERT_THAT(map.Get(DoubleValue(1.0), descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_THAT(result, + ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))); + EXPECT_THAT( + map.Get(DoubleValue(1.0), descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument)))); +} + +TEST_F(CustomMapValueTest, Interface_Find_InvalidKeyType) { + CustomMapValue map = MakeInterface(); + Value result; + ASSERT_THAT(map.Find(DoubleValue(1.0), descriptor_pool(), message_factory(), + arena(), &result), + IsOkAndHolds(false)); + EXPECT_THAT(result, + ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))); + ASSERT_THAT(map.Get(DoubleValue(1.0), descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_THAT(result, + ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))); + EXPECT_THAT( + map.Get(DoubleValue(1.0), descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument)))); +} + +TEST_F(CustomMapValueTest, Dispatcher_Find_SpecialKeys) { + CustomMapValue map = MakeDispatcher(); + Value result; + ErrorValue error_key = + ErrorValue::From(absl::CancelledError("cancelled"), arena()); + ASSERT_THAT(map.Find(error_key, descriptor_pool(), message_factory(), arena(), + &result), + IsOkAndHolds(false)); + EXPECT_THAT(result, ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled"))); + ASSERT_THAT(map.Get(error_key, descriptor_pool(), message_factory(), arena(), + &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled"))); + EXPECT_THAT(map.Get(error_key, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled")))); + + UnknownValue unknown_key; + ASSERT_THAT(map.Find(unknown_key, descriptor_pool(), message_factory(), + arena(), &result), + IsOkAndHolds(false)); + EXPECT_TRUE(result.IsUnknown()); + ASSERT_THAT(map.Get(unknown_key, descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_TRUE(result.IsUnknown()); + ASSERT_OK_AND_ASSIGN(auto get_result, map.Get(unknown_key, descriptor_pool(), + message_factory(), arena())); + EXPECT_TRUE(get_result.IsUnknown()); +} + +TEST_F(CustomMapValueTest, Interface_Find_SpecialKeys) { + CustomMapValue map = MakeInterface(); + Value result; + ErrorValue error_key = + ErrorValue::From(absl::CancelledError("cancelled"), arena()); + ASSERT_THAT(map.Find(error_key, descriptor_pool(), message_factory(), arena(), + &result), + IsOkAndHolds(false)); + EXPECT_THAT(result, ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled"))); + ASSERT_THAT(map.Get(error_key, descriptor_pool(), message_factory(), arena(), + &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled"))); + EXPECT_THAT(map.Get(error_key, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled")))); + + UnknownValue unknown_key; + ASSERT_THAT(map.Find(unknown_key, descriptor_pool(), message_factory(), + arena(), &result), + IsOkAndHolds(false)); + EXPECT_TRUE(result.IsUnknown()); + ASSERT_THAT(map.Get(unknown_key, descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_TRUE(result.IsUnknown()); + ASSERT_OK_AND_ASSIGN(auto get_result, map.Get(unknown_key, descriptor_pool(), + message_factory(), arena())); + EXPECT_TRUE(get_result.IsUnknown()); } TEST_F(CustomMapValueTest, Dispatcher_Has) { CustomMapValue map = MakeDispatcher(); - ASSERT_THAT(map.Has(StringValue("foo"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - ASSERT_THAT(map.Has(StringValue("bar"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - ASSERT_THAT(map.Has(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); } TEST_F(CustomMapValueTest, Interface_Has) { CustomMapValue map = MakeInterface(); - ASSERT_THAT(map.Has(StringValue("foo"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - ASSERT_THAT(map.Has(StringValue("bar"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - ASSERT_THAT(map.Has(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); } +TEST_F(CustomMapValueTest, Dispatcher_Has_Error) { + CustomMapValue map = MakeDispatcher(); + Value result; + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("error"), descriptor_pool(), + message_factory(), arena(), &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, + "custom error"))); + EXPECT_THAT(map.Has(StringValue::WrapUnsafe("error"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs( + absl::StatusCode::kInvalidArgument, "custom error")))); +} + +TEST_F(CustomMapValueTest, Interface_Has_Error) { + CustomMapValue map = MakeInterface(); + Value result; + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("error"), descriptor_pool(), + message_factory(), arena(), &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, + "custom error"))); + EXPECT_THAT(map.Has(StringValue::WrapUnsafe("error"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs( + absl::StatusCode::kInvalidArgument, "custom error")))); +} + +TEST_F(CustomMapValueTest, Dispatcher_Has_InvalidKeyType) { + CustomMapValue map = MakeDispatcher(); + Value result; + ASSERT_THAT(map.Has(DoubleValue(1.0), descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_THAT(result, + ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))); + EXPECT_THAT( + map.Has(DoubleValue(1.0), descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument)))); +} + +TEST_F(CustomMapValueTest, Interface_Has_InvalidKeyType) { + CustomMapValue map = MakeInterface(); + Value result; + ASSERT_THAT(map.Has(DoubleValue(1.0), descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_THAT(result, + ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))); + EXPECT_THAT( + map.Has(DoubleValue(1.0), descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument)))); +} + +TEST_F(CustomMapValueTest, Dispatcher_Has_SpecialKeys) { + CustomMapValue map = MakeDispatcher(); + Value result; + ErrorValue error_key = + ErrorValue::From(absl::CancelledError("cancelled"), arena()); + ASSERT_THAT(map.Has(error_key, descriptor_pool(), message_factory(), arena(), + &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled"))); + EXPECT_THAT(map.Has(error_key, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled")))); + + UnknownValue unknown_key; + ASSERT_THAT(map.Has(unknown_key, descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_TRUE(result.IsUnknown()); + ASSERT_OK_AND_ASSIGN(auto has_result, map.Has(unknown_key, descriptor_pool(), + message_factory(), arena())); + EXPECT_TRUE(has_result.IsUnknown()); +} + +TEST_F(CustomMapValueTest, Interface_Has_SpecialKeys) { + CustomMapValue map = MakeInterface(); + Value result; + ErrorValue error_key = + ErrorValue::From(absl::CancelledError("cancelled"), arena()); + ASSERT_THAT(map.Has(error_key, descriptor_pool(), message_factory(), arena(), + &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled"))); + EXPECT_THAT(map.Has(error_key, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled")))); + + UnknownValue unknown_key; + ASSERT_THAT(map.Has(unknown_key, descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_TRUE(result.IsUnknown()); + ASSERT_OK_AND_ASSIGN(auto has_result, map.Has(unknown_key, descriptor_pool(), + message_factory(), arena())); + EXPECT_TRUE(has_result.IsUnknown()); +} + TEST_F(CustomMapValueTest, Dispatcher_ForEach) { std::vector> entries; EXPECT_THAT( @@ -588,7 +848,7 @@ TEST_F(CustomMapValueTest, Dispatcher_NewIterator1) { EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(StringValueIs("bar")))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomMapValueTest, Interface_NewIterator1) { @@ -599,7 +859,7 @@ TEST_F(CustomMapValueTest, Interface_NewIterator1) { EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(StringValueIs("bar")))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomMapValueTest, Dispatcher_NewIterator2) { @@ -612,7 +872,7 @@ TEST_F(CustomMapValueTest, Dispatcher_NewIterator2) { iterator->Next2(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(Pair(StringValueIs("bar"), IntValueIs(1))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomMapValueTest, Interface_NewIterator2) { @@ -625,7 +885,7 @@ TEST_F(CustomMapValueTest, Interface_NewIterator2) { iterator->Next2(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(Pair(StringValueIs("bar"), IntValueIs(1))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomMapValueTest, Dispatcher) { diff --git a/common/values/custom_struct_value_test.cc b/common/values/custom_struct_value_test.cc index 32d867a4d..c4a34db72 100644 --- a/common/values/custom_struct_value_test.cc +++ b/common/values/custom_struct_value_test.cc @@ -117,7 +117,7 @@ class CustomStructValueInterfaceTest final : public CustomStructValueInterface { *result = IntValue(1); return absl::OkStatus(); } - return NoSuchFieldError(name).ToStatus(); + return common_internal::MakeNoSuchFieldError(name); } absl::Status GetFieldByNumber( @@ -134,7 +134,7 @@ class CustomStructValueInterfaceTest final : public CustomStructValueInterface { *result = IntValue(1); return absl::OkStatus(); } - return NoSuchFieldError(absl::StrCat(number)).ToStatus(); + return common_internal::MakeNoSuchFieldError(absl::StrCat(number)); } absl::StatusOr HasFieldByName(absl::string_view name) const override { @@ -144,7 +144,7 @@ class CustomStructValueInterfaceTest final : public CustomStructValueInterface { if (name == "bar") { return true; } - return NoSuchFieldError(name).ToStatus(); + return common_internal::MakeNoSuchFieldError(name); } absl::StatusOr HasFieldByNumber(int64_t number) const override { @@ -154,7 +154,7 @@ class CustomStructValueInterfaceTest final : public CustomStructValueInterface { if (number == 2) { return true; } - return NoSuchFieldError(absl::StrCat(number)).ToStatus(); + return common_internal::MakeNoSuchFieldError(absl::StrCat(number)); } absl::Status ForEachField( @@ -283,7 +283,7 @@ class CustomStructValueTest : public common_internal::ValueTest<> { *result = IntValue(1); return absl::OkStatus(); } - return NoSuchFieldError(name).ToStatus(); + return common_internal::MakeNoSuchFieldError(name); }, .get_field_by_number = [](const CustomStructValueDispatcher* absl_nonnull dispatcher, @@ -301,7 +301,7 @@ class CustomStructValueTest : public common_internal::ValueTest<> { *result = IntValue(1); return absl::OkStatus(); } - return NoSuchFieldError(absl::StrCat(number)).ToStatus(); + return common_internal::MakeNoSuchFieldError(absl::StrCat(number)); }, .has_field_by_name = [](const CustomStructValueDispatcher* absl_nonnull dispatcher, @@ -313,7 +313,7 @@ class CustomStructValueTest : public common_internal::ValueTest<> { if (name == "bar") { return true; } - return NoSuchFieldError(name).ToStatus(); + return common_internal::MakeNoSuchFieldError(name); }, .has_field_by_number = [](const CustomStructValueDispatcher* absl_nonnull dispatcher, @@ -325,7 +325,7 @@ class CustomStructValueTest : public common_internal::ValueTest<> { if (number == 2) { return true; } - return NoSuchFieldError(absl::StrCat(number)).ToStatus(); + return common_internal::MakeNoSuchFieldError(absl::StrCat(number)); }, .for_each_field = [](const CustomStructValueDispatcher* absl_nonnull dispatcher, diff --git a/common/values/duration_value.cc b/common/values/duration_value.cc index a3b41e8ea..1f3d5b723 100644 --- a/common/values/duration_value.cc +++ b/common/values/duration_value.cc @@ -75,6 +75,8 @@ absl::Status DurationValue::ConvertToJson( ABSL_DCHECK_EQ(json->GetDescriptor()->well_known_type(), google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE); + CEL_RETURN_IF_ERROR(cel::internal::ValidateDuration(NativeValue())); + ValueReflection value_reflection; CEL_RETURN_IF_ERROR(value_reflection.Initialize(json->GetDescriptor())); value_reflection.SetStringValueFromDuration(json, NativeValue()); diff --git a/common/values/duration_value.h b/common/values/duration_value.h index 1b2468b60..0c934f097 100644 --- a/common/values/duration_value.h +++ b/common/values/duration_value.h @@ -51,6 +51,12 @@ class DurationValue final : private common_internal::ValueMixin { public: static constexpr ValueKind kKind = ValueKind::kDuration; + // Constructs a `DurationValue` from an `absl::Duration`. + // + // DCHECK-fails if the value is not in the supported range. + // + // Prefer using `SafeDurationValue` or `UnsafeDurationValue` if the caller + // has already validated the value. explicit DurationValue(absl::Duration value) noexcept : DurationValue(absl::in_place, value) { ABSL_DCHECK_OK(internal::ValidateDuration(value)); diff --git a/common/values/duration_value_test.cc b/common/values/duration_value_test.cc index 29d9b0f9e..d525f2497 100644 --- a/common/values/duration_value_test.cc +++ b/common/values/duration_value_test.cc @@ -12,9 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "common/values/duration_value.h" + #include #include +#include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/time/time.h" #include "common/native_type.h" @@ -66,6 +69,16 @@ TEST_F(DurationValueTest, ConvertToJson) { EXPECT_THAT(*message, EqualsValueTextProto(R"pb(string_value: "0s")pb")); } +TEST_F(DurationValueTest, ConvertToJsonOutOfBounds) { + auto* message = NewArenaValueMessage(); + EXPECT_THAT(UnsafeDurationValue(absl::InfiniteDuration()) + .ConvertToJson(descriptor_pool(), message_factory(), message), + absl_testing::StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(UnsafeDurationValue(-absl::InfiniteDuration()) + .ConvertToJson(descriptor_pool(), message_factory(), message), + absl_testing::StatusIs(absl::StatusCode::kInvalidArgument)); +} + TEST_F(DurationValueTest, NativeTypeId) { EXPECT_EQ(NativeTypeId::Of(DurationValue(absl::Seconds(1))), NativeTypeId::For()); diff --git a/common/values/error_value.cc b/common/values/error_value.cc index 8ea6554ec..030bb03b1 100644 --- a/common/values/error_value.cc +++ b/common/values/error_value.cc @@ -13,9 +13,7 @@ // limitations under the License. #include -#include #include -#include #include "absl/base/no_destructor.h" #include "absl/base/nullability.h" @@ -46,46 +44,107 @@ const absl::Status& DefaultErrorValue() { return *value; } +absl::Status MakeNoSuchFieldError(absl::string_view field) { + return absl::NotFoundError( + absl::StrCat("no_such_field", field.empty() ? "" : " : ", field)); +} + +absl::Status MakeNoSuchKeyError(absl::string_view key) { + return absl::NotFoundError(absl::StrCat("Key not found in map : ", key)); +} + +absl::Status MakeNoSuchTypeError(absl::string_view type) { + return absl::NotFoundError(absl::StrCat("type not found: ", type)); +} + +absl::Status MakeTypeConversionError(absl::string_view from, + absl::string_view to) { + return absl::InvalidArgumentError( + absl::StrCat("type conversion error from '", from, "' to '", to, "'")); +} + +absl::Status MakeIndexOutOfBoundsError(size_t index) { + return absl::InvalidArgumentError( + absl::StrCat("index out of bounds: ", index)); +} + +absl::Status MakeIndexOutOfBoundsError(ptrdiff_t index) { + return absl::InvalidArgumentError( + absl::StrCat("index out of bounds: ", index)); +} + } // namespace -ErrorValue::ErrorValue() : ErrorValue(DefaultErrorValue()) {} +namespace common_internal { + +absl::Status MakeTypeConversionError(const Type& from, const Type& to) { + return cel::MakeTypeConversionError(from.DebugString(), to.DebugString()); +} + +absl::Status MakeTypeConversionError(absl::string_view from, + absl::string_view to) { + return cel::MakeTypeConversionError(from, to); +} + +absl::Status MakeNoSuchFieldError(absl::string_view field) { + return cel::MakeNoSuchFieldError(field); +} + +absl::Status MakeNoSuchKeyError(absl::string_view key) { + return cel::MakeNoSuchKeyError(key); +} + +absl::Status MakeIndexOutOfBoundsError(size_t index) { + return cel::MakeIndexOutOfBoundsError(index); +} + +absl::Status MakeIndexOutOfBoundsError(ptrdiff_t index) { + return cel::MakeIndexOutOfBoundsError(index); +} -ErrorValue NoSuchFieldError(absl::string_view field) { - return ErrorValue(absl::NotFoundError( - absl::StrCat("no_such_field", field.empty() ? "" : " : ", field))); +} // namespace common_internal + +ErrorValue::ErrorValue() : ErrorValue(nullptr, &DefaultErrorValue()) {} + +ErrorValue NoSuchFieldError(absl::string_view field, + google::protobuf::Arena* absl_nonnull arena) { + return ErrorValue::From(MakeNoSuchFieldError(field), arena); } -ErrorValue NoSuchKeyError(absl::string_view key) { - return ErrorValue( - absl::NotFoundError(absl::StrCat("Key not found in map : ", key))); +ErrorValue NoSuchKeyError(absl::string_view key, + google::protobuf::Arena* absl_nonnull arena) { + return ErrorValue::From(MakeNoSuchKeyError(key), arena); } -ErrorValue NoSuchTypeError(absl::string_view type) { - return ErrorValue( - absl::NotFoundError(absl::StrCat("type not found: ", type))); +ErrorValue NoSuchTypeError(absl::string_view type, + google::protobuf::Arena* absl_nonnull arena) { + return ErrorValue::From(MakeNoSuchTypeError(type), arena); } ErrorValue DuplicateKeyError() { - return ErrorValue(absl::AlreadyExistsError("duplicate key in map")); + static const absl::NoDestructor error( + absl::AlreadyExistsError("duplicate key in map")); + return ErrorValue(nullptr, &*error); } -ErrorValue TypeConversionError(absl::string_view from, absl::string_view to) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("type conversion error from '", from, "' to '", to, "'"))); +ErrorValue TypeConversionError(absl::string_view from, absl::string_view to, + google::protobuf::Arena* absl_nonnull arena) { + return ErrorValue::From(MakeTypeConversionError(from, to), arena); } -ErrorValue TypeConversionError(const Type& from, const Type& to) { - return TypeConversionError(from.DebugString(), to.DebugString()); +ErrorValue TypeConversionError(const Type& from, const Type& to, + google::protobuf::Arena* absl_nonnull arena) { + return TypeConversionError(from.DebugString(), to.DebugString(), arena); } -ErrorValue IndexOutOfBoundsError(size_t index) { - return ErrorValue( - absl::InvalidArgumentError(absl::StrCat("index out of bounds: ", index))); +ErrorValue IndexOutOfBoundsError(size_t index, + google::protobuf::Arena* absl_nonnull arena) { + return ErrorValue::From(MakeIndexOutOfBoundsError(index), arena); } -ErrorValue IndexOutOfBoundsError(ptrdiff_t index) { - return ErrorValue( - absl::InvalidArgumentError(absl::StrCat("index out of bounds: ", index))); +ErrorValue IndexOutOfBoundsError(ptrdiff_t index, + google::protobuf::Arena* absl_nonnull arena) { + return ErrorValue::From(MakeIndexOutOfBoundsError(index), arena); } bool IsNoSuchField(const ErrorValue& value) { @@ -157,38 +216,9 @@ ErrorValue ErrorValue::Clone(google::protobuf::Arena* absl_nonnull arena) const return *this; } -absl::Status ErrorValue::ToStatus() const& { +absl::Status ErrorValue::ToStatus() const { ABSL_DCHECK(*this); - - if (arena_ == nullptr) { - return *std::launder( - reinterpret_cast(&status_.val[0])); - } - return *status_.ptr; -} - -absl::Status ErrorValue::ToStatus() && { - ABSL_DCHECK(*this); - - if (arena_ == nullptr) { - return std::move( - *std::launder(reinterpret_cast(&status_.val[0]))); - } - return *status_.ptr; -} - -ErrorValue::operator bool() const { - if (arena_ == nullptr) { - return !std::launder(reinterpret_cast(&status_.val[0])) - ->ok(); - } - return status_.ptr != nullptr && !status_.ptr->ok(); -} - -void swap(ErrorValue& lhs, ErrorValue& rhs) noexcept { - ErrorValue tmp(std::move(lhs)); - lhs = std::move(rhs); - rhs = std::move(tmp); + return *status_ptr_; } } // namespace cel diff --git a/common/values/error_value.h b/common/values/error_value.h index 4e24c866b..a3cb2c3b4 100644 --- a/common/values/error_value.h +++ b/common/values/error_value.h @@ -20,7 +20,6 @@ #include #include -#include #include #include #include @@ -31,9 +30,7 @@ #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/strings/cord.h" #include "absl/strings/string_view.h" -#include "common/arena.h" #include "common/type.h" #include "common/value_kind.h" #include "common/values/values.h" @@ -45,6 +42,35 @@ namespace cel { class Value; +class ErrorValue; + +ErrorValue DuplicateKeyError(); + +namespace common_internal { +absl::Status MakeTypeConversionError(const Type& from, const Type& to); +absl::Status MakeTypeConversionError(absl::string_view from, + absl::string_view to); +absl::Status MakeNoSuchFieldError(absl::string_view field); +absl::Status MakeNoSuchKeyError(absl::string_view key); +absl::Status MakeIndexOutOfBoundsError(size_t index); +absl::Status MakeIndexOutOfBoundsError(ptrdiff_t index); +template +std::enable_if_t, std::is_unsigned, + std::negation>>, + absl::Status> +MakeIndexOutOfBoundsError(T index) { + static_assert(sizeof(T) <= sizeof(size_t)); + return MakeIndexOutOfBoundsError(static_cast(index)); +} +template +std::enable_if_t, std::is_signed, + std::negation>>, + absl::Status> +MakeIndexOutOfBoundsError(T index) { + static_assert(sizeof(T) <= sizeof(ptrdiff_t)); + return MakeIndexOutOfBoundsError(static_cast(index)); +} +} // namespace common_internal // `ErrorValue` represents values of the `ErrorType`. class ABSL_ATTRIBUTE_TRIVIAL_ABI ErrorValue final @@ -52,36 +78,30 @@ class ABSL_ATTRIBUTE_TRIVIAL_ABI ErrorValue final public: static constexpr ValueKind kKind = ValueKind::kError; - explicit ErrorValue(absl::Status value) : arena_(nullptr) { - ::new (static_cast(&status_.val[0])) absl::Status(std::move(value)); - ABSL_DCHECK(*this) << "ErrorValue requires a non-OK absl::Status"; + // Returns a new ErrorValue created from absl::Status. The resulting storage + // for the underlying representation of ErrorValue is stored on the arena. + [[nodiscard]] + static ErrorValue From(absl::Status value, + google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + ABSL_DCHECK(!value.ok()) << "ErrorValue requires a non-OK absl::Status"; + ABSL_DCHECK(arena != nullptr); + return ErrorValue( + arena, google::protobuf::Arena::Create(arena, std::move(value))); + } + + [[nodiscard]] + static ErrorValue WrapUnsafe(const absl::Status* absl_nonnull value) { + ABSL_DCHECK(!value->ok()) << "ErrorValue requires a non-OK absl::Status"; + return ErrorValue(nullptr, value); } // By default, this creates an UNKNOWN error. You should always create a more // specific error value. ErrorValue(); - ErrorValue(const ErrorValue& other) { CopyConstruct(other); } - - ErrorValue(ErrorValue&& other) noexcept { MoveConstruct(other); } - - ~ErrorValue() { Destruct(); } - - ErrorValue& operator=(const ErrorValue& other) { - if (this != &other) { - Destruct(); - CopyConstruct(other); - } - return *this; - } - - ErrorValue& operator=(ErrorValue&& other) noexcept { - if (this != &other) { - Destruct(); - MoveConstruct(other); - } - return *this; - } + ErrorValue(const ErrorValue&) = default; + ErrorValue& operator=(const ErrorValue&) = default; static constexpr ValueKind kind() { return kKind; } @@ -112,9 +132,7 @@ class ABSL_ATTRIBUTE_TRIVIAL_ABI ErrorValue final ErrorValue Clone(google::protobuf::Arena* absl_nonnull arena) const; - absl::Status ToStatus() const&; - - absl::Status ToStatus() &&; + absl::Status ToStatus() const; ABSL_DEPRECATED("Use ToStatus()") absl::Status NativeValue() const& { return ToStatus(); } @@ -122,69 +140,48 @@ class ABSL_ATTRIBUTE_TRIVIAL_ABI ErrorValue final ABSL_DEPRECATED("Use ToStatus()") absl::Status NativeValue() && { return std::move(*this).ToStatus(); } - friend void swap(ErrorValue& lhs, ErrorValue& rhs) noexcept; + friend void swap(ErrorValue& lhs, ErrorValue& rhs) noexcept { + using std::swap; + swap(lhs.arena_, rhs.arena_); + swap(lhs.status_ptr_, rhs.status_ptr_); + } - explicit operator bool() const; + explicit operator bool() const { return !status_ptr_->ok(); } private: + friend ErrorValue DuplicateKeyError(); friend class common_internal::ValueMixin; - friend struct ArenaTraits; - ErrorValue(google::protobuf::Arena* absl_nonnull arena, + ErrorValue(google::protobuf::Arena* absl_nullable arena, const absl::Status* absl_nonnull status) - : arena_(arena) { - status_.ptr = status; - } - - void CopyConstruct(const ErrorValue& other) { - arena_ = other.arena_; - if (arena_ == nullptr) { - ::new (static_cast(&status_.val[0])) absl::Status(*std::launder( - reinterpret_cast(&other.status_.val[0]))); - } else { - status_.ptr = other.status_.ptr; - } - } - - void MoveConstruct(ErrorValue& other) { - arena_ = other.arena_; - if (arena_ == nullptr) { - ::new (static_cast(&status_.val[0])) - absl::Status(std::move(*std::launder( - reinterpret_cast(&other.status_.val[0])))); - } else { - status_.ptr = other.status_.ptr; - } - } - - void Destruct() { - if (arena_ == nullptr) { - std::launder(reinterpret_cast(&status_.val[0]))->~Status(); - } - } + : arena_(arena), status_ptr_(status) {} google::protobuf::Arena* absl_nullable arena_; - union { - alignas(absl::Status) char val[sizeof(absl::Status)]; - const absl::Status* absl_nonnull ptr; - } status_; + const absl::Status* absl_nonnull status_ptr_; }; -ErrorValue NoSuchFieldError(absl::string_view field); +ErrorValue NoSuchFieldError(absl::string_view field, + google::protobuf::Arena* absl_nonnull arena); -ErrorValue NoSuchKeyError(absl::string_view key); +ErrorValue NoSuchKeyError(absl::string_view key, + google::protobuf::Arena* absl_nonnull arena); -ErrorValue NoSuchTypeError(absl::string_view type); +ErrorValue NoSuchTypeError(absl::string_view type, + google::protobuf::Arena* absl_nonnull arena); ErrorValue DuplicateKeyError(); -ErrorValue TypeConversionError(absl::string_view from, absl::string_view to); +ErrorValue TypeConversionError(absl::string_view from, absl::string_view to, + google::protobuf::Arena* absl_nonnull arena); -ErrorValue TypeConversionError(const Type& from, const Type& to); +ErrorValue TypeConversionError(const Type& from, const Type& to, + google::protobuf::Arena* absl_nonnull arena); -ErrorValue IndexOutOfBoundsError(size_t index); +ErrorValue IndexOutOfBoundsError(size_t index, + google::protobuf::Arena* absl_nonnull arena); -ErrorValue IndexOutOfBoundsError(ptrdiff_t index); +ErrorValue IndexOutOfBoundsError(ptrdiff_t index, + google::protobuf::Arena* absl_nonnull arena); // Catch other integrals and forward them to the above ones. This is needed to // avoid ambiguous overload issues for smaller integral types like `int`. @@ -192,17 +189,17 @@ template std::enable_if_t, std::is_unsigned, std::negation>>, ErrorValue> -IndexOutOfBoundsError(T index) { +IndexOutOfBoundsError(T index, google::protobuf::Arena* absl_nonnull arena) { static_assert(sizeof(T) <= sizeof(size_t)); - return IndexOutOfBoundsError(static_cast(index)); + return IndexOutOfBoundsError(static_cast(index), arena); } template std::enable_if_t, std::is_signed, std::negation>>, ErrorValue> -IndexOutOfBoundsError(T index) { +IndexOutOfBoundsError(T index, google::protobuf::Arena* absl_nonnull arena) { static_assert(sizeof(T) <= sizeof(ptrdiff_t)); - return IndexOutOfBoundsError(static_cast(index)); + return IndexOutOfBoundsError(static_cast(index), arena); } inline std::ostream& operator<<(std::ostream& out, const ErrorValue& value) { @@ -215,11 +212,16 @@ bool IsNoSuchKey(const ErrorValue& value); class ErrorValueReturn final { public: - ErrorValueReturn() = default; + explicit ErrorValueReturn(google::protobuf::Arena* absl_nonnull arena) : arena_(arena) { + ABSL_DCHECK(arena != nullptr); + } ErrorValue operator()(absl::Status status) const { - return ErrorValue(std::move(status)); + return ErrorValue::From(std::move(status), arena_); } + + private: + google::protobuf::Arena* arena_ = nullptr; }; namespace common_internal { @@ -237,8 +239,9 @@ struct ImplicitlyConvertibleStatus { } // namespace common_internal -// For use with `RETURN_IF_ERROR(...).With(cel::ErrorValueAssign(&result))` and -// `ASSIGN_OR_RETURN(..., ..., _.With(cel::ErrorValueAssign(&result)))`. +// For use with `RETURN_IF_ERROR(...).With(cel::ErrorValueAssign(&result, +// arena))` and `ASSIGN_OR_RETURN(..., ..., +// _.With(cel::ErrorValueAssign(&result, arena)))`. // // IMPORTANT: // If the returning type is `absl::Status` the result will be @@ -248,13 +251,15 @@ class ErrorValueAssign final { public: ErrorValueAssign() = delete; - explicit ErrorValueAssign(Value& value ABSL_ATTRIBUTE_LIFETIME_BOUND) - : ErrorValueAssign(std::addressof(value)) {} + ErrorValueAssign(Value& value ABSL_ATTRIBUTE_LIFETIME_BOUND, + google::protobuf::Arena* absl_nonnull arena) + : ErrorValueAssign(std::addressof(value), arena) {} - explicit ErrorValueAssign( - Value* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND) - : value_(value) { + ErrorValueAssign(Value* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND, + google::protobuf::Arena* absl_nonnull arena) + : value_(value), arena_(arena) { ABSL_DCHECK(value != nullptr); + ABSL_DCHECK(arena != nullptr); } common_internal::ImplicitlyConvertibleStatus operator()( @@ -262,13 +267,7 @@ class ErrorValueAssign final { private: Value* absl_nonnull value_; -}; - -template <> -struct ArenaTraits { - static bool trivially_destructible(const ErrorValue& value) { - return value.arena_ != nullptr; - } + google::protobuf::Arena* arena_ = nullptr; }; } // namespace cel diff --git a/common/values/error_value_test.cc b/common/values/error_value_test.cc index 343a93d19..288300f89 100644 --- a/common/values/error_value_test.cc +++ b/common/values/error_value_test.cc @@ -33,28 +33,30 @@ using ErrorValueTest = common_internal::ValueTest<>; TEST_F(ErrorValueTest, Default) { ErrorValue value; - EXPECT_THAT(value.NativeValue(), StatusIs(absl::StatusCode::kUnknown)); + EXPECT_THAT(value.ToStatus(), StatusIs(absl::StatusCode::kUnknown)); } TEST_F(ErrorValueTest, OkStatus) { - EXPECT_DEBUG_DEATH(static_cast(ErrorValue(absl::OkStatus())), _); + EXPECT_DEBUG_DEATH( + static_cast(ErrorValue::From(absl::OkStatus(), arena())), _); } TEST_F(ErrorValueTest, Kind) { - EXPECT_EQ(ErrorValue(absl::CancelledError()).kind(), ErrorValue::kKind); - EXPECT_EQ(Value(ErrorValue(absl::CancelledError())).kind(), + EXPECT_EQ(ErrorValue::From(absl::CancelledError(), arena()).kind(), + ErrorValue::kKind); + EXPECT_EQ(Value(ErrorValue::From(absl::CancelledError(), arena())).kind(), ErrorValue::kKind); } TEST_F(ErrorValueTest, DebugString) { { std::ostringstream out; - out << ErrorValue(absl::CancelledError()); + out << ErrorValue::From(absl::CancelledError(), arena()); EXPECT_THAT(out.str(), Not(IsEmpty())); } { std::ostringstream out; - out << Value(ErrorValue(absl::CancelledError())); + out << Value(ErrorValue::From(absl::CancelledError(), arena())); EXPECT_THAT(out.str(), Not(IsEmpty())); } } @@ -74,9 +76,10 @@ TEST_F(ErrorValueTest, ConvertToJson) { } TEST_F(ErrorValueTest, NativeTypeId) { - EXPECT_EQ(NativeTypeId::Of(ErrorValue(absl::CancelledError())), + EXPECT_EQ(NativeTypeId::Of(ErrorValue::From(absl::CancelledError(), arena())), NativeTypeId::For()); - EXPECT_EQ(NativeTypeId::Of(Value(ErrorValue(absl::CancelledError()))), + EXPECT_EQ(NativeTypeId::Of( + Value(ErrorValue::From(absl::CancelledError(), arena()))), NativeTypeId::For()); } diff --git a/common/values/legacy_list_value.cc b/common/values/legacy_list_value.cc index 93848ca44..50bf1e6fe 100644 --- a/common/values/legacy_list_value.cc +++ b/common/values/legacy_list_value.cc @@ -14,22 +14,327 @@ #include "common/values/legacy_list_value.h" +#include +#include +#include + #include "absl/base/nullability.h" +#include "absl/base/optimization.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/types/optional.h" +#include "common/legacy_value.h" #include "common/native_type.h" #include "common/value.h" +#include "common/values/legacy_struct_value.h" #include "common/values/list_value_builder.h" #include "common/values/values.h" #include "eval/public/cel_value.h" #include "internal/casts.h" +#include "internal/status_macros.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" namespace cel::common_internal { +class LegacyParsedRepeatedFieldListValue final + : public CustomListValueInterface, + public google::api::expr::runtime::CelList { + public: + // `arena` is expected to be the same arena as the one that the object is + // allocated on. + explicit LegacyParsedRepeatedFieldListValue(ParsedRepeatedFieldValue value, + google::protobuf::Arena* absl_nonnull arena) + : value_(std::move(value)), arena_(arena) { + ABSL_DCHECK(arena != nullptr); + } + + // CelList implementation + int size() const override { return static_cast(value_.Size()); } + + bool empty() const override { return value_.IsEmpty(); } + + google::api::expr::runtime::CelValue operator[](int index) const override { + return Get(arena_, index); + } + + google::api::expr::runtime::CelValue Get(google::protobuf::Arena* arena, + int index) const override { + if (arena == nullptr) { + arena = arena_; + } + if (ABSL_PREDICT_FALSE(index < 0 || index >= size())) { + return google::api::expr::runtime::CelValue::CreateError( + google::protobuf::Arena::Create( + arena, common_internal::MakeIndexOutOfBoundsError(index))); + } + Value result; + auto status = value_.Get( + static_cast(index), google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), arena, &result); + if (ABSL_PREDICT_FALSE(!status.ok())) { + return google::api::expr::runtime::CelValue::CreateError( + google::protobuf::Arena::Create(arena, std::move(status))); + } + return UnsafeLegacyValue(result, /*stable=*/false, arena); + } + + google::protobuf::Arena* absl_nonnull arena() const { return arena_; } + + protected: + std::string DebugString() const override { return value_.DebugString(); } + + absl::Status SerializeTo( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::io::ZeroCopyOutputStream* absl_nonnull output) const override { + return value_.SerializeTo(descriptor_pool, message_factory, output); + } + + absl::Status ConvertToJsonArray( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Message* absl_nonnull json) const override { + return value_.ConvertToJsonArray(descriptor_pool, message_factory, json); + } + + absl::Status Equal(const ListValue& other, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, + Value* absl_nonnull result) const override { + return value_.Equal(other, descriptor_pool, message_factory, arena, result); + } + + bool IsZeroValue() const override { return value_.IsZeroValue(); } + + bool IsEmpty() const override { return value_.IsEmpty(); } + + size_t Size() const override { return value_.Size(); } + + absl::Status Get(size_t index, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, + Value* absl_nonnull result) const override { + CEL_RETURN_IF_ERROR( + value_.Get(index, descriptor_pool, message_factory, arena, result)); + interop_internal::WrapLegacyFieldAccessResult(arena, result); + return absl::OkStatus(); + } + + absl::Status ForEach( + ForEachWithIndexCallback callback, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) const override { + return value_.ForEach(callback, descriptor_pool, message_factory, arena); + } + + absl::StatusOr NewIterator() const override { + return value_.NewIterator(); + } + + absl::Status Contains( + const Value& other, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, + Value* absl_nonnull result) const override { + return value_.Contains(other, descriptor_pool, message_factory, arena, + result); + } + + CustomListValue Clone(google::protobuf::Arena* absl_nonnull arena) const override { + return CustomListValue( + google::protobuf::Arena::Create( + arena, value_.Clone(arena), arena), + arena); + } + + private: + NativeTypeId GetNativeTypeId() const override { + return NativeTypeId::For(); + } + + ParsedRepeatedFieldValue value_; + google::protobuf::Arena* const arena_; +}; + +class LegacyParsedJsonListValue final + : public CustomListValueInterface, + public google::api::expr::runtime::CelList { + public: + // `arena` is expected to be the same arena as the one that the object is + // allocated on. + explicit LegacyParsedJsonListValue(ParsedJsonListValue value, + google::protobuf::Arena* absl_nonnull arena) + : value_(std::move(value)), arena_(arena) { + ABSL_DCHECK(arena != nullptr); + } + + // CelList implementation + int size() const override { return static_cast(value_.Size()); } + + bool empty() const override { return value_.IsEmpty(); } + + google::api::expr::runtime::CelValue operator[](int index) const override { + return Get(arena_, index); + } + + google::api::expr::runtime::CelValue Get(google::protobuf::Arena* arena, + int index) const override { + if (arena == nullptr) { + arena = arena_; + } + if (ABSL_PREDICT_FALSE(index < 0 || index >= size())) { + return google::api::expr::runtime::CelValue::CreateError( + google::protobuf::Arena::Create( + arena, common_internal::MakeIndexOutOfBoundsError(index))); + } + Value result; + auto status = value_.Get( + static_cast(index), google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), arena, &result); + if (ABSL_PREDICT_FALSE(!status.ok())) { + return google::api::expr::runtime::CelValue::CreateError( + google::protobuf::Arena::Create(arena, std::move(status))); + } + return UnsafeLegacyValue(result, /*stable=*/false, arena); + } + + google::protobuf::Arena* absl_nonnull arena() const { return arena_; } + + protected: + std::string DebugString() const override { return value_.DebugString(); } + + absl::Status SerializeTo( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::io::ZeroCopyOutputStream* absl_nonnull output) const override { + return value_.SerializeTo(descriptor_pool, message_factory, output); + } + + absl::Status ConvertToJsonArray( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Message* absl_nonnull json) const override { + return value_.ConvertToJsonArray(descriptor_pool, message_factory, json); + } + + absl::Status Equal(const ListValue& other, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, + Value* absl_nonnull result) const override { + return value_.Equal(other, descriptor_pool, message_factory, arena, result); + } + + bool IsZeroValue() const override { return value_.IsZeroValue(); } + + bool IsEmpty() const override { return value_.IsEmpty(); } + + size_t Size() const override { return value_.Size(); } + + absl::Status Get(size_t index, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, + Value* absl_nonnull result) const override { + CEL_RETURN_IF_ERROR( + value_.Get(index, descriptor_pool, message_factory, arena, result)); + interop_internal::WrapLegacyFieldAccessResult(arena, result); + return absl::OkStatus(); + } + + absl::Status ForEach( + ForEachWithIndexCallback callback, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) const override { + return value_.ForEach(callback, descriptor_pool, message_factory, arena); + } + + absl::StatusOr NewIterator() const override { + return value_.NewIterator(); + } + + absl::Status Contains( + const Value& other, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, + Value* absl_nonnull result) const override { + return value_.Contains(other, descriptor_pool, message_factory, arena, + result); + } + + CustomListValue Clone(google::protobuf::Arena* absl_nonnull arena) const override { + return CustomListValue(google::protobuf::Arena::Create( + arena, value_.Clone(arena), arena), + arena); + } + + private: + NativeTypeId GetNativeTypeId() const override { + return NativeTypeId::For(); + } + + ParsedJsonListValue value_; + google::protobuf::Arena* const arena_; +}; + +CustomListValue WrapLegacyParsedRepeatedField( + ParsedRepeatedFieldValue value, google::protobuf::Arena* absl_nonnull arena) { + return CustomListValue( + google::protobuf::Arena::Create( + arena, std::move(value), arena), + arena); +} + +CustomListValue WrapLegacyParsedJsonList(ParsedJsonListValue value, + google::protobuf::Arena* absl_nonnull arena) { + return CustomListValue(google::protobuf::Arena::Create( + arena, std::move(value), arena), + arena); +} + +ListValue FromLegacyList( + const google::api::expr::runtime::CelList* absl_nullable cel_list) { + if (cel_list != nullptr) { + NativeTypeId native_type_id = NativeTypeId::Of(*cel_list); + if (native_type_id == + NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast( + cel_list); + return CustomListValue(impl, impl->arena()); + } + if (native_type_id == NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast(cel_list); + return CustomListValue(impl, impl->arena()); + } + if (native_type_id == NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast(cel_list); + if (google::protobuf::Arena* arena = impl->GetOwningArena(); arena != nullptr) { + return CustomListValue(impl, arena); + } + return CustomListValue(); + } + if (native_type_id == NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast(cel_list); + if (google::protobuf::Arena* arena = impl->GetOwningArena(); arena != nullptr) { + return CustomListValue(impl, arena); + } + } + } + return LegacyListValue(cel_list); +} + absl::Status LegacyListValue::Equal( const Value& other, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, @@ -68,9 +373,21 @@ absl::optional AsLegacyListValue(const Value& value) { static_cast( cel::internal::down_cast( custom_list_value->interface()))); + } else if (native_type_id == + NativeTypeId::For()) { + return LegacyListValue(static_cast< + const google::api::expr::runtime::CelList*>( + cel::internal::down_cast( + custom_list_value->interface()))); + } else if (native_type_id == + NativeTypeId::For()) { + return LegacyListValue( + static_cast( + cel::internal::down_cast( + custom_list_value->interface()))); } } - return absl::nullopt; + return std::nullopt; } } // namespace cel::common_internal diff --git a/common/values/legacy_list_value.h b/common/values/legacy_list_value.h index caffcbc25..55a169653 100644 --- a/common/values/legacy_list_value.h +++ b/common/values/legacy_list_value.h @@ -30,6 +30,8 @@ #include "absl/types/optional.h" #include "common/value_kind.h" #include "common/values/custom_list_value.h" +#include "common/values/parsed_json_list_value.h" +#include "common/values/parsed_repeated_field_value.h" #include "common/values/values.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -160,6 +162,18 @@ LegacyListValue GetLegacyListValue(const Value& value); absl::optional AsLegacyListValue(const Value& value); +class LegacyParsedRepeatedFieldListValue; +class LegacyParsedJsonListValue; + +CustomListValue WrapLegacyParsedRepeatedField( + ParsedRepeatedFieldValue value, google::protobuf::Arena* absl_nonnull arena); + +CustomListValue WrapLegacyParsedJsonList(ParsedJsonListValue value, + google::protobuf::Arena* absl_nonnull arena); + +ListValue FromLegacyList( + const google::api::expr::runtime::CelList* absl_nullable cel_list); + } // namespace common_internal } // namespace cel diff --git a/common/values/legacy_map_value.cc b/common/values/legacy_map_value.cc index 1f370761e..bd025c51b 100644 --- a/common/values/legacy_map_value.cc +++ b/common/values/legacy_map_value.cc @@ -14,22 +14,519 @@ #include "common/values/legacy_map_value.h" +#include +#include +#include +#include +#include +#include + #include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" -#include "absl/types/optional.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "common/legacy_value.h" +#include "common/memory.h" #include "common/native_type.h" #include "common/value.h" +#include "common/value_kind.h" +#include "common/values/legacy_list_value.h" +#include "common/values/legacy_struct_value.h" #include "common/values/map_value_builder.h" #include "common/values/values.h" #include "eval/public/cel_value.h" +#include "eval/public/structs/proto_message_type_adapter.h" #include "internal/casts.h" +#include "internal/status_macros.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" namespace cel::common_internal { +namespace { + +LegacyStructValue ParsedMessageToLegacyStructValue( + const ParsedMessageValue& parsed_message) { + return LegacyStructValue( + cel::to_address(parsed_message), + &google::api::expr::runtime::GetGenericProtoTypeInfoInstance()); +} + +bool MatchesMapKeyType(const google::protobuf::FieldDescriptor* absl_nonnull key_desc, + const Value& key) { + switch (key_desc->cpp_type()) { + case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: + return key.IsBool(); + case google::protobuf::FieldDescriptor::CPPTYPE_INT32: + if (key.IsInt()) { + auto val = key.GetInt().NativeValue(); + return val >= std::numeric_limits::min() && + val <= std::numeric_limits::max(); + } + return false; + case google::protobuf::FieldDescriptor::CPPTYPE_INT64: + return key.IsInt(); + case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: + if (key.IsUint()) { + auto val = key.GetUint().NativeValue(); + return val <= std::numeric_limits::max(); + } + return false; + case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: + return key.IsUint(); + case google::protobuf::FieldDescriptor::CPPTYPE_STRING: + return key.IsString(); + default: + return false; + } +} + +absl::Status InvalidMapKeyType(absl::string_view key_type) { + return absl::InvalidArgumentError( + absl::StrCat("Invalid map key type: '", key_type, "'")); +} + +} // namespace + +class LegacyParsedMapFieldMapValue final + : public CustomMapValueInterface, + public google::api::expr::runtime::CelMap { + public: + // `arena` is expected to be the same arena as the one that the object is + // allocated on. + explicit LegacyParsedMapFieldMapValue(ParsedMapFieldValue value, + google::protobuf::Arena* absl_nonnull arena) + : value_(std::move(value)), arena_(arena) { + ABSL_DCHECK(arena != nullptr); + ABSL_DCHECK(value_.field() != nullptr); + } + + std::string DebugString() const override { return value_.DebugString(); } + + absl::Status SerializeTo( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::io::ZeroCopyOutputStream* absl_nonnull output) const override { + return value_.SerializeTo(descriptor_pool, message_factory, output); + } + + absl::Status ConvertToJsonObject( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Message* absl_nonnull json) const override { + return value_.ConvertToJsonObject(descriptor_pool, message_factory, json); + } + + absl::Status Equal(const MapValue& other, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, + Value* absl_nonnull result) const override { + return value_.Equal(other, descriptor_pool, message_factory, arena, result); + } + + bool IsZeroValue() const override { return value_.IsZeroValue(); } + + bool IsEmpty() const override { return value_.IsEmpty(); } + + size_t Size() const override { return value_.Size(); } + + absl::StatusOr Find( + const Value& key, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, + Value* absl_nonnull result) const override { + // Mimic the legacy behavior of complaining about unexpected key type. + const auto* key_field = value_.field()->message_type()->map_key(); + if (!MatchesMapKeyType(key_field, key)) { + return InvalidMapKeyType(cel::ValueKindToString(key.kind())); + } + + CEL_ASSIGN_OR_RETURN( + auto found, + value_.Find(key, descriptor_pool, message_factory, arena, result)); + if (found) { + interop_internal::WrapLegacyFieldAccessResult(arena, result); + } + return found; + } + + absl::StatusOr Has( + const Value& key, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) const override { + const auto* key_field = value_.field()->message_type()->map_key(); + if (!MatchesMapKeyType(key_field, key)) { + return InvalidMapKeyType(key_field->cpp_type_name()); + } + Value result; + CEL_RETURN_IF_ERROR( + value_.Has(key, descriptor_pool, message_factory, arena, &result)); + if (result.IsBool()) { + return result.GetBool().NativeValue(); + } + if (result.IsError()) { + return result.GetError().NativeValue(); + } + return false; + } + + absl::Status ListKeys( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, + ListValue* absl_nonnull result) const override { + return value_.ListKeys(descriptor_pool, message_factory, arena, result); + } + + absl::Status ForEach( + ForEachCallback callback, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) const override { + return value_.ForEach(callback, descriptor_pool, message_factory, arena); + } + + absl::StatusOr NewIterator() const override { + return value_.NewIterator(); + } + + CustomMapValue Clone(google::protobuf::Arena* absl_nonnull arena) const override { + return CustomMapValue(google::protobuf::Arena::Create( + arena, value_.Clone(arena), arena), + arena); + } + + // CelMap implementation + int size() const override { return static_cast(value_.Size()); } + + bool empty() const override { return value_.IsEmpty(); } + + std::optional operator[]( + google::api::expr::runtime::CelValue key) const override { + return Get(arena_, key); + } + + std::optional Get( + google::protobuf::Arena* arena, + google::api::expr::runtime::CelValue key) const override { + if (arena == nullptr) { + arena = arena_; + } + Value modern_key; + if (!ModernValue(arena, key, modern_key).ok()) { + // Legacy to modern should succeed for a valid CelValue. + return std::nullopt; + } + Value modern_val; + // Call custom map Find directly. MapValue normally handles wrapping + // non-ok result to error value types, so emulate that here. + // + // Use the descriptor pool and message factory from the value. This is not + // totally consistent with modern APIs, but this should behave the same as + // the legacy map did. + const google::protobuf::Message* msg = value_.message_; + ABSL_DCHECK(msg->GetDescriptor() != nullptr); + ABSL_DCHECK(msg->GetReflection() != nullptr); + + const google::protobuf::DescriptorPool* descriptor_pool = + msg->GetDescriptor()->file()->pool(); + google::protobuf::MessageFactory* message_factory = + msg->GetReflection()->GetMessageFactory(); + auto found = + Find(modern_key, descriptor_pool, message_factory, arena, &modern_val); + if (!found.ok()) { + return google::api::expr::runtime::CreateErrorValue(arena, + found.status()); + } + if (!(*found) && !modern_val.IsError()) { + return std::nullopt; + } + return UnsafeLegacyValue(modern_val, /*stable=*/false, arena); + } + + absl::StatusOr Has( + const google::api::expr::runtime::CelValue& key) const override { + CEL_RETURN_IF_ERROR( + google::api::expr::runtime::CelValue::CheckMapKeyType(key)); + Value modern_key; + CEL_RETURN_IF_ERROR(ModernValue(arena_, key, modern_key)); + return Has(modern_key, google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), arena_); + } + + absl::StatusOr ListKeys() + const override { + return ListKeys(arena_); + } + + absl::StatusOr ListKeys( + google::protobuf::Arena* arena) const override { + if (arena == nullptr) { + arena = arena_; + } + ListValue keys; + CEL_RETURN_IF_ERROR(value_.ListKeys( + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), arena, &keys)); + auto legacy_list = AsLegacyListValue(keys); + if (!legacy_list.has_value()) { + return absl::InternalError("failed to convert list keys to legacy list"); + } + return legacy_list->cel_list(); + } + + google::protobuf::Arena* absl_nonnull arena() const { return arena_; } + + private: + NativeTypeId GetNativeTypeId() const override { + return NativeTypeId::For(); + } + + ParsedMapFieldValue value_; + google::protobuf::Arena* const arena_; +}; + +class LegacyParsedJsonMapValue final + : public CustomMapValueInterface, + public google::api::expr::runtime::CelMap { + public: + // `arena` is expected to be the same arena as the one that the object is + // allocated on. + explicit LegacyParsedJsonMapValue(ParsedJsonMapValue value, + google::protobuf::Arena* absl_nonnull arena) + : value_(std::move(value)), arena_(arena) { + ABSL_DCHECK(arena != nullptr); + } + + std::string DebugString() const override { return value_.DebugString(); } + + absl::Status SerializeTo( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::io::ZeroCopyOutputStream* absl_nonnull output) const override { + return value_.SerializeTo(descriptor_pool, message_factory, output); + } + + absl::Status ConvertToJsonObject( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Message* absl_nonnull json) const override { + return value_.ConvertToJsonObject(descriptor_pool, message_factory, json); + } + + absl::Status Equal(const MapValue& other, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, + Value* absl_nonnull result) const override { + return value_.Equal(other, descriptor_pool, message_factory, arena, result); + } + + bool IsZeroValue() const override { return value_.IsZeroValue(); } + + bool IsEmpty() const override { return value_.IsEmpty(); } + + size_t Size() const override { return value_.Size(); } + + absl::StatusOr Find( + const Value& key, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, + Value* absl_nonnull result) const override { + if (!key.IsString()) { + return InvalidMapKeyType(cel::ValueKindToString(key.kind())); + } + CEL_ASSIGN_OR_RETURN( + auto found, + value_.Find(key, descriptor_pool, message_factory, arena, result)); + if (found && result->IsParsedMessage()) { + *result = ParsedMessageToLegacyStructValue(result->GetParsedMessage()); + } + return found; + } + + absl::StatusOr Has( + const Value& key, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) const override { + if (!key.IsString()) { + return InvalidMapKeyType(cel::ValueKindToString(key.kind())); + } + Value result; + CEL_RETURN_IF_ERROR( + value_.Has(key, descriptor_pool, message_factory, arena, &result)); + if (result.IsBool()) { + return result.GetBool().NativeValue(); + } + if (result.IsError()) { + return result.GetError().NativeValue(); + } + return false; + } + + absl::Status ListKeys( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, + ListValue* absl_nonnull result) const override { + return value_.ListKeys(descriptor_pool, message_factory, arena, result); + } + + absl::Status ForEach( + ForEachCallback callback, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) const override { + return value_.ForEach(callback, descriptor_pool, message_factory, arena); + } + + absl::StatusOr NewIterator() const override { + return value_.NewIterator(); + } + + CustomMapValue Clone(google::protobuf::Arena* absl_nonnull arena) const override { + return CustomMapValue(google::protobuf::Arena::Create( + arena, value_.Clone(arena), arena), + arena); + } + + // CelMap implementation + int size() const override { return static_cast(value_.Size()); } + + bool empty() const override { return value_.IsEmpty(); } + + std::optional operator[]( + google::api::expr::runtime::CelValue key) const override { + return Get(arena_, key); + } + + std::optional Get( + google::protobuf::Arena* arena, + google::api::expr::runtime::CelValue key) const override { + if (arena == nullptr) { + arena = arena_; + } + Value modern_key; + if (!ModernValue(arena, key, modern_key).ok()) { + // Legacy to modern should succeed for a valid CelValue. + return std::nullopt; + } + Value modern_val; + // Call custom map Find directly. MapValue normally handles wrapping + // non-ok result to error value types, so emulate that here. + // + // We know that the descriptor pool and message factory aren't needed here, + // so fine to use generated. + auto found = + Find(modern_key, google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), arena, &modern_val); + if (!found.ok()) { + return google::api::expr::runtime::CreateErrorValue(arena, + found.status()); + } + if (!(*found) && !modern_val.IsError()) { + return std::nullopt; + } + return UnsafeLegacyValue(modern_val, /*stable=*/false, arena); + } + + absl::StatusOr Has( + const google::api::expr::runtime::CelValue& key) const override { + CEL_RETURN_IF_ERROR( + google::api::expr::runtime::CelValue::CheckMapKeyType(key)); + Value modern_key; + CEL_RETURN_IF_ERROR(ModernValue(arena_, key, modern_key)); + return Has(modern_key, google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), arena_); + } + + absl::StatusOr ListKeys() + const override { + return ListKeys(arena_); + } + + absl::StatusOr ListKeys( + google::protobuf::Arena* arena) const override { + if (arena == nullptr) { + arena = arena_; + } + ListValue keys; + CEL_RETURN_IF_ERROR(value_.ListKeys( + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), arena, &keys)); + auto legacy_list = AsLegacyListValue(keys); + if (!legacy_list.has_value()) { + return absl::InternalError("failed to convert list keys to legacy list"); + } + return legacy_list->cel_list(); + } + + google::protobuf::Arena* absl_nonnull arena() const { return arena_; } + + private: + NativeTypeId GetNativeTypeId() const override { + return NativeTypeId::For(); + } + + ParsedJsonMapValue value_; + google::protobuf::Arena* const arena_; +}; + +CustomMapValue WrapLegacyParsedMapField(ParsedMapFieldValue value, + google::protobuf::Arena* absl_nonnull arena) { + return CustomMapValue(google::protobuf::Arena::Create( + arena, std::move(value), arena), + arena); +} + +CustomMapValue WrapLegacyParsedJsonMap(ParsedJsonMapValue value, + google::protobuf::Arena* absl_nonnull arena) { + return CustomMapValue(google::protobuf::Arena::Create( + arena, std::move(value), arena), + arena); +} + +MapValue FromLegacyMap( + const google::api::expr::runtime::CelMap* absl_nullable cel_map) { + if (cel_map != nullptr) { + NativeTypeId native_type_id = NativeTypeId::Of(*cel_map); + if (native_type_id == NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast( + cel_map); + return CustomMapValue(impl, impl->arena()); + } + if (native_type_id == NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast(cel_map); + return CustomMapValue(impl, impl->arena()); + } + if (native_type_id == NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast(cel_map); + if (google::protobuf::Arena* arena = impl->GetOwningArena(); arena != nullptr) { + return CustomMapValue(impl, arena); + } + return CustomMapValue(); + } + if (native_type_id == NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast(cel_map); + if (google::protobuf::Arena* arena = impl->GetOwningArena(); arena != nullptr) { + return CustomMapValue(impl, arena); + } + } + } + return LegacyMapValue(cel_map); +} + absl::Status LegacyMapValue::Equal( const Value& other, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, @@ -52,7 +549,7 @@ LegacyMapValue GetLegacyMapValue(const Value& value) { return value.variant_.Get(); } -absl::optional AsLegacyMapValue(const Value& value) { +std::optional AsLegacyMapValue(const Value& value) { if (IsLegacyMapValue(value)) { return GetLegacyMapValue(value); } @@ -68,9 +565,21 @@ absl::optional AsLegacyMapValue(const Value& value) { static_cast( cel::internal::down_cast( custom_map_value->interface()))); + } else if (native_type_id == + NativeTypeId::For()) { + return LegacyMapValue( + static_cast( + cel::internal::down_cast( + custom_map_value->interface()))); + } else if (native_type_id == + NativeTypeId::For()) { + return LegacyMapValue( + static_cast( + cel::internal::down_cast( + custom_map_value->interface()))); } } - return absl::nullopt; + return std::nullopt; } } // namespace cel::common_internal diff --git a/common/values/legacy_map_value.h b/common/values/legacy_map_value.h index c83b7fc2f..c170dea80 100644 --- a/common/values/legacy_map_value.h +++ b/common/values/legacy_map_value.h @@ -25,11 +25,12 @@ #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "common/value_kind.h" #include "common/values/custom_map_value.h" +#include "common/values/parsed_json_map_value.h" +#include "common/values/parsed_map_field_value.h" #include "common/values/values.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -178,6 +179,18 @@ LegacyMapValue GetLegacyMapValue(const Value& value); absl::optional AsLegacyMapValue(const Value& value); +class LegacyParsedMapFieldMapValue; +class LegacyParsedJsonMapValue; + +CustomMapValue WrapLegacyParsedMapField(ParsedMapFieldValue value, + google::protobuf::Arena* absl_nonnull arena); + +CustomMapValue WrapLegacyParsedJsonMap(ParsedJsonMapValue value, + google::protobuf::Arena* absl_nonnull arena); + +MapValue FromLegacyMap( + const google::api::expr::runtime::CelMap* absl_nullable cel_map); + } // namespace common_internal } // namespace cel diff --git a/common/values/legacy_struct_value.cc b/common/values/legacy_struct_value.cc index 4a91c5d42..aa342ad06 100644 --- a/common/values/legacy_struct_value.cc +++ b/common/values/legacy_struct_value.cc @@ -37,7 +37,7 @@ absl::optional AsLegacyStructValue(const Value& value) { if (IsLegacyStructValue(value)) { return GetLegacyStructValue(value); } - return absl::nullopt; + return std::nullopt; } } // namespace cel::common_internal diff --git a/common/values/legacy_struct_value_test.cc b/common/values/legacy_struct_value_test.cc new file mode 100644 index 000000000..1329232f8 --- /dev/null +++ b/common/values/legacy_struct_value_test.cc @@ -0,0 +1,630 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/values/legacy_struct_value.h" + +#include +#include +#include + +#include "google/protobuf/struct.pb.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/strings/string_view.h" +#include "common/legacy_value.h" +#include "common/value.h" +#include "common/value_testing.h" +#include "common/values/legacy_list_value.h" +#include "common/values/legacy_map_value.h" +#include "eval/public/cel_value.h" +#include "eval/public/structs/proto_message_type_adapter.h" +#include "internal/testing.h" +#include "runtime/runtime_options.h" +#include "cel/expr/conformance/proto3/test_all_types.pb.h" + +namespace cel { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::IsOkAndHolds; +using ::absl_testing::StatusIs; +using ::cel::expr::conformance::proto3::TestAllTypes; +using ::cel::test::BoolValueIs; +using ::cel::test::ErrorValueIs; +using ::cel::test::IntValueIs; +using ::cel::test::StringValueIs; +using ::google::api::expr::runtime::CelValue; +using ::testing::NotNull; + +using LegacyStructValueTest = common_internal::ValueTest<>; + +TEST_F(LegacyStructValueTest, RepeatedFieldAccess) { + TestAllTypes message; + message.add_repeated_int32(10); + message.add_repeated_int32(20); + + common_internal::LegacyStructValue struct_value( + &message, &google::api::expr::runtime::GetGenericProtoTypeInfoInstance()); + + Value field_value; + ASSERT_THAT(struct_value.GetFieldByName( + "repeated_int32", ProtoWrapperTypeOptions::kUnsetProtoDefault, + descriptor_pool(), message_factory(), arena(), &field_value), + IsOk()); + + EXPECT_TRUE(field_value.IsList()); + auto list_value = field_value.GetList(); + EXPECT_THAT(list_value.Size(), IsOkAndHolds(2)); + + // Verify legacy CelList interface + auto legacy_list = common_internal::AsLegacyListValue(field_value); + ASSERT_TRUE(legacy_list.has_value()); + const auto* cel_list = legacy_list->cel_list(); + ASSERT_THAT(cel_list, NotNull()); + EXPECT_EQ(cel_list->size(), 2); + + CelValue elem0 = cel_list->Get(arena(), 0); + ASSERT_TRUE(elem0.IsInt64()); + EXPECT_EQ(elem0.Int64OrDie(), 10); + + CelValue elem1 = cel_list->Get(arena(), 1); + ASSERT_TRUE(elem1.IsInt64()); + EXPECT_EQ(elem1.Int64OrDie(), 20); +} + +TEST_F(LegacyStructValueTest, RepeatedMessageFieldAccess) { + TestAllTypes message; + auto* elem0 = message.add_repeated_nested_message(); + elem0->set_bb(42); + + common_internal::LegacyStructValue struct_value( + &message, &google::api::expr::runtime::GetGenericProtoTypeInfoInstance()); + + Value field_value; + ASSERT_THAT(struct_value.GetFieldByName( + "repeated_nested_message", + ProtoWrapperTypeOptions::kUnsetProtoDefault, + descriptor_pool(), message_factory(), arena(), &field_value), + IsOk()); + + EXPECT_TRUE(field_value.IsList()); + auto list_value = field_value.GetList(); + EXPECT_THAT(list_value.Size(), IsOkAndHolds(1)); + + Value first_elem; + ASSERT_THAT(list_value.Get(0, descriptor_pool(), message_factory(), arena(), + &first_elem), + IsOk()); + EXPECT_TRUE(common_internal::IsLegacyStructValue(first_elem)); + + // Verify via CelList + auto legacy_list = common_internal::AsLegacyListValue(field_value); + ASSERT_TRUE(legacy_list.has_value()); + const auto* cel_list = legacy_list->cel_list(); + ASSERT_THAT(cel_list, NotNull()); + + CelValue cel_elem = cel_list->Get(arena(), 0); + ASSERT_TRUE(cel_elem.IsMessage()); + EXPECT_EQ(cel_elem.MessageOrDie()->GetDescriptor(), elem0->GetDescriptor()); + EXPECT_EQ( + static_cast(cel_elem.MessageOrDie()) + ->bb(), + 42); +} + +TEST_F(LegacyStructValueTest, MapFieldAccess) { + TestAllTypes message; + (*message.mutable_map_string_string())["hello"] = "world"; + + common_internal::LegacyStructValue struct_value( + &message, &google::api::expr::runtime::GetGenericProtoTypeInfoInstance()); + + Value field_value; + ASSERT_THAT( + struct_value.GetFieldByName( + "map_string_string", ProtoWrapperTypeOptions::kUnsetProtoDefault, + descriptor_pool(), message_factory(), arena(), &field_value), + IsOk()); + + EXPECT_TRUE(field_value.IsMap()); + auto map_value = field_value.GetMap(); + EXPECT_THAT(map_value.Size(), IsOkAndHolds(1)); + + // Verify legacy CelMap interface + auto legacy_map = common_internal::AsLegacyMapValue(field_value); + ASSERT_TRUE(legacy_map.has_value()); + const auto* cel_map = legacy_map->cel_map(); + ASSERT_THAT(cel_map, NotNull()); + EXPECT_EQ(cel_map->size(), 1); + + std::string key_str = "hello"; + CelValue cel_key = CelValue::CreateString(&key_str); + auto cel_result = cel_map->Get(arena(), cel_key); + ASSERT_TRUE(cel_result.has_value()); + ASSERT_TRUE(cel_result->IsString()); + EXPECT_EQ(cel_result->StringOrDie().value(), "world"); + + auto has_res = cel_map->Has(cel_key); + ASSERT_THAT(has_res, IsOk()); + EXPECT_TRUE(*has_res); +} + +TEST_F(LegacyStructValueTest, MapFieldKeyTypeValidation) { + TestAllTypes message; + (*message.mutable_map_int32_int32())[1] = 2; + + common_internal::LegacyStructValue struct_value( + &message, &google::api::expr::runtime::GetGenericProtoTypeInfoInstance()); + + Value field_value; + ASSERT_THAT( + struct_value.GetFieldByName( + "map_int32_int32", ProtoWrapperTypeOptions::kUnsetProtoDefault, + descriptor_pool(), message_factory(), arena(), &field_value), + IsOk()); + + auto legacy_map = common_internal::AsLegacyMapValue(field_value); + ASSERT_TRUE(legacy_map.has_value()); + const auto* cel_map = legacy_map->cel_map(); + ASSERT_THAT(cel_map, NotNull()); + + // Valid key + CelValue int_key = CelValue::CreateInt64(1); + auto has_res = cel_map->Has(int_key); + ASSERT_THAT(has_res, IsOk()); + EXPECT_TRUE(*has_res); + + // Invalid key type (string key on int32 map) + std::string str_key_val = "1"; + CelValue str_key = CelValue::CreateString(&str_key_val); + auto invalid_has_res = cel_map->Has(str_key); + EXPECT_THAT(invalid_has_res, StatusIs(absl::StatusCode::kInvalidArgument)); + + auto invalid_get_res = cel_map->Get(arena(), str_key); + ASSERT_TRUE(invalid_get_res.has_value()); + ASSERT_TRUE(invalid_get_res->IsError()); + EXPECT_THAT(*invalid_get_res->ErrorOrDie(), + StatusIs(absl::StatusCode::kInvalidArgument)); + + auto invalid_subscript_res = (*cel_map)[str_key]; + ASSERT_TRUE(invalid_subscript_res.has_value()); + ASSERT_TRUE(invalid_subscript_res->IsError()); + EXPECT_THAT(*invalid_subscript_res->ErrorOrDie(), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +TEST_F(LegacyStructValueTest, JsonStructAccess) { + TestAllTypes message; + auto* struct_field = message.mutable_single_struct(); + (*struct_field->mutable_fields())["key"].set_string_value("value"); + + common_internal::LegacyStructValue struct_value( + &message, &google::api::expr::runtime::GetGenericProtoTypeInfoInstance()); + + Value field_value; + ASSERT_THAT(struct_value.GetFieldByName( + "single_struct", ProtoWrapperTypeOptions::kUnsetProtoDefault, + descriptor_pool(), message_factory(), arena(), &field_value), + IsOk()); + + EXPECT_TRUE(field_value.IsMap()); + auto map_value = field_value.GetMap(); + EXPECT_THAT(map_value.Size(), IsOkAndHolds(1)); + + // Verify legacy CelMap interface + auto legacy_map = common_internal::AsLegacyMapValue(field_value); + ASSERT_TRUE(legacy_map.has_value()); + const auto* cel_map = legacy_map->cel_map(); + ASSERT_THAT(cel_map, NotNull()); + EXPECT_EQ(cel_map->size(), 1); + + std::string key_str = "key"; + CelValue cel_key = CelValue::CreateString(&key_str); + auto cel_result = cel_map->Get(arena(), cel_key); + ASSERT_TRUE(cel_result.has_value()); + ASSERT_TRUE(cel_result->IsString()); + EXPECT_EQ(cel_result->StringOrDie().value(), "value"); +} + +TEST_F(LegacyStructValueTest, JsonListAccess) { + TestAllTypes message; + auto* list_field = message.mutable_single_value()->mutable_list_value(); + list_field->add_values()->set_string_value("item"); + + common_internal::LegacyStructValue struct_value( + &message, &google::api::expr::runtime::GetGenericProtoTypeInfoInstance()); + + Value field_value; + ASSERT_THAT(struct_value.GetFieldByName( + "single_value", ProtoWrapperTypeOptions::kUnsetProtoDefault, + descriptor_pool(), message_factory(), arena(), &field_value), + IsOk()); + + EXPECT_TRUE(field_value.IsList()); + auto list_value = field_value.GetList(); + EXPECT_THAT(list_value.Size(), IsOkAndHolds(1)); + + // Verify legacy CelList interface + auto legacy_list = common_internal::AsLegacyListValue(field_value); + ASSERT_TRUE(legacy_list.has_value()); + const auto* cel_list = legacy_list->cel_list(); + ASSERT_THAT(cel_list, NotNull()); + EXPECT_EQ(cel_list->size(), 1); + + CelValue elem = cel_list->Get(arena(), 0); + ASSERT_TRUE(elem.IsString()); + EXPECT_EQ(elem.StringOrDie().value(), "item"); +} + +TEST_F(LegacyStructValueTest, SingularMessageAccess) { + TestAllTypes message; + message.mutable_single_nested_message()->set_bb(100); + + common_internal::LegacyStructValue struct_value( + &message, &google::api::expr::runtime::GetGenericProtoTypeInfoInstance()); + + Value field_value; + ASSERT_THAT( + struct_value.GetFieldByName( + "single_nested_message", ProtoWrapperTypeOptions::kUnsetProtoDefault, + descriptor_pool(), message_factory(), arena(), &field_value), + IsOk()); + + EXPECT_TRUE(common_internal::IsLegacyStructValue(field_value)); + auto nested_struct = common_internal::GetLegacyStructValue(field_value); + + Value bb_value; + ASSERT_THAT(nested_struct.GetFieldByName( + "bb", ProtoWrapperTypeOptions::kUnsetProtoDefault, + descriptor_pool(), message_factory(), arena(), &bb_value), + IsOk()); + EXPECT_TRUE(bb_value.IsInt()); + EXPECT_EQ(bb_value.GetInt().NativeValue(), 100); +} + +TEST_F(LegacyStructValueTest, WrapLegacyFieldAccessResultParsedRepeatedField) { + TestAllTypes message; + message.add_repeated_int32(10); + message.add_repeated_int32(20); + + const auto* field_desc = + message.GetDescriptor()->FindFieldByName("repeated_int32"); + Value val = ParsedRepeatedFieldValue(&message, field_desc, arena()); + interop_internal::WrapLegacyFieldAccessResult(arena(), &val); + + EXPECT_TRUE(val.IsList()); + auto list_val = val.GetList(); + EXPECT_THAT(list_val.IsEmpty(), IsOkAndHolds(false)); + EXPECT_FALSE(list_val.IsZeroValue()); + EXPECT_THAT(list_val.Size(), IsOkAndHolds(2)); + EXPECT_THAT(list_val.Contains(IntValue(10), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + + Value elem; + ASSERT_THAT( + list_val.Get(0, descriptor_pool(), message_factory(), arena(), &elem), + IsOk()); + EXPECT_THAT(elem, IntValueIs(10)); + + std::vector elements; + ASSERT_THAT(list_val.ForEach( + [&](const Value& v) -> absl::StatusOr { + elements.push_back(v.GetInt().NativeValue()); + return true; + }, + descriptor_pool(), message_factory(), arena()), + IsOk()); + EXPECT_THAT(elements, testing::ElementsAre(10, 20)); + + auto legacy_list = common_internal::AsLegacyListValue(val); + ASSERT_TRUE(legacy_list.has_value()); + const auto* cel_list = legacy_list->cel_list(); + ASSERT_THAT(cel_list, NotNull()); + EXPECT_EQ(cel_list->size(), 2); + EXPECT_FALSE(cel_list->empty()); + EXPECT_EQ(cel_list->Get(arena(), 0).Int64OrDie(), 10); + EXPECT_EQ((*cel_list)[1].Int64OrDie(), 20); + + Value cloned = val.Clone(arena()); + EXPECT_TRUE(cloned.IsList()); + EXPECT_THAT(cloned.GetList().Size(), IsOkAndHolds(2)); +} + +TEST_F(LegacyStructValueTest, WrapLegacyFieldAccessResultParsedJsonList) { + google::protobuf::ListValue list_proto; + list_proto.add_values()->set_string_value("item1"); + list_proto.add_values()->set_string_value("item2"); + + Value val = ParsedJsonListValue(&list_proto, arena()); + interop_internal::WrapLegacyFieldAccessResult(arena(), &val); + + EXPECT_TRUE(val.IsList()); + auto list_val = val.GetList(); + EXPECT_THAT(list_val.IsEmpty(), IsOkAndHolds(false)); + EXPECT_THAT(list_val.Size(), IsOkAndHolds(2)); + EXPECT_THAT(list_val.Contains(StringValue::WrapUnsafe("item1"), + descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + + Value elem; + ASSERT_THAT( + list_val.Get(0, descriptor_pool(), message_factory(), arena(), &elem), + IsOk()); + EXPECT_THAT(elem, StringValueIs("item1")); + + std::vector elements; + ASSERT_THAT(list_val.ForEach( + [&](const Value& v) -> absl::StatusOr { + elements.push_back(v.GetString().ToString()); + return true; + }, + descriptor_pool(), message_factory(), arena()), + IsOk()); + EXPECT_THAT(elements, testing::ElementsAre("item1", "item2")); + + auto legacy_list = common_internal::AsLegacyListValue(val); + ASSERT_TRUE(legacy_list.has_value()); + const auto* cel_list = legacy_list->cel_list(); + ASSERT_THAT(cel_list, NotNull()); + EXPECT_EQ(cel_list->size(), 2); + EXPECT_FALSE(cel_list->empty()); + EXPECT_EQ(cel_list->Get(arena(), 0).StringOrDie().value(), "item1"); + EXPECT_EQ((*cel_list)[1].StringOrDie().value(), "item2"); + + Value cloned = val.Clone(arena()); + EXPECT_TRUE(cloned.IsList()); + EXPECT_THAT(cloned.GetList().Size(), IsOkAndHolds(2)); +} + +TEST_F(LegacyStructValueTest, WrapLegacyFieldAccessResultParsedMapField) { + TestAllTypes message; + (*message.mutable_map_string_string())["key1"] = "val1"; + (*message.mutable_map_string_string())["key2"] = "val2"; + + const auto* field_desc = + message.GetDescriptor()->FindFieldByName("map_string_string"); + Value val = ParsedMapFieldValue(&message, field_desc, arena()); + interop_internal::WrapLegacyFieldAccessResult(arena(), &val); + + EXPECT_TRUE(val.IsMap()); + auto map_val = val.GetMap(); + EXPECT_THAT(map_val.IsEmpty(), IsOkAndHolds(false)); + EXPECT_FALSE(map_val.IsZeroValue()); + EXPECT_THAT(map_val.Size(), IsOkAndHolds(2)); + EXPECT_THAT(map_val.Has(StringValue::WrapUnsafe("key1"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + EXPECT_THAT(map_val.Has(StringValue::WrapUnsafe("missing"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(BoolValueIs(false))); + EXPECT_THAT( + map_val.Has(IntValue(1), descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument)))); + + Value found_val; + ASSERT_THAT(map_val.Find(StringValue::WrapUnsafe("key1"), descriptor_pool(), + message_factory(), arena(), &found_val), + IsOkAndHolds(true)); + EXPECT_THAT(found_val, StringValueIs("val1")); + + Value get_val; + ASSERT_THAT(map_val.Get(StringValue::WrapUnsafe("key2"), descriptor_pool(), + message_factory(), arena(), &get_val), + IsOk()); + EXPECT_THAT(get_val, StringValueIs("val2")); + + ListValue keys; + ASSERT_THAT( + map_val.ListKeys(descriptor_pool(), message_factory(), arena(), &keys), + IsOk()); + EXPECT_THAT(keys.Size(), IsOkAndHolds(2)); + + auto legacy_map = common_internal::AsLegacyMapValue(val); + ASSERT_TRUE(legacy_map.has_value()); + const auto* cel_map = legacy_map->cel_map(); + ASSERT_THAT(cel_map, NotNull()); + EXPECT_EQ(cel_map->size(), 2); + EXPECT_FALSE(cel_map->empty()); + + std::string k1 = "key1"; + CelValue cel_k1 = CelValue::CreateString(&k1); + auto cel_find = cel_map->Get(arena(), cel_k1); + ASSERT_TRUE(cel_find.has_value()); + EXPECT_EQ(cel_find->StringOrDie().value(), "val1"); + + auto has_res = cel_map->Has(cel_k1); + ASSERT_THAT(has_res, IsOk()); + EXPECT_TRUE(*has_res); + + Value cloned = val.Clone(arena()); + EXPECT_TRUE(cloned.IsMap()); + EXPECT_THAT(cloned.GetMap().Size(), IsOkAndHolds(2)); +} + +TEST_F(LegacyStructValueTest, WrapLegacyFieldAccessResultParsedJsonMap) { + google::protobuf::Struct struct_proto; + (*struct_proto.mutable_fields())["k1"].set_string_value("v1"); + (*struct_proto.mutable_fields())["k2"].set_string_value("v2"); + + Value val = ParsedJsonMapValue(&struct_proto, arena()); + interop_internal::WrapLegacyFieldAccessResult(arena(), &val); + + EXPECT_TRUE(val.IsMap()); + auto map_val = val.GetMap(); + EXPECT_THAT(map_val.IsEmpty(), IsOkAndHolds(false)); + EXPECT_FALSE(map_val.IsZeroValue()); + EXPECT_THAT(map_val.Size(), IsOkAndHolds(2)); + EXPECT_THAT(map_val.Has(StringValue::WrapUnsafe("k1"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + EXPECT_THAT(map_val.Has(StringValue::WrapUnsafe("missing"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(BoolValueIs(false))); + EXPECT_THAT( + map_val.Has(IntValue(1), descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument)))); + + Value found_val; + ASSERT_THAT(map_val.Find(StringValue::WrapUnsafe("k1"), descriptor_pool(), + message_factory(), arena(), &found_val), + IsOkAndHolds(true)); + EXPECT_THAT(found_val, StringValueIs("v1")); + + Value get_val; + ASSERT_THAT(map_val.Get(StringValue::WrapUnsafe("k2"), descriptor_pool(), + message_factory(), arena(), &get_val), + IsOk()); + EXPECT_THAT(get_val, StringValueIs("v2")); + + ListValue keys; + ASSERT_THAT( + map_val.ListKeys(descriptor_pool(), message_factory(), arena(), &keys), + IsOk()); + EXPECT_THAT(keys.Size(), IsOkAndHolds(2)); + + auto legacy_map = common_internal::AsLegacyMapValue(val); + ASSERT_TRUE(legacy_map.has_value()); + const auto* cel_map = legacy_map->cel_map(); + ASSERT_THAT(cel_map, NotNull()); + EXPECT_EQ(cel_map->size(), 2); + EXPECT_FALSE(cel_map->empty()); + + std::string k1 = "k1"; + CelValue cel_k1 = CelValue::CreateString(&k1); + auto cel_find = cel_map->Get(arena(), cel_k1); + ASSERT_TRUE(cel_find.has_value()); + EXPECT_EQ(cel_find->StringOrDie().value(), "v1"); + + auto has_res = cel_map->Has(cel_k1); + ASSERT_THAT(has_res, IsOk()); + EXPECT_TRUE(*has_res); + + Value cloned = val.Clone(arena()); + EXPECT_TRUE(cloned.IsMap()); + EXPECT_THAT(cloned.GetMap().Size(), IsOkAndHolds(2)); +} + +TEST_F(LegacyStructValueTest, WrapLegacyFieldAccessResultEmptyContainers) { + Value empty_list = ListValue(); + interop_internal::WrapLegacyFieldAccessResult(arena(), &empty_list); + EXPECT_TRUE(empty_list.IsList()); + EXPECT_THAT(empty_list.GetList().Size(), IsOkAndHolds(0)); + EXPECT_THAT(empty_list.GetList().IsEmpty(), IsOkAndHolds(true)); + auto legacy_list = common_internal::AsLegacyListValue(empty_list); + ASSERT_TRUE(legacy_list.has_value()); + EXPECT_EQ(legacy_list->cel_list()->size(), 0); + + Value empty_map = MapValue(); + interop_internal::WrapLegacyFieldAccessResult(arena(), &empty_map); + EXPECT_TRUE(empty_map.IsMap()); + EXPECT_THAT(empty_map.GetMap().Size(), IsOkAndHolds(0)); + EXPECT_THAT(empty_map.GetMap().IsEmpty(), IsOkAndHolds(true)); + auto legacy_map = common_internal::AsLegacyMapValue(empty_map); + ASSERT_TRUE(legacy_map.has_value()); + EXPECT_EQ(legacy_map->cel_map()->size(), 0); +} + +TEST_F(LegacyStructValueTest, ToLegacyValueParsedRepeatedField) { + TestAllTypes message; + message.add_repeated_int32(10); + message.add_repeated_int32(20); + + const auto* field_desc = + message.GetDescriptor()->FindFieldByName("repeated_int32"); + Value val = ParsedRepeatedFieldValue(&message, field_desc, arena()); + + ASSERT_OK_AND_ASSIGN(CelValue cel_val, + interop_internal::ToLegacyValue(arena(), val)); + ASSERT_TRUE(cel_val.IsList()); + EXPECT_EQ(cel_val.ListOrDie()->size(), 2); + EXPECT_EQ(cel_val.ListOrDie()->Get(arena(), 0).Int64OrDie(), 10); + EXPECT_EQ(cel_val.ListOrDie()->Get(arena(), 1).Int64OrDie(), 20); + + CelValue unsafe_cel_val = + common_internal::UnsafeLegacyValue(val, /*stable=*/false, arena()); + ASSERT_TRUE(unsafe_cel_val.IsList()); + EXPECT_EQ(unsafe_cel_val.ListOrDie()->size(), 2); +} + +TEST_F(LegacyStructValueTest, ToLegacyValueParsedJsonList) { + google::protobuf::ListValue list_proto; + list_proto.add_values()->set_string_value("item1"); + list_proto.add_values()->set_string_value("item2"); + + Value val = ParsedJsonListValue(&list_proto, arena()); + + ASSERT_OK_AND_ASSIGN(CelValue cel_val, + interop_internal::ToLegacyValue(arena(), val)); + ASSERT_TRUE(cel_val.IsList()); + EXPECT_EQ(cel_val.ListOrDie()->size(), 2); + EXPECT_EQ(cel_val.ListOrDie()->Get(arena(), 0).StringOrDie().value(), + "item1"); + EXPECT_EQ(cel_val.ListOrDie()->Get(arena(), 1).StringOrDie().value(), + "item2"); + + CelValue unsafe_cel_val = + common_internal::UnsafeLegacyValue(val, /*stable=*/false, arena()); + ASSERT_TRUE(unsafe_cel_val.IsList()); + EXPECT_EQ(unsafe_cel_val.ListOrDie()->size(), 2); +} + +TEST_F(LegacyStructValueTest, ToLegacyValueParsedMapField) { + TestAllTypes message; + (*message.mutable_map_string_string())["k1"] = "v1"; + + const auto* field_desc = + message.GetDescriptor()->FindFieldByName("map_string_string"); + Value val = ParsedMapFieldValue(&message, field_desc, arena()); + + ASSERT_OK_AND_ASSIGN(CelValue cel_val, + interop_internal::ToLegacyValue(arena(), val)); + ASSERT_TRUE(cel_val.IsMap()); + EXPECT_EQ(cel_val.MapOrDie()->size(), 1); + std::string k1 = "k1"; + CelValue cel_k1 = CelValue::CreateString(&k1); + auto res = cel_val.MapOrDie()->Get(arena(), cel_k1); + ASSERT_TRUE(res.has_value()); + EXPECT_EQ(res->StringOrDie().value(), "v1"); + + CelValue unsafe_cel_val = + common_internal::UnsafeLegacyValue(val, /*stable=*/false, arena()); + ASSERT_TRUE(unsafe_cel_val.IsMap()); + EXPECT_EQ(unsafe_cel_val.MapOrDie()->size(), 1); +} + +TEST_F(LegacyStructValueTest, ToLegacyValueParsedJsonMap) { + google::protobuf::Struct struct_proto; + (*struct_proto.mutable_fields())["k1"].set_string_value("v1"); + + Value val = ParsedJsonMapValue(&struct_proto, arena()); + + ASSERT_OK_AND_ASSIGN(CelValue cel_val, + interop_internal::ToLegacyValue(arena(), val)); + ASSERT_TRUE(cel_val.IsMap()); + EXPECT_EQ(cel_val.MapOrDie()->size(), 1); + std::string k1 = "k1"; + CelValue cel_k1 = CelValue::CreateString(&k1); + auto res = cel_val.MapOrDie()->Get(arena(), cel_k1); + ASSERT_TRUE(res.has_value()); + EXPECT_EQ(res->StringOrDie().value(), "v1"); + + CelValue unsafe_cel_val = + common_internal::UnsafeLegacyValue(val, /*stable=*/false, arena()); + ASSERT_TRUE(unsafe_cel_val.IsMap()); + EXPECT_EQ(unsafe_cel_val.MapOrDie()->size(), 1); +} + +} // namespace +} // namespace cel diff --git a/common/values/list_value.cc b/common/values/list_value.cc index 35df98c40..6dad0ad94 100644 --- a/common/values/list_value.cc +++ b/common/values/list_value.cc @@ -263,7 +263,7 @@ optional_ref ListValue::AsCustom() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional ListValue::AsCustom() && { @@ -271,7 +271,7 @@ absl::optional ListValue::AsCustom() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } const CustomListValue& ListValue::GetCustom() const& { diff --git a/common/values/list_value_builder.h b/common/values/list_value_builder.h index 91cef066d..d7af362c9 100644 --- a/common/values/list_value_builder.h +++ b/common/values/list_value_builder.h @@ -39,6 +39,9 @@ namespace common_internal { // `list_value_builder.cc`. class CompatListValue : public CustomListValueInterface, public google::api::expr::runtime::CelList { + public: + virtual google::protobuf::Arena* GetOwningArena() const = 0; + private: NativeTypeId GetNativeTypeId() const final { return NativeTypeId::For(); @@ -78,6 +81,9 @@ class MutableListValue : public CustomListValueInterface { // inheritance and `dynamic_cast`. class MutableCompatListValue : public MutableListValue, public google::api::expr::runtime::CelList { + public: + virtual google::protobuf::Arena* GetOwningArena() const = 0; + private: NativeTypeId GetNativeTypeId() const final { return NativeTypeId::For(); diff --git a/common/values/list_value_test.cc b/common/values/list_value_test.cc index 321c05249..11a2b865d 100644 --- a/common/values/list_value_test.cc +++ b/common/values/list_value_test.cc @@ -21,7 +21,6 @@ #include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" -#include "common/casting.h" #include "common/value.h" #include "common/value_testing.h" #include "internal/testing.h" @@ -91,16 +90,16 @@ TEST_F(ListValueTest, Get) { NewIntListValue(IntValue(0), IntValue(1), IntValue(2))); ASSERT_OK_AND_ASSIGN(auto element, value.Get(0, descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(element)); - ASSERT_EQ(Cast(element).NativeValue(), 0); + ASSERT_TRUE(element.Is()); + ASSERT_EQ(element.Get().NativeValue(), 0); ASSERT_OK_AND_ASSIGN( element, value.Get(1, descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(element)); - ASSERT_EQ(Cast(element).NativeValue(), 1); + ASSERT_TRUE(element.Is()); + ASSERT_EQ(element.Get().NativeValue(), 1); ASSERT_OK_AND_ASSIGN( element, value.Get(2, descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(element)); - ASSERT_EQ(Cast(element).NativeValue(), 2); + ASSERT_TRUE(element.Is()); + ASSERT_EQ(element.Get().NativeValue(), 2); EXPECT_THAT( value.Get(3, descriptor_pool(), message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument)))); @@ -112,7 +111,7 @@ TEST_F(ListValueTest, ForEach) { std::vector elements; EXPECT_THAT(value.ForEach( [&elements](const Value& element) { - elements.push_back(Cast(element).NativeValue()); + elements.push_back(element.Get().NativeValue()); return true; }, descriptor_pool(), message_factory(), arena()), @@ -126,12 +125,12 @@ TEST_F(ListValueTest, Contains) { ASSERT_OK_AND_ASSIGN(auto contained, value.Contains(IntValue(2), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(contained)); - EXPECT_TRUE(Cast(contained).NativeValue()); + ASSERT_TRUE(contained.Is()); + EXPECT_TRUE(contained.Get().NativeValue()); ASSERT_OK_AND_ASSIGN(contained, value.Contains(IntValue(3), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(contained)); - EXPECT_FALSE(Cast(contained).NativeValue()); + ASSERT_TRUE(contained.Is()); + EXPECT_FALSE(contained.Get().NativeValue()); } TEST_F(ListValueTest, NewIterator) { @@ -143,8 +142,8 @@ TEST_F(ListValueTest, NewIterator) { ASSERT_OK_AND_ASSIGN( auto element, iterator->Next(descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(element)); - elements.push_back(Cast(element).NativeValue()); + ASSERT_TRUE(element.Is()); + elements.push_back(element.Get().NativeValue()); } EXPECT_EQ(iterator->HasNext(), false); EXPECT_THAT(iterator->Next(descriptor_pool(), message_factory(), arena()), diff --git a/common/values/map_value.cc b/common/values/map_value.cc index c8bf7b785..46891d1ce 100644 --- a/common/values/map_value.cc +++ b/common/values/map_value.cc @@ -337,7 +337,7 @@ optional_ref MapValue::AsCustom() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional MapValue::AsCustom() && { @@ -345,7 +345,7 @@ absl::optional MapValue::AsCustom() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } const CustomMapValue& MapValue::GetCustom() const& { diff --git a/common/values/map_value_builder.h b/common/values/map_value_builder.h index a5a47eda9..2eb70b613 100644 --- a/common/values/map_value_builder.h +++ b/common/values/map_value_builder.h @@ -39,6 +39,9 @@ namespace common_internal { // `map_value_builder.cc`. class CompatMapValue : public CustomMapValueInterface, public google::api::expr::runtime::CelMap { + public: + virtual google::protobuf::Arena* GetOwningArena() const = 0; + private: NativeTypeId GetNativeTypeId() const final { return NativeTypeId::For(); @@ -78,6 +81,9 @@ class MutableMapValue : public CustomMapValueInterface { // inheritance and `dynamic_cast`. class MutableCompatMapValue : public MutableMapValue, public google::api::expr::runtime::CelMap { + public: + virtual google::protobuf::Arena* GetOwningArena() const = 0; + private: NativeTypeId GetNativeTypeId() const final { return NativeTypeId::For(); diff --git a/common/values/map_value_test.cc b/common/values/map_value_test.cc index f7d1c5197..5b3deca27 100644 --- a/common/values/map_value_test.cc +++ b/common/values/map_value_test.cc @@ -22,7 +22,6 @@ #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "absl/types/optional.h" -#include "common/casting.h" #include "common/value.h" #include "common/value_testing.h" #include "internal/testing.h" @@ -139,16 +138,16 @@ TEST_F(MapValueTest, Get) { std::pair{IntValue(2), DoubleValue(5.0)})); ASSERT_OK_AND_ASSIGN(auto value, map_value.Get(IntValue(0), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(value)); - ASSERT_EQ(Cast(value).NativeValue(), 3.0); + ASSERT_TRUE(value.Is()); + ASSERT_EQ(value.Get().NativeValue(), 3.0); ASSERT_OK_AND_ASSIGN(value, map_value.Get(IntValue(1), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(value)); - ASSERT_EQ(Cast(value).NativeValue(), 4.0); + ASSERT_TRUE(value.Is()); + ASSERT_EQ(value.Get().NativeValue(), 4.0); ASSERT_OK_AND_ASSIGN(value, map_value.Get(IntValue(2), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(value)); - ASSERT_EQ(Cast(value).NativeValue(), 5.0); + ASSERT_TRUE(value.Is()); + ASSERT_EQ(value.Get().NativeValue(), 5.0); EXPECT_THAT( map_value.Get(IntValue(3), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound)))); @@ -164,18 +163,18 @@ TEST_F(MapValueTest, Find) { ASSERT_OK_AND_ASSIGN(entry, map_value.Find(IntValue(0), descriptor_pool(), message_factory(), arena())); ASSERT_TRUE(entry); - ASSERT_TRUE(InstanceOf(*entry)); - ASSERT_EQ(Cast(*entry).NativeValue(), 3.0); + ASSERT_TRUE(entry->Is()); + ASSERT_EQ(entry->Get().NativeValue(), 3.0); ASSERT_OK_AND_ASSIGN(entry, map_value.Find(IntValue(1), descriptor_pool(), message_factory(), arena())); ASSERT_TRUE(entry); - ASSERT_TRUE(InstanceOf(*entry)); - ASSERT_EQ(Cast(*entry).NativeValue(), 4.0); + ASSERT_TRUE(entry->Is()); + ASSERT_EQ(entry->Get().NativeValue(), 4.0); ASSERT_OK_AND_ASSIGN(entry, map_value.Find(IntValue(2), descriptor_pool(), message_factory(), arena())); ASSERT_TRUE(entry); - ASSERT_TRUE(InstanceOf(*entry)); - ASSERT_EQ(Cast(*entry).NativeValue(), 5.0); + ASSERT_TRUE(entry->Is()); + ASSERT_EQ(entry->Get().NativeValue(), 5.0); ASSERT_OK_AND_ASSIGN(entry, map_value.Find(IntValue(3), descriptor_pool(), message_factory(), arena())); ASSERT_FALSE(entry); @@ -189,20 +188,20 @@ TEST_F(MapValueTest, Has) { std::pair{IntValue(2), DoubleValue(5.0)})); ASSERT_OK_AND_ASSIGN(auto value, map_value.Has(IntValue(0), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(value)); - ASSERT_TRUE(Cast(value).NativeValue()); + ASSERT_TRUE(value.Is()); + ASSERT_TRUE(value.Get().NativeValue()); ASSERT_OK_AND_ASSIGN(value, map_value.Has(IntValue(1), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(value)); - ASSERT_TRUE(Cast(value).NativeValue()); + ASSERT_TRUE(value.Is()); + ASSERT_TRUE(value.Get().NativeValue()); ASSERT_OK_AND_ASSIGN(value, map_value.Has(IntValue(2), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(value)); - ASSERT_TRUE(Cast(value).NativeValue()); + ASSERT_TRUE(value.Is()); + ASSERT_TRUE(value.Get().NativeValue()); ASSERT_OK_AND_ASSIGN(value, map_value.Has(IntValue(3), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(value)); - ASSERT_FALSE(Cast(value).NativeValue()); + ASSERT_TRUE(value.Is()); + ASSERT_FALSE(value.Get().NativeValue()); } TEST_F(MapValueTest, ListKeys) { @@ -217,7 +216,7 @@ TEST_F(MapValueTest, ListKeys) { std::vector keys; ASSERT_THAT(list_keys.ForEach( [&keys](const Value& element) -> bool { - keys.push_back(Cast(element).NativeValue()); + keys.push_back(element.Get().NativeValue()); return true; }, descriptor_pool(), message_factory(), arena()), @@ -235,8 +234,8 @@ TEST_F(MapValueTest, ForEach) { EXPECT_THAT(value.ForEach( [&entries](const Value& key, const Value& value) { entries.push_back( - std::pair{Cast(key).NativeValue(), - Cast(value).NativeValue()}); + std::pair{key.Get().NativeValue(), + value.Get().NativeValue()}); return true; }, descriptor_pool(), message_factory(), arena()), @@ -258,8 +257,8 @@ TEST_F(MapValueTest, NewIterator) { ASSERT_OK_AND_ASSIGN( auto element, iterator->Next(descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(element)); - keys.push_back(Cast(element).NativeValue()); + ASSERT_TRUE(element.Is()); + keys.push_back(element.Get().NativeValue()); } EXPECT_EQ(iterator->HasNext(), false); EXPECT_THAT(iterator->Next(descriptor_pool(), message_factory(), arena()), @@ -270,9 +269,10 @@ TEST_F(MapValueTest, NewIterator) { TEST_F(MapValueTest, ConvertToJson) { ASSERT_OK_AND_ASSIGN( auto value, - NewJsonMapValue(std::pair{StringValue("0"), DoubleValue(3.0)}, - std::pair{StringValue("1"), DoubleValue(4.0)}, - std::pair{StringValue("2"), DoubleValue(5.0)})); + NewJsonMapValue( + std::pair{StringValue::WrapUnsafe("0"), DoubleValue(3.0)}, + std::pair{StringValue::WrapUnsafe("1"), DoubleValue(4.0)}, + std::pair{StringValue::WrapUnsafe("2"), DoubleValue(5.0)})); auto* message = NewArenaValueMessage(); EXPECT_THAT( value.ConvertToJson(descriptor_pool(), message_factory(), message), diff --git a/common/values/message_value.cc b/common/values/message_value.cc index e06206407..028641dbe 100644 --- a/common/values/message_value.cc +++ b/common/values/message_value.cc @@ -46,7 +46,7 @@ const google::protobuf::Descriptor* absl_nonnull MessageValue::GetDescriptor() c ABSL_CHECK(*this); // Crash OK return absl::visit( absl::Overload( - [](absl::monostate) -> const google::protobuf::Descriptor* absl_nonnull { + [](std::monostate) -> const google::protobuf::Descriptor* absl_nonnull { ABSL_UNREACHABLE(); }, [](const ParsedMessageValue& alternative) @@ -58,7 +58,7 @@ const google::protobuf::Descriptor* absl_nonnull MessageValue::GetDescriptor() c std::string MessageValue::DebugString() const { return absl::visit( - absl::Overload([](absl::monostate) -> std::string { return "INVALID"; }, + absl::Overload([](std::monostate) -> std::string { return "INVALID"; }, [](const ParsedMessageValue& alternative) -> std::string { return alternative.DebugString(); }), @@ -68,7 +68,7 @@ std::string MessageValue::DebugString() const { bool MessageValue::IsZeroValue() const { ABSL_DCHECK(*this); return absl::visit( - absl::Overload([](absl::monostate) -> bool { return true; }, + absl::Overload([](std::monostate) -> bool { return true; }, [](const ParsedMessageValue& alternative) -> bool { return alternative.IsZeroValue(); }), @@ -81,7 +81,7 @@ absl::Status MessageValue::SerializeTo( google::protobuf::io::ZeroCopyOutputStream* absl_nonnull output) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `ConvertToJson` on " "an invalid `MessageValue`"); @@ -99,7 +99,7 @@ absl::Status MessageValue::ConvertToJson( google::protobuf::Message* absl_nonnull json) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `ConvertToJson` on " "an invalid `MessageValue`"); @@ -117,7 +117,7 @@ absl::Status MessageValue::ConvertToJsonObject( google::protobuf::Message* absl_nonnull json) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `ConvertToJsonObject` on " "an invalid `MessageValue`"); @@ -136,7 +136,7 @@ absl::Status MessageValue::Equal( google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `Equal` on " "an invalid `MessageValue`"); @@ -155,7 +155,7 @@ absl::Status MessageValue::GetFieldByName( google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `GetFieldByName` on " "an invalid `MessageValue`"); @@ -175,7 +175,7 @@ absl::Status MessageValue::GetFieldByNumber( google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `GetFieldByNumber` on " "an invalid `MessageValue`"); @@ -192,7 +192,7 @@ absl::StatusOr MessageValue::HasFieldByName( absl::string_view name) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::StatusOr { + [](std::monostate) -> absl::StatusOr { return absl::InternalError( "unexpected attempt to invoke `HasFieldByName` on " "an invalid `MessageValue`"); @@ -206,7 +206,7 @@ absl::StatusOr MessageValue::HasFieldByName( absl::StatusOr MessageValue::HasFieldByNumber(int64_t number) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::StatusOr { + [](std::monostate) -> absl::StatusOr { return absl::InternalError( "unexpected attempt to invoke `HasFieldByNumber` on " "an invalid `MessageValue`"); @@ -224,7 +224,7 @@ absl::Status MessageValue::ForEachField( google::protobuf::Arena* absl_nonnull arena) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `ForEachField` on " "an invalid `MessageValue`"); @@ -244,7 +244,7 @@ absl::Status MessageValue::Qualify( int* absl_nonnull count) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `Qualify` on " "an invalid `MessageValue`"); @@ -262,7 +262,7 @@ cel::optional_ref MessageValue::AsParsed() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional MessageValue::AsParsed() && { @@ -270,7 +270,7 @@ absl::optional MessageValue::AsParsed() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } const ParsedMessageValue& MessageValue::GetParsed() const& { diff --git a/common/values/mutable_list_value_test.cc b/common/values/mutable_list_value_test.cc index c08d7091c..c6aec0df4 100644 --- a/common/values/mutable_list_value_test.cc +++ b/common/values/mutable_list_value_test.cc @@ -47,7 +47,8 @@ TEST_F(MutableListValueTest, IsEmpty) { auto* mutable_list_value = NewMutableListValue(arena()); mutable_list_value->Reserve(1); EXPECT_TRUE(CustomListValue(mutable_list_value, arena()).IsEmpty()); - EXPECT_THAT(mutable_list_value->Append(StringValue("foo")), IsOk()); + EXPECT_THAT(mutable_list_value->Append(StringValue::WrapUnsafe("foo")), + IsOk()); EXPECT_FALSE(CustomListValue(mutable_list_value, arena()).IsEmpty()); } @@ -55,7 +56,8 @@ TEST_F(MutableListValueTest, Size) { auto* mutable_list_value = NewMutableListValue(arena()); mutable_list_value->Reserve(1); EXPECT_THAT(CustomListValue(mutable_list_value, arena()).Size(), 0); - EXPECT_THAT(mutable_list_value->Append(StringValue("foo")), IsOk()); + EXPECT_THAT(mutable_list_value->Append(StringValue::WrapUnsafe("foo")), + IsOk()); EXPECT_THAT(CustomListValue(mutable_list_value, arena()).Size(), 1); } @@ -73,7 +75,8 @@ TEST_F(MutableListValueTest, ForEach) { message_factory(), arena()), IsOk()); EXPECT_THAT(elements, IsEmpty()); - EXPECT_THAT(mutable_list_value->Append(StringValue("foo")), IsOk()); + EXPECT_THAT(mutable_list_value->Append(StringValue::WrapUnsafe("foo")), + IsOk()); EXPECT_THAT(CustomListValue(mutable_list_value, arena()) .ForEach(for_each_callback, descriptor_pool(), message_factory(), arena()), @@ -89,7 +92,8 @@ TEST_F(MutableListValueTest, NewIterator) { CustomListValue(mutable_list_value, arena()).NewIterator()); EXPECT_THAT(iterator->Next(descriptor_pool(), message_factory(), arena()), StatusIs(absl::StatusCode::kFailedPrecondition)); - EXPECT_THAT(mutable_list_value->Append(StringValue("foo")), IsOk()); + EXPECT_THAT(mutable_list_value->Append(StringValue::WrapUnsafe("foo")), + IsOk()); ASSERT_OK_AND_ASSIGN( iterator, CustomListValue(mutable_list_value, arena()).NewIterator()); EXPECT_TRUE(iterator->HasNext()); @@ -110,7 +114,8 @@ TEST_F(MutableListValueTest, Get) { IsOk()); EXPECT_THAT(value, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))); - EXPECT_THAT(mutable_list_value->Append(StringValue("foo")), IsOk()); + EXPECT_THAT(mutable_list_value->Append(StringValue::WrapUnsafe("foo")), + IsOk()); EXPECT_THAT( CustomListValue(mutable_list_value, arena()) .Get(0, descriptor_pool(), message_factory(), arena(), &value), diff --git a/common/values/mutable_map_value_test.cc b/common/values/mutable_map_value_test.cc index 2f08abe3f..c03e9cbd5 100644 --- a/common/values/mutable_map_value_test.cc +++ b/common/values/mutable_map_value_test.cc @@ -52,7 +52,9 @@ TEST_F(MutableMapValueTest, IsEmpty) { auto mutable_map_value = NewMutableMapValue(arena()); mutable_map_value->Reserve(1); EXPECT_TRUE(CustomMapValue(mutable_map_value, arena()).IsEmpty()); - EXPECT_THAT(mutable_map_value->Put(StringValue("foo"), IntValue(1)), IsOk()); + EXPECT_THAT( + mutable_map_value->Put(StringValue::WrapUnsafe("foo"), IntValue(1)), + IsOk()); EXPECT_FALSE(CustomMapValue(mutable_map_value, arena()).IsEmpty()); } @@ -60,7 +62,9 @@ TEST_F(MutableMapValueTest, Size) { auto mutable_map_value = NewMutableMapValue(arena()); mutable_map_value->Reserve(1); EXPECT_THAT(CustomMapValue(mutable_map_value, arena()).Size(), 0); - EXPECT_THAT(mutable_map_value->Put(StringValue("foo"), IntValue(1)), IsOk()); + EXPECT_THAT( + mutable_map_value->Put(StringValue::WrapUnsafe("foo"), IntValue(1)), + IsOk()); EXPECT_THAT(CustomMapValue(mutable_map_value, arena()).Size(), 1); } @@ -68,7 +72,9 @@ TEST_F(MutableMapValueTest, ListKeys) { auto mutable_map_value = NewMutableMapValue(arena()); mutable_map_value->Reserve(1); ListValue keys; - EXPECT_THAT(mutable_map_value->Put(StringValue("foo"), IntValue(1)), IsOk()); + EXPECT_THAT( + mutable_map_value->Put(StringValue::WrapUnsafe("foo"), IntValue(1)), + IsOk()); EXPECT_THAT( CustomMapValue(mutable_map_value, arena()) .ListKeys(descriptor_pool(), message_factory(), arena(), &keys), @@ -92,7 +98,9 @@ TEST_F(MutableMapValueTest, ForEach) { message_factory(), arena()), IsOk()); EXPECT_THAT(entries, IsEmpty()); - EXPECT_THAT(mutable_map_value->Put(StringValue("foo"), IntValue(1)), IsOk()); + EXPECT_THAT( + mutable_map_value->Put(StringValue::WrapUnsafe("foo"), IntValue(1)), + IsOk()); EXPECT_THAT(CustomMapValue(mutable_map_value, arena()) .ForEach(for_each_callback, descriptor_pool(), message_factory(), arena()), @@ -109,7 +117,9 @@ TEST_F(MutableMapValueTest, NewIterator) { EXPECT_FALSE(iterator->HasNext()); EXPECT_THAT(iterator->Next(descriptor_pool(), message_factory(), arena()), StatusIs(absl::StatusCode::kFailedPrecondition)); - EXPECT_THAT(mutable_map_value->Put(StringValue("foo"), IntValue(1)), IsOk()); + EXPECT_THAT( + mutable_map_value->Put(StringValue::WrapUnsafe("foo"), IntValue(1)), + IsOk()); ASSERT_OK_AND_ASSIGN( iterator, CustomMapValue(mutable_map_value, arena()).NewIterator()); EXPECT_TRUE(iterator->HasNext()); @@ -125,24 +135,26 @@ TEST_F(MutableMapValueTest, FindHas) { mutable_map_value->Reserve(1); Value value; EXPECT_THAT(CustomMapValue(mutable_map_value, arena()) - .Find(StringValue("foo"), descriptor_pool(), + .Find(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena(), &value), IsOkAndHolds(IsFalse())); EXPECT_THAT(value, IsNullValue()); EXPECT_THAT(CustomMapValue(mutable_map_value, arena()) - .Has(StringValue("foo"), descriptor_pool(), message_factory(), - arena(), &value), + .Has(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena(), &value), IsOk()); EXPECT_THAT(value, BoolValueIs(false)); - EXPECT_THAT(mutable_map_value->Put(StringValue("foo"), IntValue(1)), IsOk()); + EXPECT_THAT( + mutable_map_value->Put(StringValue::WrapUnsafe("foo"), IntValue(1)), + IsOk()); EXPECT_THAT(CustomMapValue(mutable_map_value, arena()) - .Find(StringValue("foo"), descriptor_pool(), + .Find(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena(), &value), IsOkAndHolds(IsTrue())); EXPECT_THAT(value, IntValueIs(1)); EXPECT_THAT(CustomMapValue(mutable_map_value, arena()) - .Has(StringValue("foo"), descriptor_pool(), message_factory(), - arena(), &value), + .Has(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena(), &value), IsOk()); EXPECT_THAT(value, BoolValueIs(true)); } diff --git a/common/values/null_value_test.cc b/common/values/null_value_test.cc index 5f244c532..ac40fef20 100644 --- a/common/values/null_value_test.cc +++ b/common/values/null_value_test.cc @@ -16,8 +16,6 @@ #include "absl/status/status_matchers.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "common/casting.h" #include "common/native_type.h" #include "common/value.h" #include "common/value_testing.h" @@ -27,8 +25,6 @@ namespace cel { namespace { using ::absl_testing::IsOk; -using ::testing::An; -using ::testing::Ne; using NullValueTest = common_internal::ValueTest<>; @@ -64,19 +60,5 @@ TEST_F(NullValueTest, NativeTypeId) { NativeTypeId::For()); } -TEST_F(NullValueTest, InstanceOf) { - EXPECT_TRUE(InstanceOf(NullValue())); - EXPECT_TRUE(InstanceOf(Value(NullValue()))); -} - -TEST_F(NullValueTest, Cast) { - EXPECT_THAT(Cast(NullValue()), An()); - EXPECT_THAT(Cast(Value(NullValue())), An()); -} - -TEST_F(NullValueTest, As) { - EXPECT_THAT(As(Value(NullValue())), Ne(absl::nullopt)); -} - } // namespace } // namespace cel diff --git a/common/values/opaque_value.cc b/common/values/opaque_value.cc index 235d268e7..4788abed0 100644 --- a/common/values/opaque_value.cc +++ b/common/values/opaque_value.cc @@ -171,14 +171,14 @@ optional_ref OpaqueValue::AsOptional() const& { if (IsOptional()) { return *reinterpret_cast(this); } - return absl::nullopt; + return std::nullopt; } absl::optional OpaqueValue::AsOptional() && { if (IsOptional()) { return std::move(*reinterpret_cast(this)); } - return absl::nullopt; + return std::nullopt; } const OptionalValue& OpaqueValue::GetOptional() const& { diff --git a/common/values/optional_value.cc b/common/values/optional_value.cc index ad0a65efb..3f4aa88f5 100644 --- a/common/values/optional_value.cc +++ b/common/values/optional_value.cc @@ -18,12 +18,14 @@ #include "absl/base/attributes.h" #include "absl/base/casts.h" +#include "absl/base/no_destructor.h" #include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" +#include "absl/types/source_location.h" #include "common/arena.h" #include "common/native_type.h" #include "common/type.h" @@ -122,200 +124,187 @@ absl::Status OptionalValueEqual( return absl::OkStatus(); } +google::protobuf::Arena* absl_nullable OptionalValueGetArenaNull( + const OpaqueValueDispatcher* absl_nonnull, OpaqueValueContent) { + return nullptr; +} + +OpaqueValue OptionalValueClone( + const OpaqueValueDispatcher* absl_nonnull dispatcher, + OpaqueValueContent content, google::protobuf::Arena* absl_nonnull arena) { + return common_internal::MakeOptionalValue(dispatcher, content); +} + +bool OptionalValueHasNoValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content) { + return false; +} + +void EmptyOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + static const absl::NoDestructor error( + absl::FailedPreconditionError("optional.none() dereference", + absl::SourceLocation())); + *result = ErrorValue::WrapUnsafe(&*error); +} + +void NullOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = NullValue(); +} + +void BoolOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = BoolValue(content.To()); +} + +void IntOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = IntValue(content.To()); +} + +void UintOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = UintValue(content.To()); +} + +void DoubleOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = DoubleValue(content.To()); +} + +void DurationOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = UnsafeDurationValue(content.To()); +} + +void TimestampOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = UnsafeTimestampValue(content.To()); +} + ABSL_CONST_INIT const OptionalValueDispatcher empty_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) - -> google::protobuf::Arena* absl_nullable { return nullptr; }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, - }, - [](const OptionalValueDispatcher* absl_nonnull dispatcher, - CustomValueContent content) -> bool { return false; }, - [](const OptionalValueDispatcher* absl_nonnull dispatcher, - CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = ErrorValue( - absl::FailedPreconditionError("optional.none() dereference")); + .clone = &OptionalValueClone, }, + &OptionalValueHasNoValue, + &EmptyOptionalValueValue, }; ABSL_CONST_INIT const OptionalValueDispatcher null_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) -> google::protobuf::Arena* absl_nullable { - return nullptr; - }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, + .clone = &OptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, CustomValueContent, - cel::Value* absl_nonnull result) -> void { *result = NullValue(); }, + &NullOptionalValueValue, }; ABSL_CONST_INIT const OptionalValueDispatcher bool_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) -> google::protobuf::Arena* absl_nullable { - return nullptr; - }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, + .clone = &OptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = BoolValue(content.To()); - }, + &BoolOptionalValueValue, }; ABSL_CONST_INIT const OptionalValueDispatcher int_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) -> google::protobuf::Arena* absl_nullable { - return nullptr; - }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, + .clone = &OptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = IntValue(content.To()); - }, + &IntOptionalValueValue, }; ABSL_CONST_INIT const OptionalValueDispatcher uint_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) -> google::protobuf::Arena* absl_nullable { - return nullptr; - }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, + .clone = &OptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = UintValue(content.To()); - }, + &UintOptionalValueValue, }; ABSL_CONST_INIT const OptionalValueDispatcher double_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) - -> google::protobuf::Arena* absl_nullable { return nullptr; }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, + .clone = &OptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, - CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = DoubleValue(content.To()); - }, + &DoubleOptionalValueValue, }; ABSL_CONST_INIT const OptionalValueDispatcher duration_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) - -> google::protobuf::Arena* absl_nullable { return nullptr; }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, + .clone = &OptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, - CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = UnsafeDurationValue(content.To()); - }, + &DurationOptionalValueValue, }; ABSL_CONST_INIT const OptionalValueDispatcher timestamp_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) - -> google::protobuf::Arena* absl_nullable { return nullptr; }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, + .clone = &OptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, - CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = UnsafeTimestampValue(content.To()); - }, + &TimestampOptionalValueValue, }; struct OptionalValueContent { @@ -323,43 +312,51 @@ struct OptionalValueContent { google::protobuf::Arena* absl_nonnull arena; }; +google::protobuf::Arena* absl_nullable GenericOptionalValueGetArena( + const OpaqueValueDispatcher* absl_nonnull, OpaqueValueContent content) { + return content.To().arena; +} + +OpaqueValue GenericOptionalValueClone( + const OpaqueValueDispatcher* absl_nonnull dispatcher, + OpaqueValueContent content, google::protobuf::Arena* absl_nonnull arena); + +void GenericOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = *content.To().value; +} + ABSL_CONST_INIT const OptionalValueDispatcher optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = - [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent content) -> google::protobuf::Arena* absl_nullable { - return content.To().arena; - }, + .get_arena = &GenericOptionalValueGetArena, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - ABSL_DCHECK(arena != nullptr); - - cel::Value* absl_nonnull result = ::new ( - arena->AllocateAligned(sizeof(cel::Value), alignof(cel::Value))) - cel::Value( - content.To().value->Clone(arena)); - if (!ArenaTraits<>::trivially_destructible(result)) { - arena->OwnDestructor(result); - } - return common_internal::MakeOptionalValue( - &optional_value_dispatcher, - OpaqueValueContent::From( - OptionalValueContent{.value = result, .arena = arena})); - }, + .clone = &GenericOptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = *content.To().value; - }, + &GenericOptionalValueValue, }; +OpaqueValue GenericOptionalValueClone( + const OpaqueValueDispatcher* absl_nonnull dispatcher, + OpaqueValueContent content, google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK(arena != nullptr); + + cel::Value* absl_nonnull result = + ::new (arena->AllocateAligned(sizeof(cel::Value), alignof(cel::Value))) + cel::Value(content.To().value->Clone(arena)); + if (!ArenaTraits<>::trivially_destructible(*result)) { + arena->OwnDestructor(result); + } + return common_internal::MakeOptionalValue( + &optional_value_dispatcher, OpaqueValueContent::From(OptionalValueContent{ + .value = result, .arena = arena})); +} + } // namespace OptionalValue OptionalValue::Of(cel::Value value, @@ -402,7 +399,7 @@ OptionalValue OptionalValue::Of(cel::Value value, cel::Value* absl_nonnull result = ::new ( arena->AllocateAligned(sizeof(cel::Value), alignof(cel::Value))) cel::Value(std::move(value)); - if (!ArenaTraits<>::trivially_destructible(result)) { + if (!ArenaTraits<>::trivially_destructible(*result)) { arena->OwnDestructor(result); } return OptionalValue(&optional_value_dispatcher, diff --git a/common/values/parsed_json_list_value.cc b/common/values/parsed_json_list_value.cc index 9acd23e3f..902deab33 100644 --- a/common/values/parsed_json_list_value.cc +++ b/common/values/parsed_json_list_value.cc @@ -41,6 +41,7 @@ #include "google/protobuf/descriptor.h" #include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/message.h" +#include "google/protobuf/message_lite.h" namespace cel { @@ -204,6 +205,12 @@ size_t ParsedJsonListValue::Size() const { if (value_ == nullptr) { return 0; } + if (const auto* generated_message = + google::protobuf::DynamicCastMessage(value_); + ABSL_PREDICT_TRUE(generated_message != nullptr)) { + return static_cast( + well_known_types::ListValueReflection::ValuesSize(*generated_message)); + } return static_cast( well_known_types::GetListValueReflectionOrDie(value_->GetDescriptor()) .ValuesSize(*value_)); @@ -220,14 +227,29 @@ absl::Status ParsedJsonListValue::Get( ABSL_DCHECK(result != nullptr); if (value_ == nullptr) { - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); + return absl::OkStatus(); + } + if (const auto* generated_message = + google::protobuf::DynamicCastMessage(value_); + ABSL_PREDICT_TRUE(generated_message != nullptr)) { + const int size = + well_known_types::ListValueReflection::ValuesSize(*generated_message); + if (ABSL_PREDICT_FALSE(index >= static_cast(size))) { + *result = IndexOutOfBoundsError(index, arena); + return absl::OkStatus(); + } + *result = common_internal::ParsedJsonValue( + &well_known_types::ListValueReflection::Values(*generated_message, + static_cast(index)), + arena); return absl::OkStatus(); } const auto reflection = well_known_types::GetListValueReflectionOrDie(value_->GetDescriptor()); if (ABSL_PREDICT_FALSE(index >= static_cast(reflection.ValuesSize(*value_)))) { - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); return absl::OkStatus(); } *result = common_internal::ParsedJsonValue( @@ -248,6 +270,22 @@ absl::Status ParsedJsonListValue::ForEach( return absl::OkStatus(); } Value scratch; + if (const auto* generated_message = + google::protobuf::DynamicCastMessage(value_); + ABSL_PREDICT_TRUE(generated_message != nullptr)) { + const int size = + well_known_types::ListValueReflection::ValuesSize(*generated_message); + for (int i = 0; i < size; ++i) { + scratch = common_internal::ParsedJsonValue( + &well_known_types::ListValueReflection::Values(*generated_message, i), + arena); + CEL_ASSIGN_OR_RETURN(auto ok, callback(static_cast(i), scratch)); + if (!ok) { + break; + } + } + return absl::OkStatus(); + } const auto reflection = well_known_types::GetListValueReflectionOrDie(value_->GetDescriptor()); const int size = reflection.ValuesSize(*value_); @@ -269,9 +307,16 @@ class ParsedJsonListValueIterator final : public ValueIterator { explicit ParsedJsonListValueIterator( const google::protobuf::Message* absl_nonnull message) : message_(message), - reflection_(well_known_types::GetListValueReflectionOrDie( - message_->GetDescriptor())), - size_(reflection_.ValuesSize(*message_)) {} + generated_( + google::protobuf::DynamicCastMessage(message_)) { + if (ABSL_PREDICT_TRUE(generated_ != nullptr)) { + size_ = well_known_types::ListValueReflection::ValuesSize(*generated_); + } else { + reflection_ = well_known_types::GetListValueReflectionOrDie( + message_->GetDescriptor()); + size_ = reflection_.ValuesSize(*message_); + } + } bool HasNext() override { return index_ < size_; } @@ -289,8 +334,14 @@ class ParsedJsonListValueIterator final : public ValueIterator { "`ValueIterator::Next` called after `ValueIterator::HasNext` " "returned false"); } - *result = common_internal::ParsedJsonValue( - &reflection_.Values(*message_, index_), arena); + if (ABSL_PREDICT_TRUE(generated_ != nullptr)) { + *result = common_internal::ParsedJsonValue( + &well_known_types::ListValueReflection::Values(*generated_, index_), + arena); + } else { + *result = common_internal::ParsedJsonValue( + &reflection_.Values(*message_, index_), arena); + } ++index_; return absl::OkStatus(); } @@ -308,8 +359,14 @@ class ParsedJsonListValueIterator final : public ValueIterator { if (index_ >= size_) { return false; } - *key_or_value = common_internal::ParsedJsonValue( - &reflection_.Values(*message_, index_), arena); + if (ABSL_PREDICT_TRUE(generated_ != nullptr)) { + *key_or_value = common_internal::ParsedJsonValue( + &well_known_types::ListValueReflection::Values(*generated_, index_), + arena); + } else { + *key_or_value = common_internal::ParsedJsonValue( + &reflection_.Values(*message_, index_), arena); + } ++index_; return true; } @@ -328,8 +385,14 @@ class ParsedJsonListValueIterator final : public ValueIterator { return false; } if (value != nullptr) { - *value = common_internal::ParsedJsonValue( - &reflection_.Values(*message_, index_), arena); + if (ABSL_PREDICT_TRUE(generated_ != nullptr)) { + *value = common_internal::ParsedJsonValue( + &well_known_types::ListValueReflection::Values(*generated_, index_), + arena); + } else { + *value = common_internal::ParsedJsonValue( + &reflection_.Values(*message_, index_), arena); + } } *key = IntValue(index_); ++index_; @@ -338,8 +401,9 @@ class ParsedJsonListValueIterator final : public ValueIterator { private: const google::protobuf::Message* absl_nonnull const message_; - const well_known_types::ListValueReflection reflection_; - const int size_; + const google::protobuf::ListValue* absl_nullable const generated_; + well_known_types::ListValueReflection reflection_; + int size_; int index_ = 0; }; @@ -365,7 +429,7 @@ absl::optional AsNumber(const Value& value) { if (auto double_value = value.AsDouble(); double_value) { return internal::Number::FromDouble(*double_value); } - return absl::nullopt; + return std::nullopt; } } // namespace diff --git a/common/values/parsed_json_list_value_test.cc b/common/values/parsed_json_list_value_test.cc index 017a24f9d..7551463f4 100644 --- a/common/values/parsed_json_list_value_test.cc +++ b/common/values/parsed_json_list_value_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include @@ -21,14 +22,15 @@ #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "common/memory.h" #include "common/type.h" #include "common/value.h" #include "common/value_kind.h" #include "common/value_testing.h" +#include "internal/parse_text_proto.h" #include "internal/testing.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" +#include "google/protobuf/arena.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" namespace cel { @@ -203,7 +205,7 @@ TEST_F(ParsedJsonListValueTest, NewIterator1) { EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(true)))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedJsonListValueTest, NewIterator2) { @@ -218,7 +220,7 @@ TEST_F(ParsedJsonListValueTest, NewIterator2) { EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(Pair(IntValueIs(1), BoolValueIs(true))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedJsonListValueTest, Contains_Dynamic) { @@ -249,12 +251,14 @@ TEST_F(ParsedJsonListValueTest, Contains_Dynamic) { EXPECT_THAT(valid_value.Contains(DoubleValue(1.0), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - EXPECT_THAT(valid_value.Contains(StringValue("bar"), descriptor_pool(), - message_factory(), arena()), - IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(valid_value.Contains(StringValue("foo"), descriptor_pool(), - message_factory(), arena()), - IsOkAndHolds(BoolValueIs(true))); + EXPECT_THAT( + valid_value.Contains(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(BoolValueIs(false))); + EXPECT_THAT( + valid_value.Contains(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); EXPECT_THAT(valid_value.Contains( ParsedJsonListValue( DynamicParseTextProto( @@ -285,5 +289,41 @@ TEST_F(ParsedJsonListValueTest, Contains_Dynamic) { IsOkAndHolds(BoolValueIs(true))); } +TEST_F(ParsedJsonListValueTest, CloneDefault) { + ParsedJsonListValue value; + EXPECT_FALSE(value.Clone(arena())); +} + +TEST_F(ParsedJsonListValueTest, CloneSameArena) { + ParsedJsonListValue value(DynamicParseTextProto( + R"pb(values { null_value: NULL_VALUE } + values { bool_value: true })pb"), + arena()); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); +} + +TEST_F(ParsedJsonListValueTest, CloneDifferentArena) { + google::protobuf::Arena other_arena; + ParsedJsonListValue value( + ::cel::internal::DynamicParseTextProto( + &other_arena, + R"pb(values { null_value: NULL_VALUE } + values { bool_value: true })pb", + descriptor_pool(), message_factory()), + &other_arena); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + EXPECT_EQ(cloned.Size(), 2); + EXPECT_THAT(cloned.Get(0, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(IsNullValue())); + EXPECT_THAT(cloned.Get(1, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); +} + } // namespace } // namespace cel diff --git a/common/values/parsed_json_map_value.cc b/common/values/parsed_json_map_value.cc index ec8c91a4f..c1d9ed6cb 100644 --- a/common/values/parsed_json_map_value.cc +++ b/common/values/parsed_json_map_value.cc @@ -28,7 +28,6 @@ #include "absl/strings/cord.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "common/allocator.h" #include "common/memory.h" #include "common/value.h" #include "common/values/parsed_json_value.h" @@ -40,7 +39,6 @@ #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/io/zero_copy_stream.h" -#include "google/protobuf/map.h" #include "google/protobuf/map_field.h" #include "google/protobuf/message.h" #include "google/protobuf/message_lite.h" @@ -205,6 +203,12 @@ size_t ParsedJsonMapValue::Size() const { if (value_ == nullptr) { return 0; } + if (const auto* generated_message = + google::protobuf::DynamicCastMessage(value_); + ABSL_PREDICT_TRUE(generated_message != nullptr)) { + return static_cast( + well_known_types::StructReflection::FieldsSize(*generated_message)); + } return static_cast( well_known_types::GetStructReflectionOrDie(value_->GetDescriptor()) .FieldsSize(*value_)); @@ -218,7 +222,7 @@ absl::Status ParsedJsonMapValue::Get( CEL_ASSIGN_OR_RETURN( bool ok, Find(key, descriptor_pool, message_factory, arena, result)); if (ABSL_PREDICT_FALSE(!ok) && !(result->IsError() || result->IsUnknown())) { - *result = NoSuchKeyError(key.DebugString()); + *result = NoSuchKeyError(key.DebugString(), arena); } return absl::OkStatus(); } @@ -239,6 +243,18 @@ absl::StatusOr ParsedJsonMapValue::Find( return false; } std::string key_scratch; + if (const auto* generated_message = + google::protobuf::DynamicCastMessage(value_); + ABSL_PREDICT_TRUE(generated_message != nullptr)) { + if (const auto* value = well_known_types::StructReflection::FindField( + *generated_message, string_key->ToStringView(&key_scratch)); + value != nullptr) { + *result = common_internal::ParsedJsonValue(value, arena); + return true; + } + *result = NullValue(); + return false; + } if (const auto* value = well_known_types::GetStructReflectionOrDie( value_->GetDescriptor()) @@ -271,6 +287,13 @@ absl::Status ParsedJsonMapValue::Has( return absl::OkStatus(); } std::string key_scratch; + if (const auto* generated_message = + google::protobuf::DynamicCastMessage(value_); + ABSL_PREDICT_TRUE(generated_message != nullptr)) { + *result = BoolValue(well_known_types::StructReflection::ContainsField( + *generated_message, string_key->ToStringView(&key_scratch))); + return absl::OkStatus(); + } if (const auto* value = well_known_types::GetStructReflectionOrDie( value_->GetDescriptor()) @@ -325,7 +348,7 @@ absl::Status ParsedJsonMapValue::ForEach( const auto map_end = reflection.EndFields(*value_); for (; map_begin != map_end; ++map_begin) { // We have to copy until `google::protobuf::MapKey` is just a view. - key_scratch = StringValue(arena, map_begin.GetKey().GetStringValue()); + key_scratch = StringValue::From(map_begin.GetKey().GetStringValue(), arena); value_scratch = common_internal::ParsedJsonValue( &map_begin.GetValueRef().GetMessageValue(), arena); CEL_ASSIGN_OR_RETURN(auto ok, callback(key_scratch, value_scratch)); diff --git a/common/values/parsed_json_map_value_test.cc b/common/values/parsed_json_map_value_test.cc index b65128076..8dfca0c70 100644 --- a/common/values/parsed_json_map_value_test.cc +++ b/common/values/parsed_json_map_value_test.cc @@ -144,15 +144,15 @@ TEST_F(ParsedJsonMapValueTest, Get_Dynamic) { valid_value.Get(BoolValue(), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound)))); - EXPECT_THAT(valid_value.Get(StringValue("foo"), descriptor_pool(), + EXPECT_THAT(valid_value.Get(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(IsNullValue())); - EXPECT_THAT(valid_value.Get(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(valid_value.Get(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); EXPECT_THAT( - valid_value.Get(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), + valid_value.Get(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound)))); } @@ -170,16 +170,16 @@ TEST_F(ParsedJsonMapValueTest, Find_Dynamic) { arena()); EXPECT_THAT(valid_value.Find(BoolValue(), descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); - EXPECT_THAT(valid_value.Find(StringValue("foo"), descriptor_pool(), - message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT(valid_value.Find(StringValue::WrapUnsafe("foo"), + descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(IsNullValue()))); - EXPECT_THAT(valid_value.Find(StringValue("bar"), descriptor_pool(), - message_factory(), arena()), + EXPECT_THAT(valid_value.Find(StringValue::WrapUnsafe("bar"), + descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(true)))); - EXPECT_THAT(valid_value.Find(StringValue("baz"), descriptor_pool(), - message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + EXPECT_THAT(valid_value.Find(StringValue::WrapUnsafe("baz"), + descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedJsonMapValueTest, Has_Dynamic) { @@ -197,13 +197,13 @@ TEST_F(ParsedJsonMapValueTest, Has_Dynamic) { EXPECT_THAT(valid_value.Has(BoolValue(), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(valid_value.Has(StringValue("foo"), descriptor_pool(), + EXPECT_THAT(valid_value.Has(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - EXPECT_THAT(valid_value.Has(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(valid_value.Has(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - EXPECT_THAT(valid_value.Has(StringValue("baz"), descriptor_pool(), + EXPECT_THAT(valid_value.Has(StringValue::WrapUnsafe("baz"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); } @@ -229,7 +229,7 @@ TEST_F(ParsedJsonMapValueTest, ListKeys_Dynamic) { EXPECT_THAT( keys.Contains(BoolValue(), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(keys.Contains(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(keys.Contains(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); EXPECT_THAT(keys.Get(0, descriptor_pool(), message_factory(), arena()), @@ -308,7 +308,7 @@ TEST_F(ParsedJsonMapValueTest, NewIterator1) { IsOkAndHolds( Optional(AnyOf(StringValueIs("foo"), StringValueIs("bar"))))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedJsonMapValueTest, NewIterator2) { @@ -333,7 +333,58 @@ TEST_F(ParsedJsonMapValueTest, NewIterator2) { AnyOf(Pair(StringValueIs("foo"), IsNullValue()), Pair(StringValueIs("bar"), BoolValueIs(true)))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); +} + +TEST_F(ParsedJsonMapValueTest, CloneDefault) { + ParsedJsonMapValue value; + EXPECT_FALSE(value.Clone(arena())); +} + +TEST_F(ParsedJsonMapValueTest, CloneSameArena) { + ParsedJsonMapValue value(DynamicParseTextProto(R"pb( + fields { + key: "foo" + value: { null_value: NULL_VALUE } + } + fields { + key: "bar" + value: { bool_value: true } + })pb"), + arena()); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); +} + +TEST_F(ParsedJsonMapValueTest, CloneDifferentArena) { + google::protobuf::Arena other_arena; + ParsedJsonMapValue value( + ::cel::internal::DynamicParseTextProto( + &other_arena, + R"pb( + fields { + key: "foo" + value: { null_value: NULL_VALUE } + } + fields { + key: "bar" + value: { bool_value: true } + })pb", + descriptor_pool(), message_factory()), + &other_arena); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + EXPECT_EQ(cloned.Size(), 2); + EXPECT_THAT(cloned.Get(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(IsNullValue())); + EXPECT_THAT(cloned.Get(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); } } // namespace diff --git a/common/values/parsed_json_value.cc b/common/values/parsed_json_value.cc index 6b10bea40..44bf872d1 100644 --- a/common/values/parsed_json_value.cc +++ b/common/values/parsed_json_value.cc @@ -17,20 +17,21 @@ #include #include +#include "google/protobuf/struct.pb.h" #include "absl/base/attributes.h" #include "absl/base/nullability.h" +#include "absl/base/optimization.h" #include "absl/functional/overload.h" #include "absl/status/status.h" #include "absl/strings/cord.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/variant.h" -#include "common/allocator.h" -#include "common/memory.h" #include "common/value.h" #include "internal/well_known_types.h" #include "google/protobuf/arena.h" #include "google/protobuf/message.h" +#include "google/protobuf/message_lite.h" namespace cel::common_internal { @@ -38,6 +39,7 @@ namespace { using ::cel::well_known_types::AsVariant; using ::cel::well_known_types::GetValueReflectionOrDie; +using ::cel::well_known_types::ValueReflection; google::protobuf::Arena* absl_nonnull MessageArenaOr( const google::protobuf::Message* absl_nonnull message, @@ -53,6 +55,45 @@ google::protobuf::Arena* absl_nonnull MessageArenaOr( Value ParsedJsonValue(const google::protobuf::Message* absl_nonnull message, google::protobuf::Arena* absl_nonnull arena) { + if (const auto* generated_message = + google::protobuf::DynamicCastMessage(message); + ABSL_PREDICT_TRUE(generated_message != nullptr)) { + const auto kind_case = ValueReflection::GetKindCase(*generated_message); + switch (kind_case) { + case google::protobuf::Value::KIND_NOT_SET: + ABSL_FALLTHROUGH_INTENDED; + case google::protobuf::Value::kNullValue: + return NullValue(); + case google::protobuf::Value::kBoolValue: + return BoolValue(ValueReflection::GetBoolValue(*generated_message)); + case google::protobuf::Value::kNumberValue: + return DoubleValue(ValueReflection::GetNumberValue(*generated_message)); + case google::protobuf::Value::kStringValue: { + absl::string_view string = + ValueReflection::GetStringValue(*generated_message); + if (string.empty()) { + return StringValue(); + } + if (google::protobuf::Arena* message_arena = generated_message->GetArena(); + message_arena != nullptr) { + return StringValue::Wrap(string, message_arena); + } + return StringValue::From(string, arena); + } + case google::protobuf::Value::kListValue: + return ParsedJsonListValue( + &ValueReflection::GetListValue(*generated_message), + MessageArenaOr(generated_message, arena)); + case google::protobuf::Value::kStructValue: + return ParsedJsonMapValue( + &ValueReflection::GetStructValue(*generated_message), + MessageArenaOr(generated_message, arena)); + default: + return ErrorValue::From(absl::InvalidArgumentError(absl::StrCat( + "unexpected value kind case: ", kind_case)), + arena); + } + } const auto reflection = GetValueReflectionOrDie(message->GetDescriptor()); const auto kind_case = reflection.GetKindCase(*message); switch (kind_case) { @@ -74,17 +115,20 @@ Value ParsedJsonValue(const google::protobuf::Message* absl_nonnull message, } if (string.data() == scratch.data() && string.size() == scratch.size()) { - return StringValue(arena, std::move(scratch)); + return StringValue::From(std::move(scratch), arena); } else { - return StringValue( - Borrower::Arena(MessageArenaOr(message, arena)), string); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return StringValue::Wrap(string, message_arena); + } + return StringValue::From(string, arena); } }, [&](absl::Cord&& cord) -> StringValue { if (cord.empty()) { return StringValue(); } - return StringValue(std::move(cord)); + return StringValue::From(std::move(cord), arena); }), AsVariant(reflection.GetStringValue(*message, scratch))); } @@ -95,8 +139,9 @@ Value ParsedJsonValue(const google::protobuf::Message* absl_nonnull message, return ParsedJsonMapValue(&reflection.GetStructValue(*message), MessageArenaOr(message, arena)); default: - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("unexpected value kind case: ", kind_case))); + return ErrorValue::From(absl::InvalidArgumentError(absl::StrCat( + "unexpected value kind case: ", kind_case)), + arena); } } diff --git a/common/values/parsed_map_field_value.cc b/common/values/parsed_map_field_value.cc index 47b737f82..7324286fc 100644 --- a/common/values/parsed_map_field_value.cc +++ b/common/values/parsed_map_field_value.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -27,12 +28,12 @@ #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/types/optional.h" #include "common/value.h" #include "common/values/values.h" #include "extensions/protobuf/internal/map_reflection.h" #include "internal/json.h" #include "internal/message_equality.h" +#include "internal/number.h" #include "internal/status_macros.h" #include "internal/well_known_types.h" #include "google/protobuf/arena.h" @@ -68,7 +69,7 @@ absl::Status ParsedMapFieldValue::SerializeTo( google::protobuf::Value message; CEL_RETURN_IF_ERROR(internal::MessageFieldToJson( *message_, field_, descriptor_pool, message_factory, &message)); - if (!message.list_value().SerializePartialToZeroCopyStream(output)) { + if (!message.struct_value().SerializePartialToZeroCopyStream(output)) { return absl::UnknownError("failed to serialize google.protobuf.Struct"); } return absl::OkStatus(); @@ -156,9 +157,8 @@ bool ParsedMapFieldValue::IsZeroValue() const { return IsEmpty(); } ParsedMapFieldValue ParsedMapFieldValue::Clone( google::protobuf::Arena* absl_nonnull arena) const { ABSL_DCHECK(arena != nullptr); - ABSL_DCHECK(*this); - if (ABSL_PREDICT_FALSE(field_ == nullptr)) { + if (ABSL_PREDICT_FALSE(!*this)) { return ParsedMapFieldValue(); } if (arena_ == arena) { @@ -187,7 +187,7 @@ size_t ParsedMapFieldValue::Size() const { namespace { -absl::optional ValueAsInt32(const Value& value) { +std::optional ValueAsInt32(const Value& value) { if (auto int_value = value.AsInt(); int_value && int_value->NativeValue() >= std::numeric_limits::min() && @@ -197,32 +197,36 @@ absl::optional ValueAsInt32(const Value& value) { uint_value && uint_value->NativeValue() <= std::numeric_limits::max()) { return static_cast(uint_value->NativeValue()); - } else if (auto double_value = value.AsDouble(); - double_value && - static_cast(static_cast( - double_value->NativeValue())) == double_value->NativeValue()) { - return static_cast(double_value->NativeValue()); + } else if (auto double_value = value.AsDouble(); double_value) { + auto number = internal::Number::FromDouble(double_value->NativeValue()); + if (number.LosslessConvertibleToInt()) { + int64_t wide_value = number.AsInt(); + if (wide_value >= std::numeric_limits::min() && + wide_value <= std::numeric_limits::max()) { + return static_cast(wide_value); + } + } } - return absl::nullopt; + return std::nullopt; } -absl::optional ValueAsInt64(const Value& value) { +std::optional ValueAsInt64(const Value& value) { if (auto int_value = value.AsInt(); int_value) { return int_value->NativeValue(); } else if (auto uint_value = value.AsUint(); uint_value && uint_value->NativeValue() <= std::numeric_limits::max()) { return static_cast(uint_value->NativeValue()); - } else if (auto double_value = value.AsDouble(); - double_value && - static_cast(static_cast( - double_value->NativeValue())) == double_value->NativeValue()) { - return static_cast(double_value->NativeValue()); + } else if (auto double_value = value.AsDouble(); double_value) { + auto number = internal::Number::FromDouble(double_value->NativeValue()); + if (number.LosslessConvertibleToInt()) { + return number.AsInt(); + } } - return absl::nullopt; + return std::nullopt; } -absl::optional ValueAsUInt32(const Value& value) { +std::optional ValueAsUInt32(const Value& value) { if (auto int_value = value.AsInt(); int_value && int_value->NativeValue() >= 0 && int_value->NativeValue() <= std::numeric_limits::max()) { @@ -231,28 +235,31 @@ absl::optional ValueAsUInt32(const Value& value) { uint_value && uint_value->NativeValue() <= std::numeric_limits::max()) { return static_cast(uint_value->NativeValue()); - } else if (auto double_value = value.AsDouble(); - double_value && - static_cast(static_cast( - double_value->NativeValue())) == double_value->NativeValue()) { - return static_cast(double_value->NativeValue()); + } else if (auto double_value = value.AsDouble(); double_value) { + auto number = internal::Number::FromDouble(double_value->NativeValue()); + if (number.LosslessConvertibleToUint()) { + uint64_t wide_value = number.AsUint(); + if (wide_value <= std::numeric_limits::max()) { + return static_cast(wide_value); + } + } } - return absl::nullopt; + return std::nullopt; } -absl::optional ValueAsUInt64(const Value& value) { +std::optional ValueAsUInt64(const Value& value) { if (auto int_value = value.AsInt(); int_value && int_value->NativeValue() >= 0) { return static_cast(int_value->NativeValue()); } else if (auto uint_value = value.AsUint(); uint_value) { return uint_value->NativeValue(); - } else if (auto double_value = value.AsDouble(); - double_value && - static_cast(static_cast( - double_value->NativeValue())) == double_value->NativeValue()) { - return static_cast(double_value->NativeValue()); + } else if (auto double_value = value.AsDouble(); double_value) { + auto number = internal::Number::FromDouble(double_value->NativeValue()); + if (number.LosslessConvertibleToUint()) { + return number.AsUint(); + } } - return absl::nullopt; + return std::nullopt; } bool ValueToProtoMapKey(const Value& key, @@ -297,8 +304,9 @@ bool ValueToProtoMapKey(const Value& key, } case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { if (auto string_key = key.AsString(); string_key) { - proto_key_scratch = string_key->NativeString(); - proto_key->SetStringValue(proto_key_scratch); + // `google::protobuf::MapKey` does not own the string, which may be stored inline + // in `key` or in `proto_key_scratch`; `key` must not be moved. + proto_key->SetStringValue(string_key->ToStringView(&proto_key_scratch)); return true; } return false; @@ -319,7 +327,7 @@ absl::Status ParsedMapFieldValue::Get( CEL_ASSIGN_OR_RETURN( bool ok, Find(key, descriptor_pool, message_factory, arena, result)); if (ABSL_PREDICT_FALSE(!ok) && !(result->IsError() || result->IsUnknown())) { - *result = ErrorValue(NoSuchKeyError(key.DebugString())); + *result = NoSuchKeyError(key.DebugString(), arena); } return absl::OkStatus(); } diff --git a/common/values/parsed_map_field_value.h b/common/values/parsed_map_field_value.h index 21d686bfd..f31f2e070 100644 --- a/common/values/parsed_map_field_value.h +++ b/common/values/parsed_map_field_value.h @@ -47,6 +47,10 @@ class ValueIterator; class ListValue; class ParsedJsonMapValue; +namespace common_internal { +class LegacyParsedMapFieldMapValue; +} // namespace common_internal + // ParsedMapFieldValue is a MapValue over a map field of a parsed protocol // buffer message. class ParsedMapFieldValue final @@ -192,6 +196,7 @@ class ParsedMapFieldValue final friend class ParsedJsonMapValue; friend class common_internal::ValueMixin; friend class common_internal::MapValueMixin; + friend class common_internal::LegacyParsedMapFieldMapValue; friend ParsedMapFieldValue UnsafeParsedMapFieldValue( const google::protobuf::Message* absl_nonnull message, const google::protobuf::FieldDescriptor* absl_nonnull field); diff --git a/common/values/parsed_map_field_value_test.cc b/common/values/parsed_map_field_value_test.cc index 271813f40..f605b5c98 100644 --- a/common/values/parsed_map_field_value_test.cc +++ b/common/values/parsed_map_field_value_test.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include #include #include @@ -21,7 +23,6 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" -#include "absl/types/optional.h" #include "common/memory.h" #include "common/type.h" #include "common/value.h" @@ -101,7 +102,7 @@ TEST_F(ParsedMapFieldValueTest, IsZeroValue) { EXPECT_TRUE(value.IsZeroValue()); } -TEST_F(ParsedMapFieldValueTest, SerializeTo) { +TEST_F(ParsedMapFieldValueTest, SerializeToEmpty) { ParsedMapFieldValue value( DynamicParseTextProto(R"pb()pb"), DynamicGetField("map_int64_int64"), arena()); @@ -111,6 +112,23 @@ TEST_F(ParsedMapFieldValueTest, SerializeTo) { EXPECT_THAT(std::move(output).Consume(), IsEmpty()); } +TEST_F(ParsedMapFieldValueTest, SerializeTo) { + ParsedMapFieldValue value( + DynamicParseTextProto( + R"pb(map_string_string { key: "foo" value: "bar" })pb"), + DynamicGetField("map_string_string"), arena()); + google::protobuf::io::CordOutputStream output; + EXPECT_THAT(value.SerializeTo(descriptor_pool(), message_factory(), &output), + IsOk()); + google::protobuf::Struct actual; + EXPECT_TRUE(actual.ParsePartialFromString(std::move(output).Consume())); + EXPECT_THAT(actual, EqualsTextProto( + R"pb(fields { + key: "foo" + value { string_value: "bar" } + })pb")); +} + TEST_F(ParsedMapFieldValueTest, ConvertToJson) { auto json = DynamicParseTextProto(R"pb()pb"); ParsedMapFieldValue value( @@ -192,15 +210,15 @@ TEST_F(ParsedMapFieldValueTest, Get) { EXPECT_THAT( value.Get(BoolValue(), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound)))); - EXPECT_THAT(value.Get(StringValue("foo"), descriptor_pool(), + EXPECT_THAT(value.Get(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(value.Get(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(value.Get(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); EXPECT_THAT( - value.Get(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), + value.Get(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound)))); } @@ -213,16 +231,88 @@ TEST_F(ParsedMapFieldValueTest, Find) { DynamicGetField("map_string_bool"), arena()); EXPECT_THAT( value.Find(BoolValue(), descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); - EXPECT_THAT(value.Find(StringValue("foo"), descriptor_pool(), + IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT(value.Find(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(false)))); - EXPECT_THAT(value.Find(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(value.Find(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(true)))); - EXPECT_THAT(value.Find(StringValue("baz"), descriptor_pool(), + EXPECT_THAT(value.Find(StringValue::WrapUnsafe("baz"), descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); +} + +TEST_F(ParsedMapFieldValueTest, FindDoubleKeyConversion) { + // Lookups on integer-keyed maps coerce double keys following CEL's numeric + // semantics ({1: 'x'}[1.0] resolves to key 1). A double that is out of range, + // negative for an unsigned key, or non-finite must be reported as absent + // rather than narrowed to the key type: converting such a double straight to + // an integer with static_cast is undefined behaviour. + ParsedMapFieldValue int64_map( + DynamicParseTextProto(R"pb( + map_int64_int64 { key: 1 value: 100 } + )pb"), + DynamicGetField("map_int64_int64"), arena()); + EXPECT_THAT(int64_map.Find(DoubleValue(1.0), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(100)))); + EXPECT_THAT(int64_map.Find(DoubleValue(1e19), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT(int64_map.Find(DoubleValue(-1e19), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT( + int64_map.Find(DoubleValue(std::numeric_limits::infinity()), + descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT( + int64_map.Find(DoubleValue(std::numeric_limits::quiet_NaN()), + descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); + + ParsedMapFieldValue uint64_map( + DynamicParseTextProto(R"pb( + map_uint64_uint64 { key: 1 value: 100 } + )pb"), + DynamicGetField("map_uint64_uint64"), arena()); + EXPECT_THAT(uint64_map.Find(DoubleValue(1.0), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(UintValueIs(100)))); + EXPECT_THAT(uint64_map.Find(DoubleValue(-1.0), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT(uint64_map.Find(DoubleValue(1e20), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); + + ParsedMapFieldValue int32_map( + DynamicParseTextProto(R"pb( + map_int32_int32 { key: 1 value: 100 } + )pb"), + DynamicGetField("map_int32_int32"), arena()); + EXPECT_THAT(int32_map.Find(DoubleValue(1.0), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(100)))); + EXPECT_THAT(int32_map.Find(DoubleValue(1e19), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); + + ParsedMapFieldValue uint32_map( + DynamicParseTextProto(R"pb( + map_uint32_uint32 { key: 1 value: 100 } + )pb"), + DynamicGetField("map_uint32_uint32"), arena()); + EXPECT_THAT(uint32_map.Find(DoubleValue(1.0), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(UintValueIs(100)))); + EXPECT_THAT(uint32_map.Find(DoubleValue(-1.0), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT(uint32_map.Find(DoubleValue(1e19), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedMapFieldValueTest, Has) { @@ -235,13 +325,13 @@ TEST_F(ParsedMapFieldValueTest, Has) { EXPECT_THAT( value.Has(BoolValue(), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(value.Has(StringValue("foo"), descriptor_pool(), + EXPECT_THAT(value.Has(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - EXPECT_THAT(value.Has(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(value.Has(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - EXPECT_THAT(value.Has(StringValue("baz"), descriptor_pool(), + EXPECT_THAT(value.Has(StringValue::WrapUnsafe("baz"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); } @@ -261,7 +351,7 @@ TEST_F(ParsedMapFieldValueTest, ListKeys) { EXPECT_THAT( keys.Contains(BoolValue(), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(keys.Contains(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(keys.Contains(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); EXPECT_THAT(keys.Get(0, descriptor_pool(), message_factory(), arena()), @@ -544,7 +634,7 @@ TEST_F(ParsedMapFieldValueTest, NewIterator1) { IsOkAndHolds( Optional(AnyOf(StringValueIs("foo"), StringValueIs("bar"))))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedMapFieldValueTest, NewIterator2) { @@ -564,7 +654,49 @@ TEST_F(ParsedMapFieldValueTest, NewIterator2) { AnyOf(Pair(StringValueIs("foo"), BoolValueIs(false)), Pair(StringValueIs("bar"), BoolValueIs(true)))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); +} + +TEST_F(ParsedMapFieldValueTest, CloneDefault) { + ParsedMapFieldValue value; + EXPECT_FALSE(value.Clone(arena())); +} + +TEST_F(ParsedMapFieldValueTest, CloneSameArena) { + ParsedMapFieldValue value( + DynamicParseTextProto(R"pb( + map_string_string { key: "foo" value: "bar" } + map_string_string { key: "bar" value: "foo" } + )pb"), + DynamicGetField("map_string_string"), arena()); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); +} + +TEST_F(ParsedMapFieldValueTest, CloneDifferentArena) { + google::protobuf::Arena other_arena; + ParsedMapFieldValue value( + ::cel::internal::DynamicParseTextProto( + &other_arena, + R"pb( + map_string_string { key: "foo" value: "bar" } + map_string_string { key: "bar" value: "foo" } + )pb", + descriptor_pool(), message_factory()), + DynamicGetField("map_string_string"), &other_arena); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + EXPECT_EQ(cloned.Size(), 2); + EXPECT_THAT(cloned.Get(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(StringValueIs("bar"))); + EXPECT_THAT(cloned.Get(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(StringValueIs("foo"))); } } // namespace diff --git a/common/values/parsed_message_value.cc b/common/values/parsed_message_value.cc index 8a2b8030d..15ce7a4cb 100644 --- a/common/values/parsed_message_value.cc +++ b/common/values/parsed_message_value.cc @@ -34,12 +34,12 @@ #include "base/attribute.h" #include "common/memory.h" #include "common/value.h" +#include "common/values/values.h" #include "extensions/protobuf/internal/qualify.h" #include "internal/empty_descriptors.h" #include "internal/json.h" #include "internal/message_equality.h" #include "internal/status_macros.h" -#include "internal/well_known_types.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -51,8 +51,6 @@ namespace cel { namespace { -using ::cel::well_known_types::ValueReflection; - template std::enable_if_t, const google::protobuf::Message* absl_nonnull> @@ -114,12 +112,8 @@ absl::Status ParsedMessageValue::ConvertToJson( ABSL_DCHECK_EQ(json->GetDescriptor()->well_known_type(), google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE); - ValueReflection value_reflection; - CEL_RETURN_IF_ERROR(value_reflection.Initialize(json->GetDescriptor())); - google::protobuf::Message* json_object = value_reflection.MutableStructValue(json); - return internal::MessageToJson(*value_, descriptor_pool, message_factory, - json_object); + json); } absl::Status ParsedMessageValue::ConvertToJsonObject( @@ -190,7 +184,7 @@ absl::Status ParsedMessageValue::GetFieldByName( field = descriptor->file()->pool()->FindExtensionByPrintableName(descriptor, name); if (field == nullptr) { - *result = NoSuchFieldError(name); + *result = NoSuchFieldError(name, arena); return absl::OkStatus(); } } @@ -211,12 +205,12 @@ absl::Status ParsedMessageValue::GetFieldByNumber( const auto* descriptor = GetDescriptor(); if (number < std::numeric_limits::min() || number > std::numeric_limits::max()) { - *result = NoSuchFieldError(absl::StrCat(number)); + *result = NoSuchFieldError(absl::StrCat(number), arena); return absl::OkStatus(); } const auto* field = descriptor->FindFieldByNumber(static_cast(number)); if (field == nullptr) { - *result = NoSuchFieldError(absl::StrCat(number)); + *result = NoSuchFieldError(absl::StrCat(number), arena); return absl::OkStatus(); } return GetField(field, unboxing_options, descriptor_pool, message_factory, @@ -231,7 +225,7 @@ absl::StatusOr ParsedMessageValue::HasFieldByName( field = descriptor->file()->pool()->FindExtensionByPrintableName(descriptor, name); if (field == nullptr) { - return NoSuchFieldError(name).NativeValue(); + return common_internal::MakeNoSuchFieldError(name); } } return HasField(field); @@ -242,11 +236,11 @@ absl::StatusOr ParsedMessageValue::HasFieldByNumber( const auto* descriptor = GetDescriptor(); if (number < std::numeric_limits::min() || number > std::numeric_limits::max()) { - return NoSuchFieldError(absl::StrCat(number)).NativeValue(); + return common_internal::MakeNoSuchFieldError(absl::StrCat(number)); } const auto* field = descriptor->FindFieldByNumber(static_cast(number)); if (field == nullptr) { - return NoSuchFieldError(absl::StrCat(number)).NativeValue(); + return common_internal::MakeNoSuchFieldError(absl::StrCat(number)); } return HasField(field); } @@ -283,27 +277,35 @@ class ParsedMessageValueQualifyState final const google::protobuf::Message* absl_nonnull message, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena) + google::protobuf::Arena* absl_nonnull arena, bool unsafe) : ProtoQualifyState(message, message->GetDescriptor(), message->GetReflection()), descriptor_pool_(descriptor_pool), message_factory_(message_factory), - arena_(arena) {} + arena_(arena), + unsafe_(unsafe) {} absl::optional& result() { return result_; } private: void SetResultFromError(absl::Status status, cel::MemoryManagerRef) override { - result_ = ErrorValue(std::move(status)); + result_ = ErrorValue::From(std::move(status), arena_); } void SetResultFromBool(bool value) override { result_ = BoolValue(value); } + // When `unsafe_` is set, the qualified message is externally managed (see + // `UnsafeParsedMessageValue()`), so borrow field values instead of copying + // them onto `arena_`. This matches `ParsedMessageValue::GetField()`. absl::Status SetResultFromField(const google::protobuf::Message* message, const google::protobuf::FieldDescriptor* field, ProtoWrapperTypeOptions unboxing_option, cel::MemoryManagerRef) override { - result_ = Value::WrapField(unboxing_option, message, field, + result_ = + unsafe_ + ? Value::WrapFieldUnsafe(unboxing_option, message, field, + descriptor_pool_, message_factory_, arena_) + : Value::WrapField(unboxing_option, message, field, descriptor_pool_, message_factory_, arena_); return absl::OkStatus(); } @@ -312,8 +314,12 @@ class ParsedMessageValueQualifyState final const google::protobuf::FieldDescriptor* field, int index, cel::MemoryManagerRef) override { - result_ = Value::WrapRepeatedField(index, message, field, descriptor_pool_, - message_factory_, arena_); + result_ = unsafe_ ? Value::WrapRepeatedFieldUnsafe(index, message, field, + descriptor_pool_, + message_factory_, arena_) + : Value::WrapRepeatedField(index, message, field, + descriptor_pool_, + message_factory_, arena_); return absl::OkStatus(); } @@ -321,14 +327,19 @@ class ParsedMessageValueQualifyState final const google::protobuf::FieldDescriptor* field, const google::protobuf::MapValueConstRef& value, cel::MemoryManagerRef) override { - result_ = Value::WrapMapFieldValue(value, message, field, descriptor_pool_, - message_factory_, arena_); + result_ = unsafe_ ? Value::WrapMapFieldValueUnsafe(value, message, field, + descriptor_pool_, + message_factory_, arena_) + : Value::WrapMapFieldValue(value, message, field, + descriptor_pool_, + message_factory_, arena_); return absl::OkStatus(); } const google::protobuf::DescriptorPool* absl_nonnull const descriptor_pool_; google::protobuf::MessageFactory* absl_nonnull const message_factory_; google::protobuf::Arena* absl_nonnull const arena_; + const bool unsafe_; absl::optional result_; }; @@ -350,8 +361,8 @@ absl::Status ParsedMessageValue::Qualify( if (ABSL_PREDICT_FALSE(qualifiers.empty())) { return absl::InvalidArgumentError("invalid select qualifier path."); } - ParsedMessageValueQualifyState qualify_state(value_, descriptor_pool, - message_factory, arena); + ParsedMessageValueQualifyState qualify_state( + value_, descriptor_pool, message_factory, arena, is_unsafe()); for (int i = 0; i < qualifiers.size() - 1; i++) { const auto& qualifier = qualifiers[i]; CEL_RETURN_IF_ERROR(qualify_state.ApplySelectQualifier( @@ -387,7 +398,7 @@ absl::Status ParsedMessageValue::GetField( ABSL_DCHECK(arena != nullptr); ABSL_DCHECK(result != nullptr); - if (arena_ == nullptr) { + if (is_unsafe()) { *result = Value::WrapFieldUnsafe(unboxing_options, value_, field, descriptor_pool, message_factory, arena); } else { diff --git a/common/values/parsed_message_value.h b/common/values/parsed_message_value.h index f3d1f7b40..1d02a4369 100644 --- a/common/values/parsed_message_value.h +++ b/common/values/parsed_message_value.h @@ -48,6 +48,10 @@ namespace cel { +namespace interop_internal { +bool IsUnsafeParsedMessageValue(const ParsedMessageValue& value); +} + class MessageValue; class StructValue; class Value; @@ -99,10 +103,8 @@ class ParsedMessageValue final return *value_; } - const google::protobuf::Message* absl_nonnull operator->() const - ABSL_ATTRIBUTE_LIFETIME_BOUND { - return value_; - } + const google::protobuf::Message* absl_nonnull operator->() const { return value_; } + const google::protobuf::Message* absl_nonnull message() const { return value_; } bool IsZeroValue() const; @@ -175,6 +177,15 @@ class ParsedMessageValue final swap(lhs.arena_, rhs.arena_); } + absl::Status GetField( + const google::protobuf::FieldDescriptor* absl_nonnull field, + ProtoWrapperTypeOptions unboxing_options, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const; + + bool HasField(const google::protobuf::FieldDescriptor* absl_nonnull field) const; + private: friend std::pointer_traits; friend class StructValue; @@ -182,10 +193,12 @@ class ParsedMessageValue final friend class common_internal::StructValueMixin; friend ParsedMessageValue UnsafeParsedMessageValue( const google::protobuf::Message* absl_nonnull value); + friend bool interop_internal::IsUnsafeParsedMessageValue( + const ParsedMessageValue& value); explicit ParsedMessageValue( const google::protobuf::Message* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND) - : value_(value), arena_(value->GetArena()) { + : value_(value), arena_(nullptr) { ABSL_DCHECK(value != nullptr); ABSL_DCHECK(!value_ || !IsWellKnownMessageType(value_->GetDescriptor())) << value_->GetTypeName() << " is a well known type"; @@ -203,18 +216,14 @@ class ParsedMessageValue final return absl::OkStatus(); } - absl::Status GetField( - const google::protobuf::FieldDescriptor* absl_nonnull field, - ProtoWrapperTypeOptions unboxing_options, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const; - - bool HasField(const google::protobuf::FieldDescriptor* absl_nonnull field) const; + bool is_unsafe() const { return arena_ == nullptr; } const google::protobuf::Message* absl_nonnull value_; - // Arena that is attributed as owning the value. May be null to indicate that - // the value is managed externally. + + // The arena attributed as Owning this value. Null if the value is created by + // UnsafeParsedMessageValue() or derived from such a value. This is used to + // identify externally managed messages and propagating the unsafe field + // access behavior. google::protobuf::Arena* absl_nullable arena_; }; diff --git a/common/values/parsed_message_value_test.cc b/common/values/parsed_message_value_test.cc index 7a84f82ba..6a1e59ead 100644 --- a/common/values/parsed_message_value_test.cc +++ b/common/values/parsed_message_value_test.cc @@ -14,10 +14,12 @@ #include +#include "google/protobuf/field_mask.pb.h" #include "google/protobuf/struct.pb.h" #include "absl/status/status_matchers.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" +#include "base/attribute.h" #include "common/memory.h" #include "common/type.h" #include "common/value.h" @@ -84,6 +86,19 @@ TEST_F(ParsedMessageValueTest, ConvertToJson) { R"pb(struct_value: {})pb")); } +TEST_F(ParsedMessageValueTest, ConvertToJsonFieldMask) { + ParsedMessageValue value = + MakeParsedMessage(R"pb(paths: "foo.bar" + paths: "baz")pb"); + google::protobuf::Message* json = + DynamicParseTextProto(R"pb()pb"); + ASSERT_THAT(value.ConvertToJson(descriptor_pool(), message_factory(), + cel::to_address(json)), + IsOk()); + EXPECT_THAT(*json, EqualsTextProto( + R"pb(string_value: "foo.bar,baz")pb")); +} + TEST_F(ParsedMessageValueTest, Equal) { MessageValue value = MakeParsedMessage(); EXPECT_THAT( @@ -108,5 +123,100 @@ TEST_F(ParsedMessageValueTest, GetFieldByNumber) { IsOkAndHolds(BoolValueIs(false))); } +// A message that is not owned by the evaluation arena, wrapped via the unsafe +// (borrowing) API, must not be deep-copied onto the arena when fields are +// selected through `Qualify()`. +TEST_F(ParsedMessageValueTest, QualifyUnsafeBorrowsMessageField) { + TestAllTypesProto3 message; // Heap-allocated, not on `arena()`. + message.mutable_standalone_message()->set_bb(42); + Value wrapped = Value::WrapMessageUnsafe(&message, descriptor_pool(), + message_factory(), arena()); + ASSERT_TRUE(wrapped.IsParsedMessage()); + + const SelectQualifier qualifiers[] = {FieldSpecifier{ + TestAllTypesProto3::kStandaloneMessageFieldNumber, "standalone_message"}}; + Value result; + int count = 0; + ASSERT_THAT(wrapped.GetParsedMessage().Qualify( + qualifiers, /*presence_test=*/false, descriptor_pool(), + message_factory(), arena(), &result, &count), + IsOk()); + ASSERT_TRUE(result.IsParsedMessage()); + EXPECT_EQ(result.GetParsedMessage().message(), &message.standalone_message()); + + ParsedMessageValue safe_wrapped(&message, arena()); + ASSERT_THAT(safe_wrapped.Qualify(qualifiers, /*presence_test=*/false, + descriptor_pool(), message_factory(), + arena(), &result, &count), + IsOk()); + ASSERT_TRUE(result.IsParsedMessage()); + EXPECT_NE(result.GetParsedMessage().message(), &message.standalone_message()); + EXPECT_EQ(result.GetParsedMessage().message()->GetArena(), arena()); +} + +TEST_F(ParsedMessageValueTest, QualifyUnsafeBorrowsRepeatedMessageField) { + TestAllTypesProto3 message; // Heap-allocated, not on `arena()`. + message.add_repeated_nested_message()->set_bb(42); + Value wrapped = Value::WrapMessageUnsafe(&message, descriptor_pool(), + message_factory(), arena()); + ASSERT_TRUE(wrapped.IsParsedMessage()); + + const SelectQualifier qualifiers[] = { + FieldSpecifier{TestAllTypesProto3::kRepeatedNestedMessageFieldNumber, + "repeated_nested_message"}, + AttributeQualifier::OfInt(0)}; + Value result; + int count = 0; + ASSERT_THAT(wrapped.GetParsedMessage().Qualify( + qualifiers, /*presence_test=*/false, descriptor_pool(), + message_factory(), arena(), &result, &count), + IsOk()); + ASSERT_TRUE(result.IsParsedMessage()); + EXPECT_EQ(result.GetParsedMessage().message(), + &message.repeated_nested_message(0)); + + ParsedMessageValue safe_wrapped(&message, arena()); + ASSERT_THAT(safe_wrapped.Qualify(qualifiers, /*presence_test=*/false, + descriptor_pool(), message_factory(), + arena(), &result, &count), + IsOk()); + ASSERT_TRUE(result.IsParsedMessage()); + EXPECT_NE(result.GetParsedMessage().message(), + &message.repeated_nested_message(0)); + EXPECT_EQ(result.GetParsedMessage().message()->GetArena(), arena()); +} + +TEST_F(ParsedMessageValueTest, QualifyUnsafeBorrowsMapMessageField) { + TestAllTypesProto3 message; // Heap-allocated, not on `arena()`. + (*message.mutable_map_string_message())["key"].set_bb(42); + Value wrapped = Value::WrapMessageUnsafe(&message, descriptor_pool(), + message_factory(), arena()); + ASSERT_TRUE(wrapped.IsParsedMessage()); + + const SelectQualifier qualifiers[] = { + FieldSpecifier{TestAllTypesProto3::kMapStringMessageFieldNumber, + "map_string_message"}, + AttributeQualifier::OfString("key")}; + Value result; + int count = 0; + ASSERT_THAT(wrapped.GetParsedMessage().Qualify( + qualifiers, /*presence_test=*/false, descriptor_pool(), + message_factory(), arena(), &result, &count), + IsOk()); + ASSERT_TRUE(result.IsParsedMessage()); + EXPECT_EQ(result.GetParsedMessage().message(), + &message.map_string_message().at("key")); + + ParsedMessageValue safe_wrapped(&message, arena()); + ASSERT_THAT(safe_wrapped.Qualify(qualifiers, /*presence_test=*/false, + descriptor_pool(), message_factory(), + arena(), &result, &count), + IsOk()); + ASSERT_TRUE(result.IsParsedMessage()); + EXPECT_NE(result.GetParsedMessage().message(), + &message.map_string_message().at("key")); + EXPECT_EQ(result.GetParsedMessage().message()->GetArena(), arena()); +} + } // namespace } // namespace cel diff --git a/common/values/parsed_repeated_field_value.cc b/common/values/parsed_repeated_field_value.cc index b990d3965..b716962cc 100644 --- a/common/values/parsed_repeated_field_value.cc +++ b/common/values/parsed_repeated_field_value.cc @@ -15,6 +15,7 @@ #include "common/values/parsed_repeated_field_value.h" #include +#include #include #include #include @@ -147,24 +148,79 @@ absl::Status ParsedRepeatedFieldValue::Equal( bool ParsedRepeatedFieldValue::IsZeroValue() const { return IsEmpty(); } +namespace { + +template +void CopyRepeatedFieldImpl(const google::protobuf::Reflection* absl_nonnull reflection, + const google::protobuf::Message& src_message, + google::protobuf::Message* absl_nonnull dst_message, + const google::protobuf::FieldDescriptor* absl_nonnull field) { + auto src_field = reflection->GetRepeatedFieldRef(src_message, field); + auto dst_field = + reflection->GetMutableRepeatedFieldRef(dst_message, field); + dst_field.CopyFrom(src_field); +} + +void CopyRepeatedField(const google::protobuf::Reflection* absl_nonnull reflection, + const google::protobuf::Message& src_message, + google::protobuf::Message* absl_nonnull dst_message, + const google::protobuf::FieldDescriptor* absl_nonnull field) { + switch (field->cpp_type()) { + case google::protobuf::FieldDescriptor::CPPTYPE_INT32: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, + field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_INT64: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, + field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, + field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, + field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, + field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, + field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_STRING: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, + field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + CopyRepeatedFieldImpl(reflection, src_message, + dst_message, field); + break; + } +} + +} // namespace + ParsedRepeatedFieldValue ParsedRepeatedFieldValue::Clone( google::protobuf::Arena* absl_nonnull arena) const { ABSL_DCHECK(arena != nullptr); - ABSL_DCHECK(*this); - if (ABSL_PREDICT_FALSE(field_ == nullptr)) { + if (ABSL_PREDICT_FALSE(!*this)) { return ParsedRepeatedFieldValue(); } if (arena_ == arena) { return *this; } - auto field = message_->GetReflection()->GetRepeatedFieldRef( - *message_, field_); auto* cloned_message = message_->New(arena); - auto cloned_field = - cloned_message->GetReflection() - ->GetMutableRepeatedFieldRef(cloned_message, field_); - cloned_field.CopyFrom(field); + CopyRepeatedField(GetReflection(), *message_, cloned_message, field_); return ParsedRepeatedFieldValue(cloned_message, field_, arena); } @@ -190,7 +246,7 @@ absl::Status ParsedRepeatedFieldValue::Get( index >= std::numeric_limits::max() || static_cast(index) >= GetReflection()->FieldSize(*message_, field_))) { - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); return absl::OkStatus(); } if (arena_ == nullptr) { diff --git a/common/values/parsed_repeated_field_value_test.cc b/common/values/parsed_repeated_field_value_test.cc index 3155e7159..bb50bee55 100644 --- a/common/values/parsed_repeated_field_value_test.cc +++ b/common/values/parsed_repeated_field_value_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include @@ -22,14 +23,15 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" -#include "absl/types/optional.h" #include "common/memory.h" #include "common/type.h" #include "common/value.h" #include "common/value_kind.h" #include "common/value_testing.h" +#include "internal/parse_text_proto.h" #include "internal/testing.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" +#include "google/protobuf/arena.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" namespace cel { @@ -45,6 +47,7 @@ using ::cel::test::DurationValueIs; using ::cel::test::ErrorValueIs; using ::cel::test::IntValueIs; using ::cel::test::IsNullValue; +using ::cel::test::StringValueIs; using ::cel::test::UintValueIs; using ::testing::_; using ::testing::ElementsAre; @@ -396,7 +399,7 @@ TEST_F(ParsedRepeatedFieldValueTest, NewIterator1) { EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(true)))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedRepeatedFieldValueTest, NewIterator2) { @@ -410,7 +413,7 @@ TEST_F(ParsedRepeatedFieldValueTest, NewIterator2) { EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(Pair(IntValueIs(1), BoolValueIs(true))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedRepeatedFieldValueTest, Contains) { @@ -435,10 +438,10 @@ TEST_F(ParsedRepeatedFieldValueTest, Contains) { EXPECT_THAT(value.Contains(DoubleValue(1.0), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(value.Contains(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(value.Contains(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(value.Contains(StringValue("foo"), descriptor_pool(), + EXPECT_THAT(value.Contains(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); EXPECT_THAT( @@ -446,5 +449,74 @@ TEST_F(ParsedRepeatedFieldValueTest, Contains) { IsOkAndHolds(BoolValueIs(false))); } +TEST_F(ParsedRepeatedFieldValueTest, CloneDefault) { + ParsedRepeatedFieldValue value; + EXPECT_FALSE(value.Clone(arena())); +} + +TEST_F(ParsedRepeatedFieldValueTest, CloneSameArena) { + ParsedRepeatedFieldValue value( + DynamicParseTextProto(R"pb(repeated_int64: 1 + repeated_int64: 2)pb"), + DynamicGetField("repeated_int64"), arena()); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); +} + +TEST_F(ParsedRepeatedFieldValueTest, CloneDifferentArenaInt64) { + google::protobuf::Arena other_arena; + ParsedRepeatedFieldValue value( + ::cel::internal::DynamicParseTextProto( + &other_arena, R"pb(repeated_int64: 1 repeated_int64: 2)pb", + descriptor_pool(), message_factory()), + DynamicGetField("repeated_int64"), &other_arena); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + EXPECT_EQ(cloned.Size(), 2); + EXPECT_THAT(cloned.Get(0, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(IntValueIs(1))); + EXPECT_THAT(cloned.Get(1, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(IntValueIs(2))); +} + +TEST_F(ParsedRepeatedFieldValueTest, CloneDifferentArenaString) { + google::protobuf::Arena other_arena; + ParsedRepeatedFieldValue value( + ::cel::internal::DynamicParseTextProto( + &other_arena, R"pb(repeated_string: "foo" repeated_string: "bar")pb", + descriptor_pool(), message_factory()), + DynamicGetField("repeated_string"), &other_arena); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + EXPECT_EQ(cloned.Size(), 2); + EXPECT_THAT(cloned.Get(0, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(StringValueIs("foo"))); + EXPECT_THAT(cloned.Get(1, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(StringValueIs("bar"))); +} + +TEST_F(ParsedRepeatedFieldValueTest, CloneDifferentArenaMessage) { + google::protobuf::Arena other_arena; + ParsedRepeatedFieldValue value( + ::cel::internal::DynamicParseTextProto( + &other_arena, + R"pb(repeated_nested_message: { bb: 1 } + repeated_nested_message: { bb: 2 })pb", + descriptor_pool(), message_factory()), + DynamicGetField("repeated_nested_message"), + &other_arena); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + EXPECT_EQ(cloned.Size(), 2); +} + } // namespace } // namespace cel diff --git a/common/values/string_value.cc b/common/values/string_value.cc index 98912d32c..b9495fa69 100644 --- a/common/values/string_value.cc +++ b/common/values/string_value.cc @@ -35,7 +35,6 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "common/internal/byte_string.h" -#include "common/internal/reference_count.h" #include "common/value.h" #include "internal/status_macros.h" #include "internal/strings.h" @@ -139,68 +138,11 @@ absl::Status StringValue::Equal( } size_t StringValue::Size() const { - return NativeValue([](const auto& alternative) -> size_t { + return value_.Visit([](const auto& alternative) -> size_t { return internal::Utf8CodePointCount(alternative); }); } -bool StringValue::IsEmpty() const { - return NativeValue( - [](const auto& alternative) -> bool { return alternative.empty(); }); -} - -bool StringValue::Equals(absl::string_view string) const { - return value_.Equals(string); -} - -bool StringValue::Equals(const absl::Cord& string) const { - return value_.Equals(string); -} - -bool StringValue::Equals(const StringValue& string) const { - return value_.Equals(string.value_); -} - -StringValue StringValue::Clone(google::protobuf::Arena* absl_nonnull arena) const { - return StringValue(value_.Clone(arena)); -} - -int StringValue::Compare(absl::string_view string) const { - return value_.Compare(string); -} - -int StringValue::Compare(const absl::Cord& string) const { - return value_.Compare(string); -} - -int StringValue::Compare(const StringValue& string) const { - return value_.Compare(string.value_); -} - -bool StringValue::StartsWith(absl::string_view string) const { - return value_.StartsWith(string); -} - -bool StringValue::StartsWith(const absl::Cord& string) const { - return value_.StartsWith(string); -} - -bool StringValue::StartsWith(const StringValue& string) const { - return value_.StartsWith(string.value_); -} - -bool StringValue::EndsWith(absl::string_view string) const { - return value_.EndsWith(string); -} - -bool StringValue::EndsWith(const absl::Cord& string) const { - return value_.EndsWith(string); -} - -bool StringValue::EndsWith(const StringValue& string) const { - return value_.EndsWith(string.value_); -} - bool StringValue::Contains(absl::string_view string) const { return value_.Visit(absl::Overload( [&](absl::string_view lhs) -> bool { @@ -246,7 +188,7 @@ absl::optional StringValue::IndexOf(absl::string_view string) const { lhs.remove_prefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; }, [&](absl::Cord lhs) -> absl::optional { int64_t code_points = 0; @@ -262,7 +204,7 @@ absl::optional StringValue::IndexOf(absl::string_view string) const { lhs.RemovePrefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; })); } @@ -282,7 +224,7 @@ absl::optional StringValue::IndexOf(const absl::Cord& string) const { lhs.remove_prefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; }, [&](absl::Cord lhs) -> absl::optional { int64_t code_points = 0; @@ -298,7 +240,7 @@ absl::optional StringValue::IndexOf(const absl::Cord& string) const { lhs.RemovePrefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; })); } @@ -332,7 +274,7 @@ absl::optional StringValue::IndexOf(absl::string_view string, lhs.remove_prefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; }, [&](absl::Cord lhs) -> absl::optional { int64_t code_points = 0; @@ -348,7 +290,7 @@ absl::optional StringValue::IndexOf(absl::string_view string, lhs.RemovePrefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; })); } @@ -372,7 +314,7 @@ absl::optional StringValue::IndexOf(const absl::Cord& string, lhs.remove_prefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; }, [&](absl::Cord lhs) -> absl::optional { int64_t code_points = 0; @@ -388,7 +330,7 @@ absl::optional StringValue::IndexOf(const absl::Cord& string, lhs.RemovePrefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; })); } @@ -421,7 +363,7 @@ absl::optional StringValue::LastIndexOf( lhs.remove_prefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; }, [&](absl::Cord lhs) -> absl::optional { @@ -439,7 +381,7 @@ absl::optional StringValue::LastIndexOf( lhs.RemovePrefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; })); } @@ -462,7 +404,7 @@ absl::optional StringValue::LastIndexOf( lhs.remove_prefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; }, [&](absl::Cord lhs) -> absl::optional { @@ -480,7 +422,7 @@ absl::optional StringValue::LastIndexOf( lhs.RemovePrefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; })); } @@ -499,7 +441,7 @@ absl::optional StringValue::LastIndexOf( absl::optional StringValue::LastIndexOf(absl::string_view string, int64_t pos) const { if (pos < 0) { - return absl::nullopt; + return std::nullopt; } return value_.Visit(absl::Overload( [&](absl::string_view lhs) -> absl::optional { @@ -517,7 +459,7 @@ absl::optional StringValue::LastIndexOf(absl::string_view string, lhs.remove_prefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; }, [&](absl::Cord lhs) -> absl::optional { @@ -535,7 +477,7 @@ absl::optional StringValue::LastIndexOf(absl::string_view string, lhs.RemovePrefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; })); } @@ -543,7 +485,7 @@ absl::optional StringValue::LastIndexOf(absl::string_view string, absl::optional StringValue::LastIndexOf(const absl::Cord& string, int64_t pos) const { if (pos < 0) { - return absl::nullopt; + return std::nullopt; } return value_.Visit(absl::Overload( [&](absl::string_view lhs) -> absl::optional { @@ -561,7 +503,7 @@ absl::optional StringValue::LastIndexOf(const absl::Cord& string, lhs.remove_prefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; }, [&](absl::Cord lhs) -> absl::optional { @@ -579,7 +521,7 @@ absl::optional StringValue::LastIndexOf(const absl::Cord& string, lhs.RemovePrefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; })); } @@ -618,8 +560,7 @@ absl::StatusOr SubstringImpl(absl::string_view string, uint64_t start) { ".substring(): is greater than .size()"); } -absl::StatusOr SubstringImpl(const absl::Cord& cord, - uint64_t start) { +absl::StatusOr SubstringImpl(const absl::Cord& cord, uint64_t start) { absl::Cord::CharIterator char_begin = cord.char_begin(); absl::Cord::CharIterator char_end = cord.char_end(); size_t size_code_points = 0; @@ -629,14 +570,14 @@ absl::StatusOr SubstringImpl(const absl::Cord& cord, size_t code_units; std::tie(code_point, code_units) = cel::internal::Utf8Decode(char_begin); if (size_code_points == start) { - return cord.Subcord(size_code_units, std::numeric_limits::max()); + return size_code_units; } absl::Cord::Advance(&char_begin, code_units); ++size_code_points; size_code_units += code_units; } if (size_code_points == start) { - return cord; + return size_code_units; } return absl::InvalidArgumentError( ".substring(): is greater than .size()"); @@ -644,15 +585,20 @@ absl::StatusOr SubstringImpl(const absl::Cord& cord, } // namespace -Value StringValue::Substring(int64_t start) const { +Value StringValue::Substring(int64_t start, + google::protobuf::Arena* absl_nonnull arena) const { if (start < 0) { - return ErrorValue(absl::InvalidArgumentError( - ".substring(): is less than 0")); + return ErrorValue::From( + absl::InvalidArgumentError( + ".substring(): is less than 0"), + arena); } if (static_cast(start) > value_.size()) { - return ErrorValue(absl::InvalidArgumentError( - ".substring(, ): or is greater than " - ".size()")); + return ErrorValue::From( + absl::InvalidArgumentError(".substring(, ): " + " or is greater than " + ".size()"), + arena); } if (start == 0) { return *this; @@ -662,7 +608,7 @@ Value StringValue::Substring(int64_t start) const { absl::StatusOr status_or_index = (SubstringImpl)(value_.GetSmall(), start); if (!status_or_index.ok()) { - return ErrorValue(std::move(status_or_index).status()); + return ErrorValue::From(std::move(status_or_index).status(), arena); } StringValue result; result.value_.rep_.header.kind = common_internal::ByteStringKind::kSmall; @@ -670,14 +616,13 @@ Value StringValue::Substring(int64_t start) const { std::memcpy(result.value_.rep_.small.data, value_.rep_.small.data + *status_or_index, result.value_.rep_.small.size); - result.value_.rep_.small.arena = value_.rep_.small.arena; return result; } case common_internal::ByteStringKind::kMedium: { absl::StatusOr status_or_index = (SubstringImpl)(value_.GetMedium(), start); if (!status_or_index.ok()) { - return ErrorValue(std::move(status_or_index).status()); + return ErrorValue::From(std::move(status_or_index).status(), arena); } StringValue result; result.value_.rep_.header.kind = common_internal::ByteStringKind::kMedium; @@ -685,17 +630,18 @@ Value StringValue::Substring(int64_t start) const { value_.rep_.medium.size - *status_or_index; result.value_.rep_.medium.data = value_.rep_.medium.data + *status_or_index; - result.value_.rep_.medium.owner = value_.rep_.medium.owner; - common_internal::StrongRef(result.value_.GetMediumReferenceCount()); + result.value_.rep_.medium.arena = value_.rep_.medium.arena; return result; } case common_internal::ByteStringKind::kLarge: { - absl::StatusOr status_or_cord = + absl::StatusOr status_or_index = (SubstringImpl)(value_.GetLarge(), start); - if (!status_or_cord.ok()) { - return ErrorValue(std::move(status_or_cord).status()); + if (!status_or_index.ok()) { + return ErrorValue::From(std::move(status_or_index).status(), arena); } - return StringValue::Wrap(*std::move(status_or_cord)); + return StringValue(common_internal::ByteString::Wrap( + value_.rep_.large.data, value_.rep_.large.offset + *status_or_index, + value_.rep_.large.size - *status_or_index, value_.rep_.large.arena)); } } } @@ -729,8 +675,9 @@ absl::StatusOr> SubstringImpl( ".size()"); } -absl::StatusOr SubstringImpl(const absl::Cord& cord, uint64_t start, - uint64_t end) { +absl::StatusOr> SubstringImpl(const absl::Cord& cord, + uint64_t start, + uint64_t end) { absl::Cord::CharIterator char_begin = cord.char_begin(); absl::Cord::CharIterator char_end = cord.char_end(); size_t size_code_points = 0; @@ -741,8 +688,7 @@ absl::StatusOr SubstringImpl(const absl::Cord& cord, uint64_t start, start_code_units = size_code_units; } if (size_code_points == end) { - return cord.Subcord(start_code_units, - size_code_points - start_code_units); + return std::pair{start_code_units, size_code_units}; } char32_t code_point; size_t code_units; @@ -752,7 +698,7 @@ absl::StatusOr SubstringImpl(const absl::Cord& cord, uint64_t start, size_code_units += code_units; } if (size_code_points == start && start == end) { - return absl::Cord(); + return std::pair{size_code_units, size_code_units}; } return absl::InvalidArgumentError( ".substring(, ): or is greater than " @@ -761,27 +707,34 @@ absl::StatusOr SubstringImpl(const absl::Cord& cord, uint64_t start, } // namespace -Value StringValue::Substring(int64_t start, int64_t end) const { +Value StringValue::Substring(int64_t start, int64_t end, + google::protobuf::Arena* absl_nonnull arena) const { if (start < 0) { - return ErrorValue(absl::InvalidArgumentError( - ".substring(, ): is less than 0")); + return ErrorValue::From( + absl::InvalidArgumentError( + ".substring(, ): is less than 0"), + arena); } if (end < start) { - return ErrorValue(absl::InvalidArgumentError( - ".substring(, ): is less than ")); + return ErrorValue::From( + absl::InvalidArgumentError( + ".substring(, ): is less than "), + arena); } if (static_cast(start) > value_.size() || static_cast(end) > value_.size()) { - return ErrorValue(absl::InvalidArgumentError( - ".substring(, ): or is greater than " - ".size()")); + return ErrorValue::From( + absl::InvalidArgumentError(".substring(, ): " + " or is greater than " + ".size()"), + arena); } switch (value_.GetKind()) { case common_internal::ByteStringKind::kSmall: { absl::StatusOr> status_or_indices = (SubstringImpl)(value_.GetSmall(), start, end); if (!status_or_indices.ok()) { - return ErrorValue(std::move(status_or_indices).status()); + return ErrorValue::From(std::move(status_or_indices).status(), arena); } StringValue result; result.value_.rep_.header.kind = common_internal::ByteStringKind::kSmall; @@ -790,14 +743,13 @@ Value StringValue::Substring(int64_t start, int64_t end) const { std::memcpy(result.value_.rep_.small.data, value_.rep_.small.data + status_or_indices->first, result.value_.rep_.small.size); - result.value_.rep_.small.arena = value_.rep_.small.arena; return result; } case common_internal::ByteStringKind::kMedium: { absl::StatusOr> status_or_indices = (SubstringImpl)(value_.GetMedium(), start, end); if (!status_or_indices.ok()) { - return ErrorValue(std::move(status_or_indices).status()); + return ErrorValue::From(std::move(status_or_indices).status(), arena); } StringValue result; result.value_.rep_.header.kind = common_internal::ByteStringKind::kMedium; @@ -805,17 +757,20 @@ Value StringValue::Substring(int64_t start, int64_t end) const { (status_or_indices->second - status_or_indices->first); result.value_.rep_.medium.data = value_.rep_.medium.data + status_or_indices->first; - result.value_.rep_.medium.owner = value_.rep_.medium.owner; - common_internal::StrongRef(result.value_.GetMediumReferenceCount()); + result.value_.rep_.medium.arena = value_.rep_.medium.arena; return result; } case common_internal::ByteStringKind::kLarge: { - absl::StatusOr status_or_cord = + absl::StatusOr> status_or_indices = (SubstringImpl)(value_.GetLarge(), start, end); - if (!status_or_cord.ok()) { - return ErrorValue(std::move(status_or_cord).status()); + if (!status_or_indices.ok()) { + return ErrorValue::From(std::move(status_or_indices).status(), arena); } - return StringValue::Wrap(*std::move(status_or_cord)); + return StringValue(common_internal::ByteString::Wrap( + value_.rep_.large.data, + value_.rep_.large.offset + status_or_indices->first, + status_or_indices->second - status_or_indices->first, + value_.rep_.large.arena)); } } } @@ -842,9 +797,9 @@ bool LowerAsciiImpl(absl::string_view in, std::string* absl_nonnull out) { return true; } -absl::Cord LowerAsciiImpl(const absl::Cord& in) { +bool LowerAsciiImpl(const absl::Cord& in, absl::Cord* absl_nonnull out) { if (in.empty()) { - return in; + return false; } size_t pos = 0; bool needs_conversion = false; @@ -856,9 +811,9 @@ absl::Cord LowerAsciiImpl(const absl::Cord& in) { pos++; } if (!needs_conversion) { - return in; + return false; } - absl::Cord out = in.Subcord(0, pos); + absl::Cord prefix = in.Subcord(0, pos); absl::Cord rest = in.Subcord(pos, in.size() - pos); std::string suffix; suffix.resize(rest.size()); @@ -866,8 +821,9 @@ absl::Cord LowerAsciiImpl(const absl::Cord& in) { for (char c : rest.Chars()) { suffix[current++] = absl::ascii_tolower(c); } - out.Append(std::move(suffix)); - return out; + prefix.Append(std::move(suffix)); + *out = std::move(prefix); + return true; } } // namespace @@ -890,8 +846,13 @@ StringValue StringValue::LowerAscii(google::protobuf::Arena* absl_nonnull arena) } return StringValue::From(std::move(out), arena); } - case common_internal::ByteStringKind::kLarge: - return StringValue::Wrap((LowerAsciiImpl)(value_.GetLarge())); + case common_internal::ByteStringKind::kLarge: { + absl::Cord out; + if (!(LowerAsciiImpl)(value_.GetLarge(), &out)) { + return *this; + } + return StringValue::From(std::move(out), arena); + } } } @@ -917,9 +878,9 @@ bool UpperAsciiImpl(absl::string_view in, std::string* absl_nonnull out) { return true; } -absl::Cord UpperAsciiImpl(const absl::Cord& in) { +bool UpperAsciiImpl(const absl::Cord& in, absl::Cord* absl_nonnull out) { if (in.empty()) { - return in; + return false; } size_t pos = 0; bool needs_conversion = false; @@ -931,9 +892,9 @@ absl::Cord UpperAsciiImpl(const absl::Cord& in) { pos++; } if (!needs_conversion) { - return in; + return false; } - absl::Cord out = in.Subcord(0, pos); + absl::Cord prefix = in.Subcord(0, pos); absl::Cord rest = in.Subcord(pos, in.size() - pos); std::string suffix; suffix.resize(rest.size()); @@ -941,15 +902,15 @@ absl::Cord UpperAsciiImpl(const absl::Cord& in) { for (char c : rest.Chars()) { suffix[current++] = absl::ascii_toupper(c); } - out.Append(std::move(suffix)); - return out; + prefix.Append(std::move(suffix)); + *out = std::move(prefix); + return true; } } // namespace StringValue StringValue::UpperAscii(google::protobuf::Arena* absl_nonnull arena) const { ABSL_DCHECK(arena != nullptr); - switch (value_.GetKind()) { case common_internal::ByteStringKind::kSmall: { std::string out; @@ -965,8 +926,13 @@ StringValue StringValue::UpperAscii(google::protobuf::Arena* absl_nonnull arena) } return StringValue::From(std::move(out), arena); } - case common_internal::ByteStringKind::kLarge: - return StringValue::Wrap((UpperAsciiImpl)(value_.GetLarge())); + case common_internal::ByteStringKind::kLarge: { + absl::Cord out; + if (!(UpperAsciiImpl)(value_.GetLarge(), &out)) { + return *this; + } + return StringValue::From(std::move(out), arena); + } } } @@ -1017,7 +983,7 @@ std::pair TrimImpl(absl::string_view string) { return {left_trim_bytes, string.size() - last_non_ws_end_bytes}; } -absl::Cord TrimImpl(const absl::Cord& cord) { +std::pair TrimImpl(const absl::Cord& cord) { size_t left_trim_bytes = 0; { absl::Cord::CharIterator begin = cord.char_begin(); @@ -1035,7 +1001,7 @@ absl::Cord TrimImpl(const absl::Cord& cord) { } if (left_trim_bytes == cord.size()) { - return absl::Cord(); + return {left_trim_bytes, 0}; } absl::Cord ltrimmed = @@ -1057,40 +1023,29 @@ absl::Cord TrimImpl(const absl::Cord& cord) { current_pos_bytes += char_len; } } - return ltrimmed.Subcord(0, last_non_ws_end_bytes); + return {left_trim_bytes, ltrimmed.size() - last_non_ws_end_bytes}; } } // namespace StringValue StringValue::Trim() const { + std::pair trims; + size_t size; switch (value_.GetKind()) { - case common_internal::ByteStringKind::kSmall: { - std::pair trims = (TrimImpl)(value_.GetSmall()); - StringValue result; - result.value_.rep_.header.kind = common_internal::ByteStringKind::kSmall; - result.value_.rep_.small.size = - value_.rep_.small.size - trims.first - trims.second; - std::memcpy(result.value_.rep_.small.data, - value_.rep_.small.data + trims.first, - result.value_.rep_.small.size); - result.value_.rep_.small.arena = value_.GetSmallArena(); - return result; - } - case common_internal::ByteStringKind::kMedium: { - std::pair trims = (TrimImpl)(value_.GetMedium()); - StringValue result; - result.value_.rep_.header.kind = common_internal::ByteStringKind::kMedium; - result.value_.rep_.medium.size = - value_.rep_.medium.size - trims.first - trims.second; - result.value_.rep_.medium.data = value_.rep_.medium.data + trims.first; - result.value_.rep_.medium.owner = value_.rep_.medium.owner; - common_internal::StrongRef(result.value_.GetMediumReferenceCount()); - return result; - } - case common_internal::ByteStringKind::kLarge: { - return StringValue::Wrap((TrimImpl)(value_.GetLarge())); - } + case common_internal::ByteStringKind::kSmall: + trims = (TrimImpl)(value_.GetSmall()); + size = value_.GetSmall().size(); + break; + case common_internal::ByteStringKind::kMedium: + trims = (TrimImpl)(value_.GetMedium()); + size = value_.GetMedium().size(); + break; + case common_internal::ByteStringKind::kLarge: + trims = (TrimImpl)(value_.GetLarge()); + size = value_.rep_.large.size; + break; } + return StringValue(value_.Substring(trims.first, size - trims.second)); } namespace { @@ -1241,8 +1196,8 @@ absl::Status StringValue::Join( string_element->AppendToString(&joined); } else { ABSL_DCHECK(!element->Is()); - *result = - ErrorValue(runtime_internal::CreateNoMatchingOverloadError("join")); + *result = ErrorValue::From( + runtime_internal::CreateNoMatchingOverloadError("join"), arena); return absl::OkStatus(); } while (true) { @@ -1256,8 +1211,8 @@ absl::Status StringValue::Join( string_element->AppendToString(&joined); } else { ABSL_DCHECK(!element->Is()); - *result = - ErrorValue(runtime_internal::CreateNoMatchingOverloadError("join")); + *result = ErrorValue::From( + runtime_internal::CreateNoMatchingOverloadError("join"), arena); return absl::OkStatus(); } } @@ -1454,13 +1409,15 @@ absl::Status StringValue::Replace(const StringValue& needle, return absl::OkStatus(); } -Value StringValue::CharAt(int64_t pos) const { +Value StringValue::CharAt(int64_t pos, + google::protobuf::Arena* absl_nonnull arena) const { if (pos < 0) { - return ErrorValue(absl::InvalidArgumentError( - ".charAt(): is less than 0")); + return ErrorValue::From(absl::InvalidArgumentError( + ".charAt(): is less than 0"), + arena); } return value_.Visit(absl::Overload( - [this, pos](absl::string_view rep) mutable -> Value { + [this, pos, arena](absl::string_view rep) mutable -> Value { while (!rep.empty()) { char32_t code_point; size_t code_units; @@ -1471,7 +1428,6 @@ Value StringValue::CharAt(int64_t pos) const { common_internal::ByteStringKind::kSmall; result.value_.rep_.small.size = cel::internal::Utf8Encode( code_point, result.value_.rep_.small.data); - result.value_.rep_.small.arena = value_.GetArena(); return result; } rep.remove_prefix(code_units); @@ -1483,10 +1439,12 @@ Value StringValue::CharAt(int64_t pos) const { if (pos == 0) { return StringValue(); } - return ErrorValue(absl::InvalidArgumentError( - ".charAt(): is greater than .size()")); + return ErrorValue::From( + absl::InvalidArgumentError(".charAt(): is " + "greater than .size()"), + arena); }, - [pos](const absl::Cord& rep) mutable -> Value { + [pos, arena](const absl::Cord& rep) mutable -> Value { absl::Cord::CharIterator begin = rep.char_begin(); absl::Cord::CharIterator end = rep.char_end(); while (begin != end) { @@ -1499,7 +1457,6 @@ Value StringValue::CharAt(int64_t pos) const { common_internal::ByteStringKind::kSmall; result.value_.rep_.small.size = cel::internal::Utf8Encode( code_point, result.value_.rep_.small.data); - result.value_.rep_.small.arena = nullptr; return result; } absl::Cord::Advance(&begin, code_units); @@ -1511,8 +1468,10 @@ Value StringValue::CharAt(int64_t pos) const { if (pos == 0) { return StringValue(); } - return ErrorValue(absl::InvalidArgumentError( - ".charAt(): is greater than .size()")); + return ErrorValue::From( + absl::InvalidArgumentError(".charAt(): is " + "greater than .size()"), + arena); })); } diff --git a/common/values/string_value.h b/common/values/string_value.h index 8045e4b3f..0959cc556 100644 --- a/common/values/string_value.h +++ b/common/values/string_value.h @@ -33,10 +33,7 @@ #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/allocator.h" -#include "common/arena.h" #include "common/internal/byte_string.h" -#include "common/memory.h" #include "common/type.h" #include "common/value_kind.h" #include "common/values/values.h" @@ -45,6 +42,16 @@ #include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/message.h" +#if ABSL_HAVE_ATTRIBUTE(enable_if) +#define CEL_COMMON_INTERNAL_REQUIRE_CONSTEXPR_STRING_VIEW(array) \ + __attribute__(( \ + enable_if(::cel::common_internal::IsWellFormedStringValueLiteral(array), \ + "Argument '" #array "' must be a 'constexpr char[N]' or a " \ + "'constexpr absl::string_view'."))) +#else +#define CEL_COMMON_INTERNAL_REQUIRE_CONSTEXPR_STRING_VIEW(array) +#endif + namespace cel { class Value; @@ -54,6 +61,14 @@ class StringValue; namespace common_internal { absl::string_view LegacyStringValue(const StringValue& value, bool stable, google::protobuf::Arena* absl_nonnull arena); +constexpr bool IsWellFormedStringValueLiteral(absl::string_view string) { + for (char c : string) { + if (c == '\0') { + return false; + } + } + return true; +} } // namespace common_internal // `StringValue` represents values of the primitive `string` type. @@ -63,75 +78,78 @@ class StringValue final : private common_internal::ValueMixin { static StringValue From(const char* absl_nullable value, google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return StringValue(common_internal::ByteString::From(value, arena)); + } static StringValue From(absl::string_view value, google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); - static StringValue From(const absl::Cord& value); + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return StringValue(common_internal::ByteString::From(value, arena)); + } + static StringValue From(const absl::Cord& value, + google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return StringValue(common_internal::ByteString::From(value, arena)); + } static StringValue From(std::string&& value, google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return StringValue( + common_internal::ByteString::From(std::move(value), arena)); + } static StringValue Wrap(absl::string_view value, google::protobuf::Arena* absl_nullable arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); - static StringValue Wrap(absl::string_view value) = delete; - static StringValue Wrap(const absl::Cord& value); - static StringValue Wrap(std::string&& value) = delete; - static StringValue Wrap(std::string&& value, - google::protobuf::Arena* absl_nullable arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) = delete; + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return StringValue(common_internal::ByteString::Wrap(value, arena)); + } + static StringValue Wrap( + const absl::Cord* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND, + google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return StringValue(common_internal::ByteString::Wrap(value, arena)); + } + static StringValue Wrap(std::nullptr_t, google::protobuf::Arena*) = delete; + static StringValue Wrap(std::string&& value, google::protobuf::Arena*) = delete; // Returns a StringValue that aliases the provided string. Caller must ensure // the provided string outlives the use of the returned StringValue. - static StringValue WrapUnsafe(absl::string_view value); + static StringValue WrapUnsafe(absl::string_view value) { + return StringValue(common_internal::ByteString::WrapUnsafe(value)); + } + static StringValue WrapUnsafe(const absl::Cord* absl_nonnull value) { + return StringValue(common_internal::ByteString::WrapUnsafe(value)); + } + static StringValue WrapUnsafe(std::nullptr_t) = delete; + + // Returns a StringValue that aliases the provided string, which must be a + // compile time constant. This is enforced during compilation when using + // Clang. + template + static StringValue Literal(const char (&value)[N]) + CEL_COMMON_INTERNAL_REQUIRE_CONSTEXPR_STRING_VIEW(value) { + static_assert(N > 0); + ABSL_DCHECK_EQ(value[N - 1], '\0'); + return WrapUnsafe(value); + } + static StringValue Literal(absl::string_view value) + CEL_COMMON_INTERNAL_REQUIRE_CONSTEXPR_STRING_VIEW(value) { + return WrapUnsafe(value); + } + static StringValue Literal(const std::string&) = delete; + static StringValue Literal(std::string&&) = delete; static StringValue Concat(const StringValue& lhs, const StringValue& rhs, google::protobuf::Arena* absl_nonnull arena ABSL_ATTRIBUTE_LIFETIME_BOUND); - ABSL_DEPRECATED("Use From") - explicit StringValue(const char* absl_nullable value) : value_(value) {} - - ABSL_DEPRECATED("Use From") - explicit StringValue(absl::string_view value) : value_(value) {} - - ABSL_DEPRECATED("Use From") - explicit StringValue(const absl::Cord& value) : value_(value) {} - - ABSL_DEPRECATED("Use From") - explicit StringValue(std::string&& value) : value_(std::move(value)) {} - - ABSL_DEPRECATED("Use From") - StringValue(Allocator<> allocator, const char* absl_nullable value) - : value_(allocator, value) {} - - ABSL_DEPRECATED("Use From") - StringValue(Allocator<> allocator, absl::string_view value) - : value_(allocator, value) {} - - ABSL_DEPRECATED("Use From") - StringValue(Allocator<> allocator, const absl::Cord& value) - : value_(allocator, value) {} - - ABSL_DEPRECATED("Use From") - StringValue(Allocator<> allocator, std::string&& value) - : value_(allocator, std::move(value)) {} - - ABSL_DEPRECATED("Use Wrap") - StringValue(Borrower borrower, absl::string_view value) - : value_(borrower, value) {} - - ABSL_DEPRECATED("Use Wrap") - StringValue(Borrower borrower, const absl::Cord& value) - : value_(borrower, value) {} - StringValue() = default; StringValue(const StringValue&) = default; StringValue(StringValue&&) = default; StringValue& operator=(const StringValue&) = default; StringValue& operator=(StringValue&&) = default; + explicit StringValue(const BytesValue& other); + constexpr ValueKind kind() const { return kKind; } absl::string_view GetTypeName() const { return StringType::kName; } @@ -157,12 +175,12 @@ class StringValue final : private common_internal::ValueMixin { Value* absl_nonnull result) const; using ValueMixin::Equal; - StringValue Clone(google::protobuf::Arena* absl_nonnull arena) const; - - bool IsZeroValue() const { - return NativeValue([](const auto& value) -> bool { return value.empty(); }); + StringValue Clone(google::protobuf::Arena* absl_nonnull arena) const { + return StringValue(value_.Clone(arena)); } + bool IsZeroValue() const { return value_.empty(); } + ABSL_DEPRECATED("Use ToString()") std::string NativeString() const { return value_.ToString(); } @@ -186,30 +204,46 @@ class StringValue final : private common_internal::ValueMixin { return value_.Visit(std::forward(visitor)); } - void swap(StringValue& other) noexcept { + friend void swap(StringValue& lhs, StringValue& rhs) noexcept { using std::swap; - swap(value_, other.value_); + swap(lhs.value_, rhs.value_); } size_t Size() const; - bool IsEmpty() const; + bool IsEmpty() const { return value_.empty(); } - bool Equals(absl::string_view string) const; - bool Equals(const absl::Cord& string) const; - bool Equals(const StringValue& string) const; + bool Equals(absl::string_view string) const { return value_.Equals(string); } + bool Equals(const absl::Cord& string) const { return value_.Equals(string); } + bool Equals(const StringValue& string) const { + return value_.Equals(string.value_); + } - int Compare(absl::string_view string) const; - int Compare(const absl::Cord& string) const; - int Compare(const StringValue& string) const; + int Compare(absl::string_view string) const { return value_.Compare(string); } + int Compare(const absl::Cord& string) const { return value_.Compare(string); } + int Compare(const StringValue& string) const { + return value_.Compare(string.value_); + } - bool StartsWith(absl::string_view string) const; - bool StartsWith(const absl::Cord& string) const; - bool StartsWith(const StringValue& string) const; + bool StartsWith(absl::string_view string) const { + return value_.StartsWith(string); + } + bool StartsWith(const absl::Cord& string) const { + return value_.StartsWith(string); + } + bool StartsWith(const StringValue& string) const { + return value_.StartsWith(string.value_); + } - bool EndsWith(absl::string_view string) const; - bool EndsWith(const absl::Cord& string) const; - bool EndsWith(const StringValue& string) const; + bool EndsWith(absl::string_view string) const { + return value_.EndsWith(string); + } + bool EndsWith(const absl::Cord& string) const { + return value_.EndsWith(string); + } + bool EndsWith(const StringValue& string) const { + return value_.EndsWith(string.value_); + } bool Contains(absl::string_view string) const; bool Contains(const absl::Cord& string) const; @@ -240,9 +274,10 @@ class StringValue final : private common_internal::ValueMixin { absl::optional LastIndexOf(const StringValue& string, int64_t pos) const; - Value Substring(int64_t start) const; + Value Substring(int64_t start, google::protobuf::Arena* absl_nonnull arena) const; - Value Substring(int64_t start, int64_t end) const; + Value Substring(int64_t start, int64_t end, + google::protobuf::Arena* absl_nonnull arena) const; // Returns a new `StringValue` with all lowercase ASCII characters // converted to lowercase. @@ -307,7 +342,7 @@ class StringValue final : private common_internal::ValueMixin { // Returns the character at `pos` as a new `StringValue`. `pos` is a // 0-based index based on Unicode code points. Returns `ErrorValue` if `pos` // is out of range. - Value CharAt(int64_t pos) const; + Value CharAt(int64_t pos, google::protobuf::Arena* absl_nonnull arena) const; absl::optional TryFlat() const ABSL_ATTRIBUTE_LIFETIME_BOUND { @@ -354,10 +389,10 @@ class StringValue final : private common_internal::ValueMixin { } private: + friend class BytesValue; friend class common_internal::ValueMixin; friend absl::string_view common_internal::LegacyStringValue( const StringValue& value, bool stable, google::protobuf::Arena* absl_nonnull arena); - friend struct ArenaTraits; explicit StringValue(common_internal::ByteString value) noexcept : value_(std::move(value)) {} @@ -365,8 +400,6 @@ class StringValue final : private common_internal::ValueMixin { common_internal::ByteString value_; }; -inline void swap(StringValue& lhs, StringValue& rhs) noexcept { lhs.swap(rhs); } - inline bool operator==(const StringValue& lhs, absl::string_view rhs) { return lhs.Equals(rhs); } @@ -423,48 +456,6 @@ inline std::ostream& operator<<(std::ostream& out, const StringValue& value) { return out << value.DebugString(); } -inline StringValue StringValue::From(const char* absl_nullable value, - google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - return From(absl::NullSafeStringView(value), arena); -} - -inline StringValue StringValue::From(absl::string_view value, - google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - ABSL_DCHECK(arena != nullptr); - - return StringValue(arena, value); -} - -inline StringValue StringValue::From(const absl::Cord& value) { - return StringValue(value); -} - -inline StringValue StringValue::From(std::string&& value, - google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - ABSL_DCHECK(arena != nullptr); - - return StringValue(arena, std::move(value)); -} - -inline StringValue StringValue::Wrap(absl::string_view value, - google::protobuf::Arena* absl_nullable arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - ABSL_DCHECK(arena != nullptr); - - return StringValue(Borrower::Arena(arena), value); -} - -inline StringValue StringValue::WrapUnsafe(absl::string_view value) { - return StringValue(common_internal::ByteString::FromExternal(value)); -} - -inline StringValue StringValue::Wrap(const absl::Cord& value) { - return StringValue(value); -} - namespace common_internal { inline absl::string_view LegacyStringValue(const StringValue& value, @@ -475,15 +466,6 @@ inline absl::string_view LegacyStringValue(const StringValue& value, } // namespace common_internal -template <> -struct ArenaTraits { - using constructible = std::true_type; - - static bool trivially_destructible(const StringValue& value) { - return ArenaTraits<>::trivially_destructible(value.value_); - } -}; - } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_VALUES_STRING_VALUE_H_ diff --git a/common/values/string_value_test.cc b/common/values/string_value_test.cc index 201724905..6bfcb83aa 100644 --- a/common/values/string_value_test.cc +++ b/common/values/string_value_test.cc @@ -22,7 +22,6 @@ #include "absl/strings/cord.h" #include "absl/strings/cord_test_helpers.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "common/native_type.h" #include "common/value.h" #include "common/value_testing.h" @@ -40,185 +39,212 @@ using ::testing::Optional; using StringValueTest = common_internal::ValueTest<>; TEST_F(StringValueTest, Kind) { - EXPECT_EQ(StringValue("foo").kind(), StringValue::kKind); - EXPECT_EQ(Value(StringValue(absl::Cord("foo"))).kind(), StringValue::kKind); + EXPECT_EQ(StringValue::WrapUnsafe("foo").kind(), StringValue::kKind); + EXPECT_EQ(Value(StringValue::From(absl::Cord("foo"), arena())).kind(), + StringValue::kKind); } TEST_F(StringValueTest, DebugString) { { std::ostringstream out; - out << StringValue("foo"); + out << StringValue::WrapUnsafe("foo"); EXPECT_EQ(out.str(), "\"foo\""); } { std::ostringstream out; - out << StringValue(absl::MakeFragmentedCord({"f", "o", "o"})); + out << StringValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), + arena()); EXPECT_EQ(out.str(), "\"foo\""); } { std::ostringstream out; - out << Value(StringValue(absl::Cord("foo"))); + out << Value(StringValue::From(absl::Cord("foo"), arena())); EXPECT_EQ(out.str(), "\"foo\""); } } TEST_F(StringValueTest, ConvertToJson) { auto* message = NewArenaValueMessage(); - EXPECT_THAT(StringValue("foo").ConvertToJson(descriptor_pool(), - message_factory(), message), + EXPECT_THAT(StringValue::WrapUnsafe("foo").ConvertToJson( + descriptor_pool(), message_factory(), message), IsOk()); EXPECT_THAT(*message, EqualsValueTextProto(R"pb(string_value: "foo")pb")); } TEST_F(StringValueTest, NativeValue) { std::string scratch; - EXPECT_EQ(StringValue("foo").NativeString(), "foo"); - EXPECT_EQ(StringValue("foo").NativeString(scratch), "foo"); - EXPECT_EQ(StringValue("foo").NativeCord(), "foo"); + EXPECT_EQ(StringValue::WrapUnsafe("foo").NativeString(), "foo"); + EXPECT_EQ(StringValue::WrapUnsafe("foo").NativeString(scratch), "foo"); + EXPECT_EQ(StringValue::WrapUnsafe("foo").NativeCord(), "foo"); } TEST_F(StringValueTest, TryFlat) { - EXPECT_THAT(StringValue("foo").TryFlat(), Optional(Eq("foo"))); + EXPECT_THAT(StringValue::WrapUnsafe("foo").TryFlat(), Optional(Eq("foo"))); EXPECT_THAT( - StringValue(absl::MakeFragmentedCord({"Hello, World!", "World, Hello!"})) + StringValue::From( + absl::MakeFragmentedCord({"Hello, World!", "World, Hello!"}), arena()) .TryFlat(), - Eq(absl::nullopt)); + Eq(std::nullopt)); } TEST_F(StringValueTest, ToString) { - EXPECT_EQ(StringValue("foo").ToString(), "foo"); - EXPECT_EQ(StringValue(absl::MakeFragmentedCord({"f", "o", "o"})).ToString(), - "foo"); + EXPECT_EQ(StringValue::WrapUnsafe("foo").ToString(), "foo"); + EXPECT_EQ( + StringValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .ToString(), + "foo"); } TEST_F(StringValueTest, CopyToString) { std::string out; - StringValue("foo").CopyToString(&out); + StringValue::WrapUnsafe("foo").CopyToString(&out); EXPECT_EQ(out, "foo"); - StringValue(absl::MakeFragmentedCord({"f", "o", "o"})).CopyToString(&out); + StringValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .CopyToString(&out); EXPECT_EQ(out, "foo"); } TEST_F(StringValueTest, AppendToString) { std::string out; - StringValue("foo").AppendToString(&out); + StringValue::WrapUnsafe("foo").AppendToString(&out); EXPECT_EQ(out, "foo"); - StringValue(absl::MakeFragmentedCord({"f", "o", "o"})).AppendToString(&out); + StringValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .AppendToString(&out); EXPECT_EQ(out, "foofoo"); } TEST_F(StringValueTest, ToCord) { - EXPECT_EQ(StringValue("foo").ToCord(), "foo"); - EXPECT_EQ(StringValue(absl::MakeFragmentedCord({"f", "o", "o"})).ToCord(), - "foo"); + EXPECT_EQ(StringValue::WrapUnsafe("foo").ToCord(), "foo"); + EXPECT_EQ( + StringValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .ToCord(), + "foo"); } TEST_F(StringValueTest, CopyToCord) { absl::Cord out; - StringValue("foo").CopyToCord(&out); + StringValue::WrapUnsafe("foo").CopyToCord(&out); EXPECT_EQ(out, "foo"); - StringValue(absl::MakeFragmentedCord({"f", "o", "o"})).CopyToCord(&out); + StringValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .CopyToCord(&out); EXPECT_EQ(out, "foo"); } TEST_F(StringValueTest, AppendToCord) { absl::Cord out; - StringValue("foo").AppendToCord(&out); + StringValue::WrapUnsafe("foo").AppendToCord(&out); EXPECT_EQ(out, "foo"); - StringValue(absl::MakeFragmentedCord({"f", "o", "o"})).AppendToCord(&out); + StringValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .AppendToCord(&out); EXPECT_EQ(out, "foofoo"); } TEST_F(StringValueTest, NativeTypeId) { - EXPECT_EQ(NativeTypeId::Of(StringValue("foo")), - NativeTypeId::For()); - EXPECT_EQ(NativeTypeId::Of(Value(StringValue(absl::Cord("foo")))), + EXPECT_EQ(NativeTypeId::Of(StringValue::WrapUnsafe("foo")), NativeTypeId::For()); + EXPECT_EQ( + NativeTypeId::Of(Value(StringValue::From(absl::Cord("foo"), arena()))), + NativeTypeId::For()); } TEST_F(StringValueTest, HashValue) { - EXPECT_EQ(absl::HashOf(StringValue("foo")), + EXPECT_EQ(absl::HashOf(StringValue::WrapUnsafe("foo")), absl::HashOf(absl::string_view("foo"))); - EXPECT_EQ(absl::HashOf(StringValue(absl::string_view("foo"))), + EXPECT_EQ(absl::HashOf(StringValue::WrapUnsafe(absl::string_view("foo"))), absl::HashOf(absl::string_view("foo"))); - EXPECT_EQ(absl::HashOf(StringValue(absl::Cord("foo"))), + EXPECT_EQ(absl::HashOf(StringValue::From(absl::Cord("foo"), arena())), absl::HashOf(absl::string_view("foo"))); } TEST_F(StringValueTest, Equality) { - EXPECT_NE(StringValue("foo"), "bar"); - EXPECT_NE("bar", StringValue("foo")); - EXPECT_NE(StringValue("foo"), StringValue("bar")); - EXPECT_NE(StringValue("foo"), absl::Cord("bar")); - EXPECT_NE(absl::Cord("bar"), StringValue("foo")); + EXPECT_NE(StringValue::WrapUnsafe("foo"), "bar"); + EXPECT_NE("bar", StringValue::WrapUnsafe("foo")); + EXPECT_NE(StringValue::WrapUnsafe("foo"), StringValue::WrapUnsafe("bar")); + EXPECT_NE(StringValue::WrapUnsafe("foo"), absl::Cord("bar")); + EXPECT_NE(absl::Cord("bar"), StringValue::WrapUnsafe("foo")); } TEST_F(StringValueTest, LessThan) { - EXPECT_LT(StringValue("bar"), "foo"); - EXPECT_LT("bar", StringValue("foo")); - EXPECT_LT(StringValue("bar"), StringValue("foo")); - EXPECT_LT(StringValue("bar"), absl::Cord("foo")); - EXPECT_LT(absl::Cord("bar"), StringValue("foo")); + EXPECT_LT(StringValue::WrapUnsafe("bar"), "foo"); + EXPECT_LT("bar", StringValue::WrapUnsafe("foo")); + EXPECT_LT(StringValue::WrapUnsafe("bar"), StringValue::WrapUnsafe("foo")); + EXPECT_LT(StringValue::WrapUnsafe("bar"), absl::Cord("foo")); + EXPECT_LT(absl::Cord("bar"), StringValue::WrapUnsafe("foo")); } TEST_F(StringValueTest, StartsWith) { EXPECT_TRUE( - StringValue("This string is large enough to not be stored inline!") - .StartsWith(StringValue("This string is large enough"))); + StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!") + .StartsWith(StringValue::WrapUnsafe("This string is large enough"))); + EXPECT_TRUE(StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!") + .StartsWith(StringValue::From( + absl::Cord("This string is large enough"), arena()))); EXPECT_TRUE( - StringValue("This string is large enough to not be stored inline!") - .StartsWith(StringValue(absl::Cord("This string is large enough")))); + StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()) + .StartsWith(StringValue::WrapUnsafe("This string is large enough"))); EXPECT_TRUE( - StringValue( - absl::Cord("This string is large enough to not be stored inline!")) - .StartsWith(StringValue("This string is large enough"))); - EXPECT_TRUE( - StringValue( - absl::Cord("This string is large enough to not be stored inline!")) - .StartsWith(StringValue(absl::Cord("This string is large enough")))); + StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()) + .StartsWith(StringValue::From( + absl::Cord("This string is large enough"), arena()))); } TEST_F(StringValueTest, EndsWith) { EXPECT_TRUE( - StringValue("This string is large enough to not be stored inline!") - .EndsWith(StringValue("to not be stored inline!"))); - EXPECT_TRUE( - StringValue("This string is large enough to not be stored inline!") - .EndsWith(StringValue(absl::Cord("to not be stored inline!")))); + StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!") + .EndsWith(StringValue::WrapUnsafe("to not be stored inline!"))); + EXPECT_TRUE(StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!") + .EndsWith(StringValue::From( + absl::Cord("to not be stored inline!"), arena()))); EXPECT_TRUE( - StringValue( - absl::Cord("This string is large enough to not be stored inline!")) - .EndsWith(StringValue("to not be stored inline!"))); + StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()) + .EndsWith(StringValue::WrapUnsafe("to not be stored inline!"))); EXPECT_TRUE( - StringValue( - absl::Cord("This string is large enough to not be stored inline!")) - .EndsWith(StringValue(absl::Cord("to not be stored inline!")))); + StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()) + .EndsWith(StringValue::From(absl::Cord("to not be stored inline!"), + arena()))); } TEST_F(StringValueTest, Contains) { + EXPECT_TRUE(StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!") + .Contains(StringValue::WrapUnsafe("string is large enough"))); + EXPECT_TRUE(StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!") + .Contains(StringValue::From( + absl::Cord("string is large enough"), arena()))); EXPECT_TRUE( - StringValue("This string is large enough to not be stored inline!") - .Contains(StringValue("string is large enough"))); - EXPECT_TRUE( - StringValue("This string is large enough to not be stored inline!") - .Contains(StringValue(absl::Cord("string is large enough")))); - EXPECT_TRUE( - StringValue( - absl::Cord("This string is large enough to not be stored inline!")) - .Contains(StringValue("string is large enough"))); + StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()) + .Contains(StringValue::WrapUnsafe("string is large enough"))); EXPECT_TRUE( - StringValue( - absl::Cord("This string is large enough to not be stored inline!")) - .Contains(StringValue(absl::Cord("string is large enough")))); + StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()) + .Contains(StringValue::From(absl::Cord("string is large enough"), + arena()))); } TEST_F(StringValueTest, IndexOf) { - StringValue big_string = - StringValue("This string is large enough to not be stored inline!"); - StringValue big_string_cord = StringValue( - absl::Cord("This string is large enough to not be stored inline!")); - StringValue small_string = StringValue("is"); - StringValue small_string_cord = StringValue(absl::Cord("is")); + StringValue big_string = StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!"); + StringValue big_string_cord = StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()); + StringValue small_string = StringValue::WrapUnsafe("is"); + StringValue small_string_cord = StringValue::From(absl::Cord("is"), arena()); EXPECT_THAT(big_string.IndexOf(small_string), Optional(Eq(2))); EXPECT_THAT(big_string.IndexOf(small_string_cord), Optional(Eq(2))); @@ -227,7 +253,7 @@ TEST_F(StringValueTest, IndexOf) { EXPECT_THAT(big_string.IndexOf("is"), Optional(Eq(2))); EXPECT_THAT(big_string_cord.IndexOf("is"), Optional(Eq(2))); - EXPECT_THAT(big_string_cord.IndexOf("not found"), Eq(absl::nullopt)); + EXPECT_THAT(big_string_cord.IndexOf("not found"), Eq(std::nullopt)); EXPECT_THAT(big_string.IndexOf(small_string, 4), Optional(Eq(12))); EXPECT_THAT(big_string.IndexOf(small_string_cord, 4), Optional(Eq(12))); @@ -237,79 +263,93 @@ TEST_F(StringValueTest, IndexOf) { EXPECT_THAT(big_string.IndexOf("is", 4), Optional(Eq(12))); EXPECT_THAT(big_string_cord.IndexOf("is", 4), Optional(Eq(12))); - EXPECT_THAT(big_string.IndexOf(small_string, 13), Eq(absl::nullopt)); - EXPECT_THAT(big_string.IndexOf(small_string_cord, 13), Eq(absl::nullopt)); - EXPECT_THAT(big_string_cord.IndexOf(small_string, 13), Eq(absl::nullopt)); - EXPECT_THAT(big_string_cord.IndexOf(small_string_cord, 13), - Eq(absl::nullopt)); + EXPECT_THAT(big_string.IndexOf(small_string, 13), Eq(std::nullopt)); + EXPECT_THAT(big_string.IndexOf(small_string_cord, 13), Eq(std::nullopt)); + EXPECT_THAT(big_string_cord.IndexOf(small_string, 13), Eq(std::nullopt)); + EXPECT_THAT(big_string_cord.IndexOf(small_string_cord, 13), Eq(std::nullopt)); EXPECT_THAT(big_string.IndexOf(absl::Cord("is"), 4), Optional(Eq(12))); EXPECT_THAT(big_string_cord.IndexOf(absl::Cord("is"), 4), Optional(Eq(12))); - EXPECT_THAT(big_string.IndexOf(absl::Cord("is"), 13), Eq(absl::nullopt)); - EXPECT_THAT(big_string_cord.IndexOf(absl::Cord("is"), 13), Eq(absl::nullopt)); + EXPECT_THAT(big_string.IndexOf(absl::Cord("is"), 13), Eq(std::nullopt)); + EXPECT_THAT(big_string_cord.IndexOf(absl::Cord("is"), 13), Eq(std::nullopt)); } TEST_F(StringValueTest, LowerAscii) { - EXPECT_EQ(StringValue("UPPER lower").LowerAscii(arena()), "upper lower"); - EXPECT_EQ(StringValue(absl::Cord("UPPER lower")).LowerAscii(arena()), + EXPECT_EQ(StringValue::WrapUnsafe("UPPER lower").LowerAscii(arena()), "upper lower"); - EXPECT_EQ(StringValue("upper lower").LowerAscii(arena()), "upper lower"); - EXPECT_EQ(StringValue(absl::Cord("upper lower")).LowerAscii(arena()), + EXPECT_EQ( + StringValue::From(absl::Cord("UPPER lower"), arena()).LowerAscii(arena()), + "upper lower"); + EXPECT_EQ(StringValue::WrapUnsafe("upper lower").LowerAscii(arena()), "upper lower"); - EXPECT_EQ(StringValue("").LowerAscii(arena()), ""); - EXPECT_EQ(StringValue(absl::Cord("")).LowerAscii(arena()), ""); + EXPECT_EQ( + StringValue::From(absl::Cord("upper lower"), arena()).LowerAscii(arena()), + "upper lower"); + EXPECT_EQ(StringValue::WrapUnsafe("").LowerAscii(arena()), ""); + EXPECT_EQ(StringValue::From(absl::Cord(""), arena()).LowerAscii(arena()), ""); const std::string kLongMixed = "A long STRING with MiXeD case to test conversion to lower case!"; const std::string kLongLower = "a long string with mixed case to test conversion to lower case!"; - EXPECT_EQ(StringValue(absl::Cord(kLongMixed)).LowerAscii(arena()), - kLongLower); + EXPECT_EQ( + StringValue::From(absl::Cord(kLongMixed), arena()).LowerAscii(arena()), + kLongLower); std::string very_long_mixed(10000, 'A'); std::string very_long_lower(10000, 'a'); + EXPECT_EQ(StringValue::From( + absl::MakeFragmentedCord({very_long_mixed.substr(0, 5000), + very_long_mixed.substr(5000)}), + arena()) + .LowerAscii(arena()), + very_long_lower); EXPECT_EQ( - StringValue(absl::MakeFragmentedCord({very_long_mixed.substr(0, 5000), - very_long_mixed.substr(5000)})) + StringValue::From(absl::MakeFragmentedCord({"hello", "WORLD"}), arena()) .LowerAscii(arena()), - very_long_lower); - EXPECT_EQ(StringValue(absl::MakeFragmentedCord({"hello", "WORLD"})) - .LowerAscii(arena()), - "helloworld"); + "helloworld"); } TEST_F(StringValueTest, UpperAscii) { - EXPECT_EQ(StringValue("UPPER lower").UpperAscii(arena()), "UPPER LOWER"); - EXPECT_EQ(StringValue(absl::Cord("UPPER lower")).UpperAscii(arena()), + EXPECT_EQ(StringValue::WrapUnsafe("UPPER lower").UpperAscii(arena()), "UPPER LOWER"); - EXPECT_EQ(StringValue("UPPER LOWER").UpperAscii(arena()), "UPPER LOWER"); - EXPECT_EQ(StringValue(absl::Cord("UPPER LOWER")).UpperAscii(arena()), + EXPECT_EQ( + StringValue::From(absl::Cord("UPPER lower"), arena()).UpperAscii(arena()), + "UPPER LOWER"); + EXPECT_EQ(StringValue::WrapUnsafe("UPPER LOWER").UpperAscii(arena()), "UPPER LOWER"); - EXPECT_EQ(StringValue("").UpperAscii(arena()), ""); - EXPECT_EQ(StringValue(absl::Cord("")).UpperAscii(arena()), ""); + EXPECT_EQ( + StringValue::From(absl::Cord("UPPER LOWER"), arena()).UpperAscii(arena()), + "UPPER LOWER"); + EXPECT_EQ(StringValue::WrapUnsafe("").UpperAscii(arena()), ""); + EXPECT_EQ(StringValue::From(absl::Cord(""), arena()).UpperAscii(arena()), ""); const std::string kLongMixed = "A long STRING with MiXeD case to test conversion to UPPER case!"; const std::string kLongUpper = "A LONG STRING WITH MIXED CASE TO TEST CONVERSION TO UPPER CASE!"; - EXPECT_EQ(StringValue(absl::Cord(kLongMixed)).UpperAscii(arena()), - kLongUpper); + EXPECT_EQ( + StringValue::From(absl::Cord(kLongMixed), arena()).UpperAscii(arena()), + kLongUpper); std::string very_long_mixed(10000, 'a'); std::string very_long_upper(10000, 'A'); + EXPECT_EQ(StringValue::From( + absl::MakeFragmentedCord({very_long_mixed.substr(0, 5000), + very_long_mixed.substr(5000)}), + arena()) + .UpperAscii(arena()), + very_long_upper); EXPECT_EQ( - StringValue(absl::MakeFragmentedCord({very_long_mixed.substr(0, 5000), - very_long_mixed.substr(5000)})) + StringValue::From(absl::MakeFragmentedCord({"HELLO", "world"}), arena()) .UpperAscii(arena()), - very_long_upper); - EXPECT_EQ(StringValue(absl::MakeFragmentedCord({"HELLO", "world"})) - .UpperAscii(arena()), - "HELLOWORLD"); + "HELLOWORLD"); } TEST_F(StringValueTest, LastIndexOf) { - StringValue big_string = - StringValue("This string is large enough to not be stored inline!"); - StringValue big_string_cord = StringValue( - absl::Cord("This string is large enough to not be stored inline!")); - StringValue small_string = StringValue("is"); - StringValue small_string_cord = StringValue(absl::Cord("is")); + StringValue big_string = StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!"); + StringValue big_string_cord = StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()); + StringValue small_string = StringValue::WrapUnsafe("is"); + StringValue small_string_cord = StringValue::From(absl::Cord("is"), arena()); EXPECT_THAT(big_string.LastIndexOf(small_string), Optional(Eq(12))); EXPECT_THAT(big_string.LastIndexOf(small_string_cord), Optional(Eq(12))); @@ -318,7 +358,7 @@ TEST_F(StringValueTest, LastIndexOf) { EXPECT_THAT(big_string.LastIndexOf("is"), Optional(Eq(12))); EXPECT_THAT(big_string_cord.LastIndexOf("is"), Optional(Eq(12))); - EXPECT_THAT(big_string_cord.LastIndexOf("not found"), Eq(absl::nullopt)); + EXPECT_THAT(big_string_cord.LastIndexOf("not found"), Eq(std::nullopt)); EXPECT_THAT(big_string.LastIndexOf(small_string, 4), Optional(Eq(2))); EXPECT_THAT(big_string.LastIndexOf(small_string_cord, 4), Optional(Eq(2))); @@ -347,12 +387,12 @@ TEST_F(StringValueTest, LastIndexOf) { TEST_F(StringValueTest, Trim) { using ::cel::test::StringValueIs; - StringValue unpadded = StringValue("no padding"); - StringValue front_padded = StringValue(" \t\r\nno padding"); - StringValue back_padded = StringValue("no padding \t\r\n"); - StringValue both_padded = StringValue(" \t\r\nno padding \t\r\n"); - StringValue whitespace = StringValue(" \t\r\n"); - StringValue empty = StringValue(""); + StringValue unpadded = StringValue::WrapUnsafe("no padding"); + StringValue front_padded = StringValue::WrapUnsafe(" \t\r\nno padding"); + StringValue back_padded = StringValue::WrapUnsafe("no padding \t\r\n"); + StringValue both_padded = StringValue::WrapUnsafe(" \t\r\nno padding \t\r\n"); + StringValue whitespace = StringValue::WrapUnsafe(" \t\r\n"); + StringValue empty = StringValue::WrapUnsafe(""); EXPECT_THAT(unpadded.Trim(), StringValueIs("no padding")); EXPECT_THAT(front_padded.Trim(), StringValueIs("no padding")); @@ -361,13 +401,17 @@ TEST_F(StringValueTest, Trim) { EXPECT_THAT(whitespace.Trim(), StringValueIs("")); EXPECT_THAT(empty.Trim(), StringValueIs("")); - StringValue unpadded_cord = StringValue(absl::Cord("no padding")); - StringValue front_padded_cord = StringValue(absl::Cord(" \t\r\nno padding")); - StringValue back_padded_cord = StringValue(absl::Cord("no padding \t\r\n")); + StringValue unpadded_cord = + StringValue::From(absl::Cord("no padding"), arena()); + StringValue front_padded_cord = + StringValue::From(absl::Cord(" \t\r\nno padding"), arena()); + StringValue back_padded_cord = + StringValue::From(absl::Cord("no padding \t\r\n"), arena()); StringValue both_padded_cord = - StringValue(absl::Cord(" \t\r\nno padding \t\r\n")); - StringValue whitespace_cord = StringValue(absl::Cord(" \t\r\n")); - StringValue empty_cord = StringValue(absl::Cord("")); + StringValue::From(absl::Cord(" \t\r\nno padding \t\r\n"), arena()); + StringValue whitespace_cord = + StringValue::From(absl::Cord(" \t\r\n"), arena()); + StringValue empty_cord = StringValue::From(absl::Cord(""), arena()); EXPECT_THAT(unpadded_cord.Trim(), StringValueIs("no padding")); EXPECT_THAT(front_padded_cord.Trim(), StringValueIs("no padding")); @@ -380,44 +424,75 @@ TEST_F(StringValueTest, Trim) { TEST_F(StringValueTest, CharAt) { using ::cel::test::ErrorValueIs; using ::cel::test::StringValueIs; - StringValue big_string = - StringValue("This string is large enough to not be stored inline!"); - StringValue big_string_cord = StringValue( - absl::Cord("This string is large enough to not be stored inline!")); - StringValue small_string = StringValue("abc"); - StringValue small_string_cord = StringValue(absl::Cord("abc")); - StringValue unicode_string = StringValue("aμc"); - StringValue unicode_string_cord = StringValue(absl::Cord("aμc")); - - EXPECT_THAT(big_string.CharAt(0), StringValueIs("T")); - EXPECT_THAT(big_string_cord.CharAt(0), StringValueIs("T")); - EXPECT_THAT(small_string.CharAt(1), StringValueIs("b")); - EXPECT_THAT(small_string_cord.CharAt(1), StringValueIs("b")); - EXPECT_THAT(unicode_string.CharAt(1), StringValueIs("μ")); - EXPECT_THAT(unicode_string_cord.CharAt(1), StringValueIs("μ")); + StringValue big_string = StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!"); + StringValue big_string_cord = StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()); + StringValue small_string = StringValue::WrapUnsafe("abc"); + StringValue small_string_cord = StringValue::From(absl::Cord("abc"), arena()); + StringValue unicode_string = StringValue::WrapUnsafe("aμc"); + StringValue unicode_string_cord = + StringValue::From(absl::Cord("aμc"), arena()); + + EXPECT_THAT(big_string.CharAt(0, arena()), StringValueIs("T")); + EXPECT_THAT(big_string_cord.CharAt(0, arena()), StringValueIs("T")); + EXPECT_THAT(small_string.CharAt(1, arena()), StringValueIs("b")); + EXPECT_THAT(small_string_cord.CharAt(1, arena()), StringValueIs("b")); + EXPECT_THAT(unicode_string.CharAt(1, arena()), StringValueIs("μ")); + EXPECT_THAT(unicode_string_cord.CharAt(1, arena()), StringValueIs("μ")); EXPECT_THAT( - big_string.CharAt(100), + big_string.CharAt(100, arena()), ErrorValueIs(absl::InvalidArgumentError( ".charAt(): is greater than .size()"))); EXPECT_THAT( - big_string_cord.CharAt(100), + big_string_cord.CharAt(100, arena()), ErrorValueIs(absl::InvalidArgumentError( ".charAt(): is greater than .size()"))); - EXPECT_THAT(big_string.CharAt(-1), + EXPECT_THAT(big_string.CharAt(-1, arena()), ErrorValueIs(absl::InvalidArgumentError( ".charAt(): is less than 0"))); - EXPECT_THAT(big_string_cord.CharAt(-1), + EXPECT_THAT(big_string_cord.CharAt(-1, arena()), ErrorValueIs(absl::InvalidArgumentError( ".charAt(): is less than 0"))); } +TEST_F(StringValueTest, Substring) { + using ::cel::test::ErrorValueIs; + using ::cel::test::StringValueIs; + + // Each '€' is three bytes, so the cord built here is 18 bytes and is stored + // as a large (cord-backed) value. The substring length must be measured in + // code units, not code points, otherwise the cord overload truncates a + // multi-byte character or underflows the length. + StringValue unicode_cord = StringValue::From(absl::Cord("€€€€€€"), arena()); + StringValue unicode_view = StringValue::WrapUnsafe("€€€€€€"); + + EXPECT_THAT(unicode_cord.Substring(0, 2, arena()), StringValueIs("€€")); + EXPECT_THAT(unicode_view.Substring(0, 2, arena()), StringValueIs("€€")); + EXPECT_THAT(unicode_cord.Substring(1, 2, arena()), StringValueIs("€")); + EXPECT_THAT(unicode_view.Substring(1, 2, arena()), StringValueIs("€")); + EXPECT_THAT(unicode_cord.Substring(2, 4, arena()), StringValueIs("€€")); + EXPECT_THAT(unicode_view.Substring(2, 4, arena()), StringValueIs("€€")); + EXPECT_THAT(unicode_cord.Substring(2, arena()), StringValueIs("€€€€")); + EXPECT_THAT(unicode_view.Substring(2, arena()), StringValueIs("€€€€")); + + EXPECT_THAT(unicode_cord.Substring(0, 7, arena()), + ErrorValueIs(absl::InvalidArgumentError( + ".substring(, ): or is " + "greater than .size()"))); + EXPECT_THAT(unicode_cord.Substring(-1, arena()), + ErrorValueIs(absl::InvalidArgumentError( + ".substring(): is less than 0"))); +} + TEST_F(StringValueTest, Join) { using ::cel::runtime_internal::CreateNoMatchingOverloadError; using ::cel::test::ErrorValueIs; using ::cel::test::StringValueIs; - StringValue separator(","); + StringValue separator = StringValue::WrapUnsafe(","); Value result; // Empty list. @@ -430,7 +505,7 @@ TEST_F(StringValueTest, Join) { // Single element list. auto list_builder1 = NewListValueBuilder(arena()); - ASSERT_THAT(list_builder1->Add(StringValue("foo")), IsOk()); + ASSERT_THAT(list_builder1->Add(StringValue::WrapUnsafe("foo")), IsOk()); auto list1 = std::move(*list_builder1).Build(); EXPECT_THAT(separator.Join(list1, descriptor_pool(), message_factory(), arena(), &result), @@ -439,9 +514,9 @@ TEST_F(StringValueTest, Join) { // Multi element list. auto list_builder2 = NewListValueBuilder(arena()); - ASSERT_THAT(list_builder2->Add(StringValue("foo")), IsOk()); - ASSERT_THAT(list_builder2->Add(StringValue("bar")), IsOk()); - ASSERT_THAT(list_builder2->Add(StringValue("baz")), IsOk()); + ASSERT_THAT(list_builder2->Add(StringValue::WrapUnsafe("foo")), IsOk()); + ASSERT_THAT(list_builder2->Add(StringValue::WrapUnsafe("bar")), IsOk()); + ASSERT_THAT(list_builder2->Add(StringValue::WrapUnsafe("baz")), IsOk()); auto list2 = std::move(*list_builder2).Build(); EXPECT_THAT(separator.Join(list2, descriptor_pool(), message_factory(), arena(), &result), @@ -459,7 +534,7 @@ TEST_F(StringValueTest, Join) { // List with string and non-string. auto list_builder4 = NewListValueBuilder(arena()); - ASSERT_THAT(list_builder4->Add(StringValue("foo")), IsOk()); + ASSERT_THAT(list_builder4->Add(StringValue::WrapUnsafe("foo")), IsOk()); ASSERT_THAT(list_builder4->Add(IntValue(1)), IsOk()); auto list4 = std::move(*list_builder4).Build(); EXPECT_THAT(separator.Join(list4, descriptor_pool(), message_factory(), @@ -472,23 +547,34 @@ TEST_F(StringValueTest, Reverse) { using ::cel::test::StringValueIs; EXPECT_THAT(StringValue().Reverse(arena()), StringValueIs("")); - EXPECT_THAT(StringValue("").Reverse(arena()), StringValueIs("")); - EXPECT_THAT(StringValue("hello").Reverse(arena()), StringValueIs("olleh")); - EXPECT_THAT(StringValue("aμc").Reverse(arena()), StringValueIs("cμa")); + EXPECT_THAT(StringValue::WrapUnsafe("").Reverse(arena()), StringValueIs("")); + EXPECT_THAT(StringValue::WrapUnsafe("hello").Reverse(arena()), + StringValueIs("olleh")); + EXPECT_THAT(StringValue::WrapUnsafe("aμc").Reverse(arena()), + StringValueIs("cμa")); EXPECT_THAT( - StringValue("This string is large enough to not be stored inline!") + StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!") .Reverse(arena()), StringValueIs("!enilni derots eb ton ot hguone egral si gnirts sihT")); - EXPECT_THAT(StringValue(absl::Cord("hello")).Reverse(arena()), + EXPECT_THAT(StringValue::From(absl::Cord("hello"), arena()).Reverse(arena()), StringValueIs("olleh")); - EXPECT_THAT(StringValue(absl::Cord("aμc")).Reverse(arena()), + EXPECT_THAT(StringValue::From(absl::Cord("aμc"), arena()).Reverse(arena()), StringValueIs("cμa")); EXPECT_THAT( - StringValue( - absl::Cord("This string is large enough to not be stored inline!")) + StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()) .Reverse(arena()), StringValueIs("!enilni derots eb ton ot hguone egral si gnirts sihT")); } +TEST_F(StringValueTest, Literal) { + using ::cel::test::StringValueIs; + + EXPECT_THAT(StringValue::Literal("Hello, World!"), + StringValueIs("Hello, World!")); +} + } // namespace } // namespace cel diff --git a/common/values/struct_value.cc b/common/values/struct_value.cc index 10238a670..8b7bb7cb1 100644 --- a/common/values/struct_value.cc +++ b/common/values/struct_value.cc @@ -39,6 +39,8 @@ #include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/message.h" +#undef GetMessage + namespace cel { StructType StructValue::GetRuntimeType() const { @@ -321,7 +323,7 @@ absl::optional StructValue::AsMessage() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional StructValue::AsMessage() && { @@ -329,7 +331,7 @@ absl::optional StructValue::AsMessage() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } optional_ref StructValue::AsParsedMessage() const& { @@ -337,7 +339,7 @@ optional_ref StructValue::AsParsedMessage() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional StructValue::AsParsedMessage() && { @@ -345,7 +347,7 @@ absl::optional StructValue::AsParsedMessage() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } MessageValue StructValue::GetMessage() const& { diff --git a/common/values/struct_value.h b/common/values/struct_value.h index d096356c7..c46442c41 100644 --- a/common/values/struct_value.h +++ b/common/values/struct_value.h @@ -54,6 +54,9 @@ #include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/message.h" +#pragma push_macro("GetMessage") +#undef GetMessage + namespace cel { class StructValue; @@ -370,4 +373,6 @@ using StructValueBuilderPtr = std::unique_ptr; } // namespace cel +#pragma pop_macro("GetMessage") + #endif // THIRD_PARTY_CEL_CPP_COMMON_VALUES_STRUCT_VALUE_H_ diff --git a/common/values/struct_value_builder.cc b/common/values/struct_value_builder.cc index 359596267..1cb117f46 100644 --- a/common/values/struct_value_builder.cc +++ b/common/values/struct_value_builder.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -75,33 +76,34 @@ absl::StatusOr> ProtoMessageCopyUsingSerialization( return absl::UnknownError( absl::StrCat("failed to parse `", to->GetTypeName(), "`")); } - return absl::nullopt; + return std::nullopt; } absl::StatusOr> ProtoMessageCopy( google::protobuf::Message* absl_nonnull to_message, const google::protobuf::Descriptor* absl_nonnull to_descriptor, - const google::protobuf::Message* absl_nonnull from_message) { + const google::protobuf::Message* absl_nonnull from_message, + google::protobuf::Arena* absl_nonnull arena) { CEL_ASSIGN_OR_RETURN(const auto* from_descriptor, GetDescriptor(*from_message)); if (to_descriptor == from_descriptor) { // Same. to_message->CopyFrom(*from_message); - return absl::nullopt; + return std::nullopt; } if (to_descriptor->full_name() == from_descriptor->full_name()) { // Same type, different descriptors. return ProtoMessageCopyUsingSerialization(to_message, from_message); } return TypeConversionError(from_descriptor->full_name(), - to_descriptor->full_name()); + to_descriptor->full_name(), arena); } absl::StatusOr> ProtoMessageFromValueImpl( const Value& value, const google::protobuf::DescriptorPool* absl_nonnull pool, google::protobuf::MessageFactory* absl_nonnull factory, well_known_types::Reflection* absl_nonnull well_known_types, - google::protobuf::Message* absl_nonnull message) { + google::protobuf::Message* absl_nonnull message, google::protobuf::Arena* absl_nonnull arena) { CEL_ASSIGN_OR_RETURN(const auto* to_desc, GetDescriptor(*message)); switch (to_desc->well_known_type()) { case google::protobuf::Descriptor::WELLKNOWNTYPE_FLOATVALUE: { @@ -110,9 +112,10 @@ absl::StatusOr> ProtoMessageFromValueImpl( message->GetDescriptor())); well_known_types->FloatValue().SetValue( message, static_cast(double_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_DOUBLEVALUE: { if (auto double_value = value.AsDouble(); double_value) { @@ -120,23 +123,26 @@ absl::StatusOr> ProtoMessageFromValueImpl( message->GetDescriptor())); well_known_types->DoubleValue().SetValue(message, double_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_INT32VALUE: { if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < std::numeric_limits::min() || int_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); + return ErrorValue::From( + absl::OutOfRangeError("int64 to int32 overflow"), arena); } CEL_RETURN_IF_ERROR(well_known_types->Int32Value().Initialize( message->GetDescriptor())); well_known_types->Int32Value().SetValue( message, static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_INT64VALUE: { if (auto int_value = value.AsInt(); int_value) { @@ -144,22 +150,25 @@ absl::StatusOr> ProtoMessageFromValueImpl( message->GetDescriptor())); well_known_types->Int64Value().SetValue(message, int_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT32VALUE: { if (auto uint_value = value.AsUint(); uint_value) { if (uint_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("uint64 to uint32 overflow")); + return ErrorValue::From( + absl::OutOfRangeError("uint64 to uint32 overflow"), arena); } CEL_RETURN_IF_ERROR(well_known_types->UInt32Value().Initialize( message->GetDescriptor())); well_known_types->UInt32Value().SetValue( message, static_cast(uint_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT64VALUE: { if (auto uint_value = value.AsUint(); uint_value) { @@ -167,9 +176,10 @@ absl::StatusOr> ProtoMessageFromValueImpl( message->GetDescriptor())); well_known_types->UInt64Value().SetValue(message, uint_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_STRINGVALUE: { if (auto string_value = value.AsString(); string_value) { @@ -177,9 +187,10 @@ absl::StatusOr> ProtoMessageFromValueImpl( message->GetDescriptor())); well_known_types->StringValue().SetValue(message, string_value->NativeCord()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_BYTESVALUE: { if (auto bytes_value = value.AsBytes(); bytes_value) { @@ -187,9 +198,10 @@ absl::StatusOr> ProtoMessageFromValueImpl( message->GetDescriptor())); well_known_types->BytesValue().SetValue(message, bytes_value->NativeCord()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_BOOLVALUE: { if (auto bool_value = value.AsBool(); bool_value) { @@ -197,9 +209,10 @@ absl::StatusOr> ProtoMessageFromValueImpl( well_known_types->BoolValue().Initialize(message->GetDescriptor())); well_known_types->BoolValue().SetValue(message, bool_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_ANY: { google::protobuf::io::CordOutputStream serialized; @@ -248,39 +261,41 @@ absl::StatusOr> ProtoMessageFromValueImpl( well_known_types->Any().SetTypeUrl(message, type_url); well_known_types->Any().SetValue(message, std::move(serialized).Consume()); - return absl::nullopt; + return std::nullopt; } case google::protobuf::Descriptor::WELLKNOWNTYPE_DURATION: { if (auto duration_value = value.AsDuration(); duration_value) { CEL_RETURN_IF_ERROR( well_known_types->Duration().Initialize(message->GetDescriptor())); - CEL_RETURN_IF_ERROR(well_known_types->Duration().SetFromAbslDuration( - message, duration_value->NativeValue())); - return absl::nullopt; + well_known_types->Duration().UnsafeSetFromAbslDuration( + message, duration_value->NativeValue()); + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_TIMESTAMP: { if (auto timestamp_value = value.AsTimestamp(); timestamp_value) { CEL_RETURN_IF_ERROR( well_known_types->Timestamp().Initialize(message->GetDescriptor())); - CEL_RETURN_IF_ERROR(well_known_types->Timestamp().SetFromAbslTime( - message, timestamp_value->NativeValue())); - return absl::nullopt; + well_known_types->Timestamp().UnsafeSetFromAbslTime( + message, timestamp_value->NativeValue()); + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE: { CEL_RETURN_IF_ERROR(value.ConvertToJson(pool, factory, message)); - return absl::nullopt; + return std::nullopt; } case google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE: { CEL_RETURN_IF_ERROR(value.ConvertToJsonArray(pool, factory, message)); - return absl::nullopt; + return std::nullopt; } case google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT: { CEL_RETURN_IF_ERROR(value.ConvertToJsonObject(pool, factory, message)); - return absl::nullopt; + return std::nullopt; } default: break; @@ -292,85 +307,96 @@ absl::StatusOr> ProtoMessageFromValueImpl( if (auto legacy_value = common_internal::AsLegacyStructValue(value); legacy_value) { const auto* from_message = legacy_value->message_ptr(); - return ProtoMessageCopy(message, to_desc, from_message); + return ProtoMessageCopy(message, to_desc, from_message, arena); } // Deal with modern values. if (auto parsed_message_value = value.AsParsedMessage(); parsed_message_value) { return ProtoMessageCopy(message, to_desc, - cel::to_address(*parsed_message_value)); + cel::to_address(*parsed_message_value), arena); } - return TypeConversionError(value.GetTypeName(), message->GetTypeName()); + return TypeConversionError(value.GetTypeName(), message->GetTypeName(), + arena); } // Converts a value to a specific protocol buffer map key. using ProtoMapKeyFromValueConverter = absl::StatusOr> (*)(const Value&, google::protobuf::MapKey&, - std::string&); + std::string&, + google::protobuf::Arena* absl_nonnull); absl::StatusOr> ProtoBoolMapKeyFromValueConverter( - const Value& value, google::protobuf::MapKey& key, std::string&) { + const Value& value, google::protobuf::MapKey& key, std::string&, + google::protobuf::Arena* absl_nonnull arena) { if (auto bool_value = value.AsBool(); bool_value) { key.SetBoolValue(bool_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "bool"); + return TypeConversionError(value.GetTypeName(), "bool", arena); } absl::StatusOr> ProtoInt32MapKeyFromValueConverter( - const Value& value, google::protobuf::MapKey& key, std::string&) { + const Value& value, google::protobuf::MapKey& key, std::string&, + google::protobuf::Arena* absl_nonnull arena) { if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < std::numeric_limits::min() || int_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); + return ErrorValue::From(absl::OutOfRangeError("int64 to int32 overflow"), + arena); } key.SetInt32Value(static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena); } absl::StatusOr> ProtoInt64MapKeyFromValueConverter( - const Value& value, google::protobuf::MapKey& key, std::string&) { + const Value& value, google::protobuf::MapKey& key, std::string&, + google::protobuf::Arena* absl_nonnull arena) { if (auto int_value = value.AsInt(); int_value) { key.SetInt64Value(int_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena); } absl::StatusOr> ProtoUInt32MapKeyFromValueConverter( - const Value& value, google::protobuf::MapKey& key, std::string&) { + const Value& value, google::protobuf::MapKey& key, std::string&, + google::protobuf::Arena* absl_nonnull arena) { if (auto uint_value = value.AsUint(); uint_value) { if (uint_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("uint64 to uint32 overflow")); + return ErrorValue::From( + absl::OutOfRangeError("uint64 to uint32 overflow"), arena); } key.SetUInt32Value(static_cast(uint_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena); } absl::StatusOr> ProtoUInt64MapKeyFromValueConverter( - const Value& value, google::protobuf::MapKey& key, std::string&) { + const Value& value, google::protobuf::MapKey& key, std::string&, + google::protobuf::Arena* absl_nonnull arena) { if (auto uint_value = value.AsUint(); uint_value) { key.SetUInt64Value(uint_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena); } absl::StatusOr> ProtoStringMapKeyFromValueConverter( - const Value& value, google::protobuf::MapKey& key, std::string& key_string) { + const Value& value, google::protobuf::MapKey& key, std::string& key_string, + google::protobuf::Arena* absl_nonnull arena) { if (auto string_value = value.AsString(); string_value) { - key_string = string_value->NativeString(); - key.SetStringValue(key_string); - return absl::nullopt; + // `google::protobuf::MapKey` does not own the string, which may be stored inline in + // `value` or in `key_string`; `value` must not be moved. + key.SetStringValue(string_value->ToStringView(&key_string)); + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "string"); + return TypeConversionError(value.GetTypeName(), "string", arena); } // Gets the converter for converting from values to protocol buffer map key. @@ -402,49 +428,51 @@ using ProtoMapValueFromValueConverter = const Value&, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef&); + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef&, + google::protobuf::Arena* absl_nonnull); absl::StatusOr> ProtoBoolMapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto bool_value = value.AsBool(); bool_value) { value_ref.SetBoolValue(bool_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "bool"); + return TypeConversionError(value.GetTypeName(), "bool", arena); } absl::StatusOr> ProtoInt32MapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < std::numeric_limits::min() || int_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); + return ErrorValue::From(absl::OutOfRangeError("int64 to int32 overflow"), + arena); } value_ref.SetInt32Value(static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena); } absl::StatusOr> ProtoInt64MapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto int_value = value.AsInt(); int_value) { value_ref.SetInt64Value(int_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena); } absl::StatusOr> @@ -452,16 +480,17 @@ ProtoUInt32MapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto uint_value = value.AsUint(); uint_value) { if (uint_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("uint64 to uint32 overflow")); + return ErrorValue::From( + absl::OutOfRangeError("uint64 to uint32 overflow"), arena); } value_ref.SetUInt32Value(static_cast(uint_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena); } absl::StatusOr> @@ -469,26 +498,26 @@ ProtoUInt64MapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto uint_value = value.AsUint(); uint_value) { value_ref.SetUInt64Value(uint_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena); } absl::StatusOr> ProtoFloatMapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto double_value = value.AsDouble(); double_value) { value_ref.SetFloatValue(double_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "double"); + return TypeConversionError(value.GetTypeName(), "double", arena); } absl::StatusOr> @@ -496,26 +525,26 @@ ProtoDoubleMapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto double_value = value.AsDouble(); double_value) { value_ref.SetDoubleValue(double_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "double"); + return TypeConversionError(value.GetTypeName(), "double", arena); } absl::StatusOr> ProtoBytesMapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto bytes_value = value.AsBytes(); bytes_value) { value_ref.SetStringValue(bytes_value->NativeString()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "bytes"); + return TypeConversionError(value.GetTypeName(), "bytes", arena); } absl::StatusOr> @@ -523,43 +552,45 @@ ProtoStringMapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto string_value = value.AsString(); string_value) { value_ref.SetStringValue(string_value->NativeString()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "string"); + return TypeConversionError(value.GetTypeName(), "string", arena); } absl::StatusOr> ProtoNullMapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (value.IsNull() || value.IsInt()) { value_ref.SetEnumValue(0); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "google.protobuf.NullValue"); + return TypeConversionError(value.GetTypeName(), "google.protobuf.NullValue", + arena); } absl::StatusOr> ProtoEnumMapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull field, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < std::numeric_limits::min() || int_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); + return ErrorValue::From(absl::OutOfRangeError("int64 to int32 overflow"), + arena); } value_ref.SetEnumValue(static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "enum"); + return TypeConversionError(value.GetTypeName(), "enum", arena); } absl::StatusOr> @@ -568,9 +599,9 @@ ProtoMessageMapValueFromValueConverter( const google::protobuf::DescriptorPool* absl_nonnull pool, google::protobuf::MessageFactory* absl_nonnull factory, well_known_types::Reflection* absl_nonnull well_known_types, - google::protobuf::MapValueRef& value_ref) { + google::protobuf::MapValueRef& value_ref, google::protobuf::Arena* absl_nonnull arena) { return ProtoMessageFromValueImpl(value, pool, factory, well_known_types, - value_ref.MutableMessageValue()); + value_ref.MutableMessageValue(), arena); } // Gets the converter for converting from values to protocol buffer map value. @@ -620,7 +651,8 @@ using ProtoRepeatedFieldFromValueMutator = google::protobuf::MessageFactory* absl_nonnull, well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull, google::protobuf::Message* absl_nonnull, - const google::protobuf::FieldDescriptor* absl_nonnull, const Value&); + const google::protobuf::FieldDescriptor* absl_nonnull, const Value&, + google::protobuf::Arena* absl_nonnull); absl::StatusOr> ProtoBoolRepeatedFieldFromValueMutator( @@ -629,12 +661,13 @@ ProtoBoolRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto bool_value = value.AsBool(); bool_value) { reflection->AddBool(message, field, bool_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "bool"); + return TypeConversionError(value.GetTypeName(), "bool", arena); } absl::StatusOr> @@ -644,17 +677,19 @@ ProtoInt32RepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < std::numeric_limits::min() || int_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); + return ErrorValue::From(absl::OutOfRangeError("int64 to int32 overflow"), + arena); } reflection->AddInt32(message, field, static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena); } absl::StatusOr> @@ -664,12 +699,13 @@ ProtoInt64RepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto int_value = value.AsInt(); int_value) { reflection->AddInt64(message, field, int_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena); } absl::StatusOr> @@ -679,16 +715,18 @@ ProtoUInt32RepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto uint_value = value.AsUint(); uint_value) { if (uint_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("uint64 to uint32 overflow")); + return ErrorValue::From( + absl::OutOfRangeError("uint64 to uint32 overflow"), arena); } reflection->AddUInt32(message, field, static_cast(uint_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena); } absl::StatusOr> @@ -698,12 +736,13 @@ ProtoUInt64RepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto uint_value = value.AsUint(); uint_value) { reflection->AddUInt64(message, field, uint_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena); } absl::StatusOr> @@ -713,13 +752,14 @@ ProtoFloatRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto double_value = value.AsDouble(); double_value) { reflection->AddFloat(message, field, static_cast(double_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "double"); + return TypeConversionError(value.GetTypeName(), "double", arena); } absl::StatusOr> @@ -729,12 +769,13 @@ ProtoDoubleRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto double_value = value.AsDouble(); double_value) { reflection->AddDouble(message, field, double_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "double"); + return TypeConversionError(value.GetTypeName(), "double", arena); } absl::StatusOr> @@ -744,12 +785,13 @@ ProtoBytesRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto bytes_value = value.AsBytes(); bytes_value) { reflection->AddString(message, field, bytes_value->NativeString()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "bytes"); + return TypeConversionError(value.GetTypeName(), "bytes", arena); } absl::StatusOr> @@ -759,12 +801,13 @@ ProtoStringRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto string_value = value.AsString(); string_value) { reflection->AddString(message, field, string_value->NativeString()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "string"); + return TypeConversionError(value.GetTypeName(), "string", arena); } absl::StatusOr> @@ -774,12 +817,13 @@ ProtoNullRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (value.IsNull() || value.IsInt()) { reflection->AddEnumValue(message, field, 0); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "null_type"); + return TypeConversionError(value.GetTypeName(), "null_type", arena); } absl::StatusOr> @@ -789,19 +833,21 @@ ProtoEnumRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { const auto* enum_descriptor = field->enum_type(); if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < std::numeric_limits::min() || int_value->NativeValue() > std::numeric_limits::max()) { return TypeConversionError(value.GetTypeName(), - enum_descriptor->full_name()); + enum_descriptor->full_name(), arena); } reflection->AddEnumValue(message, field, static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), enum_descriptor->full_name()); + return TypeConversionError(value.GetTypeName(), enum_descriptor->full_name(), + arena); } absl::StatusOr> @@ -811,10 +857,22 @@ ProtoMessageRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull well_known_types, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { + // If the value is null and the target repeated field is anything except + // google.protobuf.{Any,ListValue,Struct,Value}, it should be pruned. + if (value.IsNull()) { + const auto well_known_type = field->message_type()->well_known_type(); + if (well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_ANY) { + return std::nullopt; + } + } auto* element = reflection->AddMessage(message, field, factory); auto result = ProtoMessageFromValueImpl(value, pool, factory, - well_known_types, element); + well_known_types, element, arena); if (!result.ok() || result->has_value()) { reflection->RemoveLast(message, field); } @@ -886,7 +944,7 @@ class MessageValueBuilderImpl { if (field == nullptr) { field = descriptor_pool_->FindExtensionByPrintableName(descriptor_, name); if (field == nullptr) { - return NoSuchFieldError(name); + return NoSuchFieldError(name, arena_); } } return SetField(field, std::move(value)); @@ -896,12 +954,12 @@ class MessageValueBuilderImpl { Value value) { if (number < std::numeric_limits::min() || number > std::numeric_limits::max()) { - return NoSuchFieldError(absl::StrCat(number)); + return NoSuchFieldError(absl::StrCat(number), arena_); } const auto* field = descriptor_->FindFieldByNumber(static_cast(number)); if (field == nullptr) { - return NoSuchFieldError(absl::StrCat(number)); + return NoSuchFieldError(absl::StrCat(number), arena_); } return SetField(field, std::move(value)); } @@ -920,7 +978,7 @@ class MessageValueBuilderImpl { const google::protobuf::FieldDescriptor* absl_nonnull field, Value value) { auto map_value = value.AsMap(); if (!map_value) { - return TypeConversionError(value.GetTypeName(), "map"); + return TypeConversionError(value.GetTypeName(), "map", arena_); } CEL_ASSIGN_OR_RETURN(auto key_converter, GetProtoMapKeyFromValueConverter( @@ -941,10 +999,23 @@ class MessageValueBuilderImpl { google::protobuf::MapKey proto_key; CEL_ASSIGN_OR_RETURN( error_value, - (*key_converter)(entry_key, proto_key, proto_key_string)); + (*key_converter)(entry_key, proto_key, proto_key_string, arena_)); if (error_value) { return false; } + if (map_value_field->cpp_type() == + google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE && + entry_value.IsNull()) { + auto well_known_type = + map_value_field->message_type()->well_known_type(); + if (well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_ANY && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE && + well_known_type != + google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT) { + return true; + } + } google::protobuf::MapValueRef proto_value; extensions::protobuf_internal::InsertOrLookupMapValue( *reflection_, message_, *field, proto_key, &proto_value); @@ -952,7 +1023,7 @@ class MessageValueBuilderImpl { error_value, (*value_converter)(entry_value, map_value_field, descriptor_pool_, message_factory_, &well_known_types_, - proto_value)); + proto_value, arena_)); if (error_value) { return false; } @@ -969,7 +1040,8 @@ class MessageValueBuilderImpl { const google::protobuf::FieldDescriptor* absl_nonnull field, Value value) { auto list_value = value.AsList(); if (!list_value) { - return TypeConversionError(value.GetTypeName(), "list").NativeValue(); + return TypeConversionError(value.GetTypeName(), "list", arena_) + .NativeValue(); } CEL_ASSIGN_OR_RETURN(auto accessor, GetProtoRepeatedFieldFromValueMutator(field)); @@ -978,10 +1050,20 @@ class MessageValueBuilderImpl { CEL_RETURN_IF_ERROR(list_value->ForEach( [this, field, accessor, &error_value](const Value& element) -> absl::StatusOr { + if (field->message_type() != nullptr && element.IsNull()) { + auto well_known_type = field->message_type()->well_known_type(); + if (well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_ANY && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE && + well_known_type != + google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT) { + return true; + } + } CEL_ASSIGN_OR_RETURN(error_value, (*accessor)(descriptor_pool_, message_factory_, &well_known_types_, reflection_, - message_, field, element)); + message_, field, element, arena_)); return !error_value; }, descriptor_pool_, message_factory_, arena_)); @@ -994,63 +1076,64 @@ class MessageValueBuilderImpl { case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { if (auto bool_value = value.AsBool(); bool_value) { reflection_->SetBool(message_, field, bool_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "bool"); + return TypeConversionError(value.GetTypeName(), "bool", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < std::numeric_limits::min() || int_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); + return ErrorValue::From( + absl::OutOfRangeError("int64 to int32 overflow"), arena_); } reflection_->SetInt32(message_, field, static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { if (auto int_value = value.AsInt(); int_value) { reflection_->SetInt64(message_, field, int_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { if (auto uint_value = value.AsUint(); uint_value) { if (uint_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue( - absl::OutOfRangeError("uint64 to uint32 overflow")); + return ErrorValue::From( + absl::OutOfRangeError("uint64 to uint32 overflow"), arena_); } reflection_->SetUInt32( message_, field, static_cast(uint_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { if (auto uint_value = value.AsUint(); uint_value) { reflection_->SetUInt64(message_, field, uint_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { if (auto double_value = value.AsDouble(); double_value) { reflection_->SetFloat(message_, field, double_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "double"); + return TypeConversionError(value.GetTypeName(), "double", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { if (auto double_value = value.AsDouble(); double_value) { reflection_->SetDouble(message_, field, double_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "double"); + return TypeConversionError(value.GetTypeName(), "double", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { if (field->type() == google::protobuf::FieldDescriptor::TYPE_BYTES) { @@ -1062,9 +1145,9 @@ class MessageValueBuilderImpl { [this, field](const absl::Cord& cord) { reflection_->SetString(message_, field, cord); })); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "bytes"); + return TypeConversionError(value.GetTypeName(), "bytes", arena_); } if (auto string_value = value.AsString(); string_value) { string_value->NativeValue(absl::Overload( @@ -1074,35 +1157,35 @@ class MessageValueBuilderImpl { [this, field](const absl::Cord& cord) { reflection_->SetString(message_, field, cord); })); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "string"); + return TypeConversionError(value.GetTypeName(), "string", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { if (field->enum_type()->full_name() == "google.protobuf.NullValue") { if (value.IsNull() || value.IsInt()) { reflection_->SetEnumValue(message_, field, 0); - return absl::nullopt; + return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "null_type"); + return TypeConversionError(value.GetTypeName(), "null_type", arena_); } if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() >= std::numeric_limits::min() && int_value->NativeValue() <= std::numeric_limits::max()) { reflection_->SetEnumValue( message_, field, static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } } return TypeConversionError(value.GetTypeName(), - field->enum_type()->full_name()); + field->enum_type()->full_name(), arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { switch (field->message_type()->well_known_type()) { case google::protobuf::Descriptor::WELLKNOWNTYPE_BOOLVALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto bool_value = value.AsBool(); bool_value) { CEL_RETURN_IF_ERROR(well_known_types_.BoolValue().Initialize( @@ -1111,15 +1194,16 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), bool_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_INT32VALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < @@ -1134,15 +1218,16 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_INT64VALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto int_value = value.AsInt(); int_value) { CEL_RETURN_IF_ERROR(well_known_types_.Int64Value().Initialize( @@ -1151,15 +1236,16 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), int_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT32VALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto uint_value = value.AsUint(); uint_value) { if (uint_value->NativeValue() > @@ -1172,15 +1258,16 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), static_cast(uint_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT64VALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto uint_value = value.AsUint(); uint_value) { CEL_RETURN_IF_ERROR(well_known_types_.UInt64Value().Initialize( @@ -1189,15 +1276,16 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), uint_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_FLOATVALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto double_value = value.AsDouble(); double_value) { CEL_RETURN_IF_ERROR(well_known_types_.FloatValue().Initialize( @@ -1206,15 +1294,16 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), static_cast(double_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_DOUBLEVALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto double_value = value.AsDouble(); double_value) { CEL_RETURN_IF_ERROR(well_known_types_.DoubleValue().Initialize( @@ -1223,15 +1312,16 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), double_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_BYTESVALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto bytes_value = value.AsBytes(); bytes_value) { CEL_RETURN_IF_ERROR(well_known_types_.BytesValue().Initialize( @@ -1240,15 +1330,16 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), bytes_value->NativeCord()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_STRINGVALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto string_value = value.AsString(); string_value) { CEL_RETURN_IF_ERROR(well_known_types_.StringValue().Initialize( @@ -1257,66 +1348,69 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), string_value->NativeCord()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_DURATION: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto duration_value = value.AsDuration(); duration_value) { CEL_RETURN_IF_ERROR(well_known_types_.Duration().Initialize( field->message_type())); - CEL_RETURN_IF_ERROR( - well_known_types_.Duration().SetFromAbslDuration( - reflection_->MutableMessage(message_, field, - message_factory_), - duration_value->NativeValue())); - return absl::nullopt; + + well_known_types_.Duration().UnsafeSetFromAbslDuration( + reflection_->MutableMessage(message_, field, + message_factory_), + duration_value->NativeValue()); + return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_TIMESTAMP: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto timestamp_value = value.AsTimestamp(); timestamp_value) { CEL_RETURN_IF_ERROR(well_known_types_.Timestamp().Initialize( field->message_type())); - CEL_RETURN_IF_ERROR(well_known_types_.Timestamp().SetFromAbslTime( + well_known_types_.Timestamp().UnsafeSetFromAbslTime( reflection_->MutableMessage(message_, field, message_factory_), - timestamp_value->NativeValue())); - return absl::nullopt; + timestamp_value->NativeValue()); + return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE: { CEL_RETURN_IF_ERROR( value.ConvertToJson(descriptor_pool_, message_factory_, reflection_->MutableMessage( message_, field, message_factory_))); - return absl::nullopt; + return std::nullopt; } case google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE: { CEL_RETURN_IF_ERROR(value.ConvertToJsonArray( descriptor_pool_, message_factory_, reflection_->MutableMessage(message_, field, message_factory_))); - return absl::nullopt; + return std::nullopt; } case google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT: { CEL_RETURN_IF_ERROR(value.ConvertToJsonObject( descriptor_pool_, message_factory_, reflection_->MutableMessage(message_, field, message_factory_))); - return absl::nullopt; + return std::nullopt; } case google::protobuf::Descriptor::WELLKNOWNTYPE_ANY: { // Probably not correct, need to use the parent/common one. @@ -1370,18 +1464,19 @@ class MessageValueBuilderImpl { well_known_types_.Any().SetValue( reflection_->MutableMessage(message_, field, message_factory_), std::move(serialized).Consume()); - return absl::nullopt; + return std::nullopt; } default: if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } break; } return ProtoMessageFromValueImpl( value, descriptor_pool_, message_factory_, &well_known_types_, - reflection_->MutableMessage(message_, field, message_factory_)); + reflection_->MutableMessage(message_, field, message_factory_), + arena_); } default: return absl::InternalError( diff --git a/common/values/timestamp_value.cc b/common/values/timestamp_value.cc index 7d3a347e8..0aff5f3d7 100644 --- a/common/values/timestamp_value.cc +++ b/common/values/timestamp_value.cc @@ -75,6 +75,8 @@ absl::Status TimestampValue::ConvertToJson( ABSL_DCHECK_EQ(json->GetDescriptor()->well_known_type(), google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE); + CEL_RETURN_IF_ERROR(cel::internal::ValidateTimestamp(NativeValue())); + ValueReflection value_reflection; CEL_RETURN_IF_ERROR(value_reflection.Initialize(json->GetDescriptor())); value_reflection.SetStringValueFromTimestamp(json, NativeValue()); diff --git a/common/values/timestamp_value.h b/common/values/timestamp_value.h index acc202300..87eae0ed0 100644 --- a/common/values/timestamp_value.h +++ b/common/values/timestamp_value.h @@ -52,6 +52,12 @@ class TimestampValue final public: static constexpr ValueKind kKind = ValueKind::kTimestamp; + // Constructs a `TimestampValue` from an `absl::Time`. + // + // DCHECK-fails if the value is not in the supported range. + // + // Prefer using `SafeTimestampValue` or `UnsafeTimestampValue` if the caller + // has already validated the value. explicit TimestampValue(absl::Time value) noexcept : TimestampValue(absl::in_place, value) { ABSL_DCHECK_OK(internal::ValidateTimestamp(value)); diff --git a/common/values/timestamp_value_test.cc b/common/values/timestamp_value_test.cc index 142e6511d..ff34c8000 100644 --- a/common/values/timestamp_value_test.cc +++ b/common/values/timestamp_value_test.cc @@ -12,8 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "common/values/timestamp_value.h" + #include +#include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/time/time.h" #include "common/native_type.h" @@ -56,6 +59,16 @@ TEST_F(TimestampValueTest, ConvertToJson) { R"pb(string_value: "1970-01-01T00:00:00Z")pb")); } +TEST_F(TimestampValueTest, ConvertToJsonOutOfBounds) { + auto* message = NewArenaValueMessage(); + EXPECT_THAT(UnsafeTimestampValue(absl::InfiniteFuture()) + .ConvertToJson(descriptor_pool(), message_factory(), message), + absl_testing::StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(UnsafeTimestampValue(absl::InfinitePast()) + .ConvertToJson(descriptor_pool(), message_factory(), message), + absl_testing::StatusIs(absl::StatusCode::kInvalidArgument)); +} + TEST_F(TimestampValueTest, NativeTypeId) { EXPECT_EQ( NativeTypeId::Of(TimestampValue(absl::UnixEpoch() + absl::Seconds(1))), diff --git a/common/values/unknown_value.cc b/common/values/unknown_value.cc index 1cb8a7674..dd07b251c 100644 --- a/common/values/unknown_value.cc +++ b/common/values/unknown_value.cc @@ -12,10 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include +#include +#include +#include + #include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" +#include "base/attribute.h" +#include "base/attribute_set.h" +#include "base/function_result.h" +#include "base/function_result_set.h" +#include "common/internal/unknowns.h" +#include "common/unknown.h" #include "common/value.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -63,4 +76,88 @@ absl::Status UnknownValue::Equal( return absl::OkStatus(); } +[[nodiscard]] +AttributeSet UnknownValue::ToAttributeSet() const { + AttributeSet attribute_set; + if (rep_.attributes == nullptr || rep_.attributes->size == 0) { + return attribute_set; + } + std::vector path; + path.resize(rep_.attributes->max_depth - 1); + for (uint32_t i = 0; i < rep_.attributes->size; ++i) { + const common_internal::UnknownAttributeNode* leaf = + rep_.attributes->nodes[i]; + const size_t num_qualifiers = leaf->depth() - 1; + size_t j = num_qualifiers; + while (leaf->parent() != rep_.root->Attributes()) { + path[--j] = leaf; + leaf = leaf->parent(); + } + // leaf is now the ancestor which is a child of root_, also known as our + // variable name. + std::vector qualifiers; + qualifiers.reserve(num_qualifiers); + for (j = 0; j < num_qualifiers; ++j) { + qualifiers.push_back(AttributeQualifier(path[j]->key())); + } + attribute_set.Add( + Attribute(std::string(leaf->key().GetString()), std::move(qualifiers))); + } + return attribute_set; +} + +FunctionResultSet UnknownValue::ToFunctionResultSet() const { + FunctionResultSet function_set; + if (rep_.functions == nullptr || rep_.functions->size == 0) { + return function_set; + } + for (size_t i = 0; i < rep_.functions->size; ++i) { + function_set.Add(FunctionResult(rep_.functions->nodes[i]->key())); + } + return function_set; +} + +namespace common_internal { + +UnknownValue MakeUnknownValue(const Unknown& value, + google::protobuf::Arena* absl_nonnull arena) { + if (value.unknown_attributes().empty() && + value.unknown_function_results().empty()) { + return UnknownValue(); + } + common_internal::UnknownTree tree(arena); + common_internal::UnknownRoot* root = tree.Root(); + common_internal::UnknownAttributeSet* attributes = nullptr; + common_internal::UnknownFunctionSet* functions = nullptr; + if (!value.unknown_attributes().empty()) { + attributes = common_internal::UnknownAttributeSet::Allocate( + value.unknown_attributes().size(), arena); + attributes->max_depth = 0; + size_t i = 0; + for (const auto& attribute : value.unknown_attributes()) { + common_internal::UnknownAttributeNode* node = root->Attributes()->Step( + &tree, AttributeQualifierView::OfString(attribute.variable_name())); + for (const auto& attribute_qualifier : attribute.qualifier_path()) { + node = node->Step(&tree, attribute_qualifier); + } + attributes->max_depth = + std::max(attributes->max_depth, static_cast(node->depth())); + attributes->nodes[i++] = node; + } + std::sort(attributes->nodes, attributes->nodes + attributes->size); + } + if (!value.unknown_function_results().empty()) { + functions = common_internal::UnknownFunctionSet::Allocate( + value.unknown_function_results().size(), arena); + size_t i = 0; + for (const auto& function_result : value.unknown_function_results()) { + functions->nodes[i++] = root->SetFunction(&tree, function_result.name()); + } + std::sort(functions->nodes, functions->nodes + functions->size); + } + return MakeUnknownValue(root, attributes, functions); +} + +} // namespace common_internal + } // namespace cel diff --git a/common/values/unknown_value.h b/common/values/unknown_value.h index 9e8ddaae0..13b952f79 100644 --- a/common/values/unknown_value.h +++ b/common/values/unknown_value.h @@ -20,12 +20,13 @@ #include #include -#include #include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/strings/string_view.h" +#include "base/attribute_set.h" +#include "common/internal/unknowns.h" #include "common/type.h" #include "common/unknown.h" #include "common/value_kind.h" @@ -40,13 +41,40 @@ namespace cel { class Value; class UnknownValue; -// `UnknownValue` represents values of the primitive `duration` type. +namespace common_internal { +[[nodiscard]] +UnknownValue MakeUnknownValue(const Unknown& value, + google::protobuf::Arena* absl_nonnull arena); +[[nodiscard]] +UnknownValue MakeUnknownValue( + const common_internal::UnknownRoot* root, + const common_internal::UnknownAttributeSet* attributes, + const common_internal::UnknownFunctionSet* functions); +[[nodiscard]] +Unknown ToUnknown(const UnknownValue& value); +using UnknownValueRep = UnknownSet; +[[nodiscard]] +UnknownValue MakeUnknownValue(const UnknownValueRep& rep); +[[nodiscard]] +const UnknownValueRep& GetUnknownValueRep( + const UnknownValue& value ABSL_ATTRIBUTE_LIFETIME_BOUND); +[[nodiscard]] +UnknownValueRep& GetUnknownValueRep( + UnknownValue& value ABSL_ATTRIBUTE_LIFETIME_BOUND); +} // namespace common_internal + +// `UnknownValue` represents matches to unknown attribute patterns encountered +// during evaluation. class UnknownValue final : private common_internal::ValueMixin { public: static constexpr ValueKind kKind = ValueKind::kUnknown; - explicit UnknownValue(Unknown unknown) : unknown_(std::move(unknown)) {} - + // There is currently no supported way for clients create "well-formed" + // unknown values, only the evaluator may create them. The reasons for this + // are performance and correctness, by hiding the internal representation we + // can easily change it as necessary. If it became absolutely necessary due to + // an appropriate use case, we can introduce a way for clients to create them + // as well without exposing the internal implementation details. UnknownValue() = default; UnknownValue(const UnknownValue&) = default; UnknownValue(UnknownValue&&) = default; @@ -80,42 +108,77 @@ class UnknownValue final : private common_internal::ValueMixin { bool IsZeroValue() const { return false; } - void swap(UnknownValue& other) noexcept { - using std::swap; - swap(unknown_, other.unknown_); - } - - const Unknown& NativeValue() const& ABSL_ATTRIBUTE_LIFETIME_BOUND { - return unknown_; - } - - Unknown NativeValue() && { - Unknown unknown = std::move(unknown_); - return unknown; - } + [[nodiscard]] + AttributeSet ToAttributeSet() const; - const AttributeSet& attribute_set() const { - return unknown_.unknown_attributes(); - } - - const FunctionResultSet& function_result_set() const { - return unknown_.unknown_function_results(); - } + [[nodiscard]] + FunctionResultSet ToFunctionResultSet() const; private: + friend UnknownValue common_internal::MakeUnknownValue( + const common_internal::UnknownRoot* root, + const common_internal::UnknownAttributeSet* attributes, + const common_internal::UnknownFunctionSet* functions); + friend Unknown common_internal::ToUnknown(const UnknownValue& value); + friend const common_internal::UnknownValueRep& + common_internal::GetUnknownValueRep(const UnknownValue& value); + friend common_internal::UnknownValueRep& common_internal::GetUnknownValueRep( + UnknownValue& value); + friend UnknownValue common_internal::MakeUnknownValue( + const common_internal::UnknownValueRep& rep); friend class common_internal::ValueMixin; - Unknown unknown_; -}; + UnknownValue(const common_internal::UnknownRoot* root, + const common_internal::UnknownAttributeSet* attributes, + const common_internal::UnknownFunctionSet* functions) + : rep_{root, attributes, functions} {} -inline void swap(UnknownValue& lhs, UnknownValue& rhs) noexcept { - lhs.swap(rhs); -} + explicit UnknownValue(const common_internal::UnknownValueRep& rep) + : rep_(rep) {} + + common_internal::UnknownValueRep rep_; +}; inline std::ostream& operator<<(std::ostream& out, const UnknownValue& value) { return out << value.DebugString(); } +namespace common_internal { + +const FunctionResultSet& GetEmptyFunctionResults(); + +[[nodiscard]] +inline UnknownValue MakeUnknownValue( + const common_internal::UnknownRoot* root, + const common_internal::UnknownAttributeSet* attributes, + const common_internal::UnknownFunctionSet* functions) { + return UnknownValue(root, attributes, functions); +} + +[[nodiscard]] +inline Unknown ToUnknown(const UnknownValue& value) { + return Unknown(value.ToAttributeSet(), value.ToFunctionResultSet()); +} + +[[nodiscard]] +inline UnknownValue MakeUnknownValue(const UnknownValueRep& rep) { + return UnknownValue(rep); +} + +[[nodiscard]] +inline const UnknownValueRep& GetUnknownValueRep( + const UnknownValue& value ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return value.rep_; +} + +[[nodiscard]] +inline UnknownValueRep& GetUnknownValueRep( + UnknownValue& value ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return value.rep_; +} + +} // namespace common_internal + } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_VALUES_UNKNOWN_VALUE_H_ diff --git a/common/values/value_builder.cc b/common/values/value_builder.cc index 979837411..526bd4adc 100644 --- a/common/values/value_builder.cc +++ b/common/values/value_builder.cc @@ -36,7 +36,6 @@ #include "absl/types/optional.h" #include "absl/types/span.h" #include "common/allocator.h" -#include "common/arena.h" #include "common/legacy_value.h" #include "common/native_type.h" #include "common/type.h" @@ -206,12 +205,6 @@ class ListValueBuilderImpl final : public ListValueBuilder { elements_.Construct(arena); } - ~ListValueBuilderImpl() override { - if (!elements_trivially_destructible_) { - elements_.Destruct(); - } - } - absl::Status Add(Value value) override { CEL_RETURN_IF_ERROR(CheckListElement(value)); UnsafeAdd(std::move(value)); @@ -221,10 +214,6 @@ class ListValueBuilderImpl final : public ListValueBuilder { void UnsafeAdd(Value value) override { ABSL_DCHECK_OK(CheckListElement(value)); elements_->emplace_back(std::move(value)); - if (elements_trivially_destructible_) { - elements_trivially_destructible_ = - ArenaTraits<>::trivially_destructible(elements_->back()); - } } size_t Size() const override { return elements_->size(); } @@ -243,7 +232,6 @@ class ListValueBuilderImpl final : public ListValueBuilder { private: google::protobuf::Arena* absl_nonnull const arena_; internal::Manual elements_; - bool elements_trivially_destructible_ = true; }; class CompatListValueImpl final : public CompatListValue { @@ -251,6 +239,10 @@ class CompatListValueImpl final : public CompatListValue { explicit CompatListValueImpl(ValueVector&& elements) : elements_(std::move(elements)) {} + google::protobuf::Arena* GetOwningArena() const override { + return elements_.get_allocator().arena(); + } + std::string DebugString() const override { return absl::StrCat("[", absl::StrJoin(elements_, ", ", ValueFormatter{}), "]"); @@ -309,7 +301,7 @@ class CompatListValueImpl final : public CompatListValue { } if (ABSL_PREDICT_FALSE(index < 0 || index >= size())) { return CelValue::CreateError(google::protobuf::Arena::Create( - arena, IndexOutOfBoundsError(index).ToStatus())); + arena, common_internal::MakeIndexOutOfBoundsError(index))); } return common_internal::UnsafeLegacyValue( elements_[index], @@ -326,7 +318,7 @@ class CompatListValueImpl final : public CompatListValue { google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const override { if (index >= elements_.size()) { - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); } else { *result = elements_[index]; } @@ -337,19 +329,6 @@ class CompatListValueImpl final : public CompatListValue { const ValueVector elements_; }; -} // namespace - -} // namespace common_internal - -template <> -struct ArenaTraits { - using always_trivially_destructible = std::true_type; -}; - -namespace common_internal { - -namespace { - ListValue ListValueBuilderImpl::Build() && { if (elements_->empty()) { return ListValue(); @@ -376,10 +355,6 @@ const CompatListValue* absl_nonnull ListValueBuilderImpl::BuildCompatAt( void* absl_nonnull address) && { CompatListValueImpl* absl_nonnull impl = ::new (address) CompatListValueImpl(std::move(*elements_)); - if (!elements_trivially_destructible_) { - arena_->OwnDestructor(impl); - elements_trivially_destructible_ = true; - } return impl; } @@ -388,6 +363,10 @@ class MutableCompatListValueImpl final : public MutableCompatListValue { explicit MutableCompatListValueImpl(google::protobuf::Arena* absl_nonnull arena) : elements_(arena) {} + google::protobuf::Arena* GetOwningArena() const override { + return elements_.get_allocator().arena(); + } + std::string DebugString() const override { return absl::StrCat("[", absl::StrJoin(elements_, ", ", ValueFormatter{}), "]"); @@ -446,7 +425,7 @@ class MutableCompatListValueImpl final : public MutableCompatListValue { } if (ABSL_PREDICT_FALSE(index < 0 || index >= size())) { return CelValue::CreateError(google::protobuf::Arena::Create( - arena, IndexOutOfBoundsError(index).ToStatus())); + arena, common_internal::MakeIndexOutOfBoundsError(index))); } return common_internal::UnsafeLegacyValue( elements_[index], /*stable=*/false, @@ -458,14 +437,6 @@ class MutableCompatListValueImpl final : public MutableCompatListValue { absl::Status Append(Value value) const override { CEL_RETURN_IF_ERROR(CheckListElement(value)); elements_.emplace_back(std::move(value)); - if (elements_trivially_destructible_) { - elements_trivially_destructible_ = - ArenaTraits<>::trivially_destructible(elements_.back()); - if (!elements_trivially_destructible_) { - elements_.get_allocator().arena()->OwnDestructor( - const_cast(this)); - } - } return absl::OkStatus(); } @@ -478,7 +449,7 @@ class MutableCompatListValueImpl final : public MutableCompatListValue { google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const override { if (index >= elements_.size()) { - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); } else { *result = elements_[index]; } @@ -487,24 +458,10 @@ class MutableCompatListValueImpl final : public MutableCompatListValue { private: mutable ValueVector elements_; - mutable bool elements_trivially_destructible_ = true; }; } // namespace -} // namespace common_internal - -template <> -struct ArenaTraits { - using constructible = std::true_type; - - using always_trivially_destructible = std::true_type; -}; - -namespace common_internal { - -namespace {} // namespace - absl::StatusOr MakeCompatListValue( const CustomListValue& value, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, @@ -797,8 +754,8 @@ absl::StatusOr ValueToJsonString(const Value& value) { case ValueKind::kString: return value.GetString().NativeString(); default: - return TypeConversionError(value.GetRuntimeType(), StringType()) - .ToStatus(); + return common_internal::MakeTypeConversionError(value.GetRuntimeType(), + StringType()); } } @@ -949,12 +906,6 @@ class MapValueBuilderImpl final : public MapValueBuilder { map_.Construct(arena_); } - ~MapValueBuilderImpl() override { - if (!entries_trivially_destructible_) { - map_.Destruct(); - } - } - absl::Status Put(Value key, Value value) override { CEL_RETURN_IF_ERROR(CheckMapKey(key)); CEL_RETURN_IF_ERROR(CheckMapValue(value)); @@ -968,11 +919,6 @@ class MapValueBuilderImpl final : public MapValueBuilder { void UnsafePut(Value key, Value value) override { auto insertion = map_->insert({std::move(key), std::move(value)}); ABSL_DCHECK(insertion.second); - if (entries_trivially_destructible_) { - entries_trivially_destructible_ = - ArenaTraits<>::trivially_destructible(insertion.first->first) && - ArenaTraits<>::trivially_destructible(insertion.first->second); - } } size_t Size() const override { return map_->size(); } @@ -988,13 +934,16 @@ class MapValueBuilderImpl final : public MapValueBuilder { private: google::protobuf::Arena* absl_nonnull const arena_; internal::Manual map_; - bool entries_trivially_destructible_ = true; }; class CompatMapValueImpl final : public CompatMapValue { public: explicit CompatMapValueImpl(ValueFlatHashMap&& map) : map_(std::move(map)) {} + google::protobuf::Arena* GetOwningArena() const override { + return map_.get_allocator().arena(); + } + std::string DebugString() const override { return absl::StrCat("{", absl::StrJoin(map_, ", ", ValueFormatter{}), "}"); } @@ -1055,14 +1004,14 @@ class CompatMapValueImpl final : public CompatMapValue { CelValue key) const override { if (auto status = CelValue::CheckMapKeyType(key); !status.ok()) { status.IgnoreError(); - return absl::nullopt; + return std::nullopt; } if (auto it = map_.find(key); it != map_.end()) { return common_internal::UnsafeLegacyValue( it->second, /*stable=*/true, arena != nullptr ? arena : map_.get_allocator().arena()); } - return absl::nullopt; + return std::nullopt; } absl::StatusOr Has(const CelValue& key) const override { @@ -1147,18 +1096,18 @@ const CompatMapValue* absl_nonnull MapValueBuilderImpl::BuildCompat() && { CompatMapValueImpl* absl_nonnull impl = ::new (arena_->AllocateAligned( sizeof(CompatMapValueImpl), alignof(CompatMapValueImpl))) CompatMapValueImpl(std::move(*map_)); - if (!entries_trivially_destructible_) { - arena_->OwnDestructor(impl); - entries_trivially_destructible_ = true; - } return impl; } -class TrivialMutableMapValueImpl final : public MutableCompatMapValue { +class MutableCompatMapValueImpl final : public MutableCompatMapValue { public: - explicit TrivialMutableMapValueImpl(google::protobuf::Arena* absl_nonnull arena) + explicit MutableCompatMapValueImpl(google::protobuf::Arena* absl_nonnull arena) : map_(arena) {} + google::protobuf::Arena* GetOwningArena() const override { + return map_.get_allocator().arena(); + } + std::string DebugString() const override { return absl::StrCat("{", absl::StrJoin(map_, ", ", ValueFormatter{}), "}"); } @@ -1219,14 +1168,14 @@ class TrivialMutableMapValueImpl final : public MutableCompatMapValue { CelValue key) const override { if (auto status = CelValue::CheckMapKeyType(key); !status.ok()) { status.IgnoreError(); - return absl::nullopt; + return std::nullopt; } if (auto it = map_.find(key); it != map_.end()) { return common_internal::UnsafeLegacyValue( it->second, /*stable=*/false, arena != nullptr ? arena : map_.get_allocator().arena()); } - return absl::nullopt; + return std::nullopt; } absl::StatusOr Has(const CelValue& key) const override { @@ -1253,15 +1202,6 @@ class TrivialMutableMapValueImpl final : public MutableCompatMapValue { } auto insertion = map_.insert({std::move(key), std::move(value)}); ABSL_DCHECK(insertion.second); - if (entries_trivially_destructible_) { - entries_trivially_destructible_ = - ArenaTraits<>::trivially_destructible(insertion.first->first) && - ArenaTraits<>::trivially_destructible(insertion.first->second); - if (!entries_trivially_destructible_) { - map_.get_allocator().arena()->OwnDestructor( - const_cast(this)); - } - } return absl::OkStatus(); } @@ -1308,7 +1248,6 @@ class TrivialMutableMapValueImpl final : public MutableCompatMapValue { } mutable ValueFlatHashMap map_; - mutable bool entries_trivially_destructible_ = true; mutable absl::once_flag keys_once_; alignas(CompatListValueImpl) mutable char keys_[sizeof(CompatListValueImpl)]; }; @@ -1335,9 +1274,9 @@ absl::StatusOr MakeCompatMapValue( MutableMapValue* absl_nonnull NewMutableMapValue( google::protobuf::Arena* absl_nonnull arena) { - return ::new (arena->AllocateAligned(sizeof(TrivialMutableMapValueImpl), - alignof(TrivialMutableMapValueImpl))) - TrivialMutableMapValueImpl(arena); + return ::new (arena->AllocateAligned(sizeof(MutableCompatMapValueImpl), + alignof(MutableCompatMapValueImpl))) + MutableCompatMapValueImpl(arena); } bool IsMutableMapValue(const Value& value) { diff --git a/common/values/value_variant.cc b/common/values/value_variant.cc deleted file mode 100644 index 1c287239c..000000000 --- a/common/values/value_variant.cc +++ /dev/null @@ -1,537 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/values/value_variant.h" - -#include -#include -#include -#include - -#include "absl/base/optimization.h" -#include "absl/log/absl_check.h" -#include "common/values/bytes_value.h" -#include "common/values/error_value.h" -#include "common/values/string_value.h" -#include "common/values/unknown_value.h" -#include "common/values/values.h" - -namespace cel::common_internal { - -void ValueVariant::SlowCopyConstruct(const ValueVariant& other) noexcept { - ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); - - switch (index_) { - case ValueIndex::kBytes: - ::new (static_cast(&raw_[0])) BytesValue(*other.At()); - break; - case ValueIndex::kString: - ::new (static_cast(&raw_[0])) - StringValue(*other.At()); - break; - case ValueIndex::kError: - ::new (static_cast(&raw_[0])) ErrorValue(*other.At()); - break; - case ValueIndex::kUnknown: - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - break; - default: - ABSL_UNREACHABLE(); - } -} - -void ValueVariant::SlowMoveConstruct(ValueVariant& other) noexcept { - ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); - - switch (index_) { - case ValueIndex::kBytes: - ::new (static_cast(&raw_[0])) - BytesValue(std::move(*other.At())); - break; - case ValueIndex::kString: - ::new (static_cast(&raw_[0])) - StringValue(std::move(*other.At())); - break; - case ValueIndex::kError: - ::new (static_cast(&raw_[0])) - ErrorValue(std::move(*other.At())); - break; - case ValueIndex::kUnknown: - ::new (static_cast(&raw_[0])) - UnknownValue(std::move(*other.At())); - break; - default: - ABSL_UNREACHABLE(); - } -} - -void ValueVariant::SlowDestruct() noexcept { - ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); - - switch (index_) { - case ValueIndex::kBytes: - At()->~BytesValue(); - break; - case ValueIndex::kString: - At()->~StringValue(); - break; - case ValueIndex::kError: - At()->~ErrorValue(); - break; - case ValueIndex::kUnknown: - At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } -} - -void ValueVariant::SlowCopyAssign(const ValueVariant& other, bool trivial, - bool other_trivial) noexcept { - ABSL_DCHECK(!trivial || !other_trivial); - - if (trivial) { - switch (other.index_) { - case ValueIndex::kBytes: - ::new (static_cast(&raw_[0])) - BytesValue(*other.At()); - break; - case ValueIndex::kString: - ::new (static_cast(&raw_[0])) - StringValue(*other.At()); - break; - case ValueIndex::kError: - ::new (static_cast(&raw_[0])) - ErrorValue(*other.At()); - break; - case ValueIndex::kUnknown: - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - break; - default: - ABSL_UNREACHABLE(); - } - index_ = other.index_; - kind_ = other.kind_; - flags_ = other.flags_; - } else if (other_trivial) { - switch (index_) { - case ValueIndex::kBytes: - At()->~BytesValue(); - break; - case ValueIndex::kString: - At()->~StringValue(); - break; - case ValueIndex::kError: - At()->~ErrorValue(); - break; - case ValueIndex::kUnknown: - At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } - FastCopyAssign(other); - } else { - switch (index_) { - case ValueIndex::kBytes: - switch (other.index_) { - case ValueIndex::kBytes: - *At() = *other.At(); - break; - case ValueIndex::kString: - At()->~BytesValue(); - ::new (static_cast(&raw_[0])) - StringValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kError: - At()->~BytesValue(); - ::new (static_cast(&raw_[0])) - ErrorValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kUnknown: - At()->~BytesValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; - case ValueIndex::kString: - switch (other.index_) { - case ValueIndex::kBytes: - At()->~StringValue(); - ::new (static_cast(&raw_[0])) - BytesValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kString: - *At() = *other.At(); - break; - case ValueIndex::kError: - At()->~StringValue(); - ::new (static_cast(&raw_[0])) - ErrorValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kUnknown: - At()->~StringValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; - case ValueIndex::kError: - switch (other.index_) { - case ValueIndex::kBytes: - At()->~ErrorValue(); - ::new (static_cast(&raw_[0])) - BytesValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kString: - At()->~ErrorValue(); - ::new (static_cast(&raw_[0])) - StringValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kError: - *At() = *other.At(); - break; - case ValueIndex::kUnknown: - At()->~ErrorValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; - case ValueIndex::kUnknown: - switch (other.index_) { - case ValueIndex::kBytes: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - BytesValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kString: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - StringValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kError: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - ErrorValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kUnknown: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; - default: - ABSL_UNREACHABLE(); - } - flags_ = other.flags_; - } -} - -void ValueVariant::SlowMoveAssign(ValueVariant& other, bool trivial, - bool other_trivial) noexcept { - ABSL_DCHECK(!trivial || !other_trivial); - - if (trivial) { - switch (other.index_) { - case ValueIndex::kBytes: - ::new (static_cast(&raw_[0])) - BytesValue(std::move(*other.At())); - break; - case ValueIndex::kString: - ::new (static_cast(&raw_[0])) - StringValue(std::move(*other.At())); - break; - case ValueIndex::kError: - ::new (static_cast(&raw_[0])) - ErrorValue(std::move(*other.At())); - break; - case ValueIndex::kUnknown: - ::new (static_cast(&raw_[0])) - UnknownValue(std::move(*other.At())); - break; - default: - ABSL_UNREACHABLE(); - } - index_ = other.index_; - kind_ = other.kind_; - flags_ = other.flags_; - } else if (other_trivial) { - switch (index_) { - case ValueIndex::kBytes: - At()->~BytesValue(); - break; - case ValueIndex::kString: - At()->~StringValue(); - break; - case ValueIndex::kError: - At()->~ErrorValue(); - break; - case ValueIndex::kUnknown: - At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } - FastMoveAssign(other); - } else { - switch (index_) { - case ValueIndex::kBytes: - switch (other.index_) { - case ValueIndex::kBytes: - *At() = std::move(*other.At()); - break; - case ValueIndex::kString: - At()->~BytesValue(); - ::new (static_cast(&raw_[0])) - StringValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kError: - At()->~BytesValue(); - ::new (static_cast(&raw_[0])) - ErrorValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kUnknown: - At()->~BytesValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; - case ValueIndex::kString: - switch (other.index_) { - case ValueIndex::kBytes: - At()->~StringValue(); - ::new (static_cast(&raw_[0])) - BytesValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kString: - *At() = std::move(*other.At()); - break; - case ValueIndex::kError: - At()->~StringValue(); - ::new (static_cast(&raw_[0])) - ErrorValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kUnknown: - At()->~StringValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; - case ValueIndex::kError: - switch (other.index_) { - case ValueIndex::kBytes: - At()->~ErrorValue(); - ::new (static_cast(&raw_[0])) - BytesValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kString: - At()->~ErrorValue(); - ::new (static_cast(&raw_[0])) - StringValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kError: - *At() = std::move(*other.At()); - break; - case ValueIndex::kUnknown: - At()->~ErrorValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; - case ValueIndex::kUnknown: - switch (other.index_) { - case ValueIndex::kBytes: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - BytesValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kString: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - StringValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kError: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - ErrorValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kUnknown: - *At() = std::move(*other.At()); - break; - default: - ABSL_UNREACHABLE(); - } - break; - default: - ABSL_UNREACHABLE(); - } - flags_ = other.flags_; - } -} - -void ValueVariant::SlowSwap(ValueVariant& lhs, ValueVariant& rhs, - bool lhs_trivial, bool rhs_trivial) noexcept { - using std::swap; - ABSL_DCHECK(!lhs_trivial || !rhs_trivial); - - if (lhs_trivial) { - alignas(ValueVariant) std::byte tmp[sizeof(ValueVariant)]; - // This is acceptable. We know that both are trivially copyable at runtime. - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(tmp, std::addressof(lhs), sizeof(ValueVariant)); - switch (rhs.index_) { - case ValueIndex::kBytes: - ::new (static_cast(&lhs.raw_[0])) - BytesValue(*rhs.At()); - rhs.At()->~BytesValue(); - break; - case ValueIndex::kString: - ::new (static_cast(&lhs.raw_[0])) - StringValue(*rhs.At()); - rhs.At()->~StringValue(); - break; - case ValueIndex::kError: - ::new (static_cast(&lhs.raw_[0])) - ErrorValue(*rhs.At()); - rhs.At()->~ErrorValue(); - break; - case ValueIndex::kUnknown: - ::new (static_cast(&lhs.raw_[0])) - UnknownValue(*rhs.At()); - rhs.At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } - lhs.index_ = rhs.index_; - lhs.kind_ = rhs.kind_; - lhs.flags_ = rhs.flags_; - // This is acceptable. We know that both are trivially copyable at runtime. - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(std::addressof(rhs), tmp, sizeof(ValueVariant)); - } else if (rhs_trivial) { - alignas(ValueVariant) std::byte tmp[sizeof(ValueVariant)]; - // This is acceptable. We know that both are trivially copyable at runtime. - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(tmp, std::addressof(rhs), sizeof(ValueVariant)); - switch (lhs.index_) { - case ValueIndex::kBytes: - ::new (static_cast(&rhs.raw_[0])) - BytesValue(*lhs.At()); - lhs.At()->~BytesValue(); - break; - case ValueIndex::kString: - ::new (static_cast(&rhs.raw_[0])) - StringValue(*lhs.At()); - lhs.At()->~StringValue(); - break; - case ValueIndex::kError: - ::new (static_cast(&rhs.raw_[0])) - ErrorValue(*lhs.At()); - lhs.At()->~ErrorValue(); - break; - case ValueIndex::kUnknown: - ::new (static_cast(&rhs.raw_[0])) - UnknownValue(*lhs.At()); - lhs.At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } - rhs.index_ = lhs.index_; - rhs.kind_ = lhs.kind_; - rhs.flags_ = lhs.flags_; - // This is acceptable. We know that both are trivially copyable at runtime. - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(std::addressof(lhs), tmp, sizeof(ValueVariant)); - } else { - ValueVariant tmp = std::move(lhs); - lhs = std::move(rhs); - rhs = std::move(tmp); - } -} - -} // namespace cel::common_internal diff --git a/common/values/value_variant.h b/common/values/value_variant.h index b05511e3c..aaf0766d2 100644 --- a/common/values/value_variant.h +++ b/common/values/value_variant.h @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -27,8 +26,6 @@ #include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/meta/type_traits.h" -#include "absl/utility/utility.h" -#include "common/arena.h" #include "common/value_kind.h" #include "common/values/bool_value.h" #include "common/values/bytes_value.h" @@ -86,27 +83,12 @@ enum class ValueIndex : uint8_t { kParsedMessage, kCustomStruct, kOpaque, - - // Keep non-trivial alternatives together to aid in compiling optimizations. kBytes, kString, kError, kUnknown, }; -// Used by ValueVariant to indicate pre-computed behaviors. -enum class ValueFlags : uint32_t { - kNone = 0, - kNonTrivial = 1, -}; - -ABSL_ATTRIBUTE_ALWAYS_INLINE inline constexpr ValueFlags operator&( - ValueFlags lhs, ValueFlags rhs) { - return static_cast( - static_cast>(lhs) & - static_cast>(rhs)); -} - // Traits specialized by each alternative. // // ValueIndex ValueAlternative::kIndex @@ -131,166 +113,90 @@ template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kNull; static constexpr ValueKind kKind = NullValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const NullValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kBool; static constexpr ValueKind kKind = BoolValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const BoolValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kInt; static constexpr ValueKind kKind = IntValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const IntValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kUint; static constexpr ValueKind kKind = UintValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const UintValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kDouble; static constexpr ValueKind kKind = DoubleValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const DoubleValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kDuration; static constexpr ValueKind kKind = DurationValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const DurationValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kTimestamp; static constexpr ValueKind kKind = TimestampValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const TimestampValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kType; static constexpr ValueKind kKind = TypeValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const TypeValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kLegacyList; static constexpr ValueKind kKind = LegacyListValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const LegacyListValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedJsonList; static constexpr ValueKind kKind = ParsedJsonListValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const ParsedJsonListValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedRepeatedField; static constexpr ValueKind kKind = ParsedRepeatedFieldValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags( - const ParsedRepeatedFieldValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kCustomList; static constexpr ValueKind kKind = CustomListValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const CustomListValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kLegacyMap; static constexpr ValueKind kKind = LegacyMapValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const LegacyMapValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedJsonMap; static constexpr ValueKind kKind = ParsedJsonMapValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const ParsedJsonMapValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedMapField; static constexpr ValueKind kKind = ParsedMapFieldValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const ParsedMapFieldValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> @@ -298,104 +204,54 @@ struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kCustomMap; static constexpr ValueKind kKind = CustomMapValue::kKind; static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const CustomMapValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kLegacyStruct; static constexpr ValueKind kKind = LegacyStructValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const LegacyStructValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedMessage; static constexpr ValueKind kKind = ParsedMessageValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const ParsedMessageValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kCustomStruct; static constexpr ValueKind kKind = CustomStructValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const CustomStructValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kOpaque; static constexpr ValueKind kKind = OpaqueValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const OpaqueValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kBytes; static constexpr ValueKind kKind = BytesValue::kKind; - static constexpr bool kAlwaysTrivial = false; - - static ValueFlags Flags(const BytesValue* absl_nonnull alternative) { - return ArenaTraits::trivially_destructible(*alternative) - ? ValueFlags::kNone - : ValueFlags::kNonTrivial; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kString; static constexpr ValueKind kKind = StringValue::kKind; - static constexpr bool kAlwaysTrivial = false; - - static ValueFlags Flags(const StringValue* absl_nonnull alternative) { - return ArenaTraits::trivially_destructible(*alternative) - ? ValueFlags::kNone - : ValueFlags::kNonTrivial; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kError; static constexpr ValueKind kKind = ErrorValue::kKind; - static constexpr bool kAlwaysTrivial = false; - - static ValueFlags Flags(const ErrorValue* absl_nonnull alternative) { - return ArenaTraits::trivially_destructible(*alternative) - ? ValueFlags::kNone - : ValueFlags::kNonTrivial; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kUnknown; static constexpr ValueKind kKind = UnknownValue::kKind; - static constexpr bool kAlwaysTrivial = false; - - static constexpr ValueFlags Flags(const UnknownValue* absl_nonnull) { - return ValueFlags::kNonTrivial; - } }; template @@ -413,93 +269,33 @@ inline constexpr bool IsValueAlternativeV = IsValueAlternative::value; inline constexpr size_t kValueVariantAlign = 8; inline constexpr size_t kValueVariantSize = 24; -// Hand-rolled variant used by cel::Value which exhibits up to a 25% performance -// improvement compared to using std::variant. -// -// The implementation abuses the fact that most alternatives are trivially -// copyable and some are conditionally trivially copyable at runtime. For the -// fast path, we perform raw byte copying. For the slow path, we fallback to a -// non-inlined function. The compiler is typically smart enough to inline the -// fast path and emit efficient instructions for the raw byte copying (usually -// two instructions). It also uses switch for visiting, which most compilers can -// optimize better compared to a function pointer table (which libc++ currently -// uses and Clang currently does not optimize well). -class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI - ValueVariant final { +// Hand-rolled variant used by cel::Value which avoids std::variant and its +// lack of support for an unchecked get. +class alignas(kValueVariantAlign) ValueVariant { public: ValueVariant() = default; - - ValueVariant(const ValueVariant& other) noexcept - : index_(other.index_), kind_(other.kind_), flags_(other.flags_) { - if ((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone) { - std::memcpy(raw_, other.raw_, sizeof(raw_)); - } else { - SlowCopyConstruct(other); - } - } - - ValueVariant(ValueVariant&& other) noexcept - : index_(other.index_), kind_(other.kind_), flags_(other.flags_) { - if ((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone) { - std::memcpy(raw_, other.raw_, sizeof(raw_)); - } else { - SlowMoveConstruct(other); - } - } - - ~ValueVariant() { - if ((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial) { - SlowDestruct(); - } - } - - ValueVariant& operator=(const ValueVariant& other) noexcept { - if (this != &other) { - const bool trivial = - (flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - const bool other_trivial = - (other.flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - if (trivial && other_trivial) { - FastCopyAssign(other); - } else { - SlowCopyAssign(other, trivial, other_trivial); - } - } - return *this; - } - - ValueVariant& operator=(ValueVariant&& other) noexcept { - if (this != &other) { - const bool trivial = - (flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - const bool other_trivial = - (other.flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - if (trivial && other_trivial) { - FastMoveAssign(other); - } else { - SlowMoveAssign(other, trivial, other_trivial); - } - } - return *this; - } + ValueVariant(const ValueVariant&) = default; + ValueVariant& operator=(const ValueVariant&) = default; template - explicit ValueVariant(absl::in_place_type_t, Args&&... args) + explicit ValueVariant(std::in_place_type_t, Args&&... args) : index_(ValueAlternative::kIndex), kind_(ValueAlternative::kKind) { static_assert(alignof(T) <= kValueVariantAlign); static_assert(sizeof(T) <= kValueVariantSize); - flags_ = ValueAlternative::Flags(::new (static_cast(&raw_[0])) - T(std::forward(args)...)); + ::new (static_cast(&raw_[0])) T(std::forward(args)...); } template >>> explicit ValueVariant(T&& value) - : ValueVariant(absl::in_place_type>, + : ValueVariant(std::in_place_type>, std::forward(value)) {} - ValueKind kind() const { return kind_; } + [[nodiscard]] + ValueKind kind() const { + return kind_; + } template void Assign(T&& value) { @@ -508,67 +304,35 @@ class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI static_assert(alignof(U) <= kValueVariantAlign); static_assert(sizeof(U) <= kValueVariantSize); - if constexpr (ValueAlternative::kAlwaysTrivial) { - if ((flags_ & ValueFlags::kNonTrivial) != ValueFlags::kNone) { - SlowDestruct(); - } - index_ = ValueAlternative::kIndex; - kind_ = ValueAlternative::kKind; - flags_ = ValueAlternative::Flags(::new (static_cast(&raw_[0])) - U(std::forward(value))); - } else { - // U is not always trivial. See if the current active alternative is U. If - // it is, we can just do a simple assignment without having to destruct - // first. Otherwise fallback to destruct and construct. - if (index_ == ValueAlternative::kIndex) { - *At() = std::forward(value); - flags_ = ValueAlternative::Flags(At()); - } else { - if ((flags_ & ValueFlags::kNonTrivial) != ValueFlags::kNone) { - SlowDestruct(); - } - index_ = ValueAlternative::kIndex; - kind_ = ValueAlternative::kKind; - flags_ = ValueAlternative::Flags(::new (static_cast(&raw_[0])) - U(std::forward(value))); - } - } + index_ = ValueAlternative::kIndex; + kind_ = ValueAlternative::kKind; + ::new (static_cast(&raw_[0])) U(std::forward(value)); } template + [[nodiscard]] bool Is() const { return index_ == ValueAlternative::kIndex; } template - T& Get() & ABSL_ATTRIBUTE_LIFETIME_BOUND { + [[nodiscard]] + T& Get() ABSL_ATTRIBUTE_LIFETIME_BOUND { ABSL_DCHECK(Is()); return *At(); } template - const T& Get() const& ABSL_ATTRIBUTE_LIFETIME_BOUND { + [[nodiscard]] + const T& Get() const ABSL_ATTRIBUTE_LIFETIME_BOUND { ABSL_DCHECK(Is()); return *At(); } template - T&& Get() && ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_DCHECK(Is()); - - return std::move(*At()); - } - - template - const T&& Get() const&& ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_DCHECK(Is()); - - return std::move(*At()); - } - - template + [[nodiscard]] T* absl_nullable As() ABSL_ATTRIBUTE_LIFETIME_BOUND { if (Is()) { return At(); @@ -577,6 +341,7 @@ class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI } template + [[nodiscard]] const T* absl_nullable As() const ABSL_ATTRIBUTE_LIFETIME_BOUND { if (Is()) { return At(); @@ -585,187 +350,119 @@ class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI } template - ABSL_ATTRIBUTE_ALWAYS_INLINE decltype(auto) Visit(Visitor&& visitor) & { - return std::as_const(*this).Visit(std::forward(visitor)); - } - - template - decltype(auto) Visit(Visitor&& visitor) const& { + decltype(auto) Visit(Visitor&& visitor) const { switch (index_) { case ValueIndex::kNull: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kBool: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kInt: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kUint: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kDouble: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kDuration: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kTimestamp: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kType: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyList: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedJsonList: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedRepeatedField: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomList: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyMap: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedJsonMap: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedMapField: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomMap: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyStruct: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedMessage: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomStruct: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kOpaque: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kBytes: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kString: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kError: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kUnknown: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); } } template - decltype(auto) Visit(Visitor&& visitor) && { + decltype(auto) Visit(Visitor&& visitor) { switch (index_) { case ValueIndex::kNull: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kBool: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kInt: - return std::forward(visitor)(std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kUint: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kDouble: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kDuration: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kTimestamp: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kType: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyList: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedJsonList: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedRepeatedField: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomList: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyMap: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedJsonMap: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedMapField: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomMap: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyStruct: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedMessage: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomStruct: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kOpaque: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kBytes: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kString: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kError: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kUnknown: - return std::forward(visitor)( - std::move(*this).Get()); - } - } - - template - ABSL_ATTRIBUTE_ALWAYS_INLINE decltype(auto) Visit(Visitor&& visitor) const&& { - return Visit(std::forward(visitor)); - } - - friend void swap(ValueVariant& lhs, ValueVariant& rhs) noexcept { - if (&lhs != &rhs) { - const bool lhs_trivial = - (lhs.flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - const bool rhs_trivial = - (rhs.flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - if (lhs_trivial && rhs_trivial) { -// We validated the instances can be copied byte-wise at runtime, but compilers -// warn since this is not safe in the general case. -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" -#elif defined(__clang__) && __clang_major__ >= 20 -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wnontrivial-memcall" -#endif - alignas(ValueVariant) std::byte tmp[sizeof(ValueVariant)]; - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(tmp, std::addressof(lhs), sizeof(ValueVariant)); - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(std::addressof(lhs), std::addressof(rhs), - sizeof(ValueVariant)); - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(std::addressof(rhs), tmp, sizeof(ValueVariant)); -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#elif defined(__clang__) && __clang_major__ >= 20 -#pragma clang diagnostic pop -#endif - } else { - SlowSwap(lhs, rhs, lhs_trivial, rhs_trivial); - } + return std::forward(visitor)(*At()); } } private: - friend struct cel::ArenaTraits; - template + [[nodiscard]] ABSL_ATTRIBUTE_ALWAYS_INLINE T* absl_nonnull At() ABSL_ATTRIBUTE_LIFETIME_BOUND { + static_assert(IsValueAlternativeV); static_assert(alignof(T) <= kValueVariantAlign); static_assert(sizeof(T) <= kValueVariantSize); @@ -773,59 +470,23 @@ class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI } template + [[nodiscard]] ABSL_ATTRIBUTE_ALWAYS_INLINE const T* absl_nonnull At() const ABSL_ATTRIBUTE_LIFETIME_BOUND { + static_assert(IsValueAlternativeV); static_assert(alignof(T) <= kValueVariantAlign); static_assert(sizeof(T) <= kValueVariantSize); return std::launder(reinterpret_cast(&raw_[0])); } - ABSL_ATTRIBUTE_ALWAYS_INLINE void FastCopyAssign( - const ValueVariant& other) noexcept { - index_ = other.index_; - kind_ = other.kind_; - flags_ = other.flags_; - std::memcpy(raw_, other.raw_, sizeof(raw_)); - } - - ABSL_ATTRIBUTE_ALWAYS_INLINE void FastMoveAssign( - ValueVariant& other) noexcept { - FastCopyAssign(other); - } - - void SlowCopyConstruct(const ValueVariant& other) noexcept; - - void SlowMoveConstruct(ValueVariant& other) noexcept; - - void SlowDestruct() noexcept; - - void SlowCopyAssign(const ValueVariant& other, bool trivial, - bool other_trivial) noexcept; - - void SlowMoveAssign(ValueVariant& other, bool ntrivial, - bool other_trivial) noexcept; - - static void SlowSwap(ValueVariant& lhs, ValueVariant& rhs, bool lhs_trivial, - bool rhs_trivial) noexcept; - ValueIndex index_ = ValueIndex::kNull; ValueKind kind_ = ValueKind::kNull; - ValueFlags flags_ = ValueFlags::kNone; alignas(kValueVariantAlign) std::byte raw_[kValueVariantSize]; }; } // namespace common_internal -template <> -struct ArenaTraits { - static bool trivially_destructible( - const common_internal::ValueVariant& value) { - return (value.flags_ & common_internal::ValueFlags::kNonTrivial) == - common_internal::ValueFlags::kNone; - } -}; - } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_VALUES_VALUE_VARIANT_H_ diff --git a/common/values/value_variant_test.cc b/common/values/value_variant_test.cc index 1fd3629aa..1c0416d4c 100644 --- a/common/values/value_variant_test.cc +++ b/common/values/value_variant_test.cc @@ -63,16 +63,16 @@ struct DefaultValue { template <> struct DefaultValue { BytesValue operator()() const { - return BytesValue( - absl::Cord("Some somewhat large string that is not storable inline!")); + return BytesValue::WrapUnsafe( + "Some somewhat large string that is not storable inline!"); } }; template <> struct DefaultValue { StringValue operator()() const { - return StringValue( - absl::Cord("Some somewhat large string that is not storable inline!")); + return StringValue::WrapUnsafe( + "Some somewhat large string that is not storable inline!"); } }; @@ -110,6 +110,7 @@ TYPED_TEST(ValueVariantTest, MoveAssign) { } TYPED_TEST(ValueVariantTest, Swap) { + using std::swap; using Left = typename TypeParam::first_type; using Right = typename TypeParam::second_type; diff --git a/compiler/BUILD b/compiler/BUILD index 170f1068b..2e3f7a2ba 100644 --- a/compiler/BUILD +++ b/compiler/BUILD @@ -25,12 +25,15 @@ cc_library( "//checker:type_checker", "//checker:type_checker_builder", "//checker:validation_result", + "//common:source", "//parser:options", "//parser:parser_interface", "//validator", + "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:string_view", + "@com_google_protobuf//:protobuf", ], ) @@ -40,10 +43,12 @@ cc_library( hdrs = ["compiler_factory.h"], deps = [ ":compiler", + "//checker:type_check_issue", "//checker:type_checker", "//checker:type_checker_builder", "//checker:type_checker_builder_factory", "//checker:validation_result", + "//common:ast", "//common:source", "//internal:noop_delete", "//internal:status_macros", diff --git a/compiler/compiler.h b/compiler/compiler.h index 48fa4e0b1..58c60027b 100644 --- a/compiler/compiler.h +++ b/compiler/compiler.h @@ -19,6 +19,7 @@ #include #include +#include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" @@ -26,9 +27,11 @@ #include "checker/type_checker.h" #include "checker/type_checker_builder.h" #include "checker/validation_result.h" +#include "common/source.h" #include "parser/options.h" #include "parser/parser_interface.h" #include "validator/validator.h" +#include "google/protobuf/arena.h" namespace cel { @@ -95,6 +98,8 @@ struct CompilerLibrarySubset { struct CompilerOptions { ParserOptions parser_options; CheckerOptions checker_options; + // If true, parse errors will be adapted to issues where possible. + bool adapt_parser_errors = false; }; // Interface for CEL CompilerBuilder objects. @@ -125,11 +130,26 @@ class Compiler { public: virtual ~Compiler() = default; - virtual absl::StatusOr Compile( - absl::string_view source, absl::string_view description) const = 0; + absl::StatusOr Compile( + const Source& source, google::protobuf::Arena* absl_nullable arena) const { + return CompileImpl(source, arena); + } + + absl::StatusOr Compile(const Source& source) const { + return CompileImpl(source, nullptr); + } + + absl::StatusOr Compile( + absl::string_view source, absl::string_view description, + google::protobuf::Arena* absl_nullable arena) const; absl::StatusOr Compile(absl::string_view source) const { - return Compile(source, ""); + return Compile(source, "", nullptr); + } + + absl::StatusOr Compile( + absl::string_view source, absl::string_view description) const { + return Compile(source, description, nullptr); } // Accessor for the underlying type checker. @@ -149,8 +169,27 @@ class Compiler { // The returned builder does not share state with the compiler and may be // modified independently. virtual std::unique_ptr ToBuilder() const = 0; + + protected: + virtual absl::StatusOr CompileImpl( + const Source& source, google::protobuf::Arena* absl_nullable arena) const = 0; }; +inline absl::StatusOr Compiler::Compile( + absl::string_view source, absl::string_view description, + google::protobuf::Arena* absl_nullable arena) const { + absl::StatusOr> source_obj = + GetParser().PrepareSource(source, description); + if (!source_obj.ok()) { + return source_obj.status(); + } + absl::StatusOr result = CompileImpl(**source_obj, arena); + if (result.ok()) { + result->SetSource(std::move(*source_obj)); + } + return result; +} + } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMPILER_COMPILER_INTERFACE_H_ diff --git a/compiler/compiler_factory.cc b/compiler/compiler_factory.cc index 3e9871706..a8a44461c 100644 --- a/compiler/compiler_factory.cc +++ b/compiler/compiler_factory.cc @@ -17,22 +17,26 @@ #include #include #include +#include #include "absl/container/flat_hash_set.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "checker/type_check_issue.h" #include "checker/type_checker.h" #include "checker/type_checker_builder.h" #include "checker/type_checker_builder_factory.h" #include "checker/validation_result.h" +#include "common/ast.h" #include "common/source.h" #include "compiler/compiler.h" #include "internal/status_macros.h" #include "parser/parser.h" #include "parser/parser_interface.h" #include "validator/validator.h" +#include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" namespace cel { @@ -44,47 +48,63 @@ class CompilerImpl : public Compiler { CompilerImpl(std::unique_ptr type_checker, std::unique_ptr parser, // Copy the validator in case builder is reused. - Validator validator) + Validator validator, CompilerOptions options) : type_checker_(std::move(type_checker)), parser_(std::move(parser)), - validator_(std::move(validator)) {} - - absl::StatusOr Compile( - absl::string_view expression, - absl::string_view description) const override { - CEL_ASSIGN_OR_RETURN(auto source, - cel::NewSource(expression, std::string(description))); - CEL_ASSIGN_OR_RETURN(auto ast, parser_->Parse(*source)); + validator_(std::move(validator)), + options_(options) {} + + std::unique_ptr ToBuilder() const override; + + const TypeChecker& GetTypeChecker() const override { return *type_checker_; } + const Parser& GetParser() const override { return *parser_; } + const Validator& GetValidator() const override { return validator_; } + + protected: + absl::StatusOr CompileImpl( + const Source& source, google::protobuf::Arena* arena) const override { + std::vector parse_issues; + absl::StatusOr> ast = + parser_->Parse(source, &parse_issues); + if (!ast.ok()) { + if (!options_.adapt_parser_errors || + ast.status().code() != absl::StatusCode::kInvalidArgument || + parse_issues.empty()) { + return ast.status(); + } + std::vector check_issues; + check_issues.reserve(parse_issues.size()); + for (const auto& issue : parse_issues) { + check_issues.push_back(TypeCheckIssue::CreateError( + issue.location(), std::string(issue.message()))); + } + return ValidationResult(std::move(check_issues)); + } CEL_ASSIGN_OR_RETURN(ValidationResult result, - type_checker_->Check(std::move(ast))); + type_checker_->Check(*std::move(ast), arena)); - result.SetSource(std::move(source)); if (!validator_.validations().empty()) { validator_.UpdateValidationResult(result); } return result; } - std::unique_ptr ToBuilder() const override; - - const TypeChecker& GetTypeChecker() const override { return *type_checker_; } - const Parser& GetParser() const override { return *parser_; } - const Validator& GetValidator() const override { return validator_; } - private: std::unique_ptr type_checker_; std::unique_ptr parser_; Validator validator_; + CompilerOptions options_; }; class CompilerBuilderImpl : public CompilerBuilder { public: CompilerBuilderImpl(std::unique_ptr type_checker_builder, std::unique_ptr parser_builder, - Validator validator = Validator()) + Validator validator, CompilerOptions options) : type_checker_builder_(std::move(type_checker_builder)), parser_builder_(std::move(parser_builder)), - validator_(std::move(validator)) {} + validator_(std::move(validator)), + options_(options) {} absl::Status AddLibrary(CompilerLibrary library) override { if (!library.id.empty()) { @@ -145,23 +165,23 @@ class CompilerBuilderImpl : public CompilerBuilder { absl::StatusOr> Build() override { CEL_ASSIGN_OR_RETURN(auto parser, parser_builder_->Build()); CEL_ASSIGN_OR_RETURN(auto type_checker, type_checker_builder_->Build()); - return std::make_unique(std::move(type_checker), - std::move(parser), validator_); + return std::make_unique( + std::move(type_checker), std::move(parser), validator_, options_); } private: std::unique_ptr type_checker_builder_; std::unique_ptr parser_builder_; Validator validator_; + CompilerOptions options_; absl::flat_hash_set library_ids_; absl::flat_hash_set subsets_; }; std::unique_ptr CompilerImpl::ToBuilder() const { - auto builder = std::make_unique( - type_checker_->ToBuilder(), parser_->ToBuilder(), validator_); - return builder; + return std::make_unique( + type_checker_->ToBuilder(), parser_->ToBuilder(), validator_, options_); } } // namespace @@ -178,7 +198,8 @@ absl::StatusOr> NewCompilerBuilder( auto parser_builder = NewParserBuilder(options.parser_options); return std::make_unique(std::move(type_checker_builder), - std::move(parser_builder)); + std::move(parser_builder), + Validator(), options); } } // namespace cel diff --git a/compiler/compiler_factory_test.cc b/compiler/compiler_factory_test.cc index d217e4cc7..f90cc4862 100644 --- a/compiler/compiler_factory_test.cc +++ b/compiler/compiler_factory_test.cc @@ -37,6 +37,7 @@ #include "parser/parser_interface.h" #include "testutil/baseline_tests.h" #include "validator/timestamp_literal_validator.h" +#include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" namespace cel { @@ -390,5 +391,74 @@ TEST(CompilerFactoryTest, ToBuilderWorks) { EXPECT_TRUE(result.IsValid()); } +TEST(CompilerFactoryTest, SpecifyArenaKeepsResolvedTypes) { + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(cel::internal::GetSharedTestingDescriptorPool())); + + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->AddLibrary(OptionalCompilerLibrary()), IsOk()); + + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(ValidationResult result, + compiler->Compile("[[1, 2, 3]][?0]", "", &arena)); + ASSERT_OK_AND_ASSIGN(auto ast, result.ReleaseAst()); + auto it = result.GetResolvedTypeMap().find(ast->root_expr().id()); + ASSERT_TRUE(it != result.GetResolvedTypeMap().end()); + EXPECT_TRUE( + it->second.IsOptional() && + it->second.GetOptional().GetParameter().IsList() && + it->second.GetOptional().GetParameter().GetList().GetElement().IsInt()); +} + +TEST(CompilerFactoryTest, ReturnsIssuesFromParser) { + CompilerOptions opts; + opts.adapt_parser_errors = true; + ASSERT_OK_AND_ASSIGN( + auto builder, NewCompilerBuilder( + cel::internal::GetSharedTestingDescriptorPool(), opts)); + + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult result, compiler->Compile("a +")); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.GetIssues(), testing::Not(testing::IsEmpty())); +} + +TEST(CompilerFactoryTest, CompileSourceOverload) { + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(cel::internal::GetSharedTestingDescriptorPool())); + + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("1 + 2")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, compiler->Compile(*source)); + + EXPECT_TRUE(result.IsValid()); +} + +TEST(CompilerFactoryTest, CodepointLimitExceeded) { + CompilerOptions options; + options.parser_options.expression_size_codepoint_limit = 10; + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(cel::internal::GetSharedTestingDescriptorPool(), + options)); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + EXPECT_THAT( + compiler->Compile("123456789012345", "test.cel"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("expression is larger than codepoint limit 10"))); + + ASSERT_OK_AND_ASSIGN(ValidationResult result, + compiler->Compile("1234567890", "test.cel")); + EXPECT_TRUE(result.IsValid()); +} + } // namespace } // namespace cel diff --git a/compiler/optional_test.cc b/compiler/optional_test.cc index 699c69f76..f5091b9fc 100644 --- a/compiler/optional_test.cc +++ b/compiler/optional_test.cc @@ -111,11 +111,66 @@ optional.of( TestCase{ .expr = "optional.of('foo').optMap(x, x)", .expected_ast = R"( -_?_:_( +__comprehension__( + // Variable + #unused, + // Target + []~list(dyn), + // Accumulator + @target, + // Init optional.of( "foo"~string - )~optional_type(string)^optional_of.hasValue()~bool^optional_hasValue, + )~optional_type(string)^optional_of, + // LoopCondition + false~bool, + // LoopStep + @target~optional_type(string)^@target, + // Result + _?_:_( + @target~optional_type(string)^@target.hasValue()~bool^optional_hasValue, + optional.of( + __comprehension__( + // Variable + #unused, + // Target + []~list(dyn), + // Accumulator + x, + // Init + @target~optional_type(string)^@target.value()~string^optional_value, + // LoopCondition + false~bool, + // LoopStep + x~string^x, + // Result + x~string^x)~string + )~optional_type(string)^optional_of, + optional.none()~optional_type(string)^optional_none + )~optional_type(string)^conditional)~optional_type(string) +)", + }, + TestCase{ + .expr = "optional.of('foo').optFlatMap(x, optional.of(x))", + .expected_ast = R"( +__comprehension__( + // Variable + #unused, + // Target + []~list(dyn), + // Accumulator + @target, + // Init optional.of( + "foo"~string + )~optional_type(string)^optional_of, + // LoopCondition + false~bool, + // LoopStep + @target~optional_type(string)^@target, + // Result + _?_:_( + @target~optional_type(string)^@target.hasValue()~bool^optional_hasValue, __comprehension__( // Variable #unused, @@ -124,48 +179,17 @@ _?_:_( // Accumulator x, // Init - optional.of( - "foo"~string - )~optional_type(string)^optional_of.value()~string^optional_value, + @target~optional_type(string)^@target.value()~string^optional_value, // LoopCondition false~bool, // LoopStep x~string^x, // Result - x~string^x)~string - )~optional_type(string)^optional_of, - optional.none()~optional_type(string)^optional_none -)~optional_type(string)^conditional -)", - }, - TestCase{ - .expr = "optional.of('foo').optFlatMap(x, optional.of(x))", - .expected_ast = R"( -_?_:_( - optional.of( - "foo"~string - )~optional_type(string)^optional_of.hasValue()~bool^optional_hasValue, - __comprehension__( - // Variable - #unused, - // Target - []~list(dyn), - // Accumulator - x, - // Init - optional.of( - "foo"~string - )~optional_type(string)^optional_of.value()~string^optional_value, - // LoopCondition - false~bool, - // LoopStep - x~string^x, - // Result - optional.of( - x~string^x - )~optional_type(string)^optional_of)~optional_type(string), - optional.none()~optional_type(string)^optional_none -)~optional_type(string)^conditional + optional.of( + x~string^x + )~optional_type(string)^optional_of)~optional_type(string), + optional.none()~optional_type(string)^optional_none + )~optional_type(string)^conditional)~optional_type(string) )", }, TestCase{ diff --git a/conformance/BUILD b/conformance/BUILD index 139739891..ddef1eaf8 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -25,6 +25,7 @@ cc_library( srcs = ["service.cc"], hdrs = ["service.h"], deps = [ + "//checker:checker_options", "//checker:optional", "//checker:standard_library", "//checker:type_checker_builder", @@ -32,7 +33,6 @@ cc_library( "//common:ast", "//common:ast_proto", "//common:decl_proto_v1alpha1", - "//common:expr", "//common:source", "//common:value", "//common/internal:value_conversion", @@ -48,17 +48,18 @@ cc_library( "//extensions:comprehensions_v2_functions", "//extensions:comprehensions_v2_macros", "//extensions:encoders", + "//extensions:lists_functions", "//extensions:math_ext", "//extensions:math_ext_decls", "//extensions:math_ext_macros", "//extensions:proto_ext", "//extensions:select_optimization", + "//extensions:set_membership_optimization", + "//extensions:sets_functions", "//extensions:strings", "//extensions/protobuf:enum_adapter", "//internal:status_macros", "//parser", - "//parser:macro", - "//parser:macro_expr_factory", "//parser:macro_registry", "//parser:options", "//parser:standard_macros", @@ -67,21 +68,22 @@ cc_library( "//runtime:constant_folding", "//runtime:optional_types", "//runtime:reference_resolver", + "//runtime:regex_precompilation", "//runtime:runtime_options", "//runtime:standard_runtime_builder_factory", + "//testutil:test_macros", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", "@com_google_googleapis//google/api/expr/conformance/v1alpha1:conformance_cc_proto", "@com_google_googleapis//google/api/expr/v1alpha1:checked_cc_proto", "@com_google_googleapis//google/rpc:code_cc_proto", + "@com_google_googleapis//google/rpc:status_cc_proto", "@com_google_protobuf//:duration_cc_proto", "@com_google_protobuf//:empty_cc_proto", "@com_google_protobuf//:protobuf", @@ -97,6 +99,7 @@ cc_library( deps = [ ":service", ":utils", + "//internal:runfiles", "//internal:testing_no_main", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/log:absl_check", @@ -144,6 +147,7 @@ _ALL_TESTS = [ "@com_google_cel_spec//tests/simple:testdata/fp_math.textproto", "@com_google_cel_spec//tests/simple:testdata/integer_math.textproto", "@com_google_cel_spec//tests/simple:testdata/lists.textproto", + "@com_google_cel_spec//tests/simple:testdata/lists_ext.textproto", "@com_google_cel_spec//tests/simple:testdata/logic.textproto", "@com_google_cel_spec//tests/simple:testdata/macros.textproto", "@com_google_cel_spec//tests/simple:testdata/macros2.textproto", @@ -161,6 +165,7 @@ _ALL_TESTS = [ "@com_google_cel_spec//tests/simple:testdata/unknowns.textproto", "@com_google_cel_spec//tests/simple:testdata/wrappers.textproto", "@com_google_cel_spec//tests/simple:testdata/block_ext.textproto", + "@com_google_cel_spec//tests/simple:testdata/sets_ext.textproto", "@com_google_cel_spec//tests/simple:testdata/type_deduction.textproto", ] @@ -170,22 +175,31 @@ _TESTS_TO_SKIP = [ "timestamps/duration_converters/get_milliseconds", # Broken test cases which should be supported. - # TODO(issues/112): Unbound functions result in empty eval response. - "basic/functions/unbound", - "basic/functions/unbound_is_runtime_error", - # TODO(issues/97): Parse-only qualified variable lookup "x.y" with binding "x.y" or "y" within container "x" fails - "fields/qualified_identifier_resolution/qualified_ident,map_field_select,ident_with_longest_prefix_check,qualified_identifier_resolution_unchecked", - "namespace/qualified/self_eval_qualified_lookup", - "namespace/namespace/self_eval_container_lookup,self_eval_container_lookup_unchecked", - # TODO(issues/117): Integer overflow on enum assignments should error. - "enums/legacy_proto2/select_big,select_neg", + # NOTE: These tests have `disable_check: true` in their definition, making them force parse-only behavior even in checked mode test runs. + "fields/qualified_identifier_resolution/qualified_identifier_resolution_unchecked", + "namespace/namespace/self_eval_container_lookup_unchecked", # Skip until fixed. - "wrappers/field_mask/to_json", - "wrappers/empty/to_json", "fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous", - "parse/receiver_function_names", + + # Skip until duration/timestamp range and proto wrapper validation is updated for spec parity. + "timestamps/timestamp_arithmetic/add_duration_to_duration_max", + "timestamps/timestamp_arithmetic/subtract_duration_from_duration_min", + "timestamps/timestamp_arithmetic/subtract_max_duration_from_max_time", + "timestamps/timestamp_range/sub_time_duration_over", + "timestamps/timestamp_range/sub_time_duration_under", + "wrappers/duration/nanos_over", + "wrappers/duration/nanos_under", + "wrappers/duration/negative_seconds_positive_nanos", + "wrappers/duration/positive_seconds_negative_nanos", + "wrappers/duration/seconds_over", + "wrappers/duration/seconds_under", + "wrappers/timestamp/nanos_over", + "wrappers/timestamp/nanos_under", + "wrappers/timestamp/negative_seconds_negative_nanos", + "wrappers/timestamp/seconds_over", + "wrappers/timestamp/seconds_under", # Future features for CEL 1.0 # TODO(issues/119): Strong typing support for enums, specified but not implemented. @@ -202,6 +216,18 @@ _TESTS_TO_SKIP = [ "conversions/string/double_hard", ] +_TESTS_TO_SKIP_PARSE_ONLY = [ + # TODO(issues/97): Parse-only qualified variable lookup "x.y" with binding "x.y" or "y" within container "x" fails. + "namespace/namespace_shadowing/basic", + "namespace/namespace_shadowing/comprehension_shadowing_namespaced_selector_disambiguation", + "fields/qualified_identifier_resolution/qualified_ident", + "fields/qualified_identifier_resolution/map_field_select", + "fields/qualified_identifier_resolution/ident_with_longest_prefix_check", + "namespace/qualified/self_eval_qualified_lookup", + "namespace/namespace/self_eval_container_lookup", + "type_deductions", +] + _TESTS_TO_SKIP_MODERN = _TESTS_TO_SKIP _TESTS_TO_SKIP_MODERN_DASHBOARD = [ @@ -221,17 +247,14 @@ _TESTS_TO_SKIP_LEGACY = _TESTS_TO_SKIP + [ "proto3/set_null/list_value", "proto3/set_null/single_struct", - # cel.@block + # no optional support for legacy types "block_ext/basic/optional_list", "block_ext/basic/optional_map", "block_ext/basic/optional_map_chained", "block_ext/basic/optional_message", -] -_TESTS_TO_SKIP_CHECKED = [ - # block is a post-check optimization that inserts internal variables. The C++ type checker - # needs support for a proper optimizer for this to work. - "block_ext", + # lists_ext not implemented for legacy evaluator + "lists_ext", ] _TESTS_TO_SKIP_LEGACY_DASHBOARD = [ @@ -245,19 +268,19 @@ _TESTS_TO_SKIP_LEGACY_DASHBOARD = [ ] # Generates a bunch of `cc_test` whose names follow the pattern -# `conformance_(...)_{arena|refcount}_{optimized|unoptimized}_{recursive|iterative}`. +# `conformance_(...)_{pratt|antlr}_{optimized|unoptimized}`. gen_conformance_tests( name = "conformance_parse_only", data = _ALL_TESTS, modern = True, - skip_tests = _TESTS_TO_SKIP_MODERN + ["type_deductions"], + skip_tests = _TESTS_TO_SKIP_MODERN + _TESTS_TO_SKIP_PARSE_ONLY, ) gen_conformance_tests( name = "conformance_legacy_parse_only", data = _ALL_TESTS, modern = False, - skip_tests = _TESTS_TO_SKIP_LEGACY + ["type_deductions"], + skip_tests = _TESTS_TO_SKIP_LEGACY + _TESTS_TO_SKIP_PARSE_ONLY, ) gen_conformance_tests( @@ -265,7 +288,7 @@ gen_conformance_tests( checked = True, data = _ALL_TESTS, modern = True, - skip_tests = _TESTS_TO_SKIP_MODERN + _TESTS_TO_SKIP_CHECKED, + skip_tests = _TESTS_TO_SKIP_MODERN, ) gen_conformance_tests( @@ -273,7 +296,7 @@ gen_conformance_tests( checked = True, data = _ALL_TESTS, modern = False, - skip_tests = _TESTS_TO_SKIP_LEGACY + _TESTS_TO_SKIP_CHECKED, + skip_tests = _TESTS_TO_SKIP_LEGACY, ) # select optimization is only supported for checked expressions. @@ -281,22 +304,40 @@ gen_conformance_tests( name = "conformance_legacy_select_opt", checked = True, data = _ALL_TESTS, + extended_optimizations = True, modern = False, - select_opt = True, - skip_tests = _TESTS_TO_SKIP_LEGACY + _TESTS_TO_SKIP_CHECKED, + skip_tests = _TESTS_TO_SKIP_LEGACY, ) gen_conformance_tests( name = "conformance_select_opt", checked = True, data = _ALL_TESTS, + extended_optimizations = True, + modern = True, + skip_tests = _TESTS_TO_SKIP_MODERN, +) + +gen_conformance_tests( + name = "conformance_variadic", + checked = True, + data = _ALL_TESTS, + enable_variadic_logical_operators = True, modern = True, - select_opt = True, - skip_tests = _TESTS_TO_SKIP_MODERN + _TESTS_TO_SKIP_CHECKED, + skip_tests = _TESTS_TO_SKIP_MODERN, +) + +gen_conformance_tests( + name = "conformance_legacy_variadic", + checked = True, + data = _ALL_TESTS, + enable_variadic_logical_operators = True, + modern = False, + skip_tests = _TESTS_TO_SKIP_LEGACY, ) # Generates a bunch of `cc_test` whose names follow the pattern -# `conformance_dashboard_..._{arena|refcount}_{optimized|unoptimized}_{recursive|iterative}`. +# `conformance_dashboard_..._{pratt|antlr}_{optimized|unoptimized}`. gen_conformance_tests( name = "conformance_dashboard_parse_only", dashboard = True, diff --git a/conformance/policy/BUILD b/conformance/policy/BUILD new file mode 100644 index 000000000..50ea27534 --- /dev/null +++ b/conformance/policy/BUILD @@ -0,0 +1,124 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load( + "//conformance/policy:policy_conformance_test.bzl", + "cel_policy_conformance_test", +) + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "policy_conformance_test_lib", + testonly = True, + srcs = ["policy_conformance_test.cc"], + deps = [ + "//common:ast", + "//common:source", + "//common:value", + "//common/internal:value_conversion", + "//compiler", + "//env", + "//env:config", + "//env:env_runtime", + "//env:env_std_extensions", + "//env:env_yaml", + "//env:runtime_std_extensions", + "//extensions/protobuf:enum_adapter", + "//internal:runfiles", + "//internal:status_macros", + "//internal:testing_descriptor_pool", + "//internal:testing_no_main", + "//policy:cel_policy", + "//policy:cel_policy_parser", + "//policy:cel_policy_validation_result", + "//policy:compiler", + "//policy:test_util", + "//policy:yaml_policy_parser", + "//runtime", + "//runtime:activation", + "//runtime:bind_proto_to_activation", + "//runtime:function_adapter", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_cel_spec//proto/cel/expr:value_cc_proto", + "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto", + "@com_google_protobuf//:protobuf", + ], + alwayslink = True, +) + +_ALL_TESTS = [ + "@cel_policy//conformance:agent_tool_execution_governance", + "@cel_policy//conformance:aggregate", + "@cel_policy//conformance:aggregate_explicit_list_output", + "@cel_policy//conformance:aggregate_explicit_optional_none", + "@cel_policy//conformance:aggregate_nested_all_rules_fall_through_to_empty", + "@cel_policy//conformance:aggregate_nested_explicit_list_double_wrapping", + "@cel_policy//conformance:aggregate_nested_multi_branch_match", + "@cel_policy//conformance:aggregate_nested_single_conditional_match", + "@cel_policy//conformance:aggregate_shadowed_variables", + "@cel_policy//conformance:first_match_nested_aggregate", + "@cel_policy//conformance:k8s", + "@cel_policy//conformance:limits", + "@cel_policy//conformance:nested_rule", + "@cel_policy//conformance:nested_rule2", + "@cel_policy//conformance:nested_rule3", + "@cel_policy//conformance:nested_rule4", + "@cel_policy//conformance:nested_rule5", + "@cel_policy//conformance:nested_rule6", + "@cel_policy//conformance:nested_rule7", + "@cel_policy//conformance:nested_rules_variable_shadowing", + "@cel_policy//conformance:pb", + "@cel_policy//conformance:required_labels", + "@cel_policy//conformance:restricted_destinations", + "@cel_policy//conformance:unconditional_rules", + "@cel_policy//conformance:unnest", + "@cel_policy//conformance:variable_type_propagation", + "@cel_policy//conformance:compile_error_aggregate_false_condition", + "@cel_policy//conformance:compile_error_aggregate_heterogeneous_outputs", + "@cel_policy//conformance:compile_error_aggregate_nested_mixed_semantics", + "@cel_policy//conformance:compile_error_aggregate_subrule_heterogeneous_outputs", + "@cel_policy//conformance:compile_error_aggregate_unreachable_in_nested_first_match", + "@cel_policy//conformance:compile_error_compose_conflicting_output", + "@cel_policy//conformance:compile_error_compose_conflicting_subrule", + "@cel_policy//conformance:compile_error_unreachable", + "@cel_policy//conformance:compile_error_unreachable_under_unconditional_aggregate_subrule", + "@cel_policy//conformance:compile_error_duplicate_variable", + "@cel_policy//conformance:compile_error_import", + "@cel_policy//conformance:compile_error_incompatible_outputs", + "@cel_policy//conformance:compile_error_syntax", + "@cel_policy//conformance:compile_error_undeclared_reference", +] + +cel_policy_conformance_test( + name = "policy_conformance_test", + skip_tests = [ + # TODO(b/506179116): Fix these. + # Need to add k8s custom yaml parser and mock runtime. + "k8s", + # Expects Java reverse-order type mismatch error message. + "compile_error_aggregate_heterogeneous_outputs", + # Expects "rule creates unreachable outputs" on the exhaustive subrule, + # whereas C++ reports "match creates unreachable outputs" on the + # subsequent unreachable match. + "compile_error_unreachable_under_unconditional_aggregate_subrule", + ], + test_files = _ALL_TESTS, +) diff --git a/conformance/policy/policy_conformance_test.bzl b/conformance/policy/policy_conformance_test.bzl new file mode 100644 index 000000000..e57ac7be4 --- /dev/null +++ b/conformance/policy/policy_conformance_test.bzl @@ -0,0 +1,45 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains build rules for generating policy conformance test targets. +""" + +load("@rules_cc//cc:cc_test.bzl", "cc_test") + +def cel_policy_conformance_test(name, test_files, skip_tests = [], **kwargs): + """Generates a policy conformance test target. + + Args: + name: Name of the test target. + test_files: List of targets or files representing the test data. + skip_tests: List of test cases to skip. + **kwargs: Additional arguments passed to the underlying cc_test. + """ + args = ["--gunit_fail_if_no_test_linked"] + paths = ["$(rlocationpath {})".format(f) for f in test_files] + args.append("--test_bundles=\"{}\"".format(",".join(paths))) + + if skip_tests: + args.append("--skip_tests=" + ",".join(skip_tests)) + + cc_test( + name = name, + data = test_files, + deps = [ + "//conformance/policy:policy_conformance_test_lib", + ], + args = args, + **kwargs + ) diff --git a/conformance/policy/policy_conformance_test.cc b/conformance/policy/policy_conformance_test.cc new file mode 100644 index 000000000..2f05aed04 --- /dev/null +++ b/conformance/policy/policy_conformance_test.cc @@ -0,0 +1,724 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include + +#include "cel/expr/eval.pb.h" +#include "absl/flags/flag.h" +#include "absl/log/absl_check.h" +#include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" +#include "common/ast.h" +#include "common/internal/value_conversion.h" +#include "common/source.h" +#include "common/value.h" +#include "compiler/compiler.h" +#include "env/config.h" +#include "env/env.h" +#include "env/env_runtime.h" +#include "env/env_std_extensions.h" +#include "env/env_yaml.h" +#include "env/runtime_std_extensions.h" +#include "extensions/protobuf/enum_adapter.h" +#include "internal/runfiles.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" +#include "policy/cel_policy_validation_result.h" +#include "policy/compiler.h" +#include "policy/test_util.h" +#include "policy/yaml_policy_parser.h" +#include "runtime/activation.h" +#include "runtime/bind_proto_to_activation.h" +#include "runtime/function_adapter.h" +#include "runtime/runtime.h" +#include "cel/expr/conformance/test/suite.pb.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/dynamic_message.h" +#include "google/protobuf/message.h" + +ABSL_FLAG(std::vector, test_bundles, {}, + "Space or comma separated list of test bundle runfiles paths."); +ABSL_FLAG(std::vector, skip_tests, {}, + "Comma-separated list of tests to skip."); + +namespace cel { +namespace { + +using ::absl_testing::IsOk; +using ::cel::expr::conformance::test::TestSuite; +using ::cel::internal::GetSharedTestingDescriptorPool; +using ::testing::HasSubstr; + +struct BundleSections { + absl::string_view config_content; + absl::string_view policy_content; + absl::string_view tests_content; +}; + +absl::string_view TrimDoc(absl::string_view doc) { + absl::ConsumeSuffix(&doc, "\n"); + absl::ConsumeSuffix(&doc, "\r"); + return doc; +} + +absl::StatusOr ParseYamlBundle( + absl::string_view bundle_content) { + BundleSections sections; + std::vector docs; + absl::string_view remaining = bundle_content; + + size_t next_line = remaining.find('\n'); + while (next_line != absl::string_view::npos) { + if (absl::StartsWith(remaining.substr(next_line), "\n---\r\n")) { + docs.push_back(TrimDoc(remaining.substr(0, next_line))); + remaining = remaining.substr(next_line + 5); + next_line = remaining.find('\n'); + continue; + } + + if (absl::StartsWith(remaining.substr(next_line), "\n---\n")) { + docs.push_back(TrimDoc(remaining.substr(0, next_line))); + remaining = remaining.substr(next_line + 4); + next_line = remaining.find('\n'); + continue; + } + + next_line = remaining.find('\n', next_line + 1); + } + + if (remaining.empty()) { + return absl::InvalidArgumentError("Empty document in yaml bundle"); + } + docs.push_back(remaining); + + if (docs.size() == 3) { + sections.config_content = docs[0]; + sections.policy_content = docs[1]; + sections.tests_content = docs[2]; + } else if (docs.size() == 2) { + sections.policy_content = docs[0]; + sections.tests_content = docs[1]; + } else { + return absl::InvalidArgumentError( + absl::StrCat("Invalid number of sections: ", docs.size())); + } + + return sections; +} + +// Implementations for extension functions referenced in conformance tests. +cel::Value LocationCode(const cel::StringValue& ip, + const google::protobuf::DescriptorPool* pool, + google::protobuf::MessageFactory* factory, google::protobuf::Arena* arena) { + std::string ip_str = ip.ToString(); + if (ip_str == "10.0.0.1") return cel::StringValue::WrapUnsafe("us"); + if (ip_str == "10.0.0.2") return cel::StringValue::WrapUnsafe("de"); + return cel::StringValue::WrapUnsafe("ir"); +} + +bool MapContainsKey(const cel::MapValue& map_val, absl::string_view key, + const google::protobuf::DescriptorPool* pool, + google::protobuf::MessageFactory* factory, google::protobuf::Arena* arena) { + cel::Value has_val; + if (!map_val + .Has(cel::StringValue::WrapUnsafe(key), pool, factory, arena, + &has_val) + .ok()) { + return false; + } + return has_val.IsTrue(); +} + +cel::Value HasCreditCard(const cel::Value& arg, + const google::protobuf::DescriptorPool* pool, + google::protobuf::MessageFactory* factory, + google::protobuf::Arena* arena) { + if (!arg.IsMap()) { + return cel::BoolValue(false); + } + return cel::BoolValue( + MapContainsKey(arg.GetMap(), "cc", pool, factory, arena)); +} + +cel::Value HasEmailOrPhone(const cel::Value& arg, + const google::protobuf::DescriptorPool* pool, + google::protobuf::MessageFactory* factory, + google::protobuf::Arena* arena) { + if (!arg.IsMap()) { + return cel::BoolValue(false); + } + const cel::MapValue& map_val = arg.GetMap(); + return cel::BoolValue( + MapContainsKey(map_val, "email", pool, factory, arena) || + MapContainsKey(map_val, "phone", pool, factory, arena)); +} + +// TODO(uncreated-issue/92): This should be migrated to use the testrunner utility +// after adding support for reading the yaml specification for envs/tests. +class InputEvaluator { + public: + static absl::StatusOr> Create( + const std::shared_ptr& pool) { + cel::Env env; + env.SetDescriptorPool(pool); + cel::RegisterStandardExtensions(env); + + cel::EnvRuntime env_runtime; + env_runtime.SetDescriptorPool(pool); + cel::RegisterStandardExtensions(env_runtime); + env_runtime.mutable_runtime_options().enable_qualified_type_identifiers = + true; + + // Enable default extensions (optional, bindings) + cel::Config config; + CEL_RETURN_IF_ERROR(config.AddExtensionConfig( + "optional", cel::Config::ExtensionConfig::kLatest)); + CEL_RETURN_IF_ERROR(config.AddExtensionConfig( + "bindings", cel::Config::ExtensionConfig::kLatest)); + env.SetConfig(config); + env_runtime.SetConfig(config); + + auto compiler_builder_or = env.NewCompilerBuilder(); + CEL_ASSIGN_OR_RETURN(auto compiler_builder, std::move(compiler_builder_or)); + compiler_builder->GetParserBuilder().GetOptions().enable_optional_syntax = + true; + CEL_ASSIGN_OR_RETURN(auto compiler, compiler_builder->Build()); + + auto runtime_builder_or = env_runtime.CreateRuntimeBuilder(); + CEL_ASSIGN_OR_RETURN(auto runtime_builder, std::move(runtime_builder_or)); + + // Register conformance enums + for (const auto& enum_name : + {"cel.expr.conformance.proto2.GlobalEnum", + "cel.expr.conformance.proto3.GlobalEnum", + "cel.expr.conformance.proto2.TestAllTypes.NestedEnum", + "cel.expr.conformance.proto3.TestAllTypes.NestedEnum"}) { + auto* enum_desc = pool->FindEnumTypeByName(enum_name); + if (enum_desc != nullptr) { + CEL_RETURN_IF_ERROR(cel::extensions::RegisterProtobufEnum( + runtime_builder.type_registry(), enum_desc)); + } + } + + CEL_ASSIGN_OR_RETURN(auto runtime, std::move(runtime_builder).Build()); + + return absl::WrapUnique( + new InputEvaluator(std::move(compiler), std::move(runtime))); + } + + absl::StatusOr Evaluate( + absl::string_view expr_str, google::protobuf::Arena* arena, + google::protobuf::MessageFactory* message_factory) const { + CEL_ASSIGN_OR_RETURN(auto validation_result, compiler_->Compile(expr_str)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError( + absl::StrCat("Failed to compile input expr: ", expr_str)); + } + CEL_ASSIGN_OR_RETURN(auto ast, validation_result.ReleaseAst()); + CEL_ASSIGN_OR_RETURN( + auto program, + runtime_->CreateProgram(std::make_unique(std::move(*ast)))); + cel::Activation activation; + EvaluateOptions options; + options.message_factory = message_factory; + CEL_ASSIGN_OR_RETURN(auto result, + program->Evaluate(arena, activation, options)); + return result.Clone(arena); + } + + private: + InputEvaluator(std::unique_ptr compiler, + std::unique_ptr runtime) + : compiler_(std::move(compiler)), runtime_(std::move(runtime)) {} + + std::unique_ptr compiler_; + std::unique_ptr runtime_; +}; + +absl::StatusOr EvaluateInputValue( + const cel::expr::conformance::test::InputValue& input_val, + const InputEvaluator& evaluator, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, google::protobuf::Arena* arena) { + if (input_val.has_expr()) { + return evaluator.Evaluate(input_val.expr(), arena, message_factory); + } + if (input_val.has_value()) { + return cel::test::FromExprValue(input_val.value(), descriptor_pool, + message_factory, arena); + } + return absl::InvalidArgumentError("Empty InputValue"); +} + +class CelValueMatcherImpl + : public testing::MatcherInterface { + public: + CelValueMatcherImpl(cel::Value expected_val, + const google::protobuf::DescriptorPool* pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena) + : expected_val_(std::move(expected_val)), + pool_(pool), + message_factory_(message_factory), + arena_(arena) {} + + bool MatchAndExplain(const cel::Value& actual_val, + testing::MatchResultListener* listener) const override { + cel::Value actual = actual_val; + if (actual.IsOptional() && !expected_val_.IsOptional()) { + auto opt_val = actual.AsOptional(); + if (opt_val->HasValue()) { + actual = opt_val->Value(); + } + } + cel::Value eq_result; + auto eq_status = actual.Equal(expected_val_, pool_, message_factory_, + arena_, &eq_result); + if (!eq_status.ok()) { + *listener << "equality check failed with status: " << eq_status; + return false; + } + if (!eq_result.IsTrue()) { + *listener << "expected: " << expected_val_.DebugString() + << "\nactual: " << actual.DebugString(); + return false; + } + return true; + } + + void DescribeTo(std::ostream* os) const override { + *os << "is equal to " << expected_val_.DebugString(); + } + + void DescribeNegationTo(std::ostream* os) const override { + *os << "is not equal to " << expected_val_.DebugString(); + } + + private: + cel::Value expected_val_; + const google::protobuf::DescriptorPool* pool_; + google::protobuf::MessageFactory* message_factory_; + google::protobuf::Arena* arena_; +}; + +absl::StatusOr> MakeExpectedValueMatcher( + const cel::expr::conformance::test::TestOutput& output, + const InputEvaluator& input_evaluator, const google::protobuf::DescriptorPool* pool, + google::protobuf::MessageFactory* message_factory, google::protobuf::Arena* arena) { + cel::Value expected_val; + if (output.has_result_expr()) { + CEL_ASSIGN_OR_RETURN( + expected_val, + input_evaluator.Evaluate(output.result_expr(), arena, message_factory)); + } else if (output.has_result_value()) { + CEL_ASSIGN_OR_RETURN(expected_val, + cel::test::FromExprValue(output.result_value(), pool, + message_factory, arena)); + } else { + return absl::InvalidArgumentError("Unsupported output kind"); + } + return testing::Matcher( + new CelValueMatcherImpl(expected_val, pool, message_factory, arena)); +} + +bool ShouldRunTest(absl::string_view test_name, + const std::vector& skip_tests) { + for (const std::string& skip : skip_tests) { + if (absl::StartsWith(test_name, skip)) { + return false; + } + } + return true; +} + +absl::Status PopulateActivation( + const cel::expr::conformance::test::TestCase& test, + const InputEvaluator& input_evaluator, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + absl::string_view context_msg_type_name, google::protobuf::Arena* arena, + Activation& activation) { + if (!test.has_input_context()) { + for (const auto& [var_name, input_val] : test.input()) { + CEL_ASSIGN_OR_RETURN( + auto val, + EvaluateInputValue(input_val, input_evaluator, descriptor_pool, + message_factory, arena)); + activation.InsertOrAssignValue(var_name, std::move(val)); + } + return absl::OkStatus(); + } + + const auto& input_context = test.input_context(); + const google::protobuf::Message* context_message = nullptr; + + if (input_context.has_context_message()) { + const google::protobuf::Any& any_msg = input_context.context_message(); + const google::protobuf::Descriptor* msg_descriptor = + descriptor_pool->FindMessageTypeByName(context_msg_type_name); + if (msg_descriptor == nullptr) { + return absl::NotFoundError(absl::StrCat( + "Failed to find message descriptor for: ", context_msg_type_name)); + } + const google::protobuf::Message* prototype = + message_factory->GetPrototype(msg_descriptor); + if (prototype == nullptr) { + return absl::NotFoundError( + absl::StrCat("Failed to get prototype for: ", context_msg_type_name)); + } + auto* buf = prototype->New(arena); + if (!any_msg.UnpackTo(buf)) { + return absl::InvalidArgumentError(absl::StrCat( + "Failed to unpack context message to ", context_msg_type_name)); + } + context_message = buf; + } else if (input_context.has_context_expr() && + !context_msg_type_name.empty()) { + CEL_ASSIGN_OR_RETURN(cel::Value evaluated_val, + input_evaluator.Evaluate(input_context.context_expr(), + arena, message_factory)); + + if (!evaluated_val.IsParsedMessage()) { + return absl::InvalidArgumentError( + absl::StrCat("Context expression did not evaluate to a message: ", + input_context.context_expr())); + } + if (evaluated_val.GetParsedMessage().GetDescriptor()->full_name() != + context_msg_type_name) { + return absl::InvalidArgumentError(absl::StrCat( + "Context expression evaluated to a message of type ", + evaluated_val.GetParsedMessage().GetDescriptor()->full_name(), + " which does not match the expected type ", context_msg_type_name)); + } + context_message = static_cast( + evaluated_val.GetParsedMessage().operator->()); + } + if (context_message == nullptr) { + return absl::InvalidArgumentError( + "Failed to resolve context message for test case"); + } + + return cel::BindProtoToActivation( + *context_message, cel::BindProtoUnsetFieldBehavior::kBindDefaultValue, + descriptor_pool, message_factory, arena, &activation); +} + +class PolicyTestSuiteRunner { + public: + PolicyTestSuiteRunner(std::string suite_name, + std::unique_ptr compiler, + std::unique_ptr runtime, + std::shared_ptr policy_source, + CelPolicyValidationResult compile_result, + std::shared_ptr pool, + std::shared_ptr message_factory, + std::shared_ptr input_evaluator, + std::string context_msg_type_name, + bool expect_compile_fail = false) + : suite_name_(std::move(suite_name)), + compiler_(std::move(compiler)), + runtime_(std::move(runtime)), + policy_source_(std::move(policy_source)), + compile_result_(std::move(compile_result)), + pool_(std::move(pool)), + message_factory_(std::move(message_factory)), + input_evaluator_(std::move(input_evaluator)), + context_msg_type_name_(std::move(context_msg_type_name)), + expect_compile_fail_(expect_compile_fail) {} + + void RunTest(const cel::expr::conformance::test::TestCase& test, + absl::string_view full_test_name) { + const auto& output = test.output(); + + if (expect_compile_fail_) { + ASSERT_FALSE(compile_result_.IsValid()) + << "Expected compilation to fail in " << full_test_name; + ASSERT_TRUE(output.has_eval_error()) + << "Expected eval_error to be present in compile error test " + << full_test_name; + std::string err_msg = compile_result_.FormatIssues(); + for (const auto& expected_err : output.eval_error().errors()) { + EXPECT_THAT(err_msg, HasSubstr(expected_err.message())) + << "Did not find expected compile time error"; + } + return; + } + + // Compilation should have succeeded for evaluation tests + ASSERT_TRUE(compile_result_.IsValid()) + << "Compilation has validation errors in " << full_test_name << ": " + << compile_result_.FormatIssues(); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime_->CreateProgram(std::make_unique( + *compile_result_.GetAst()))); + + // Parse Inputs and evaluate them + google::protobuf::Arena arena; + Activation activation; + ASSERT_THAT(PopulateActivation(test, *input_evaluator_, pool_.get(), + message_factory_.get(), + context_msg_type_name_, &arena, activation), + IsOk()); + + // Evaluate Policy + auto eval_result_or = program->Evaluate(&arena, activation); + ASSERT_THAT(eval_result_or.status(), IsOk()) + << "Evaluation failed in " << full_test_name; + cel::Value actual_val = *eval_result_or; + + ASSERT_OK_AND_ASSIGN( + auto matcher, + MakeExpectedValueMatcher(output, *input_evaluator_, pool_.get(), + message_factory_.get(), &arena)); + + // Apply matcher to the output of evaluation + EXPECT_THAT(actual_val, matcher) << "Test failed: " << full_test_name; + } + + private: + std::string suite_name_; + std::unique_ptr compiler_; + std::unique_ptr runtime_; + std::shared_ptr policy_source_; + CelPolicyValidationResult compile_result_; + std::shared_ptr pool_; + std::shared_ptr message_factory_; + std::shared_ptr input_evaluator_; + std::string context_msg_type_name_; + bool expect_compile_fail_; +}; + +class CelPolicyTest : public testing::Test { + public: + explicit CelPolicyTest(std::shared_ptr runner, + cel::expr::conformance::test::TestCase test_case, + std::string full_test_name, bool skip) + : runner_(std::move(runner)), + test_case_(std::move(test_case)), + full_test_name_(std::move(full_test_name)), + skip_(skip) {} + + void TestBody() override { + if (skip_) { + GTEST_SKIP() << "Skipping test: " << full_test_name_; + } + EXPECT_NO_FATAL_FAILURE(runner_->RunTest(test_case_, full_test_name_)); + } + + private: + std::shared_ptr runner_; + cel::expr::conformance::test::TestCase test_case_; + std::string full_test_name_; + bool skip_; +}; + +absl::Status RegisterTestSuite( + absl::string_view suite_name, const BundleSections& sections, + const std::shared_ptr& input_evaluator, + const std::shared_ptr& pool, + const std::shared_ptr& message_factory, + const std::vector& skip_tests) { + // Check if the entire suite should be skipped (prefix match) + for (const auto& skip : skip_tests) { + if (suite_name == skip || + absl::StartsWith(suite_name, absl::StrCat(skip, "/"))) { + std::cout << "[ SKIPPED SUITE ] " << suite_name << std::endl; + return absl::OkStatus(); + } + } + + if (sections.policy_content.empty() || sections.tests_content.empty()) { + return absl::OkStatus(); + } + + // Parse Environment Config + cel::Config config; + if (!sections.config_content.empty()) { + CEL_ASSIGN_OR_RETURN( + config, cel::EnvConfigFromYaml(std::string(sections.config_content))); + } + + // Enable default extensions (optional, bindings) in the config + CEL_RETURN_IF_ERROR(config.AddExtensionConfig( + "optional", cel::Config::ExtensionConfig::kLatest)); + CEL_RETURN_IF_ERROR(config.AddExtensionConfig( + "bindings", cel::Config::ExtensionConfig::kLatest)); + + // Set up compiler & runtime environments + cel::Env env; + env.SetDescriptorPool(pool); + cel::RegisterStandardExtensions(env); + env.SetConfig(config); + + cel::EnvRuntime env_runtime; + env_runtime.SetDescriptorPool(pool); + cel::RegisterStandardExtensions(env_runtime); + env_runtime.SetConfig(config); + env_runtime.mutable_runtime_options().enable_qualified_type_identifiers = + true; + + CEL_ASSIGN_OR_RETURN(auto compiler_builder, env.NewCompilerBuilder()); + compiler_builder->GetParserBuilder().GetOptions().enable_optional_syntax = + true; + + CEL_ASSIGN_OR_RETURN(auto compiler, compiler_builder->Build()); + + CEL_ASSIGN_OR_RETURN(auto runtime_builder, + env_runtime.CreateRuntimeBuilder()); + + // Register conformance enums + for (const auto& enum_name : + {"cel.expr.conformance.proto2.GlobalEnum", + "cel.expr.conformance.proto3.GlobalEnum", + "cel.expr.conformance.proto2.TestAllTypes.NestedEnum", + "cel.expr.conformance.proto3.TestAllTypes.NestedEnum"}) { + auto* enum_desc = pool->FindEnumTypeByName(enum_name); + if (enum_desc != nullptr) { + CEL_RETURN_IF_ERROR(cel::extensions::RegisterProtobufEnum( + runtime_builder.type_registry(), enum_desc)); + } + } + + // Register custom functions in runtime + CEL_RETURN_IF_ERROR( + (cel::UnaryFunctionAdapter:: + RegisterGlobalOverload("locationCode", LocationCode, + runtime_builder.function_registry()))); + CEL_RETURN_IF_ERROR( + (cel::UnaryFunctionAdapter:: + RegisterGlobalOverload("hasCreditCard", HasCreditCard, + runtime_builder.function_registry()))); + CEL_RETURN_IF_ERROR( + (cel::UnaryFunctionAdapter:: + RegisterGlobalOverload("hasEmailOrPhone", HasEmailOrPhone, + runtime_builder.function_registry()))); + + CEL_ASSIGN_OR_RETURN(auto runtime, std::move(runtime_builder).Build()); + + // Parse Policy + CEL_ASSIGN_OR_RETURN(auto source, + cel::NewSource(sections.policy_content, "policy.yaml")); + auto policy_source = std::make_shared(std::move(source)); + CEL_ASSIGN_OR_RETURN(CelPolicyParseResult parse_result, + cel::ParseYamlCelPolicy(policy_source)); + if (!parse_result.IsValid()) { + return absl::InvalidArgumentError( + absl::StrCat("Failed to parse policy.yaml in ", suite_name, + "\nIssues:\n", parse_result.FormattedIssues())); + } + const CelPolicy* policy = parse_result.GetPolicy(); + + // Compile Policy (unexpected non-ok status represents a bug) + CEL_ASSIGN_OR_RETURN(CelPolicyValidationResult compile_result, + CompilePolicy(*compiler, *policy)); + + TestSuite test_suite; + CEL_ASSIGN_OR_RETURN( + test_suite, cel::test::ParsePolicyTestSuiteYaml(sections.tests_content)); + + std::string suite_name_str(suite_name); + auto runner = std::make_shared( + suite_name_str, std::move(compiler), std::move(runtime), + std::move(policy_source), std::move(compile_result), pool, + message_factory, input_evaluator, config.GetContextType(), + /*expect_compile_fail=*/absl::StrContains(suite_name, "compile_error")); + + for (const auto& section : test_suite.sections()) { + std::string section_name = section.name(); + for (const auto& test : section.tests()) { + std::string test_name = test.name(); + std::string full_test_name = + absl::StrCat(suite_name, "/", section_name, "/", test_name); + + bool skip = !ShouldRunTest(full_test_name, skip_tests); + + testing::RegisterTest( + suite_name_str.c_str(), + absl::StrCat(section_name, "/", test_name).c_str(), nullptr, + test_name.c_str(), __FILE__, __LINE__, + [runner, test, full_test_name, skip]() -> CelPolicyTest* { + return new CelPolicyTest(runner, test, full_test_name, skip); + }); + } + } + return absl::OkStatus(); +} + +void RegisterAllTests() { + std::vector bundle_paths = absl::GetFlag(FLAGS_test_bundles); + std::vector skip_tests = absl::GetFlag(FLAGS_skip_tests); + + ABSL_CHECK(!bundle_paths.empty()) + << "No test bundles specified in --test_bundles flag."; + + std::shared_ptr pool = + GetSharedTestingDescriptorPool(); + auto message_factory = + std::make_shared(pool.get()); + message_factory->SetDelegateToGeneratedFactory(true); + auto evaluator_or = InputEvaluator::Create(pool); + ABSL_CHECK_OK(evaluator_or.status()) << "Failed to create input evaluator"; + std::shared_ptr evaluator = std::move(evaluator_or.value()); + + for (const std::string& bundle_path : bundle_paths) { + std::string abs_path = cel::internal::ResolveRunfilesPath(bundle_path); + ABSL_CHECK(!abs_path.empty()) + << "Could not resolve runfile path for test bundle: " << bundle_path; + + std::string bundle_content; + ABSL_CHECK_OK(cel::internal::GetFileContents(abs_path, &bundle_content)) + << "Failed to read bundle file: " << abs_path; + + auto sections_or = ParseYamlBundle(bundle_content); + ABSL_CHECK_OK(sections_or.status()) + << "Failed to parse bundle file: " << abs_path; + + absl::string_view filename = bundle_path; + size_t last_slash = filename.find_last_of("/\\"); + if (last_slash != absl::string_view::npos) { + filename = filename.substr(last_slash + 1); + } + absl::string_view suite_view = filename; + absl::ConsumeSuffix(&suite_view, "_bundle.yaml"); + std::string suite_name = std::string(suite_view); + + ABSL_CHECK_OK(RegisterTestSuite(suite_name, sections_or.value(), evaluator, + pool, message_factory, skip_tests)); + } +} + +} // namespace +} // namespace cel + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + + cel::RegisterAllTests(); + return RUN_ALL_TESTS(); +} diff --git a/conformance/run.bzl b/conformance/run.bzl index 4fcf325c6..c56db3ae3 100644 --- a/conformance/run.bzl +++ b/conformance/run.bzl @@ -47,41 +47,45 @@ def _expand_tests_to_skip(tests_to_skip): result.append(test_to_skip[0:slash] + part) return result -def _conformance_test_name(name, optimize, recursive): +def _conformance_test_name(name, pratt, optimize): return "_".join( [ name, + "pratt" if pratt else "antlr", "optimized" if optimize else "unoptimized", - "recursive" if recursive else "iterative", ], ) -def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, skip_tests, dashboard): +def _conformance_test_args(modern, optimize, extended_optimizations, skip_check, dashboard, enable_variadic_logical_operators, pratt): args = [] if modern: args.append("--modern") if optimize: args.append("--opt") - if select_opt: - args.append("--select_optimization") - if recursive: - args.append("--recursive") + if extended_optimizations: + args.append("--extended_optimizations") if skip_check: args.append("--skip_check") else: args.append("--noskip_check") - args.append("--skip_tests={}".format(",".join(_expand_tests_to_skip(skip_tests)))) if dashboard: args.append("--dashboard") + if enable_variadic_logical_operators: + args.append("--enable_variadic_logical_operators") + if pratt: + args.append("--enable_pratt_parser") + else: + args.append("--noenable_pratt_parser") return args -def _conformance_test(name, data, modern, optimize, recursive, select_opt, skip_check, skip_tests, tags, dashboard): +def _conformance_test(name, data, modern, optimize, extended_optimizations, skip_check, skip_tests, tags, dashboard, enable_variadic_logical_operators, pratt): cc_test( - name = _conformance_test_name(name, optimize, recursive), - args = _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, skip_tests, dashboard) + ["$(location " + test + ")" for test in data] + select( + name = _conformance_test_name(name, pratt, optimize), + args = _conformance_test_args(modern, optimize, extended_optimizations, skip_check, dashboard, enable_variadic_logical_operators, pratt) + ["$(rlocationpath {})".format(test) for test in data], + env = select( { - "@platforms//os:windows": ["--skip_tests={}".format(",".join(skip_tests + _TESTS_TO_SKIP_WINDOWS))], - "//conditions:default": ["--skip_tests={}".format(",".join(skip_tests))], + "@platforms//os:windows": {"CEL_SKIP_TESTS": ",".join(skip_tests + _TESTS_TO_SKIP_WINDOWS)}, + "//conditions:default": {"CEL_SKIP_TESTS": ",".join(skip_tests)}, }, ), data = data, @@ -89,36 +93,39 @@ def _conformance_test(name, data, modern, optimize, recursive, select_opt, skip_ tags = tags, ) -def gen_conformance_tests(name, data, modern = False, checked = False, select_opt = False, dashboard = False, skip_tests = [], tags = []): +def gen_conformance_tests(name, data, modern = False, checked = False, extended_optimizations = False, dashboard = False, skip_tests = [], tags = [], enable_variadic_logical_operators = False): """Generates conformance tests. Args: name: prefix for all tests + data: textproto targets describing conformance tests modern: run using modern APIs checked: whether to apply type checking - data: textproto targets describing conformance tests + extended_optimizations: enable extended optimizations (select and set membership) + dashboard: enable dashboard mode skip_tests: tests to skip in the format of the cel-spec test runner. See documentation in github.com/google/cel-spec/tests/simple/simple_test.go tags: tags added to the generated targets - dashboard: enable dashboard mode + enable_variadic_logical_operators: enable variadic logical operators """ skip_check = not checked tests = [] - for optimize in (True, False): - for recursive in (True, False): - test_name = _conformance_test_name(name, optimize, recursive) + for pratt in (True, False): + for optimize in (True, False): + test_name = _conformance_test_name(name, pratt, optimize) tests.append(test_name) _conformance_test( name, data, modern = modern, optimize = optimize, - recursive = recursive, - select_opt = select_opt, + extended_optimizations = extended_optimizations, skip_check = skip_check, skip_tests = _expand_tests_to_skip(skip_tests), tags = tags, dashboard = dashboard, + enable_variadic_logical_operators = enable_variadic_logical_operators, + pratt = pratt, ) native.test_suite( name = name, diff --git a/conformance/run.cc b/conformance/run.cc index d5a919d76..67cd7c694 100644 --- a/conformance/run.cc +++ b/conformance/run.cc @@ -42,11 +42,13 @@ #include "absl/strings/cord.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" #include "absl/strings/string_view.h" #include "absl/strings/strip.h" #include "absl/types/span.h" #include "conformance/service.h" #include "conformance/utils.h" +#include "internal/runfiles.h" #include "internal/testing.h" #include "cel/expr/conformance/test/simple.pb.h" #include "google/protobuf/io/zero_copy_stream_impl.h" @@ -57,18 +59,19 @@ ABSL_FLAG(bool, opt, false, "Enable optimizations (constant folding)"); ABSL_FLAG( bool, modern, false, "Use modern cel::Value APIs implementation of the conformance service."); -ABSL_FLAG(bool, recursive, false, - "Enable recursive plans. Depth limited to slightly more than the " - "default nesting limit."); ABSL_FLAG(std::vector, skip_tests, {}, "Tests to skip"); ABSL_FLAG(bool, dashboard, false, "Dashboard mode, ignore test failures"); ABSL_FLAG(bool, skip_check, true, "Skip type checking the expressions"); -ABSL_FLAG(bool, select_optimization, false, "Enable select optimization."); +ABSL_FLAG(bool, extended_optimizations, false, + "Enable extended optimizations."); +ABSL_FLAG(bool, enable_variadic_logical_operators, false, + "Enable parsing logical AND & OR operators as a single flat variadic " + "call."); +ABSL_FLAG(bool, enable_pratt_parser, true, + "Enable manual (Pratt) parser instead of ANTLR parser."); namespace { -using ::testing::IsEmpty; - using cel::expr::conformance::test::SimpleTest; using cel::expr::conformance::test::SimpleTestFile; using google::api::expr::conformance::v1alpha1::CheckRequest; @@ -77,6 +80,7 @@ using google::api::expr::conformance::v1alpha1::EvalRequest; using google::api::expr::conformance::v1alpha1::EvalResponse; using google::api::expr::conformance::v1alpha1::ParseRequest; using google::api::expr::conformance::v1alpha1::ParseResponse; +using ::testing::IsEmpty; google::rpc::Code ToGrpcCode(absl::StatusCode code) { return static_cast(code); @@ -258,8 +262,10 @@ NewConformanceServiceFromFlags() { cel_conformance::ConformanceServiceOptions{ .optimize = absl::GetFlag(FLAGS_opt), .modern = absl::GetFlag(FLAGS_modern), - .recursive = absl::GetFlag(FLAGS_recursive), - .select_optimization = absl::GetFlag(FLAGS_select_optimization), + .extended_optimizations = absl::GetFlag(FLAGS_extended_optimizations), + .enable_variadic_logical_operators = + absl::GetFlag(FLAGS_enable_variadic_logical_operators), + .enable_pratt_parser = absl::GetFlag(FLAGS_enable_pratt_parser), }); ABSL_CHECK_OK(status_or_service); return std::shared_ptr( @@ -273,9 +279,17 @@ int main(int argc, char** argv) { { auto service = NewConformanceServiceFromFlags(); auto tests_to_skip = absl::GetFlag(FLAGS_skip_tests); + if (const char* env_skip = std::getenv("CEL_SKIP_TESTS"); + env_skip != nullptr) { + for (absl::string_view test : + absl::StrSplit(env_skip, ',', absl::SkipEmpty())) { + tests_to_skip.push_back(std::string(test)); + } + } for (int argi = 1; argi < argc; argi++) { + std::string path = cel::internal::ResolveRunfilesPath(argv[argi]); ABSL_CHECK_OK(RegisterTestsFromFile(service, tests_to_skip, - absl::string_view(argv[argi]))); + absl::string_view(path))); } } int exit_code = RUN_ALL_TESTS(); diff --git a/conformance/service.cc b/conformance/service.cc index 3edc214e6..3d5fef919 100644 --- a/conformance/service.cc +++ b/conformance/service.cc @@ -14,7 +14,6 @@ #include "conformance/service.h" -#include #include #include #include @@ -26,21 +25,16 @@ #include "google/api/expr/v1alpha1/eval.pb.h" #include "google/api/expr/v1alpha1/syntax.pb.h" #include "google/api/expr/v1alpha1/value.pb.h" -#include "google/protobuf/duration.pb.h" -#include "google/protobuf/empty.pb.h" -#include "google/protobuf/struct.pb.h" -#include "google/protobuf/timestamp.pb.h" #include "google/rpc/code.pb.h" +#include "google/rpc/status.pb.h" #include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" -#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/strings/strip.h" -#include "absl/types/optional.h" -#include "absl/types/span.h" +#include "checker/checker_options.h" #include "checker/optional.h" #include "checker/standard_library.h" #include "checker/type_checker_builder.h" @@ -48,7 +42,6 @@ #include "common/ast.h" #include "common/ast_proto.h" #include "common/decl_proto_v1alpha1.h" -#include "common/expr.h" #include "common/internal/value_conversion.h" #include "common/source.h" #include "common/value.h" @@ -64,16 +57,17 @@ #include "extensions/comprehensions_v2_functions.h" #include "extensions/comprehensions_v2_macros.h" #include "extensions/encoders.h" +#include "extensions/lists_functions.h" #include "extensions/math_ext.h" #include "extensions/math_ext_decls.h" #include "extensions/math_ext_macros.h" #include "extensions/proto_ext.h" #include "extensions/protobuf/enum_adapter.h" #include "extensions/select_optimization.h" +#include "extensions/set_membership_optimization.h" +#include "extensions/sets_functions.h" #include "extensions/strings.h" #include "internal/status_macros.h" -#include "parser/macro.h" -#include "parser/macro_expr_factory.h" #include "parser/macro_registry.h" #include "parser/options.h" #include "parser/parser.h" @@ -82,9 +76,11 @@ #include "runtime/constant_folding.h" #include "runtime/optional_types.h" #include "runtime/reference_resolver.h" +#include "runtime/regex_precompilation.h" #include "runtime/runtime.h" #include "runtime/runtime_options.h" #include "runtime/standard_runtime_builder_factory.h" +#include "testutil/test_macros.h" #include "cel/expr/conformance/proto2/test_all_types.pb.h" #include "cel/expr/conformance/proto2/test_all_types_extensions.pb.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" @@ -106,109 +102,6 @@ namespace google::api::expr::runtime { namespace { -bool IsCelNamespace(const cel::Expr& target) { - return target.has_ident_expr() && target.ident_expr().name() == "cel"; -} - -absl::optional CelBlockMacroExpander(cel::MacroExprFactory& factory, - cel::Expr& target, - absl::Span args) { - if (!IsCelNamespace(target)) { - return absl::nullopt; - } - cel::Expr& bindings_arg = args[0]; - if (!bindings_arg.has_list_expr()) { - return factory.ReportErrorAt( - bindings_arg, "cel.block requires the first arg to be a list literal"); - } - return factory.NewCall("cel.@block", args); -} - -absl::optional CelIndexMacroExpander(cel::MacroExprFactory& factory, - cel::Expr& target, - absl::Span args) { - if (!IsCelNamespace(target)) { - return absl::nullopt; - } - cel::Expr& index_arg = args[0]; - if (!index_arg.has_const_expr() || !index_arg.const_expr().has_int_value()) { - return factory.ReportErrorAt( - index_arg, "cel.index requires a single non-negative int constant arg"); - } - int64_t index = index_arg.const_expr().int_value(); - if (index < 0) { - return factory.ReportErrorAt( - index_arg, "cel.index requires a single non-negative int constant arg"); - } - return factory.NewIdent(absl::StrCat("@index", index)); -} - -absl::optional CelIterVarMacroExpander( - cel::MacroExprFactory& factory, cel::Expr& target, - absl::Span args) { - if (!IsCelNamespace(target)) { - return absl::nullopt; - } - cel::Expr& depth_arg = args[0]; - if (!depth_arg.has_const_expr() || !depth_arg.const_expr().has_int_value() || - depth_arg.const_expr().int_value() < 0) { - return factory.ReportErrorAt( - depth_arg, "cel.iterVar requires two non-negative int constant args"); - } - cel::Expr& unique_arg = args[1]; - if (!unique_arg.has_const_expr() || - !unique_arg.const_expr().has_int_value() || - unique_arg.const_expr().int_value() < 0) { - return factory.ReportErrorAt( - unique_arg, "cel.iterVar requires two non-negative int constant args"); - } - return factory.NewIdent( - absl::StrCat("@it:", depth_arg.const_expr().int_value(), ":", - unique_arg.const_expr().int_value())); -} - -absl::optional CelAccuVarMacroExpander( - cel::MacroExprFactory& factory, cel::Expr& target, - absl::Span args) { - if (!IsCelNamespace(target)) { - return absl::nullopt; - } - cel::Expr& depth_arg = args[0]; - if (!depth_arg.has_const_expr() || !depth_arg.const_expr().has_int_value() || - depth_arg.const_expr().int_value() < 0) { - return factory.ReportErrorAt( - depth_arg, "cel.accuVar requires two non-negative int constant args"); - } - cel::Expr& unique_arg = args[1]; - if (!unique_arg.has_const_expr() || - !unique_arg.const_expr().has_int_value() || - unique_arg.const_expr().int_value() < 0) { - return factory.ReportErrorAt( - unique_arg, "cel.accuVar requires two non-negative int constant args"); - } - return factory.NewIdent( - absl::StrCat("@ac:", depth_arg.const_expr().int_value(), ":", - unique_arg.const_expr().int_value())); -} - -absl::Status RegisterCelBlockMacros(cel::MacroRegistry& registry) { - CEL_ASSIGN_OR_RETURN(auto block_macro, - cel::Macro::Receiver("block", 2, CelBlockMacroExpander)); - CEL_RETURN_IF_ERROR(registry.RegisterMacro(block_macro)); - CEL_ASSIGN_OR_RETURN(auto index_macro, - cel::Macro::Receiver("index", 1, CelIndexMacroExpander)); - CEL_RETURN_IF_ERROR(registry.RegisterMacro(index_macro)); - CEL_ASSIGN_OR_RETURN( - auto iter_var_macro, - cel::Macro::Receiver("iterVar", 2, CelIterVarMacroExpander)); - CEL_RETURN_IF_ERROR(registry.RegisterMacro(iter_var_macro)); - CEL_ASSIGN_OR_RETURN( - auto accu_var_macro, - cel::Macro::Receiver("accuVar", 2, CelAccuVarMacroExpander)); - CEL_RETURN_IF_ERROR(registry.RegisterMacro(accu_var_macro)); - return absl::OkStatus(); -} - google::rpc::Code ToGrpcCode(absl::StatusCode code) { return static_cast(code); } @@ -236,13 +129,17 @@ cel::expr::Expr ExtractExpr( absl::Status LegacyParse(const conformance::v1alpha1::ParseRequest& request, conformance::v1alpha1::ParseResponse& response, - bool enable_optional_syntax) { + bool enable_optional_syntax, + bool enable_variadic_logical_operators, + bool enable_pratt_parser) { if (request.cel_source().empty()) { return absl::InvalidArgumentError("no source code"); } cel::ParserOptions options; options.enable_optional_syntax = enable_optional_syntax; options.enable_quoted_identifiers = true; + options.enable_variadic_logical_operators = enable_variadic_logical_operators; + options.enable_pratt_parser = enable_pratt_parser; cel::MacroRegistry macros; CEL_RETURN_IF_ERROR(cel::RegisterStandardMacros(macros, options)); CEL_RETURN_IF_ERROR( @@ -250,7 +147,8 @@ absl::Status LegacyParse(const conformance::v1alpha1::ParseRequest& request, CEL_RETURN_IF_ERROR(cel::extensions::RegisterBindingsMacros(macros, options)); CEL_RETURN_IF_ERROR(cel::extensions::RegisterMathMacros(macros, options)); CEL_RETURN_IF_ERROR(cel::extensions::RegisterProtoMacros(macros, options)); - CEL_RETURN_IF_ERROR(RegisterCelBlockMacros(macros)); + CEL_RETURN_IF_ERROR(cel::extensions::RegisterListsMacros(macros, options)); + CEL_RETURN_IF_ERROR(cel::test::RegisterTestMacros(macros)); CEL_ASSIGN_OR_RETURN(auto source, cel::NewSource(request.cel_source(), request.source_location())); CEL_ASSIGN_OR_RETURN(auto parsed_expr, @@ -278,13 +176,18 @@ absl::Status CheckImpl(google::protobuf::Arena* arena, CEL_ASSIGN_OR_RETURN(source, cel::NewSource(location)); } + cel::CheckerOptions checker_options; + checker_options.enable_cross_numeric_comparisons = true; CEL_ASSIGN_OR_RETURN( std::unique_ptr builder, - cel::CreateTypeCheckerBuilder(google::protobuf::DescriptorPool::generated_pool())); + cel::CreateTypeCheckerBuilder(google::protobuf::DescriptorPool::generated_pool(), + checker_options)); if (!request.no_std_env()) { CEL_RETURN_IF_ERROR(builder->AddLibrary(cel::StandardCheckerLibrary())); CEL_RETURN_IF_ERROR(builder->AddLibrary(cel::OptionalCheckerLibrary())); + CEL_RETURN_IF_ERROR( + builder->AddLibrary(cel::extensions::BindingsCheckerLibrary())); CEL_RETURN_IF_ERROR( builder->AddLibrary(cel::extensions::StringsCheckerLibrary())); CEL_RETURN_IF_ERROR( @@ -293,6 +196,10 @@ absl::Status CheckImpl(google::protobuf::Arena* arena, builder->AddLibrary(cel::extensions::EncodersCheckerLibrary())); CEL_RETURN_IF_ERROR( builder->AddLibrary(cel::extensions::ComprehensionsV2CheckerLibrary())); + CEL_RETURN_IF_ERROR( + builder->AddLibrary(cel::extensions::ListsCheckerLibrary())); + CEL_RETURN_IF_ERROR( + builder->AddLibrary(cel::extensions::SetsCheckerLibrary())); } for (const auto& decl : request.type_env()) { @@ -302,7 +209,7 @@ absl::Status CheckImpl(google::protobuf::Arena* arena, auto fn_decl, cel::FunctionDeclFromV1Alpha1Proto( name, decl.function(), google::protobuf::DescriptorPool::generated_pool(), arena)); - CEL_RETURN_IF_ERROR(builder->AddFunction(std::move(fn_decl))); + CEL_RETURN_IF_ERROR(builder->MergeFunction(std::move(fn_decl))); } else if (decl.has_ident()) { CEL_ASSIGN_OR_RETURN( auto var_decl, cel::VariableDeclFromV1Alpha1Proto( @@ -342,7 +249,8 @@ absl::Status CheckImpl(google::protobuf::Arena* arena, class LegacyConformanceServiceImpl : public ConformanceServiceInterface { public: static absl::StatusOr> Create( - bool optimize, bool recursive, bool select_optimization) { + bool optimize, bool recursive, bool extended_optimizations, + bool enable_variadic_logical_operators, bool enable_pratt_parser) { static auto* constant_arena = new Arena(); google::protobuf::LinkMessageReflection< @@ -386,10 +294,11 @@ class LegacyConformanceServiceImpl : public ConformanceServiceInterface { std::cerr << "Enabling optimizations" << std::endl; options.constant_folding = true; options.constant_arena = constant_arena; + options.enable_typed_field_access = true; } - if (select_optimization) { - std::cerr << "Enabling select optimizations" << std::endl; + if (extended_optimizations) { + std::cerr << "Enabling extended optimizations" << std::endl; options.enable_select_optimization = true; } @@ -418,15 +327,19 @@ class LegacyConformanceServiceImpl : public ConformanceServiceInterface { builder->GetRegistry(), options)); CEL_RETURN_IF_ERROR(cel::extensions::RegisterMathExtensionFunctions( builder->GetRegistry(), options)); + CEL_RETURN_IF_ERROR(cel::extensions::RegisterSetsFunctions( + builder->GetRegistry(), options)); - return absl::WrapUnique( - new LegacyConformanceServiceImpl(std::move(builder))); + return absl::WrapUnique(new LegacyConformanceServiceImpl( + std::move(builder), enable_variadic_logical_operators, + enable_pratt_parser)); } void Parse(const conformance::v1alpha1::ParseRequest& request, conformance::v1alpha1::ParseResponse& response) override { auto status = - LegacyParse(request, response, /*enable_optional_syntax=*/false); + LegacyParse(request, response, /*enable_optional_syntax=*/false, + enable_variadic_logical_operators_, enable_pratt_parser_); if (!status.ok()) { auto* issue = response.add_issues(); issue->set_code(ToGrpcCode(status.code())); @@ -524,17 +437,23 @@ class LegacyConformanceServiceImpl : public ConformanceServiceInterface { } private: - explicit LegacyConformanceServiceImpl( - std::unique_ptr builder) - : builder_(std::move(builder)) {} + LegacyConformanceServiceImpl(std::unique_ptr builder, + bool enable_variadic_logical_operators, + bool enable_pratt_parser) + : builder_(std::move(builder)), + enable_variadic_logical_operators_(enable_variadic_logical_operators), + enable_pratt_parser_(enable_pratt_parser) {} std::unique_ptr builder_; + bool enable_variadic_logical_operators_; + bool enable_pratt_parser_; }; class ModernConformanceServiceImpl : public ConformanceServiceInterface { public: static absl::StatusOr> Create( - bool optimize, bool recursive, bool select_optimization) { + bool optimize, bool recursive, bool extended_optimizations, + bool enable_variadic_logical_operators, bool enable_pratt_parser) { google::protobuf::LinkMessageReflection< cel::expr::conformance::proto3::TestAllTypes>(); google::protobuf::LinkMessageReflection< @@ -577,13 +496,17 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { } return absl::WrapUnique(new ModernConformanceServiceImpl( - options, optimize, select_optimization)); + options, optimize, extended_optimizations, + enable_variadic_logical_operators, enable_pratt_parser)); } absl::StatusOr> Setup( absl::string_view container) { RuntimeOptions options(options_); options.container = std::string(container); + if (enable_optimizations_) { + options.enable_typed_field_access = true; + } CEL_ASSIGN_OR_RETURN( auto builder, CreateStandardRuntimeBuilder( google::protobuf::DescriptorPool::generated_pool(), options)); @@ -591,11 +514,14 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { if (enable_optimizations_) { CEL_RETURN_IF_ERROR(cel::extensions::EnableConstantFolding( builder, google::protobuf::MessageFactory::generated_factory())); + CEL_RETURN_IF_ERROR(cel::extensions::EnableRegexPrecompilation(builder)); } CEL_RETURN_IF_ERROR(cel::EnableReferenceResolver( builder, cel::ReferenceResolverEnabled::kAlways)); - if (enable_select_optimization_) { + if (enable_extended_optimizations_) { CEL_RETURN_IF_ERROR(cel::extensions::EnableSelectOptimization(builder)); + CEL_RETURN_IF_ERROR( + cel::extensions::EnableSetMembershipOptimization(builder)); } auto& type_registry = builder.type_registry(); @@ -622,6 +548,10 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { builder.function_registry(), options)); CEL_RETURN_IF_ERROR(cel::extensions::RegisterMathExtensionFunctions( builder.function_registry(), options)); + CEL_RETURN_IF_ERROR(cel::extensions::RegisterListsFunctions( + builder.function_registry(), options)); + CEL_RETURN_IF_ERROR(cel::extensions::RegisterSetsFunctions( + builder.function_registry(), options)); return std::move(builder).Build(); } @@ -629,7 +559,8 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { void Parse(const conformance::v1alpha1::ParseRequest& request, conformance::v1alpha1::ParseResponse& response) override { auto status = - LegacyParse(request, response, /*enable_optional_syntax=*/true); + LegacyParse(request, response, /*enable_optional_syntax=*/true, + enable_variadic_logical_operators_, enable_pratt_parser_); if (!status.ok()) { auto* issue = response.add_issues(); issue->set_code(ToGrpcCode(status.code())); @@ -720,10 +651,14 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { private: ModernConformanceServiceImpl(const RuntimeOptions& options, bool enable_optimizations, - bool enable_select_optimization) + bool enable_extended_optimizations, + bool enable_variadic_logical_operators, + bool enable_pratt_parser) : options_(options), enable_optimizations_(enable_optimizations), - enable_select_optimization_(enable_select_optimization) {} + enable_extended_optimizations_(enable_extended_optimizations), + enable_variadic_logical_operators_(enable_variadic_logical_operators), + enable_pratt_parser_(enable_pratt_parser) {} static absl::StatusOr> Plan( const cel::Runtime& runtime, @@ -753,7 +688,9 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { RuntimeOptions options_; bool enable_optimizations_; - bool enable_select_optimization_; + bool enable_extended_optimizations_; + bool enable_variadic_logical_operators_; + bool enable_pratt_parser_; }; } // namespace @@ -766,10 +703,12 @@ absl::StatusOr> NewConformanceService(const ConformanceServiceOptions& options) { if (options.modern) { return google::api::expr::runtime::ModernConformanceServiceImpl::Create( - options.optimize, options.recursive, options.select_optimization); + options.optimize, options.recursive, options.extended_optimizations, + options.enable_variadic_logical_operators, options.enable_pratt_parser); } else { return google::api::expr::runtime::LegacyConformanceServiceImpl::Create( - options.optimize, options.recursive, options.select_optimization); + options.optimize, options.recursive, options.extended_optimizations, + options.enable_variadic_logical_operators, options.enable_pratt_parser); } } diff --git a/conformance/service.h b/conformance/service.h index 2dd2abf32..1a3c28c3d 100644 --- a/conformance/service.h +++ b/conformance/service.h @@ -45,7 +45,9 @@ struct ConformanceServiceOptions { bool modern; bool arena; bool recursive; - bool select_optimization; + bool extended_optimizations; + bool enable_variadic_logical_operators = false; + bool enable_pratt_parser = true; }; absl::StatusOr> diff --git a/env/BUILD b/env/BUILD index 55297b190..0c17d6305 100644 --- a/env/BUILD +++ b/env/BUILD @@ -28,6 +28,7 @@ cc_library( "type_info.h", ], deps = [ + "//common:ast", "//common:constant", "//common:type", "//common:type_kind", @@ -52,7 +53,9 @@ cc_library( ":config", "//checker:type_checker_builder", "//common:constant", + "//common:container", "//common:decl", + "//common:signature", "//common:type", "//compiler", "//compiler:compiler_factory", @@ -119,13 +122,16 @@ cc_library( features = ["-use_header_modules"], deps = [ ":config", + "//common:ast", "//common:constant", + "//common:signature", "//internal:status_macros", "//internal:strings", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -177,9 +183,11 @@ cc_test( ":config", "//common:type", "//common:type_proto", + "//common/ast:metadata", "//internal:proto_matchers", "//internal:testing", "//internal:testing_descriptor_pool", + "@com_google_absl//absl/status", "@com_google_protobuf//:protobuf", ], ) @@ -200,7 +208,6 @@ cc_test( "//common:type", "//common:value", "//compiler", - "//internal:proto_matchers", "//internal:status_macros", "//internal:testing", "//internal:testing_descriptor_pool", @@ -240,7 +247,6 @@ cc_test( "//common:value", "//compiler", "//extensions:math_ext", - "//internal:status_macros", "//internal:testing", "//internal:testing_descriptor_pool", "//runtime", diff --git a/env/config.h b/env/config.h index 10b23d030..68e4a1dd9 100644 --- a/env/config.h +++ b/env/config.h @@ -32,11 +32,23 @@ class Config { void SetName(std::string name) { name_ = std::move(name); } std::string GetName() const { return name_; } + void SetContextType(std::string context_type) { + context_type_ = std::move(context_type); + } + std::string GetContextType() const { return context_type_; } + struct ContainerConfig { std::string name; - // TODO(uncreated-issue/87): add support for aliases and abbreviations. + std::vector abbreviations; + struct Alias { + std::string alias; + std::string qualified_name; + }; + std::vector aliases; - bool IsEmpty() const { return name.empty(); } + bool IsEmpty() const { + return name.empty() && abbreviations.empty() && aliases.empty(); + } }; void SetContainerConfig(ContainerConfig container_config) { @@ -143,6 +155,7 @@ class Config { private: std::string name_; + std::string context_type_; ContainerConfig container_config_; std::vector extension_configs_; StandardLibraryConfig standard_library_config_; diff --git a/env/config_test.cc b/env/config_test.cc index df0d6f875..8cfc3cf7f 100644 --- a/env/config_test.cc +++ b/env/config_test.cc @@ -88,6 +88,34 @@ INSTANTIATE_TEST_SUITE_P( StandardLibraryConfigTestCase{ .standard_library_config = {}, }, + StandardLibraryConfigTestCase{ + .standard_library_config = + { + .included_functions = {{"_+_", "add_int64"}, + {"_+_", "add_list"}}, + }, + }, + StandardLibraryConfigTestCase{ + .standard_library_config = + { + .included_functions = {{"_+_", "add(int,int)"}, + {"_+_", "add(list,list)"}}, + }, + }, + StandardLibraryConfigTestCase{ + .standard_library_config = + { + .excluded_functions = {{"_+_", "add_int64"}, + {"_+_", "add_list"}}, + }, + }, + StandardLibraryConfigTestCase{ + .standard_library_config = + { + .excluded_functions = {{"_+_", "add(int,int)"}, + {"_+_", "add(list,list)"}}, + }, + }, StandardLibraryConfigTestCase{ .standard_library_config = { @@ -106,6 +134,15 @@ INSTANTIATE_TEST_SUITE_P( .expected_error = "Cannot set both included and excluded functions.", }, + StandardLibraryConfigTestCase{ + .standard_library_config = + { + .included_functions = {{"_+_", "add(int,int)"}}, + .excluded_functions = {{"_-_", ""}}, + }, + .expected_error = + "Cannot set both included and excluded functions.", + }, StandardLibraryConfigTestCase{ .standard_library_config = { @@ -114,6 +151,15 @@ INSTANTIATE_TEST_SUITE_P( .expected_error = "Cannot include function '_+_' and also its " "specific overload 'add_list'", }, + StandardLibraryConfigTestCase{ + .standard_library_config = + { + .included_functions = {{"_+_", ""}, + {"_+_", "add(int,int)"}}, + }, + .expected_error = "Cannot include function '_+_' and also its " + "specific overload 'add(int,int)'", + }, StandardLibraryConfigTestCase{ .standard_library_config = { @@ -121,6 +167,15 @@ INSTANTIATE_TEST_SUITE_P( }, .expected_error = "Cannot exclude function '_+_' and also its " "specific overload 'add_list'", + }, + StandardLibraryConfigTestCase{ + .standard_library_config = + { + .excluded_functions = {{"_+_", ""}, + {"_+_", "add(int,int)"}}, + }, + .expected_error = "Cannot exclude function '_+_' and also its " + "specific overload 'add(int,int)'", })); TEST(VariableConfigTest, VariableConfig) { diff --git a/env/env.cc b/env/env.cc index 5a4198497..dd71323d5 100644 --- a/env/env.cc +++ b/env/env.cc @@ -24,7 +24,9 @@ #include "absl/strings/string_view.h" #include "checker/type_checker_builder.h" #include "common/constant.h" +#include "common/container.h" #include "common/decl.h" +#include "common/signature.h" #include "common/type.h" #include "compiler/compiler.h" #include "compiler/compiler_factory.h" @@ -56,21 +58,45 @@ bool ShouldIncludeMacro(const Config::StandardLibraryConfig& config, bool ShouldIncludeFunction(const Config::StandardLibraryConfig& config, absl::string_view function, - absl::string_view overload_id) { - if (config.excluded_functions.contains( - std::make_pair(std::string(function), std::string(overload_id))) || - config.excluded_functions.contains( - std::make_pair(std::string(function), ""))) { - return false; + const OverloadDecl& overload) { + if (config.excluded_functions.empty() && config.included_functions.empty()) { + return true; + } + + if (!config.excluded_functions.empty()) { + if (config.excluded_functions.contains(std::make_pair( + std::string(function), std::string(overload.id()))) || + config.excluded_functions.contains( + std::make_pair(std::string(function), ""))) { + return false; + } + absl::StatusOr signature = + MakeOverloadSignature(function, overload.args(), overload.member()); + if (signature.ok() && config.excluded_functions.contains(std::make_pair( + std::string(function), *std::move(signature)))) { + return false; + } } - if (!config.included_functions.empty() && - !config.included_functions.contains( - std::make_pair(std::string(function), "")) && - !config.included_functions.contains( - std::make_pair(std::string(function), std::string(overload_id)))) { + + if (!config.included_functions.empty()) { + if (config.included_functions.contains(std::make_pair( + std::string(function), std::string(overload.id()))) || + config.included_functions.contains( + std::make_pair(std::string(function), ""))) { + return true; + } + // Ok to call MakeOverloadSignature() again, because in practice either + // included or excluded functions may be specified, but not both. + absl::StatusOr signature = + MakeOverloadSignature(function, overload.args(), overload.member()); + if (signature.ok() && config.included_functions.contains(std::make_pair( + std::string(function), *std::move(signature)))) { + return true; + } return false; } - return true; + + return true; // Never reached } absl::StatusOr MakeStdlibSubset( @@ -86,9 +112,8 @@ absl::StatusOr MakeStdlibSubset( }; subset.should_include_overload = [&standard_library_config]( absl::string_view function, - absl::string_view overload_id) { - return ShouldIncludeFunction(standard_library_config, function, - overload_id); + const OverloadDecl& overload) { + return ShouldIncludeFunction(standard_library_config, function, overload); }; return subset; } @@ -121,16 +146,32 @@ absl::StatusOr FunctionConfigToFunctionDecl( Env::Env() { compiler_options_.parser_options.enable_quoted_identifiers = true; + compiler_options_.adapt_parser_errors = true; } -absl::StatusOr> Env::NewCompilerBuilder() { +absl::StatusOr> Env::NewCompilerBuilder() + const { CEL_ASSIGN_OR_RETURN( std::unique_ptr compiler_builder, cel::NewCompilerBuilder(descriptor_pool_, compiler_options_)); cel::TypeCheckerBuilder& checker_builder = compiler_builder->GetCheckerBuilder(); - checker_builder.set_container(config_.GetContainerConfig().name); + ExpressionContainer container; + CEL_RETURN_IF_ERROR( + container.SetContainer(config_.GetContainerConfig().name)); + for (const auto& abbr : config_.GetContainerConfig().abbreviations) { + CEL_RETURN_IF_ERROR(container.AddAbbreviation(abbr)); + } + + if (!config_.GetContextType().empty()) { + CEL_RETURN_IF_ERROR( + checker_builder.AddContextDeclaration(config_.GetContextType())); + } + for (const auto& alias : config_.GetContainerConfig().aliases) { + CEL_RETURN_IF_ERROR(container.AddAlias(alias.alias, alias.qualified_name)); + } + checker_builder.SetExpressionContainer(std::move(container)); if (!config_.GetStandardLibraryConfig().disable) { CEL_RETURN_IF_ERROR( @@ -174,7 +215,7 @@ absl::StatusOr> Env::NewCompilerBuilder() { return compiler_builder; } -absl::StatusOr> Env::NewCompiler() { +absl::StatusOr> Env::NewCompiler() const { CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler_builder, NewCompilerBuilder()); return compiler_builder->Build(); diff --git a/env/env.h b/env/env.h index 9830b67d7..c4a23e2cb 100644 --- a/env/env.h +++ b/env/env.h @@ -59,10 +59,12 @@ class Env { void SetConfig(const Config& config) { config_ = config; } - absl::StatusOr> NewCompilerBuilder(); + CompilerOptions& GetCompilerOptions() { return compiler_options_; } + + absl::StatusOr> NewCompilerBuilder() const; // Shortcut for NewCompilerBuilder() followed by Build(). - absl::StatusOr> NewCompiler(); + absl::StatusOr> NewCompiler() const; private: cel::env_internal::ExtensionRegistry extension_registry_; diff --git a/env/env_runtime.cc b/env/env_runtime.cc index 33e0747cc..0f58d80b9 100644 --- a/env/env_runtime.cc +++ b/env/env_runtime.cc @@ -41,32 +41,38 @@ void EnvRuntime::RegisterExtensionFunctions( name, alias, version, std::move(function_registration_callback)); } -absl::StatusOr EnvRuntime::CreateRuntimeBuilder() { +absl::StatusOr EnvRuntime::CreateRuntimeBuilder() const { + return CreateRuntimeBuilder(runtime_options_); +} + +absl::StatusOr EnvRuntime::CreateRuntimeBuilder( + const RuntimeOptions& options) const { + RuntimeOptions runtime_options = options; const std::vector& extension_configs = config_.GetExtensionConfigs(); const Config::ExtensionConfig* optional_extension_config = nullptr; for (const Config::ExtensionConfig& extension_config : extension_configs) { if (extension_config.name == "optional") { optional_extension_config = &extension_config; - runtime_options_.enable_qualified_type_identifiers = true; + runtime_options.enable_qualified_type_identifiers = true; break; } } CEL_ASSIGN_OR_RETURN( RuntimeBuilder runtime_builder, - cel::CreateRuntimeBuilder(descriptor_pool_, runtime_options_)); + cel::CreateRuntimeBuilder(descriptor_pool_, runtime_options)); if (!config_.GetStandardLibraryConfig().disable) { CEL_RETURN_IF_ERROR(RegisterStandardFunctions( - runtime_builder.function_registry(), runtime_options_)); + runtime_builder.function_registry(), runtime_options)); } // Register optional extension functions first, because other extensions // depend on it (e.g. regex). if (optional_extension_config != nullptr) { CEL_RETURN_IF_ERROR(extension_registry_.RegisterExtensionFunctions( - runtime_builder, runtime_options_, optional_extension_config->name, + runtime_builder, runtime_options, optional_extension_config->name, optional_extension_config->version)); } @@ -75,14 +81,20 @@ absl::StatusOr EnvRuntime::CreateRuntimeBuilder() { continue; } CEL_RETURN_IF_ERROR(extension_registry_.RegisterExtensionFunctions( - runtime_builder, runtime_options_, extension_config.name, + runtime_builder, runtime_options, extension_config.name, extension_config.version)); } return runtime_builder; } -absl::StatusOr> EnvRuntime::NewRuntime() { - CEL_ASSIGN_OR_RETURN(RuntimeBuilder runtime_builder, CreateRuntimeBuilder()); +absl::StatusOr> EnvRuntime::NewRuntime() const { + return NewRuntime(runtime_options_); +} + +absl::StatusOr> EnvRuntime::NewRuntime( + const RuntimeOptions& options) const { + CEL_ASSIGN_OR_RETURN(RuntimeBuilder runtime_builder, + CreateRuntimeBuilder(options)); return std::move(runtime_builder).Build(); } diff --git a/env/env_runtime.h b/env/env_runtime.h index 63473c295..f8a80eb12 100644 --- a/env/env_runtime.h +++ b/env/env_runtime.h @@ -62,10 +62,14 @@ class EnvRuntime { RuntimeOptions& mutable_runtime_options() { return runtime_options_; } - absl::StatusOr CreateRuntimeBuilder(); + absl::StatusOr CreateRuntimeBuilder() const; + absl::StatusOr CreateRuntimeBuilder( + const RuntimeOptions& options) const; // Shortcut for CreateRuntimeBuilder() followed by Build(). - absl::StatusOr> NewRuntime(); + absl::StatusOr> NewRuntime() const; + absl::StatusOr> NewRuntime( + const RuntimeOptions& options) const; private: cel::env_internal::RuntimeExtensionRegistry& GetRuntimeExtensionRegistry() { diff --git a/env/env_test.cc b/env/env_test.cc index 076eb57bc..00143a857 100644 --- a/env/env_test.cc +++ b/env/env_test.cc @@ -280,6 +280,15 @@ INSTANTIATE_TEST_SUITE_P( .expected_invalid_expressions = {"1 + 2", "[1, 2, 3] + [4, 5, 6]", "'hello' + 'world'"}, }, + StandardLibraryConfigTestCase{ + .standard_library_config = + {.excluded_functions = {{"_+_", "_+_(bytes,bytes)"}, + {"_+_", "_+_(list<~A>,list<~A>)"}, + {"_+_", "_+_(string,string)"}}}, + .expected_valid_expressions = {"1 + 2"}, + .expected_invalid_expressions = {"[1, 2, 3] + [4, 5, 6]", + "'hello' + 'world'"}, + }, StandardLibraryConfigTestCase{ .standard_library_config = {.excluded_functions = {{"_+_", "add_bytes"}, @@ -294,6 +303,13 @@ INSTANTIATE_TEST_SUITE_P( .expected_valid_expressions = {"1 + 2", "[1, 2, 3] + [4, 5, 6]", "'hello' + 'world'"}, }, + StandardLibraryConfigTestCase{ + .standard_library_config = + {.included_functions = {{"_+_", "_+_(int,int)"}, + {"_+_", "_+_(list<~A>,list<~A>)"}}}, + .expected_valid_expressions = {"1 + 2", "[1, 2, 3] + [4, 5, 6]"}, + .expected_invalid_expressions = {"'hello' + 'world'"}, + }, StandardLibraryConfigTestCase{ .standard_library_config = {.included_functions = {{"_+_", "add_int64"}, @@ -314,6 +330,55 @@ TEST(ContainerConfigTest, ContainerConfig) { EXPECT_THAT(result.GetIssues(), IsEmpty()) << result.FormatError(); } +TEST(ContainerConfigTest, ContainerConfigWithAbbreviations) { + Env env; + env.SetDescriptorPool(internal::GetSharedTestingDescriptorPool()); + Config config; + config.SetContainerConfig( + {.name = "cel.expr.conformance", + .abbreviations = {"cel.expr.conformance.proto2.TestAllTypes"}}); + env.SetConfig(config); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, env.NewCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, compiler->Compile("TestAllTypes{}")); + + EXPECT_THAT(result.GetIssues(), IsEmpty()) << result.FormatError(); +} + +TEST(ContainerConfigTest, ContainerConfigWithAliases) { + Env env; + env.SetDescriptorPool(internal::GetSharedTestingDescriptorPool()); + Config config; + config.SetContainerConfig( + {.name = "cel.expr.conformance", + .aliases = { + {.alias = "MyTestType", + .qualified_name = "cel.expr.conformance.proto2.TestAllTypes"}}}); + env.SetConfig(config); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, env.NewCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, compiler->Compile("MyTestType{}")); + + EXPECT_THAT(result.GetIssues(), IsEmpty()) << result.FormatError(); +} + +TEST(ContextVariableConfigTest, Basic) { + Env env; + env.SetDescriptorPool(internal::GetSharedTestingDescriptorPool()); + Config config; + config.SetContextType("cel.expr.conformance.proto3.TestAllTypes"); + env.SetConfig(config); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, env.NewCompiler()); + + // Top-level fields of TestAllTypes like "single_int32" should resolve + // successfully. + ASSERT_OK_AND_ASSIGN(auto result, compiler->Compile("single_int32 > 10")); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + + ASSERT_OK_AND_ASSIGN(auto result_invalid, + compiler->Compile("non_existent_field > 10")); + EXPECT_THAT(result_invalid.GetIssues(), Not(IsEmpty())); +} + struct VariableConfigWithValueTestCase { Config::VariableConfig variable_config; std::string validate_type_expr; diff --git a/env/env_yaml.cc b/env/env_yaml.cc index 4ba16ea84..281cf3ff1 100644 --- a/env/env_yaml.cc +++ b/env/env_yaml.cc @@ -26,6 +26,7 @@ #include "absl/base/no_destructor.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/escaping.h" @@ -35,8 +36,11 @@ #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" +#include "common/ast.h" #include "common/constant.h" +#include "common/signature.h" #include "env/config.h" +#include "env/type_info.h" #include "internal/status_macros.h" #include "internal/strings.h" #include "yaml-cpp/emitter.h" @@ -117,8 +121,8 @@ absl::StatusOr GetBinary(absl::string_view yaml, return binary; } else { return YamlError(yaml, node, - "Node '" + GetString(yaml, node) + - "' is not a valid Base64 encoded binary"); + absl::StrCat("Node '", GetString(yaml, node), + "' is not a valid Base64 encoded binary")); } } @@ -131,10 +135,22 @@ absl::StatusOr GetBool(absl::string_view yaml, absl::string_view key, return node.as(); } catch (YAML::Exception& e) { return YamlError(yaml, node, - "Node '" + std::string(key) + "' is not a boolean"); + absl::StrCat("Node '", key, "' is not a boolean")); } } +// Returns the key in the map `node` that has the given `value_node` as its +// value. If no such key exists, returns `value_node` itself. +YAML::Node GetContextNodeForKeyValue(const YAML::Node& node, + const YAML::Node& value_node) { + for (const auto& kv : node) { + if (kv.second.IsDefined() && kv.second.is(value_node)) { + return kv.first; + } + } + return value_node; +} + absl::Status ParseName(Config& config, absl::string_view yaml, const YAML::Node& root) { const YAML::Node name = root["name"]; @@ -150,12 +166,72 @@ absl::Status ParseName(Config& config, absl::string_view yaml, absl::Status ParseContainerConfig(Config& config, absl::string_view yaml, const YAML::Node& root) { const YAML::Node container = root["container"]; - if (container.IsDefined()) { - if (!container.IsScalar()) { - return YamlError(yaml, container, "Node 'container' is not a string"); - } + if (!container.IsDefined()) { + return absl::OkStatus(); + } + + if (container.IsScalar()) { config.SetContainerConfig({.name = GetString(yaml, container)}); + return absl::OkStatus(); + } + + if (!container.IsMap()) { + return YamlError(yaml, container, + "Node 'container' is neither a string nor a map"); + } + + Config::ContainerConfig container_config; + + const YAML::Node name = container["name"]; + if (name.IsDefined()) { + if (!name.IsScalar()) { + return YamlError(yaml, name, "Node 'name' in container is not a string"); + } + container_config.name = GetString(yaml, name); + } + + const YAML::Node abbreviations = container["abbreviations"]; + if (abbreviations.IsDefined()) { + if (!abbreviations.IsSequence()) { + return YamlError(yaml, abbreviations, + "Node 'abbreviations' is not a sequence"); + } + for (const YAML::Node& abbr : abbreviations) { + if (!abbr.IsScalar()) { + return YamlError(yaml, abbr, "Abbreviation is not a string"); + } + container_config.abbreviations.push_back(GetString(yaml, abbr)); + } + } + + const YAML::Node aliases = container["aliases"]; + if (aliases.IsDefined()) { + if (!aliases.IsSequence()) { + return YamlError(yaml, aliases, "Node 'aliases' is not a sequence"); + } + for (const YAML::Node& alias_node : aliases) { + if (!alias_node.IsMap()) { + return YamlError(yaml, alias_node, "Alias entry is not a map"); + } + const YAML::Node alias_key = alias_node["alias"]; + const YAML::Node qualified_name_key = alias_node["qualified_name"]; + + if (!alias_key.IsDefined() || !alias_key.IsScalar()) { + return YamlError(yaml, alias_node, + "Alias entry missing 'alias' string"); + } + if (!qualified_name_key.IsDefined() || !qualified_name_key.IsScalar()) { + return YamlError(yaml, alias_node, + "Alias entry missing 'qualified_name' string"); + } + + container_config.aliases.push_back( + {.alias = GetString(yaml, alias_key), + .qualified_name = GetString(yaml, qualified_name_key)}); + } } + + config.SetContainerConfig(std::move(container_config)); return absl::OkStatus(); } @@ -347,7 +423,22 @@ absl::Status ParseStandardLibraryConfig(Config& config, absl::string_view yaml, absl::StatusOr ParseTypeInfo(const YAML::Node& node, absl::string_view yaml) { Config::TypeInfo type_config; + const YAML::Node type = node["type"]; const YAML::Node type_name = node["type_name"]; + if (type.IsDefined() && type_name.IsDefined()) { + return YamlError(yaml, GetContextNodeForKeyValue(node, type_name), + "Node 'type' and 'type_name' are mutually exclusive"); + } + + if (type.IsDefined()) { + if (!type.IsScalar()) { + return YamlError(yaml, type, "Node 'type' is not a string"); + } + CEL_ASSIGN_OR_RETURN(auto type_spec, ParseTypeSpec(GetString(yaml, type))); + CEL_ASSIGN_OR_RETURN(auto type_config, TypeSpecToTypeInfo(type_spec)); + return type_config; + } + if (!type_name.IsDefined()) { return type_config; } @@ -529,8 +620,14 @@ absl::Status ParseVariableConfigs(Config& config, absl::string_view yaml, } variable_config.description = GetString(yaml, description); } - - CEL_ASSIGN_OR_RETURN(auto type_info, ParseTypeInfo(variable, yaml)); + const YAML::Node type = variable["type"]; + Config::TypeInfo type_info; + if (type.IsDefined() && !type.IsScalar()) { + // Old format, type spec is in 'type' instead of directly embedded. + CEL_ASSIGN_OR_RETURN(type_info, ParseTypeInfo(variable["type"], yaml)); + } else { + CEL_ASSIGN_OR_RETURN(type_info, ParseTypeInfo(variable, yaml)); + } ConstantKindCase constant_kind_case = GetConstantKindCase(type_info.name); std::string value_str; YAML::Node value = variable["value"]; @@ -567,7 +664,8 @@ absl::Status ParseVariableConfigs(Config& config, absl::string_view yaml, } absl::StatusOr ParseFunctionOverloadConfig( - absl::string_view yaml, const YAML::Node& overload) { + absl::string_view yaml, const YAML::Node& overload, + absl::string_view function_name) { Config::FunctionOverloadConfig overload_config; if (!overload || !overload.IsMap()) { return YamlError(yaml, overload, "Function overload is not a map"); @@ -594,40 +692,91 @@ absl::StatusOr ParseFunctionOverloadConfig( } } + const YAML::Node signature_node = overload["signature"]; const YAML::Node target = overload["target"]; - if (target.IsDefined()) { - if (!target.IsMap()) { - return YamlError(yaml, target, "Function overload target is not a map"); + const YAML::Node args = overload["args"]; + if (signature_node.IsDefined()) { + if (!signature_node.IsScalar()) { + return YamlError(yaml, signature_node, + "Function overload signature is not a string"); } - CEL_ASSIGN_OR_RETURN(Config::TypeInfo type_info, - ParseTypeInfo(target, yaml)); - overload_config.is_member_function = true; - overload_config.parameters.push_back(type_info); - } - const YAML::Node args = overload["args"]; - if (args.IsDefined()) { - if (!args.IsSequence()) { - return YamlError(yaml, args, "Function overload args is not a sequence"); + if (target.IsDefined()) { + return YamlError(yaml, GetContextNodeForKeyValue(overload, target), + "Function overload signature and target are mutually " + "exclusive"); + } + if (args.IsDefined()) { + return YamlError(yaml, GetContextNodeForKeyValue(overload, args), + "Function overload signature and args are mutually " + "exclusive"); + } + + std::string signature = GetString(yaml, signature_node); + CEL_ASSIGN_OR_RETURN(ParsedFunctionOverload parsed_signature, + ParseFunctionSignature(signature)); + if (parsed_signature.function_name != function_name) { + return YamlError(yaml, signature_node, + absl::StrCat("Function overload name \"", + parsed_signature.function_name, + "\" does not match function name \"", + function_name, "\"")); } - for (const YAML::Node& arg : args) { - if (!arg.IsMap()) { - return YamlError(yaml, arg, "Function overload arg is not a map"); + overload_config.is_member_function = parsed_signature.is_member; + if (overload_config.overload_id.empty()) { + overload_config.overload_id = signature; + } + if (!parsed_signature.signature_type.has_function()) { + return absl::InternalError(absl::StrCat( + "Function overload signature has no function type: ", signature)); + } + const FunctionTypeSpec& function_type_spec = + parsed_signature.signature_type.function(); + for (const auto& arg : function_type_spec.arg_types()) { + CEL_ASSIGN_OR_RETURN(auto type_info, TypeSpecToTypeInfo(arg)); + overload_config.parameters.push_back(std::move(type_info)); + } + } else { + if (target.IsDefined()) { + if (!target.IsMap()) { + return YamlError(yaml, target, "Function overload target is not a map"); } CEL_ASSIGN_OR_RETURN(Config::TypeInfo type_info, - ParseTypeInfo(arg, yaml)); + ParseTypeInfo(target, yaml)); + overload_config.is_member_function = true; overload_config.parameters.push_back(type_info); } - } + if (args.IsDefined()) { + if (!args.IsSequence()) { + return YamlError(yaml, args, + "Function overload args is not a sequence"); + } + for (const YAML::Node& arg : args) { + if (!arg.IsMap()) { + return YamlError(yaml, arg, "Function overload arg is not a map"); + } + CEL_ASSIGN_OR_RETURN(Config::TypeInfo type_info, + ParseTypeInfo(arg, yaml)); + overload_config.parameters.push_back(type_info); + } + } + } const YAML::Node return_type = overload["return"]; if (return_type.IsDefined()) { - if (!return_type.IsMap()) { - return YamlError(yaml, return_type, - "Function overload return type is not a map"); + if (return_type.IsScalar()) { + CEL_ASSIGN_OR_RETURN(auto type_spec, + ParseTypeSpec(GetString(yaml, return_type))); + CEL_ASSIGN_OR_RETURN(overload_config.return_type, + TypeSpecToTypeInfo(type_spec)); + } else if (return_type.IsMap()) { + CEL_ASSIGN_OR_RETURN(overload_config.return_type, + ParseTypeInfo(return_type, yaml)); + } else { + return YamlError( + yaml, return_type, + "Function overload return type is neither a string nor a map"); } - CEL_ASSIGN_OR_RETURN(overload_config.return_type, - ParseTypeInfo(return_type, yaml)); } return overload_config; } @@ -668,8 +817,9 @@ absl::Status ParseFunctionConfigs(Config& config, absl::string_view yaml, } for (const YAML::Node& overload : overloads) { - CEL_ASSIGN_OR_RETURN(Config::FunctionOverloadConfig overload_config, - ParseFunctionOverloadConfig(yaml, overload)); + CEL_ASSIGN_OR_RETURN( + Config::FunctionOverloadConfig overload_config, + ParseFunctionOverloadConfig(yaml, overload, function_config.name)); function_config.overload_configs.push_back(std::move(overload_config)); } } @@ -686,7 +836,44 @@ void EmitContainerConfig(const Config& env_config, YAML::Emitter& out) { } out << YAML::Key << "container"; - out << YAML::Value << YAML::DoubleQuoted << container_config.name; + if (container_config.abbreviations.empty() && + container_config.aliases.empty()) { + out << YAML::Value << YAML::DoubleQuoted << container_config.name; + } else { + out << YAML::Value << YAML::BeginMap; + if (!container_config.name.empty()) { + out << YAML::Key << "name" << YAML::Value << YAML::DoubleQuoted + << container_config.name; + } + if (!container_config.abbreviations.empty()) { + std::vector sorted_abbrs = container_config.abbreviations; + absl::c_sort(sorted_abbrs); + out << YAML::Key << "abbreviations" << YAML::Value << YAML::BeginSeq; + for (const auto& abbr : sorted_abbrs) { + out << YAML::Value << YAML::DoubleQuoted << abbr; + } + out << YAML::EndSeq; + } + if (!container_config.aliases.empty()) { + std::vector sorted_aliases = + container_config.aliases; + absl::c_sort(sorted_aliases, [](const Config::ContainerConfig::Alias& a, + const Config::ContainerConfig::Alias& b) { + return a.alias < b.alias; + }); + out << YAML::Key << "aliases" << YAML::Value << YAML::BeginSeq; + for (const auto& alias : sorted_aliases) { + out << YAML::BeginMap; + out << YAML::Key << "alias" << YAML::Value << YAML::DoubleQuoted + << alias.alias; + out << YAML::Key << "qualified_name" << YAML::Value + << YAML::DoubleQuoted << alias.qualified_name; + out << YAML::EndMap; + } + out << YAML::EndSeq; + } + out << YAML::EndMap; + } } void EmitExtensionConfigs(const Config& env_config, YAML::Emitter& out) { @@ -796,26 +983,42 @@ void EmitStandardLibraryConfig(const Config& env_config, YAML::Emitter& out) { out << YAML::EndMap; } -void EmitTypeInfo(const Config::TypeInfo& type_info, YAML::Emitter& out) { +void EmitTypeInfo(const Config::TypeInfo& type_info, YAML::Emitter& out, + const EnvConfigToYamlOptions& options) { // Note: the map is already started when this is called, so we don't emit // BeginMap here or EndMap at the end. - out << YAML::Key << "type_name"; - out << YAML::Value << YAML::DoubleQuoted << type_info.name; - if (type_info.is_type_param) { - out << YAML::Key << "is_type_param" << YAML::Value << true; - } - if (!type_info.params.empty()) { - out << YAML::Key << "params" << YAML::Value << YAML::BeginSeq; - for (const Config::TypeInfo& param : type_info.params) { - out << YAML::BeginMap; - EmitTypeInfo(param, out); - out << YAML::EndMap; + bool signature_generated = false; + if (options.use_type_signatures) { + absl::StatusOr type_spec = TypeInfoToTypeSpec(type_info); + if (type_spec.ok()) { + absl::StatusOr signature = MakeTypeSpecSignature(*type_spec); + if (signature.ok()) { + out << YAML::Key << "type"; + out << YAML::Value << YAML::DoubleQuoted << *signature; + signature_generated = true; + } + } + } + if (!signature_generated) { + out << YAML::Key << "type_name"; + out << YAML::Value << YAML::DoubleQuoted << type_info.name; + if (type_info.is_type_param) { + out << YAML::Key << "is_type_param" << YAML::Value << true; + } + if (!type_info.params.empty()) { + out << YAML::Key << "params" << YAML::Value << YAML::BeginSeq; + for (const Config::TypeInfo& param : type_info.params) { + out << YAML::BeginMap; + EmitTypeInfo(param, out, options); + out << YAML::EndMap; + } + out << YAML::EndSeq; } - out << YAML::EndSeq; } } -void EmitVariableConfigs(const Config& env_config, YAML::Emitter& out) { +void EmitVariableConfigs(const Config& env_config, YAML::Emitter& out, + const EnvConfigToYamlOptions& options) { const auto& variable_configs = env_config.GetVariableConfigs(); if (variable_configs.empty()) { return; @@ -839,7 +1042,7 @@ void EmitVariableConfigs(const Config& env_config, YAML::Emitter& out) { out << YAML::Key << "description"; out << YAML::Value << YAML::DoubleQuoted << variable_config.description; } - EmitTypeInfo(variable_config.type_info, out); + EmitTypeInfo(variable_config.type_info, out, options); if (variable_config.value.has_value()) { const Constant& constant = variable_config.value; switch (constant.kind_case()) { @@ -894,51 +1097,101 @@ void EmitVariableConfigs(const Config& env_config, YAML::Emitter& out) { } void EmitFunctionOverloadConfig( - const Config::FunctionOverloadConfig& overload_config, YAML::Emitter& out) { + absl::string_view function_name, + const Config::FunctionOverloadConfig& overload_config, YAML::Emitter& out, + const EnvConfigToYamlOptions& options) { out << YAML::BeginMap; - out << YAML::Key << "id"; - out << YAML::Value << YAML::DoubleQuoted << overload_config.overload_id; - if (overload_config.is_member_function) { - out << YAML::Key << "target" << YAML::Value; - out << YAML::BeginMap; - if (overload_config.parameters.empty()) { - // This should never happen, but if it does, emit a dynamic type. - EmitTypeInfo({.name = "dyn"}, out); - } else { - EmitTypeInfo(overload_config.parameters[0], out); + bool signature_generated = false; + std::string signature_str; + if (options.use_type_signatures) { + bool param_type_spec_generated = true; + std::vector params; + params.reserve(overload_config.parameters.size()); + for (const auto& parameter : overload_config.parameters) { + absl::StatusOr type_spec = TypeInfoToTypeSpec(parameter); + if (!type_spec.ok()) { + param_type_spec_generated = false; + break; + } + params.push_back(std::move(*type_spec)); } - out << YAML::EndMap; - if (overload_config.parameters.size() > 1) { - out << YAML::Key << "args"; - out << YAML::Value << YAML::BeginSeq; - for (size_t i = 1; i < overload_config.parameters.size(); ++i) { - out << YAML::BeginMap; - EmitTypeInfo(overload_config.parameters[i], out); - out << YAML::EndMap; + if (param_type_spec_generated) { + absl::StatusOr signature = MakeOverloadSignature( + function_name, params, overload_config.is_member_function); + if (signature.ok()) { + signature_str = std::move(*signature); + signature_generated = true; } - out << YAML::EndSeq; } - } else { - if (!overload_config.parameters.empty()) { - out << YAML::Key << "args"; - out << YAML::Value << YAML::BeginSeq; - for (const Config::TypeInfo& parameter : overload_config.parameters) { - out << YAML::BeginMap; - EmitTypeInfo(parameter, out); - out << YAML::EndMap; + } + if (!overload_config.overload_id.empty()) { + if (!signature_generated || overload_config.overload_id != signature_str) { + out << YAML::Key << "id"; + out << YAML::Value << YAML::DoubleQuoted << overload_config.overload_id; + } + } + if (signature_generated) { + out << YAML::Key << "signature"; + out << YAML::Value << YAML::DoubleQuoted << signature_str; + } + if (!signature_generated) { + if (overload_config.is_member_function) { + out << YAML::Key << "target" << YAML::Value; + out << YAML::BeginMap; + if (overload_config.parameters.empty()) { + // This should never happen, but if it does, emit a dynamic type. + EmitTypeInfo({.name = "dyn"}, out, options); + } else { + EmitTypeInfo(overload_config.parameters[0], out, options); + } + out << YAML::EndMap; + if (overload_config.parameters.size() > 1) { + out << YAML::Key << "args"; + out << YAML::Value << YAML::BeginSeq; + for (size_t i = 1; i < overload_config.parameters.size(); ++i) { + out << YAML::BeginMap; + EmitTypeInfo(overload_config.parameters[i], out, options); + out << YAML::EndMap; + } + out << YAML::EndSeq; + } + } else { + if (!overload_config.parameters.empty()) { + out << YAML::Key << "args"; + out << YAML::Value << YAML::BeginSeq; + for (const Config::TypeInfo& parameter : overload_config.parameters) { + out << YAML::BeginMap; + EmitTypeInfo(parameter, out, options); + out << YAML::EndMap; + } + out << YAML::EndSeq; } - out << YAML::EndSeq; } } - out << YAML::Key << "return"; - out << YAML::Value << YAML::BeginMap; - EmitTypeInfo(overload_config.return_type, out); - out << YAML::EndMap; - + bool return_type_signature_generated = false; + if (options.use_type_signatures) { + absl::StatusOr type_spec = + TypeInfoToTypeSpec(overload_config.return_type); + if (type_spec.ok()) { + absl::StatusOr signature = MakeTypeSpecSignature(*type_spec); + if (signature.ok()) { + out << YAML::Key << "return"; + out << YAML::Value << YAML::DoubleQuoted << *signature; + return_type_signature_generated = true; + } + } + } + if (!return_type_signature_generated) { + out << YAML::Key << "return"; + out << YAML::Value << YAML::BeginMap; + EmitTypeInfo(overload_config.return_type, out, options); + out << YAML::EndMap; + } out << YAML::EndMap; } -void EmitFunctionConfigs(const Config& env_config, YAML::Emitter& out) { +void EmitFunctionConfigs(const Config& env_config, YAML::Emitter& out, + const EnvConfigToYamlOptions& options) { const std::vector& function_configs = env_config.GetFunctionConfigs(); if (function_configs.empty()) { @@ -988,7 +1241,8 @@ void EmitFunctionConfigs(const Config& env_config, YAML::Emitter& out) { out << YAML::Key << "overloads" << YAML::Value << YAML::BeginSeq; for (const Config::FunctionOverloadConfig& overload_config : sorted_overloads) { - EmitFunctionOverloadConfig(overload_config, out); + EmitFunctionOverloadConfig(function_config.name, overload_config, out, + options); } out << YAML::EndSeq; } @@ -996,6 +1250,34 @@ void EmitFunctionConfigs(const Config& env_config, YAML::Emitter& out) { } out << YAML::EndSeq; } + +absl::Status ParseContextVariableConfig(Config& config, absl::string_view yaml, + const YAML::Node& root) { + const YAML::Node context_variable = root["context_variable"]; + if (!context_variable.IsDefined()) { + return absl::OkStatus(); + } + if (!context_variable.IsMap()) { + return YamlError(yaml, context_variable, + "Node 'context_variable' is not a map"); + } + + const YAML::Node type_name = context_variable["type_name"]; + const YAML::Node type = context_variable["type"]; + const YAML::Node* type_node = nullptr; + if (type.IsDefined() && type.IsScalar()) { + type_node = &type; + } else if (type_name.IsDefined() && type_name.IsScalar()) { + type_node = &type_name; + } else { + return YamlError(yaml, context_variable, + "Node 'context_variable' does not have a valid type"); + } + ABSL_DCHECK(type_node != nullptr); + config.SetContextType(GetString(yaml, *type_node)); + return absl::OkStatus(); +} + } // namespace absl::StatusOr EnvConfigFromYaml(const std::string& yaml) { @@ -1014,12 +1296,14 @@ absl::StatusOr EnvConfigFromYaml(const std::string& yaml) { CEL_RETURN_IF_ERROR(ParseContainerConfig(config, yaml, root)); CEL_RETURN_IF_ERROR(ParseExtensionConfigs(config, yaml, root)); CEL_RETURN_IF_ERROR(ParseStandardLibraryConfig(config, yaml, root)); + CEL_RETURN_IF_ERROR(ParseContextVariableConfig(config, yaml, root)); CEL_RETURN_IF_ERROR(ParseVariableConfigs(config, yaml, root)); CEL_RETURN_IF_ERROR(ParseFunctionConfigs(config, yaml, root)); return config; } -void EnvConfigToYaml(const Config& env_config, std::ostream& os) { +void EnvConfigToYaml(const Config& env_config, std::ostream& os, + const EnvConfigToYamlOptions& options) { YAML::Emitter out(os); out.SetIndent(2); out << YAML::BeginMap; @@ -1030,8 +1314,8 @@ void EnvConfigToYaml(const Config& env_config, std::ostream& os) { EmitContainerConfig(env_config, out); EmitExtensionConfigs(env_config, out); EmitStandardLibraryConfig(env_config, out); - EmitVariableConfigs(env_config, out); - EmitFunctionConfigs(env_config, out); + EmitVariableConfigs(env_config, out, options); + EmitFunctionConfigs(env_config, out, options); out << YAML::EndMap; } diff --git a/env/env_yaml.h b/env/env_yaml.h index c96b45933..7bf7bf6b4 100644 --- a/env/env_yaml.h +++ b/env/env_yaml.h @@ -31,8 +31,43 @@ namespace cel { // expensive expressions. absl::StatusOr EnvConfigFromYaml(const std::string& yaml); +struct EnvConfigToYamlOptions { + // Whether to use type and overload signatures instead of arg/return types in + // the output YAML. + // Example of type signature: "map>" vs + // type_name: "map" + // params: + // - type_name: "int" + // - type_name: "A" + // params: + // - type_name: "B" + // is_type_param: true + // + // Example of overload signature config: + // name: "foo" + // overloads: + // - signature: "timestamp.foo(A<~B>)" + // return: "int" + // vs + // name: "foo" + // overloads: + // - id: "foo_id" + // target: + // type_name: "timestamp" + // args: + // - type_name: "A" + // params: + // - type_name: "B" + // is_type_param: true + // return: + // type_name: "int" + // TODO(uncreated-issue/91): default to true after all dependencies are updated + bool use_type_signatures = false; +}; + // EnvConfigToYaml serializes an environment configuration as a YAML string. -void EnvConfigToYaml(const Config& env_config, std::ostream& os); +void EnvConfigToYaml(const Config& env_config, std::ostream& os, + const EnvConfigToYamlOptions& options = {}); } // namespace cel diff --git a/env/env_yaml_test.cc b/env/env_yaml_test.cc index 25cc63206..c5bd1b787 100644 --- a/env/env_yaml_test.cc +++ b/env/env_yaml_test.cc @@ -55,6 +55,31 @@ TEST(EnvYamlTest, ParseContainerConfig) { Field(&Config::ContainerConfig::name, "test.container")); } +TEST(EnvYamlTest, ParseContainerConfig_AlternativeSyntax) { + ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( + container: + name: test.container + abbreviations: + - abbr1.Abbr1 + - abbr2.Abbr2 + aliases: + - alias: alias1 + qualified_name: qual.name1 + - alias: alias2 + qualified_name: qual.name2 + )yaml")); + + const auto& container_config = config.GetContainerConfig(); + EXPECT_EQ(container_config.name, "test.container"); + EXPECT_THAT(container_config.abbreviations, + UnorderedElementsAre("abbr1.Abbr1", "abbr2.Abbr2")); + ASSERT_THAT(container_config.aliases, SizeIs(2)); + EXPECT_EQ(container_config.aliases[0].alias, "alias1"); + EXPECT_EQ(container_config.aliases[0].qualified_name, "qual.name1"); + EXPECT_EQ(container_config.aliases[1].alias, "alias2"); + EXPECT_EQ(container_config.aliases[1].qualified_name, "qual.name2"); +} + TEST(EnvYamlTest, ParseExtensionConfigs) { ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( extensions: @@ -128,7 +153,7 @@ TEST(EnvYamlTest, ParseStdlibConfig_InclusionStyle) { - name: "_+_" overloads: - id: add_bytes - - id: add_list + - id: "_+_(list<~A>,list<~A>)" - name: "matches" - name: "timestamp" overloads: @@ -141,7 +166,7 @@ TEST(EnvYamlTest, ParseStdlibConfig_InclusionStyle) { EXPECT_THAT( stdlib_config.included_functions, UnorderedElementsAre(std::make_pair("_+_", "add_bytes"), - std::make_pair("_+_", "add_list"), + std::make_pair("_+_", "_+_(list<~A>,list<~A>)"), std::make_pair("matches", ""), std::make_pair("timestamp", "string_to_timestamp"))) << " Actual stdlib config: " << stdlib_config; @@ -170,6 +195,69 @@ TEST(EnvYamlTest, ParseVariableConfigs) { } TEST(EnvYamlTest, ParseVariableConfigWithTypeParams) { + ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( + variables: + - name: "dict" + type: "map" + )yaml")); + + const Config::VariableConfig& variable_config = + config.GetVariableConfigs()[0]; + EXPECT_EQ(variable_config.name, "dict"); + const auto& type_info = variable_config.type_info; + EXPECT_EQ(type_info.name, "map"); + EXPECT_FALSE(type_info.is_type_param); + EXPECT_THAT(type_info.params, SizeIs(2)); + EXPECT_EQ(type_info.params[0].name, "string"); + EXPECT_FALSE(type_info.params[0].is_type_param); + EXPECT_THAT(type_info.params[0].params, IsEmpty()); + EXPECT_EQ(type_info.params[1].name, "A"); + EXPECT_TRUE(type_info.params[1].is_type_param); + EXPECT_THAT(type_info.params[1].params, IsEmpty()); +} + +TEST(EnvYamlTest, ParseContextVariableConfig) { + ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( + context_variable: + type_name: "cel.expr.conformance.proto3.TestAllTypes" + )yaml")); + + EXPECT_EQ(config.GetContextType(), + "cel.expr.conformance.proto3.TestAllTypes"); +} + +TEST(EnvYamlTest, ParseContextVariableConfigAlternativeSyntax) { + ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( + context_variable: + type: "cel.expr.conformance.proto3.TestAllTypes" + )yaml")); + + EXPECT_EQ(config.GetContextType(), + "cel.expr.conformance.proto3.TestAllTypes"); +} + +TEST(EnvYamlTest, ParseContextVariableMalformedContextVariable) { + EXPECT_THAT(EnvConfigFromYaml(R"yaml( + context_variable: 123 + + )yaml"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Node 'context_variable' is not a map"))); +} + +TEST(EnvYamlTest, ParseContextVariableMalformedContextVariable2) { + EXPECT_THAT( + EnvConfigFromYaml(R"yaml( + context_variable: + type: + foo: bar + )yaml"), + StatusIs( + absl::StatusCode::kInvalidArgument, + HasSubstr("Node 'context_variable' does not have a valid type"))); +} + +TEST(EnvYamlTest, ParseVariableConfigWithTypeParamsLegacySyntax) { ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( variables: - name: "dict" @@ -195,8 +283,26 @@ TEST(EnvYamlTest, ParseVariableConfigWithTypeParams) { EXPECT_THAT(type_info.params[1].params, IsEmpty()); } +TEST(EnvYamlTest, ParseVariableConfigWithNestedRuleOldFormat) { + ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( + variables: + - name: "x" + type: + type_name: "int" + )yaml")); + + ASSERT_THAT(config.GetVariableConfigs(), SizeIs(1)); + const Config::VariableConfig& variable_config = + config.GetVariableConfigs()[0]; + EXPECT_EQ(variable_config.name, "x"); + const auto& type_info = variable_config.type_info; + EXPECT_EQ(type_info.name, "int"); + EXPECT_FALSE(type_info.is_type_param); + EXPECT_THAT(type_info.params, IsEmpty()); +} + struct ParseConstantTestCase { - std::string type_name; + std::string type; std::string value; std::string expected_error; // Empty if no error. Constant expected_constant; @@ -211,10 +317,10 @@ TEST_P(EnvYamlParseConstantTest, EnvYamlParseConstant) { R"yaml( variables: - name: "const" - type_name: "%s" + type: "%s" value: %s )yaml", - param.type_name, param.value); + param.type, param.value); absl::StatusOr status_or_config = EnvConfigFromYaml(yaml); if (!param.expected_error.empty()) { EXPECT_THAT(status_or_config, StatusIs(absl::StatusCode::kInvalidArgument, @@ -226,8 +332,7 @@ TEST_P(EnvYamlParseConstantTest, EnvYamlParseConstant) { const Config::VariableConfig& variable_config = config.GetVariableConfigs()[0]; EXPECT_EQ(variable_config.name, "const"); - EXPECT_EQ(variable_config.type_info.name, param.type_name) - << " yaml: " << yaml; + EXPECT_EQ(variable_config.type_info.name, param.type) << " yaml: " << yaml; EXPECT_EQ(variable_config.value, param.expected_constant) << " yaml: " << yaml; } @@ -235,119 +340,119 @@ TEST_P(EnvYamlParseConstantTest, EnvYamlParseConstant) { std::vector GetParseConstantTestCases() { return { ParseConstantTestCase{ - .type_name = "null", + .type = "null", .value = "\"\"", .expected_constant = Constant(nullptr), }, ParseConstantTestCase{ - .type_name = "null", + .type = "null", .value = "anything", .expected_error = "Failed to parse null constant", }, ParseConstantTestCase{ - .type_name = "bool", + .type = "bool", .value = "TRUE", .expected_constant = Constant(true), }, ParseConstantTestCase{ - .type_name = "bool", + .type = "bool", .value = "false", .expected_constant = Constant(false), }, ParseConstantTestCase{ - .type_name = "bool", + .type = "bool", .value = "yes", .expected_error = "Failed to parse bool constant", }, ParseConstantTestCase{ - .type_name = "int", + .type = "int", .value = "42", .expected_constant = Constant(int64_t{42}), }, ParseConstantTestCase{ - .type_name = "int", + .type = "int", .value = "41.999", .expected_error = "Failed to parse int constant", }, ParseConstantTestCase{ - .type_name = "uint", + .type = "uint", .value = "42", .expected_constant = Constant(uint64_t{42}), }, ParseConstantTestCase{ - .type_name = "uint", + .type = "uint", .value = "42u", .expected_constant = Constant(uint64_t{42}), }, ParseConstantTestCase{ - .type_name = "uint", + .type = "uint", .value = "-1", .expected_error = "Failed to parse uint constant", }, ParseConstantTestCase{ - .type_name = "double", + .type = "double", .value = "42.42", .expected_constant = Constant(42.42), }, ParseConstantTestCase{ - .type_name = "double", + .type = "double", .value = "abc", .expected_error = "Failed to parse double constant", }, ParseConstantTestCase{ - .type_name = "bytes", + .type = "bytes", .value = "abc", .expected_constant = Constant(BytesConstant("abc")), }, ParseConstantTestCase{ - .type_name = "bytes", + .type = "bytes", .value = "b\"\\xFF\\x00\\x01\"", .expected_constant = Constant(BytesConstant(absl::string_view("\xff\x00\x01", 3))), }, ParseConstantTestCase{ - .type_name = "bytes", + .type = "bytes", .value = "!!binary /wAB", .expected_constant = Constant(BytesConstant(absl::string_view("\xff\x00\x01", 3))), }, ParseConstantTestCase{ - .type_name = "bytes", + .type = "bytes", .value = "!!binary YWJj=", .expected_error = "Node 'YWJj=' is not a valid Base64 encoded binary", }, ParseConstantTestCase{ - .type_name = "bytes", + .type = "bytes", .value = "abc", .expected_constant = Constant(BytesConstant("abc")), }, ParseConstantTestCase{ - .type_name = "string", + .type = "string", .value = "abc", .expected_constant = Constant(StringConstant("abc")), }, ParseConstantTestCase{ - .type_name = "string", + .type = "string", .value = "\"\\\"abc\\\"\"", .expected_constant = Constant(StringConstant("\"abc\"")), }, ParseConstantTestCase{ - .type_name = "duration", + .type = "duration", .value = "1s", .expected_constant = Constant(absl::Seconds(1)), }, ParseConstantTestCase{ - .type_name = "duration", + .type = "duration", .value = "abc", .expected_error = "Failed to parse duration constant", }, ParseConstantTestCase{ - .type_name = "timestamp", + .type = "timestamp", .value = "2023-01-01T00:00:00Z", .expected_constant = Constant(absl::FromUnixSeconds(1672531200)), }, ParseConstantTestCase{ - .type_name = "timestamp", + .type = "timestamp", .value = "abc", .expected_error = "Failed to parse timestamp constant", }, @@ -414,6 +519,53 @@ TEST_P(EnvYamlParseFunctionTest, EnvYamlParseFunction) { std::vector GetParseFunctionTestCases() { return { + ParseFunctionTestCase{ + .yaml = R"yaml( + functions: + - name: "isEmpty" + description: |- + determines whether a list is empty, + or a string has no characters + overloads: + - signature: "google.protobuf.StringValue.isEmpty()" + examples: + - "''.isEmpty() // true" + return: "bool" + - signature: "list<~T>.isEmpty()" + examples: + - "[].isEmpty() // true" + - "[1].isEmpty() // false" + return: "bool" + )yaml", + .expected_function_config = + { + .name = "isEmpty", + .description = "determines whether a list is empty,\nor a " + "string has no characters", + .overload_configs = + { + Config::FunctionOverloadConfig{ + .overload_id = + "google.protobuf.StringValue.isEmpty()", + .examples = {"''.isEmpty() // true"}, + .is_member_function = true, + .parameters = {{.name = "string_wrapper"}}, + .return_type = {.name = "bool"}, + }, + Config::FunctionOverloadConfig{ + .overload_id = "list<~T>.isEmpty()", + .examples = {"[].isEmpty() // true", + "[1].isEmpty() // false"}, + .is_member_function = true, + .parameters = {{.name = "list", + .params = {{.name = "T", + .is_type_param = + true}}}}, + .return_type = {.name = "bool"}, + }, + }, + }, + }, ParseFunctionTestCase{ .yaml = R"yaml( functions: @@ -470,6 +622,35 @@ std::vector GetParseFunctionTestCases() { }, }, }, + ParseFunctionTestCase{ + .yaml = R"yaml( + functions: + - name: "contains" + overloads: + - signature: "contains(list<~T>, ~T)" + examples: + - "contains([1, 2, 3], 2) // true" + return: "bool" + )yaml", + .expected_function_config = + { + .name = "contains", + .overload_configs = + { + Config::FunctionOverloadConfig{ + .overload_id = "contains(list<~T>, ~T)", + .examples = {"contains([1, 2, 3], 2) // true"}, + .is_member_function = false, + .parameters = + {{.name = "list", + .params = {{.name = "T", + .is_type_param = true}}}, + {.name = "T", .is_type_param = true}}, + .return_type = {.name = "bool"}, + }, + }, + }, + }, ParseFunctionTestCase{ .yaml = R"yaml( functions: @@ -550,9 +731,78 @@ INSTANTIATE_TEST_SUITE_P( container: - error: "error" )yaml", - .expected_error = "3:19: Node 'container' is not a string\n" - "| - error: \"error\"\n" - "| ^", + .expected_error = + "3:19: Node 'container' is neither a string nor a map\n" + "| - error: \"error\"\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + container: + name: [] + )yaml", + .expected_error = "3:25: Node 'name' in container is not a string\n" + "| name: []\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + container: + abbreviations: "abbr" + )yaml", + .expected_error = "3:34: Node 'abbreviations' is not a sequence\n" + "| abbreviations: \"abbr\"\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + container: + abbreviations: + - [] + )yaml", + .expected_error = "4:21: Abbreviation is not a string\n" + "| - []\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + container: + aliases: "not a sequence" + )yaml", + .expected_error = "3:28: Node 'aliases' is not a sequence\n" + "| aliases: \"not a sequence\"\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + container: + aliases: + - "not a map" + )yaml", + .expected_error = "4:21: Alias entry is not a map\n" + "| - \"not a map\"\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + container: + aliases: + - qualified_name: "qual" + )yaml", + .expected_error = "4:21: Alias entry missing 'alias' string\n" + "| - qualified_name: \"qual\"\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + container: + aliases: + - alias: "my_alias" + )yaml", + .expected_error = "4:21: Alias entry missing" + " 'qualified_name' string\n" + "| - alias: \"my_alias\"\n" + "| ^", }, ParseTestCase{ .yaml = R"yaml( @@ -771,6 +1021,18 @@ INSTANTIATE_TEST_SUITE_P( "| is_type_param: maybe\n" "| ^", }, + ParseTestCase{ + .yaml = R"yaml( + variables: + - name: "foo" + type_name: "opaque" + type: "opaque" + )yaml", + .expected_error = "4:19: Node 'type' and 'type_name'" + " are mutually exclusive\n" + "| type_name: \"opaque\"\n" + "| ^", + }, ParseTestCase{ .yaml = R"yaml( variables: @@ -871,12 +1133,65 @@ INSTANTIATE_TEST_SUITE_P( - name: "foo" overloads: - id: "foo_int64" - return: "to sender" + return: [1] )yaml", .expected_error = "6:31: Function overload return type" - " is not a map\n" - "| return: \"to sender\"\n" + " is neither a string nor a map\n" + "| return: [1]\n" "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + functions: + - name: "foo" + overloads: + - id: "foo_int64" + signature: "bar()" + )yaml", + .expected_error = "6:34: Function overload name \"bar\" " + "does not match function name \"foo\"\n" + "| signature: \"bar()\"\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + functions: + - name: "foo" + overloads: + - signature: [ "foo()" ] + )yaml", + .expected_error = + "5:34: Function overload signature is not a string\n" + "| - signature: [ \"foo()\" ]\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + functions: + - name: "foo" + overloads: + - signature: "foo()" + target: + type_name: "int" + )yaml", + .expected_error = "6:23: Function overload signature and target " + "are mutually exclusive\n" + "| target:\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + functions: + - name: "foo" + overloads: + - signature: "foo()" + args: + - type_name: "int" + )yaml", + .expected_error = "6:23: Function overload signature and args are " + "mutually exclusive\n" + "| args:\n" + "| ^", })); std::string Unindent(std::string_view yaml) { @@ -905,6 +1220,7 @@ std::string Unindent(std::string_view yaml) { struct ExportTestCase { absl::StatusOr config; std::string expected_yaml; + std::string expected_alt_yaml; }; class EnvYamlExportTest : public testing::TestWithParam {}; @@ -913,10 +1229,18 @@ TEST_P(EnvYamlExportTest, EnvYamlExport) { const ExportTestCase& param = GetParam(); ASSERT_OK_AND_ASSIGN(Config config, param.config); std::stringstream ss; - EnvConfigToYaml(config, ss); + EnvConfigToYaml(config, ss, {.use_type_signatures = true}); std::string yaml_output = Unindent(ss.str()); std::string expected_yaml = Unindent(param.expected_yaml); EXPECT_EQ(yaml_output, expected_yaml); + + if (!param.expected_alt_yaml.empty()) { + std::stringstream alt_ss; + EnvConfigToYaml(config, alt_ss, {.use_type_signatures = false}); + std::string alt_yaml_output = Unindent(alt_ss.str()); + std::string expected_alt_yaml = Unindent(param.expected_alt_yaml); + EXPECT_EQ(alt_yaml_output, expected_alt_yaml); + } } std::vector GetExportTestCases() { @@ -946,6 +1270,33 @@ std::vector GetExportTestCases() { container: "test.container" )yaml", }, + ExportTestCase{ + .config = []() -> absl::StatusOr { + Config config; + config.SetName("test.env"); + config.SetContainerConfig( + {.name = "test.container", + .abbreviations = {"foo", "bar"}, + .aliases = { + {.alias = "foo", .qualified_name = "test.foo"}, + {.alias = "bar", .qualified_name = "test.bar"}, + }}); + return config; + }(), + .expected_yaml = R"yaml( + name: "test.env" + container: + name: "test.container" + abbreviations: + - "bar" + - "foo" + aliases: + - alias: "bar" + qualified_name: "test.bar" + - alias: "foo" + qualified_name: "test.foo" + )yaml", + }, ExportTestCase{ .config = []() -> absl::StatusOr { Config config; @@ -1058,9 +1409,9 @@ std::vector GetExportTestCases() { .included_functions = { std::make_pair("timestamp", "string_to_timestamp"), - std::make_pair("_+_", "add_list"), + std::make_pair("_+_", "_+_(list<~A>,list<~A>)"), std::make_pair("matches", ""), - std::make_pair("_+_", "add_bytes"), + std::make_pair("_+_", "_+_(bytes,bytes)"), }, })); return config; @@ -1070,8 +1421,8 @@ std::vector GetExportTestCases() { include_functions: - name: "_+_" overloads: - - id: "add_bytes" - - id: "add_list" + - id: "_+_(bytes,bytes)" + - id: "_+_(list<~A>,list<~A>)" - name: "matches" - name: "timestamp" overloads: @@ -1090,7 +1441,7 @@ std::vector GetExportTestCases() { .expected_yaml = R"yaml( variables: - name: "foo" - type_name: "null" + type: "null" )yaml", }, ExportTestCase{ @@ -1103,6 +1454,12 @@ std::vector GetExportTestCases() { return config; }(), .expected_yaml = R"yaml( + variables: + - name: "foo" + type: "bool" + value: true + )yaml", + .expected_alt_yaml = R"yaml( variables: - name: "foo" type_name: "bool" @@ -1119,6 +1476,12 @@ std::vector GetExportTestCases() { return config; }(), .expected_yaml = R"yaml( + variables: + - name: "foo" + type: "int" + value: 42 + )yaml", + .expected_alt_yaml = R"yaml( variables: - name: "foo" type_name: "int" @@ -1137,7 +1500,7 @@ std::vector GetExportTestCases() { .expected_yaml = R"yaml( variables: - name: "foo" - type_name: "uint" + type: "uint" value: 777 )yaml", }, @@ -1153,7 +1516,7 @@ std::vector GetExportTestCases() { .expected_yaml = R"yaml( variables: - name: "foo" - type_name: "double" + type: "double" value: 0.75 )yaml", }, @@ -1170,7 +1533,7 @@ std::vector GetExportTestCases() { .expected_yaml = R"yaml( variables: - name: "foo" - type_name: "bytes" + type: "bytes" value: b"\xff\x00\x01" )yaml", }, @@ -1188,7 +1551,7 @@ std::vector GetExportTestCases() { .expected_yaml = R"yaml( variables: - name: "foo" - type_name: "string" + type: "string" value: "'single' \"double\"" )yaml", }, @@ -1203,6 +1566,12 @@ std::vector GetExportTestCases() { return config; }(), .expected_yaml = R"yaml( + variables: + - name: "foo" + type: "duration" + value: 1h2m3s + )yaml", + .expected_alt_yaml = R"yaml( variables: - name: "foo" type_name: "duration" @@ -1219,6 +1588,12 @@ std::vector GetExportTestCases() { return config; }(), .expected_yaml = R"yaml( + variables: + - name: "foo" + type: "timestamp" + value: 2026-01-02T03:04:05Z + )yaml", + .expected_alt_yaml = R"yaml( variables: - name: "foo" type_name: "timestamp" @@ -1237,7 +1612,7 @@ std::vector GetExportTestCases() { .expected_yaml = R"yaml( variables: - name: "foo" - type_name: "google.expr.proto3.test.TestAllTypes" + type: "google.expr.proto3.test.TestAllTypes" )yaml", }, ExportTestCase{ @@ -1252,6 +1627,11 @@ std::vector GetExportTestCases() { return config; }(), .expected_yaml = R"yaml( + variables: + - name: "foo" + type: "A" + )yaml", + .expected_alt_yaml = R"yaml( variables: - name: "foo" type_name: "A" @@ -1281,12 +1661,22 @@ std::vector GetExportTestCases() { {.overload_id = "foo_overload_id", .is_member_function = true, .parameters = {{.name = "timestamp"}, - {.name = "A", .params = {{.name = "B"}}}}, + {.name = "A", + .params = {{.name = "B", + .is_type_param = true}}}}, .return_type = {.name = "int"}}, }})); return config; }(), .expected_yaml = R"yaml( + functions: + - name: "foo" + overloads: + - id: "foo_overload_id" + signature: "timestamp.foo(A<~B>)" + return: "int" + )yaml", + .expected_alt_yaml = R"yaml( functions: - name: "foo" overloads: @@ -1297,6 +1687,7 @@ std::vector GetExportTestCases() { - type_name: "A" params: - type_name: "B" + is_type_param: true return: type_name: "int" )yaml", @@ -1306,6 +1697,7 @@ std::vector GetExportTestCases() { Config config; CEL_RETURN_IF_ERROR(config.AddFunctionConfig( {.name = "foo", + .description = "my desc", .overload_configs = { {.overload_id = "foo_overload_a", .parameters = {{.name = "timestamp"}}, @@ -1321,6 +1713,19 @@ std::vector GetExportTestCases() { .expected_yaml = R"yaml( functions: - name: "foo" + description: "my desc" + overloads: + - id: "foo_overload_b" + signature: "foo(double,A)" + return: "string" + - id: "foo_overload_a" + signature: "foo(timestamp)" + return: "list" + )yaml", + .expected_alt_yaml = R"yaml( + functions: + - name: "foo" + description: "my desc" overloads: - id: "foo_overload_b" args: @@ -1339,15 +1744,55 @@ std::vector GetExportTestCases() { - type_name: "int" )yaml", }, + ExportTestCase{ + .config = []() -> absl::StatusOr { + Config config; + CEL_RETURN_IF_ERROR(config.AddFunctionConfig( + {.name = "foo", + .overload_configs = { + {.overload_id = "timestamp.foo(A<~B>)", + .is_member_function = true, + .parameters = {{.name = "timestamp"}, + {.name = "A", + .params = {{.name = "B", + .is_type_param = true}}}}, + .return_type = {.name = "int"}}, + }})); + return config; + }(), + .expected_yaml = R"yaml( + functions: + - name: "foo" + overloads: + - signature: "timestamp.foo(A<~B>)" + return: "int" + )yaml", + .expected_alt_yaml = R"yaml( + functions: + - name: "foo" + overloads: + - id: "timestamp.foo(A<~B>)" + target: + type_name: "timestamp" + args: + - type_name: "A" + params: + - type_name: "B" + is_type_param: true + return: + type_name: "int" + )yaml", + }, }; }; INSTANTIATE_TEST_SUITE_P(EnvYamlExportTest, EnvYamlExportTest, ::testing::ValuesIn(GetExportTestCases())); -class EnvYamlRoundTripTest : public testing::TestWithParam {}; +class EnvYamlStructuredRoundTripTest + : public testing::TestWithParam {}; -TEST_P(EnvYamlRoundTripTest, EnvYamlRoundTrip) { +TEST_P(EnvYamlStructuredRoundTripTest, EnvYamlRoundTrip) { const std::string& yaml = Unindent(GetParam()); ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(yaml)); @@ -1356,7 +1801,7 @@ TEST_P(EnvYamlRoundTripTest, EnvYamlRoundTrip) { EXPECT_EQ(ss.str(), yaml); } -std::vector GetRoundTripTestCases() { +std::vector GetStructuredRoundTripTestCases() { return { R"yaml( stdlib: @@ -1385,6 +1830,18 @@ std::vector GetRoundTripTestCases() { overloads: - id: "string_to_timestamp" )yaml", + R"yaml( + container: + name: "test.container" + abbreviations: + - "abbr1.Abbr1" + - "abbr2.Abbr2" + aliases: + - alias: "alias1" + qualified_name: "qual.name1" + - alias: "alias2" + qualified_name: "qual.name2" + )yaml", R"yaml( extensions: - name: "bindings" @@ -1403,74 +1860,90 @@ std::vector GetRoundTripTestCases() { overloads: - id: "string_to_timestamp" )yaml", + R"yaml( + functions: + - name: "bar" + - name: "foo" + )yaml", + }; +} + +INSTANTIATE_TEST_SUITE_P( + EnvYamlStructuredRoundTripTest, EnvYamlStructuredRoundTripTest, + ::testing::ValuesIn(GetStructuredRoundTripTestCases())); + +class EnvYamlSignatureRoundTripTest + : public testing::TestWithParam {}; + +TEST_P(EnvYamlSignatureRoundTripTest, EnvYamlRoundTrip) { + const std::string& yaml = Unindent(GetParam()); + ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(yaml)); + + std::stringstream ss; + EnvConfigToYaml(config, ss, {.use_type_signatures = true}); + EXPECT_EQ(ss.str(), yaml); +} + +std::vector GetSignatureRoundTripTestCases() { + return { R"yaml( variables: - name: "a" - type_name: "null" + type: "null" - name: "b" - type_name: "bool" + type: "bool" value: true - name: "c" - type_name: "int" + type: "int" value: 42 - name: "d" - type_name: "uint" + type: "uint" value: 777 - name: "e" - type_name: "double" + type: "double" value: 0.75 - name: "f" - type_name: "bytes" + type: "bytes" value: b"\xff\x00\x01" - name: "g" - type_name: "string" + type: "string" value: "plain 'single' \"double\"" - name: "h" - type_name: "duration" + type: "duration" value: 1h2m3s - name: "i" - type_name: "timestamp" + type: "timestamp" value: 2026-01-02T03:04:05Z )yaml", R"yaml( functions: - - name: "bar" - name: "foo" + overloads: + - id: "foo_overload_id" + signature: "timestamp.foo(A<~B>)" + return: "int" )yaml", R"yaml( functions: - name: "foo" overloads: - id: "foo_overload_id" - target: - type_name: "timestamp" - args: - - type_name: "A" - params: - - type_name: "B" - return: - type_name: "int" + signature: "foo(timestamp,A<~B>)" + return: "list" )yaml", R"yaml( functions: - name: "foo" overloads: - - id: "foo_overload_id" - args: - - type_name: "timestamp" - - type_name: "A" - params: - - type_name: "B" - return: - type_name: "list" - params: - - type_name: "int" + - signature: "timestamp.foo(A<~B>)" + return: "int" )yaml", }; } -INSTANTIATE_TEST_SUITE_P(EnvYamlRoundTripTest, EnvYamlRoundTripTest, - ::testing::ValuesIn(GetRoundTripTestCases())); +INSTANTIATE_TEST_SUITE_P(EnvYamlSignatureRoundTripTest, + EnvYamlSignatureRoundTripTest, + ::testing::ValuesIn(GetSignatureRoundTripTestCases())); } // namespace } // namespace cel diff --git a/env/type_info.cc b/env/type_info.cc index a5b47b6f1..f49fab9f4 100644 --- a/env/type_info.cc +++ b/env/type_info.cc @@ -14,13 +14,17 @@ #include "env/type_info.h" +#include #include +#include #include #include "absl/base/no_destructor.h" #include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" +#include "common/ast.h" #include "common/type.h" #include "common/type_kind.h" #include "env/config.h" @@ -180,5 +184,227 @@ absl::StatusOr TypeInfoToType( return DynType(); } } +absl::StatusOr TypeInfoToTypeSpec(const Config::TypeInfo& type_info) { + if (type_info.is_type_param) { + return TypeSpec(ParamTypeSpec(type_info.name)); + } + + std::optional type_kind = TypeNameToTypeKind(type_info.name); + if (!type_kind.has_value()) { + if (type_info.params.empty()) { + return TypeSpec(MessageTypeSpec(type_info.name)); + } else { + std::vector param_specs; + param_specs.reserve(type_info.params.size()); + for (const Config::TypeInfo& param : type_info.params) { + CEL_ASSIGN_OR_RETURN(TypeSpec param_spec, TypeInfoToTypeSpec(param)); + param_specs.push_back(std::move(param_spec)); + } + return TypeSpec(AbstractType(type_info.name, std::move(param_specs))); + } + } + + switch (*type_kind) { + case TypeKind::kNull: + return TypeSpec(NullTypeSpec()); + case TypeKind::kBool: + return TypeSpec(PrimitiveType::kBool); + case TypeKind::kInt: + return TypeSpec(PrimitiveType::kInt64); + case TypeKind::kUint: + return TypeSpec(PrimitiveType::kUint64); + case TypeKind::kDouble: + return TypeSpec(PrimitiveType::kDouble); + case TypeKind::kString: + return TypeSpec(PrimitiveType::kString); + case TypeKind::kBytes: + return TypeSpec(PrimitiveType::kBytes); + case TypeKind::kTimestamp: + return TypeSpec(WellKnownTypeSpec::kTimestamp); + case TypeKind::kDuration: + return TypeSpec(WellKnownTypeSpec::kDuration); + case TypeKind::kList: { + if (!type_info.params.empty()) { + CEL_ASSIGN_OR_RETURN(TypeSpec elem_type, + TypeInfoToTypeSpec(type_info.params[0])); + return TypeSpec( + ListTypeSpec(std::make_unique(std::move(elem_type)))); + } else { + return TypeSpec(ListTypeSpec()); + } + } + case TypeKind::kMap: { + if (type_info.params.empty()) { + return TypeSpec(MapTypeSpec()); + } + CEL_ASSIGN_OR_RETURN(TypeSpec key_type, + TypeInfoToTypeSpec(type_info.params[0])); + if (type_info.params.size() > 1) { + CEL_ASSIGN_OR_RETURN(TypeSpec value_type, + TypeInfoToTypeSpec(type_info.params[1])); + return TypeSpec( + MapTypeSpec(std::make_unique(std::move(key_type)), + std::make_unique(std::move(value_type)))); + } + return TypeSpec(MapTypeSpec( + std::make_unique(std::move(key_type)), nullptr)); + } + case TypeKind::kDyn: + return TypeSpec(DynTypeSpec()); + case TypeKind::kAny: + return TypeSpec(WellKnownTypeSpec::kAny); + case TypeKind::kBoolWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBool)); + case TypeKind::kIntWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kInt64)); + case TypeKind::kUintWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kUint64)); + case TypeKind::kDoubleWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kDouble)); + case TypeKind::kStringWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kString)); + case TypeKind::kBytesWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBytes)); + case TypeKind::kType: { + if (type_info.params.empty()) { + return TypeSpec(std::make_unique(DynTypeSpec())); + } + CEL_ASSIGN_OR_RETURN(TypeSpec type_param, + TypeInfoToTypeSpec(type_info.params[0])); + return TypeSpec(std::make_unique(std::move(type_param))); + } + default: + return TypeSpec(DynTypeSpec()); + } +} + +absl::StatusOr TypeSpecToTypeInfo(const TypeSpec& type_spec) { + Config::TypeInfo type_info; + + if (type_spec.has_dyn()) { + type_info.name = "dyn"; + } else if (type_spec.has_null()) { + type_info.name = "null"; + } else if (type_spec.has_primitive()) { + switch (type_spec.primitive()) { + case PrimitiveType::kBool: + type_info.name = "bool"; + break; + case PrimitiveType::kInt64: + type_info.name = "int"; + break; + case PrimitiveType::kUint64: + type_info.name = "uint"; + break; + case PrimitiveType::kDouble: + type_info.name = "double"; + break; + case PrimitiveType::kString: + type_info.name = "string"; + break; + case PrimitiveType::kBytes: + type_info.name = "bytes"; + break; + default: + return absl::InvalidArgumentError("Unspecified primitive type"); + } + } else if (type_spec.has_wrapper()) { + switch (type_spec.wrapper()) { + case PrimitiveType::kBool: + type_info.name = "bool_wrapper"; + break; + case PrimitiveType::kInt64: + type_info.name = "int_wrapper"; + break; + case PrimitiveType::kUint64: + type_info.name = "uint_wrapper"; + break; + case PrimitiveType::kDouble: + type_info.name = "double_wrapper"; + break; + case PrimitiveType::kString: + type_info.name = "string_wrapper"; + break; + case PrimitiveType::kBytes: + type_info.name = "bytes_wrapper"; + break; + default: + return absl::InvalidArgumentError("Unspecified wrapper type"); + } + } else if (type_spec.has_well_known()) { + switch (type_spec.well_known()) { + case WellKnownTypeSpec::kAny: + type_info.name = "any"; + break; + case WellKnownTypeSpec::kTimestamp: + type_info.name = "timestamp"; + break; + case WellKnownTypeSpec::kDuration: + type_info.name = "duration"; + break; + default: + return absl::InvalidArgumentError("Unspecified well known type"); + } + } else if (type_spec.has_list_type()) { + type_info.name = "list"; + const ListTypeSpec& list_type = type_spec.list_type(); + if (list_type.has_elem_type() && list_type.elem_type().is_specified()) { + CEL_ASSIGN_OR_RETURN(Config::TypeInfo param, + TypeSpecToTypeInfo(list_type.elem_type())); + type_info.params.push_back(std::move(param)); + } + } else if (type_spec.has_map_type()) { + type_info.name = "map"; + const MapTypeSpec& map_type = type_spec.map_type(); + bool has_key = + map_type.has_key_type() && map_type.key_type().is_specified(); + bool has_value = + map_type.has_value_type() && map_type.value_type().is_specified(); + if (has_key || has_value) { + if (has_key) { + CEL_ASSIGN_OR_RETURN(Config::TypeInfo param, + TypeSpecToTypeInfo(map_type.key_type())); + type_info.params.push_back(std::move(param)); + } else { + type_info.params.push_back(Config::TypeInfo{.name = "dyn"}); + } + if (has_value) { + CEL_ASSIGN_OR_RETURN(Config::TypeInfo param_value, + TypeSpecToTypeInfo(map_type.value_type())); + type_info.params.push_back(std::move(param_value)); + } else { + type_info.params.push_back(Config::TypeInfo{.name = "dyn"}); + } + } + } else if (type_spec.has_message_type()) { + type_info.name = type_spec.message_type().type(); + } else if (type_spec.has_type_param()) { + type_info.name = type_spec.type_param().type(); + type_info.is_type_param = true; + } else if (type_spec.has_type()) { + type_info.name = "type"; + CEL_ASSIGN_OR_RETURN(Config::TypeInfo param, + TypeSpecToTypeInfo(type_spec.type())); + type_info.params.push_back(std::move(param)); + } else if (type_spec.has_abstract_type()) { + type_info.name = type_spec.abstract_type().name(); + for (const TypeSpec& param_spec : + type_spec.abstract_type().parameter_types()) { + CEL_ASSIGN_OR_RETURN(Config::TypeInfo param, + TypeSpecToTypeInfo(param_spec)); + type_info.params.push_back(std::move(param)); + } + } else if (type_spec.has_error()) { + return absl::InvalidArgumentError( + "ErrorType cannot be converted to TypeInfo"); + } else if (type_spec.has_function()) { + return absl::InvalidArgumentError( + "FunctionType cannot be converted to TypeInfo"); + } else { + return absl::InvalidArgumentError("Unknown TypeSpec kind"); + } + + return type_info; +} } // namespace cel diff --git a/env/type_info.h b/env/type_info.h index bb3cfde43..3f802ce1a 100644 --- a/env/type_info.h +++ b/env/type_info.h @@ -16,6 +16,7 @@ #define THIRD_PARTY_CEL_CPP_ENV_TYPE_INFO_H_ #include "absl/status/statusor.h" +#include "common/ast.h" #include "common/type.h" #include "env/config.h" #include "google/protobuf/arena.h" @@ -30,6 +31,12 @@ absl::StatusOr TypeInfoToType( const Config::TypeInfo& type_info, const google::protobuf::DescriptorPool* descriptor_pool, google::protobuf::Arena* arena); +// Converts a Config::TypeInfo to a cel::TypeSpec. +absl::StatusOr TypeInfoToTypeSpec(const Config::TypeInfo& type_info); + +// Converts a cel::TypeSpec to a Config::TypeInfo. +absl::StatusOr TypeSpecToTypeInfo(const TypeSpec& type_spec); + } // namespace cel #endif // THIRD_PARTY_CEL_CPP_ENV_TYPE_INFO_H_ diff --git a/env/type_info_test.cc b/env/type_info_test.cc index 015d8a928..f9d46f9a9 100644 --- a/env/type_info_test.cc +++ b/env/type_info_test.cc @@ -14,9 +14,14 @@ #include "env/type_info.h" +#include +#include +#include #include #include +#include "absl/status/status.h" +#include "common/ast/metadata.h" #include "common/type.h" #include "common/type_proto.h" #include "env/config.h" @@ -28,9 +33,27 @@ #include "google/protobuf/text_format.h" namespace cel { + +std::ostream& operator<<(std::ostream& os, const Config::TypeInfo& type_info) { + if (type_info.is_type_param) { + os << "?"; + } + os << type_info.name; + if (!type_info.params.empty()) { + os << "<"; + for (size_t i = 0; i < type_info.params.size(); ++i) { + if (i > 0) os << ", "; + os << type_info.params[i]; + } + os << ">"; + } + return os; +} + namespace { using absl_testing::IsOk; +using absl_testing::StatusIs; using testing::ValuesIn; struct TestCase { @@ -127,5 +150,151 @@ std::vector GetTestCases() { INSTANTIATE_TEST_SUITE_P(TypeInfoTest, TypeInfoTest, ValuesIn(GetTestCases())); +bool TypeInfoEqImpl(const Config::TypeInfo& actual, + const Config::TypeInfo& expected) { + if (actual.name != expected.name) return false; + if (actual.is_type_param != expected.is_type_param) return false; + if (actual.params.size() != expected.params.size()) return false; + for (size_t i = 0; i < actual.params.size(); ++i) { + if (!TypeInfoEqImpl(actual.params[i], expected.params[i])) return false; + } + return true; +} + +MATCHER_P(TypeInfoEq, expected, "") { return TypeInfoEqImpl(arg, expected); } + +struct TypeSpecTestCase { + TypeSpec type_spec; + Config::TypeInfo expected_type_info; +}; + +using TypeSpecToTypeInfoTest = testing::TestWithParam; + +TEST_P(TypeSpecToTypeInfoTest, Convert) { + const TypeSpecTestCase& param = GetParam(); + ASSERT_OK_AND_ASSIGN(Config::TypeInfo actual_type_info, + TypeSpecToTypeInfo(param.type_spec)); + EXPECT_THAT(actual_type_info, TypeInfoEq(param.expected_type_info)); +} + +std::vector GetTypeSpecTestCases() { + return { + TypeSpecTestCase{ + .type_spec = TypeSpec(PrimitiveType::kInt64), + .expected_type_info = {.name = "int"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec( + ListTypeSpec(std::make_unique(PrimitiveType::kInt64))), + .expected_type_info = {.name = "list", + .params = {Config::TypeInfo{.name = "int"}}}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(ListTypeSpec()), + .expected_type_info = {.name = "list"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec( + MapTypeSpec(std::make_unique(PrimitiveType::kString), + std::make_unique(PrimitiveType::kInt64))), + .expected_type_info = {.name = "map", + .params = {Config::TypeInfo{.name = "string"}, + Config::TypeInfo{.name = "int"}}}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(MapTypeSpec()), + .expected_type_info = {.name = "map"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec( + MessageTypeSpec("cel.expr.conformance.proto2.TestAllTypes")), + .expected_type_info = + {.name = "cel.expr.conformance.proto2.TestAllTypes"}, + }, + TypeSpecTestCase{ + .type_spec = + TypeSpec(AbstractType("A", {TypeSpec(ParamTypeSpec("B"))})), + .expected_type_info = {.name = "A", + .params = {Config::TypeInfo{ + .name = "B", .is_type_param = true}}}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(WellKnownTypeSpec::kAny), + .expected_type_info = {.name = "any"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(WellKnownTypeSpec::kTimestamp), + .expected_type_info = {.name = "timestamp"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kDouble)), + .expected_type_info = {.name = "double_wrapper"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec( + std::make_unique(WellKnownTypeSpec::kDuration)), + .expected_type_info = {.name = "type", + .params = {Config::TypeInfo{.name = + "duration"}}}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(std::make_unique(DynTypeSpec())), + .expected_type_info = {.name = "type", + .params = {Config::TypeInfo{.name = "dyn"}}}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(DynTypeSpec{}), + .expected_type_info = {.name = "dyn"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(NullTypeSpec{}), + .expected_type_info = {.name = "null"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec( + MapTypeSpec(std::make_unique(PrimitiveType::kString), + std::make_unique(DynTypeSpec()))), + .expected_type_info = {.name = "map", + .params = {Config::TypeInfo{.name = "string"}, + Config::TypeInfo{.name = "dyn"}}}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec( + MapTypeSpec(std::make_unique(DynTypeSpec()), + std::make_unique(PrimitiveType::kInt64))), + .expected_type_info = {.name = "map", + .params = {Config::TypeInfo{.name = "dyn"}, + Config::TypeInfo{.name = "int"}}}, + }, + }; +} + +INSTANTIATE_TEST_SUITE_P(TypeSpecToTypeInfoTest, TypeSpecToTypeInfoTest, + ValuesIn(GetTypeSpecTestCases())); + +using TypeInfoToTypeSpecTest = testing::TestWithParam; + +TEST_P(TypeInfoToTypeSpecTest, Convert) { + const TypeSpecTestCase& param = GetParam(); + ASSERT_OK_AND_ASSIGN(TypeSpec actual_type_spec, + TypeInfoToTypeSpec(param.expected_type_info)); + EXPECT_EQ(actual_type_spec, param.type_spec); +} + +INSTANTIATE_TEST_SUITE_P(TypeInfoToTypeSpecTest, TypeInfoToTypeSpecTest, + ValuesIn(GetTypeSpecTestCases())); + +TEST(TypeSpecToTypeInfoTest, ErrorConversions) { + EXPECT_THAT(TypeSpecToTypeInfo(TypeSpec(ErrorTypeSpec::kValue)), + StatusIs(absl::StatusCode::kInvalidArgument, + "ErrorType cannot be converted to TypeInfo")); + EXPECT_THAT(TypeSpecToTypeInfo(TypeSpec(FunctionTypeSpec())), + StatusIs(absl::StatusCode::kInvalidArgument, + "FunctionType cannot be converted to TypeInfo")); + EXPECT_THAT( + TypeSpecToTypeInfo(TypeSpec(UnsetTypeSpec())), + StatusIs(absl::StatusCode::kInvalidArgument, "Unknown TypeSpec kind")); +} + } // namespace } // namespace cel diff --git a/eval/README.md b/eval/README.md index ee6fd0798..bc805cb29 100644 --- a/eval/README.md +++ b/eval/README.md @@ -2,5 +2,246 @@ A C++ implementation of a [Common Expression Language][1] evaluator. +## Migrating to cel::Value APIs -[1]: https://github.com/google/cel-spec +New users should prefer using the `cel::Value` APIs ("Modern"). The +`google::api::expr::runtime::CelValue` APIs ("Legacy") are not formally +deprecated at this time, but they now incur some overhead and will not be +updated to support all new features. Internally, both flavors use the same +underlying implementation based on the cel::Value representation. + +### When to Migrate + +If your integration is stable and you don't need access to newer features there +is no need to migrate at this time. + +If you need to add support for optionals or other custom opaque or struct types +you must migrate. + +If you need to interactively evaluate from source, you should migrate to avoid +extra proto serialization costs. + +### How to Migrate + +#### 1. Header, Namespace, and Build Target Mapping + +Concept | Legacy API (`google::api::expr::runtime`) | Modern API (`cel`) +:----------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :----------------- +**Namespace** | `google::api::expr::runtime` | `cel` +**Runtime & Program Creation** | `#include "eval/public/cel_expression.h"`
`#include "eval/public/cel_expr_builder_factory.h"`
Target: `//eval/public:cel_expression` | `#include "runtime/runtime.h"`
`#include "runtime/standard_runtime_builder_factory.h"`
Targets: `//runtime`, `//runtime:standard_runtime_builder_factory` +**Value Representation** | `#include "eval/public/cel_value.h"`
Target: `//eval/public:cel_value` | `#include "common/value.h"`
Target: `//common:value` +**Activation & Variables** | `#include "eval/public/activation.h"`
Target: `//eval/public:activation` | `#include "runtime/activation.h"`
Target: `//runtime:activation` +**Proto Activation Binding** | `#include "eval/public/activation_bind_helper.h"`
Target: `//eval/public:activation_bind_helper` | `#include "runtime/bind_proto_to_activation.h"`
Target: `//runtime:bind_proto_to_activation` +**Function Adapters** | `#include "eval/public/cel_function_adapter.h"` | `#include "runtime/function_adapter.h"` +**Interoperability Adapters** | N/A | `#include "common/legacy_value.h"`
Target: `//common:legacy_value` + +#### 2. Runtime Creation and AST Planning (`CelExpressionBuilder` vs `cel::Runtime`) + +In the legacy API, `CelExpressionBuilder` created expression plans from +`google::api::expr::CheckedExpr` protobufs. In the modern API, `cel::Runtime` +creates reusable `cel::Program` instances directly from `cel::Ast` objects +without requiring protobuf serialization round-trips. + +**Legacy (`google::api::expr::runtime::CelExpressionBuilder`)**: + +```cpp +using ::google::api::expr::runtime::CelExpressionBuilder; +using ::google::api::expr::runtime::CreateCelExpressionBuilder; +using ::google::api::expr::runtime::InterpreterOptions; +using ::google::api::expr::runtime::RegisterBuiltinFunctions; + +InterpreterOptions options; +std::unique_ptr builder = + CreateCelExpressionBuilder(descriptor_pool, message_factory, options); +CEL_RETURN_IF_ERROR(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + +// Requires serializing or passing a google::api::expr::CheckedExpr proto +CEL_ASSIGN_OR_RETURN(std::unique_ptr plan, + builder->CreateExpression(&checked_expr_proto)); +``` + +**Modern (`cel::Runtime`)**: + +```cpp +#include "runtime/runtime.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" + +cel::RuntimeOptions options; +CEL_ASSIGN_OR_RETURN( + cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder(descriptor_pool, options)); +CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + +// Create program directly from a cel::Ast (e.g., from compiler->Compile(expr)) +CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); +``` + +*Note on AST Conversion*: If your architecture persists or receives serialized +`google::api::expr::CheckedExpr` protobufs, convert them to `cel::Ast` using +`cel::CreateAstFromCheckedExpr(checked_expr_proto)` (`#include +"common/ast_proto.h"`, target +`//common:ast_proto`). + +#### 3. Creating, Inspecting, and Unpacking Values (`CelValue` vs `cel::Value`) + +**Legacy (`google::api::expr::runtime::CelValue`)**: + +```cpp +#include "eval/public/cel_value.h" + +using ::google::api::expr::runtime::CelValue; + +// Creation +CelValue bool_val = CelValue::CreateBool(true); +CelValue int_val = CelValue::CreateInt64(42); +CelValue str_val = CelValue::CreateStringView("hello"); + +// Inspection & Unpacking +if (bool_val.IsBool()) { + bool b; + bool_val.GetValue(&b); +} +``` + +**Modern (`cel::Value`)**: + +```cpp +#include "common/value.h" + +// Creation +cel::Value bool_val = cel::BoolValue(true); +cel::Value int_val = cel::IntValue(42); +cel::Value str_val = cel::StringValue("hello"); + +// Inspection & Unpacking +if (bool_val.IsBool()) { + bool b = bool_val.GetBool().NativeValue(); +} else if (str_val.IsString()) { + std::string s = str_val.GetString().ToString(); +} else if (str_val.IsError()) { + absl::Status status = str_val.GetError().ToStatus(); +} +``` + +#### 4. Activation and Protobuf Binding (`BindProtoToActivation`) + +The modern equivalent of `activation_bind_helper.h` is +`runtime/bind_proto_to_activation.h`. + +**Legacy**: + +```cpp +#include "eval/public/activation.h" +#include "eval/public/activation_bind_helper.h" + +using ::google::api::expr::runtime::Activation; +using ::google::api::expr::runtime::BindProtoToActivation; +using ::google::api::expr::runtime::ProtoUnsetFieldOptions; + +Activation activation; +activation.InsertValue("bool_var", CelValue::CreateBool(true)); +CEL_RETURN_IF_ERROR(BindProtoToActivation( + &context_message, &arena, &activation, ProtoUnsetFieldOptions::kBindDefault)); +``` + +**Modern**: + +```cpp +#include "runtime/activation.h" +#include "runtime/bind_proto_to_activation.h" + +cel::Activation activation; +activation.InsertOrAssignValue("bool_var", cel::BoolValue(true)); +CEL_RETURN_IF_ERROR(cel::BindProtoToActivation( + context_message, cel::BindProtoUnsetFieldBehavior::kBindDefaultValue, + descriptor_pool, message_factory, &arena, &activation)); +``` + +#### 5. Program Evaluation + +Notice the parameter order change when calling `Evaluate`: `(&arena, +activation)` instead of `(activation, &arena)`. + +**Legacy**: + +```cpp +CEL_ASSIGN_OR_RETURN(CelValue result, plan->Evaluate(activation, &arena)); +``` + +**Modern**: + +```cpp +CEL_ASSIGN_OR_RETURN(cel::Value result, program->Evaluate(&arena, activation)); +``` + +#### 6. Custom Extension Functions (`FunctionAdapter`) + +**Legacy (`google::api::expr::runtime::FunctionAdapter`)**: + +```cpp +#include "eval/public/cel_function_adapter.h" + +using AdapterT = google::api::expr::runtime::FunctionAdapter< + absl::StatusOr, const CelMap*, CelValue::StringHolder, CelValue>; +CEL_RETURN_IF_ERROR(AdapterT::CreateAndRegister( + "contains", /*receiver_style=*/true, &ContainsFunc, builder->GetRegistry())); +``` + +**Modern (`cel::FunctionAdapter` / `cel::{arity}FunctionAdapter`)**: + +```cpp +#include "runtime/function_adapter.h" + +using AdapterT = cel::TernaryFunctionAdapter< + absl::StatusOr, const cel::MapValue&, const cel::StringValue&, const cel::Value&>; +CEL_RETURN_IF_ERROR(AdapterT::RegisterMemberOverload( + "contains", &ContainsFunc, runtime_builder.function_registry())); +``` + +### Other behavior notes + +There is limited support for interoperability when migrating incrementally. You +can use `#include "common/legacy_value.h"` (build target +`//common:legacy_value`) to adapt compatible value types +across legacy and modern boundaries: + +* **Adapting Legacy `CelValue` to Modern `cel::Value`**: + + ```cpp + #include "common/legacy_value.h" + + CEL_ASSIGN_OR_RETURN(cel::Value modern_val, + cel::ModernValue(&arena, legacy_cel_val)); + ``` + +* **Adapting Modern `cel::Value` to Legacy `CelValue`**: + + ```cpp + #include "common/legacy_value.h" + + CEL_ASSIGN_OR_RETURN(google::api::expr::runtime::CelValue legacy_val, + cel::LegacyValue(&arena, modern_val)); + ``` + +Legacy style extension functions use the interop helpers implicitly and can be +used with modern APIs. `google::api::expr::runtime::CelFunction` is also a +`cel::Function`. + +```cpp +std::unique_ptr ext_func = ...; + +cel::FunctionDescriptor descriptor = ext_func->descriptor(); +CEL_RETURN_IF_ERROR( + runtime_builder.function_registry().Register( + descriptor, std::move(ext_func))); +``` + +Important: The interop code does not support adapting new types introduced with +`cel::Value` to `google::api::expr::runtime::CelValue`. This means you should +not use any of the legacy APIs if your expressions refer to values with an +optional, custom (non-protobuf) struct, or custom opaque type. + +[1]: https://github.com/cel-expr/cel-spec diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index ed8e4d20c..5395f2e4b 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -31,13 +31,13 @@ cc_library( ":resolver", "//base:ast", "//base:data", + "//common:ast", "//common:expr", - "//common:native_type", + "//common:type", + "//common:type_spec_resolver", "//common:value", - "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", - "//eval/eval:trace_step", - "//internal:casts", + "//eval/eval:expression_step_logic", "//runtime:runtime_options", "//runtime/internal:issue_collector", "//runtime/internal:runtime_env", @@ -50,7 +50,6 @@ cc_library( "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:variant", "@com_google_protobuf//:protobuf", ], @@ -63,13 +62,10 @@ cc_test( ":flat_expr_builder_extensions", ":resolver", "//common:expr", - "//common:native_type", + "//common:type", "//common:value", - "//eval/eval:const_value_step", - "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", - "//eval/eval:function_step", - "//internal:status_macros", + "//eval/eval:expression_step_logic", "//internal:testing", "//runtime:function_registry", "//runtime:runtime_issue", @@ -101,7 +97,6 @@ cc_library( "//base:ast", "//base:builtins", "//base:data", - "//common:allocator", "//common:ast", "//common:ast_traverse", "//common:ast_visitor", @@ -110,25 +105,15 @@ cc_library( "//common:kind", "//common:type", "//common:value", - "//eval/eval:comprehension_step", - "//eval/eval:const_value_step", "//eval/eval:container_access_step", - "//eval/eval:create_list_step", "//eval/eval:create_map_step", "//eval/eval:create_struct_step", - "//eval/eval:direct_expression_step", - "//eval/eval:equality_steps", "//eval/eval:evaluator_core", - "//eval/eval:function_step", - "//eval/eval:ident_step", - "//eval/eval:jump_step", - "//eval/eval:lazy_init_step", - "//eval/eval:logic_step", + "//eval/eval:expression_step_logic", "//eval/eval:optional_or_step", "//eval/eval:select_step", "//eval/eval:shadowable_value_step", "//eval/eval:ternary_step", - "//eval/eval:trace_step", "//internal:status_macros", "//runtime:function_registry", "//runtime:runtime_issue", @@ -138,8 +123,8 @@ cc_library( "//runtime/internal:issue_collector", "//runtime/internal:runtime_env", "@com_google_absl//absl/algorithm:container", - "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/cleanup", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/container:node_hash_map", @@ -149,7 +134,6 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_absl//absl/types:variant", "@com_google_protobuf//:protobuf", @@ -165,15 +149,28 @@ cc_test( ":cel_expression_builder_flat_impl", ":constant_folding", ":flat_expr_builder", + ":flat_expr_builder_extensions", ":qualified_reference_resolver", "//base:builtins", + "//checker:validation_result", + "//common:ast", + "//common:decl", + "//common:expr", "//common:function_descriptor", "//common:kind", + "//common:type", "//common:value", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", + "//eval/eval:cel_expression_flat_impl", + "//eval/eval:evaluator_core", + "//eval/eval:expression_step_logic", "//eval/public:activation", + "//eval/public:ast_rewrite", + "//eval/public:base_activation", "//eval/public:builtin_func_registrar", "//eval/public:cel_attribute", - "//eval/public:cel_builtins", "//eval/public:cel_expr_builder_factory", "//eval/public:cel_expression", "//eval/public:cel_function", @@ -182,6 +179,7 @@ cc_test( "//eval/public:cel_options", "//eval/public:cel_value", "//eval/public:portable_cel_function_adapter", + "//eval/public:source_position", "//eval/public:unknown_attribute_set", "//eval/public:unknown_set", "//eval/public/containers:container_backed_map_impl", @@ -189,16 +187,17 @@ cc_test( "//eval/public/structs:cel_proto_wrapper", "//eval/public/testing:matchers", "//eval/testutil:test_message_cc_proto", + "//extensions/protobuf:ast_converters", "//internal:proto_matchers", "//internal:status_macros", "//internal:testing", "//parser", + "//parser:options", "//runtime:function", "//runtime:function_adapter", "//runtime:runtime_options", "//runtime:standard_functions", "//runtime/internal:runtime_env_testing", - "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", @@ -210,7 +209,6 @@ cc_test( "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", - "@com_google_protobuf//:field_mask_cc_proto", "@com_google_protobuf//:protobuf", ], ) @@ -224,6 +222,11 @@ cc_test( ":cel_expression_builder_flat_impl", ":comprehension_vulnerability_check", ":flat_expr_builder", + "//base:ast", + "//common:source", + "//common:value", + "//common:value_testing", + "//eval/eval:evaluator_core", "//eval/public:activation", "//eval/public:builtin_func_registrar", "//eval/public:cel_attribute", @@ -233,10 +236,15 @@ cc_test( "//eval/public/containers:container_backed_list_impl", "//eval/public/testing:matchers", "//eval/testutil:test_message_cc_proto", + "//extensions:bindings_ext", "//internal:testing", "//parser", + "//parser:parser_interface", + "//runtime:activation", "//runtime:runtime_options", + "//runtime/internal:runtime_env", "//runtime/internal:runtime_env_testing", + "//testutil:test_macros", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", @@ -256,9 +264,7 @@ cc_library( deps = [ ":flat_expr_builder", "//base:ast", - "//common:native_type", "//eval/eval:cel_expression_flat_impl", - "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", "//eval/public:cel_expression", "//eval/public:cel_function_registry", @@ -287,9 +293,6 @@ cc_test( ], deps = [ ":cel_expression_builder_flat_impl", - ":constant_folding", - ":regex_precompilation_optimization", - "//eval/eval:cel_expression_flat_impl", "//eval/public:activation", "//eval/public:builtin_func_registrar", "//eval/public:cel_expression", @@ -298,7 +301,6 @@ cc_test( "//eval/public:portable_cel_function_adapter", "//eval/public/containers:container_backed_map_impl", "//eval/public/structs:cel_proto_wrapper", - "//eval/public/structs:protobuf_descriptor_type_provider", "//eval/public/testing:matchers", "//extensions:bindings_ext", "//internal:status_macros", @@ -335,12 +337,10 @@ cc_library( "//common:constant", "//common:expr", "//common:value", - "//eval/eval:const_value_step", "//eval/eval:evaluator_core", "//internal:status_macros", "//runtime:activation", "//runtime/internal:convert_constant", - "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -360,8 +360,6 @@ cc_test( "//base:ast", "//common:expr", "//common:value", - "//eval/eval:const_value_step", - "//eval/eval:create_list_step", "//eval/eval:create_map_step", "//eval/eval:evaluator_core", "//extensions/protobuf:ast_converters", @@ -409,7 +407,6 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", ], ) @@ -500,17 +497,21 @@ cc_test( ], deps = [ ":cel_expression_builder_flat_impl", - "//base:builtins", "//eval/public:activation", + "//eval/public:builtin_func_registrar", "//eval/public:cel_attribute", "//eval/public:cel_expression", "//eval/public:cel_value", "//eval/public:unknown_attribute_set", "//eval/public:unknown_set", "//internal:testing", + "//parser", + "//parser:options", "//runtime:runtime_options", "//runtime/internal:runtime_env_testing", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings", "@com_google_protobuf//:protobuf", ], @@ -543,15 +544,11 @@ cc_library( ":flat_expr_builder_extensions", "//base:builtins", "//common:ast", - "//common:casting", "//common:expr", "//common:native_type", "//common:value", - "//eval/eval:compiler_constant_step", - "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", "//eval/eval:regex_match_step", - "//internal:casts", "//internal:re2_options", "//internal:status_macros", "@com_google_absl//absl/base:nullability", @@ -559,7 +556,6 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", "@com_googlesource_code_re2//:re2", ], ) diff --git a/eval/compiler/cel_expression_builder_flat_impl.cc b/eval/compiler/cel_expression_builder_flat_impl.cc index 98ecc6aae..f519d9393 100644 --- a/eval/compiler/cel_expression_builder_flat_impl.cc +++ b/eval/compiler/cel_expression_builder_flat_impl.cc @@ -27,9 +27,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "base/ast.h" -#include "common/native_type.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/public/cel_expression.h" #include "extensions/protobuf/ast_converters.h" @@ -96,14 +94,6 @@ CelExpressionBuilderFlatImpl::CreateExpressionImpl( warnings->push_back(issue.ToStatus()); } } - if (flat_expr_builder_.options().max_recursion_depth != 0 && - !impl.subexpressions().empty() && - // mainline expression is exactly one recursive step. - impl.subexpressions().front().size() == 1 && - impl.subexpressions().front().front()->GetNativeTypeId() == - cel::NativeTypeId::For()) { - return CelExpressionRecursiveImpl::Create(env_, std::move(impl)); - } return std::make_unique(env_, std::move(impl)); } diff --git a/eval/compiler/cel_expression_builder_flat_impl_test.cc b/eval/compiler/cel_expression_builder_flat_impl_test.cc index 9802d2a05..b48fbd959 100644 --- a/eval/compiler/cel_expression_builder_flat_impl_test.cc +++ b/eval/compiler/cel_expression_builder_flat_impl_test.cc @@ -30,9 +30,6 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "eval/compiler/constant_folding.h" -#include "eval/compiler/regex_precompilation_optimization.h" -#include "eval/eval/cel_expression_flat_impl.h" #include "eval/public/activation.h" #include "eval/public/builtin_func_registrar.h" #include "eval/public/cel_expression.h" @@ -72,8 +69,6 @@ using ::google::api::expr::parser::ParseWithMacros; using ::testing::_; using ::testing::Contains; using ::testing::HasSubstr; -using ::testing::IsNull; -using ::testing::NotNull; TEST(CelExpressionBuilderFlatImplTest, Error) { Expr expr; @@ -100,14 +95,14 @@ TEST(CelExpressionBuilderFlatImplTest, ParsedExpr) { EXPECT_THAT(result, test::IsCelInt64(3)); } -struct RecursiveTestCase { +struct TestCase { std::string test_name; std::string expr; test::CelValueMatcher matcher; std::string pb_expr; }; -class RecursivePlanTest : public ::testing::TestWithParam { +class PlanTest : public ::testing::TestWithParam { protected: absl::Status SetupBuilder(CelExpressionBuilderFlatImpl& builder) { builder.GetTypeRegistry()->RegisterEnum("TestEnum", @@ -146,7 +141,7 @@ class RecursivePlanTest : public ::testing::TestWithParam { } }; -absl::StatusOr ParseTestCase(const RecursiveTestCase& test_case) { +absl::StatusOr ParseTestCase(const TestCase& test_case) { static const std::vector* kMacros = []() { auto* result = new std::vector(Macro::AllMacros()); absl::c_copy(cel::extensions::bindings_macros(), @@ -166,107 +161,14 @@ absl::StatusOr ParseTestCase(const RecursiveTestCase& test_case) { return absl::InvalidArgumentError("No expression provided"); } -TEST_P(RecursivePlanTest, ParsedExprRecursiveImpl) { - const RecursiveTestCase& test_case = GetParam(); - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, ParseTestCase(test_case)); - cel::RuntimeOptions options; - options.container = "cel.expr.conformance.proto3"; - google::protobuf::Arena arena; - // Unbounded. - options.max_recursion_depth = -1; - CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); - - ASSERT_OK(SetupBuilder(builder)); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr plan, - builder.CreateExpression(&parsed_expr.expr(), - &parsed_expr.source_info())); - - EXPECT_THAT(dynamic_cast(plan.get()), - NotNull()); - - Activation activation; - - ASSERT_OK(SetupActivation(activation, &arena)); - - ASSERT_OK_AND_ASSIGN(CelValue result, plan->Evaluate(activation, &arena)); - EXPECT_THAT(result, test_case.matcher); -} - -TEST_P(RecursivePlanTest, ParsedExprRecursiveOptimizedImpl) { - const RecursiveTestCase& test_case = GetParam(); - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, ParseTestCase(test_case)); - cel::RuntimeOptions options; - options.container = "cel.expr.conformance.proto3"; - google::protobuf::Arena arena; - // Unbounded. - options.max_recursion_depth = -1; - options.enable_comprehension_list_append = true; - CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); - - ASSERT_OK(SetupBuilder(builder)); - - builder.flat_expr_builder().AddProgramOptimizer( - cel::runtime_internal::CreateConstantFoldingOptimizer()); - builder.flat_expr_builder().AddProgramOptimizer( - CreateRegexPrecompilationExtension(options.regex_max_program_size)); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr plan, - builder.CreateExpression(&parsed_expr.expr(), - &parsed_expr.source_info())); - - EXPECT_THAT(dynamic_cast(plan.get()), - NotNull()); - - Activation activation; - - ASSERT_OK(SetupActivation(activation, &arena)); - - ASSERT_OK_AND_ASSIGN(CelValue result, plan->Evaluate(activation, &arena)); - EXPECT_THAT(result, test_case.matcher); -} - -TEST_P(RecursivePlanTest, ParsedExprRecursiveTraceSupport) { - const RecursiveTestCase& test_case = GetParam(); - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, ParseTestCase(test_case)); - cel::RuntimeOptions options; - options.container = "cel.expr.conformance.proto3"; - google::protobuf::Arena arena; - auto cb = [](int64_t id, const CelValue& value, google::protobuf::Arena* arena) { - return absl::OkStatus(); - }; - // Unbounded. - options.max_recursion_depth = -1; - options.enable_recursive_tracing = true; - CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); - - ASSERT_OK(SetupBuilder(builder)); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr plan, - builder.CreateExpression(&parsed_expr.expr(), - &parsed_expr.source_info())); - - EXPECT_THAT(dynamic_cast(plan.get()), - NotNull()); - - Activation activation; - - ASSERT_OK(SetupActivation(activation, &arena)); - - ASSERT_OK_AND_ASSIGN(CelValue result, plan->Trace(activation, &arena, cb)); - EXPECT_THAT(result, test_case.matcher); -} - -TEST_P(RecursivePlanTest, Disabled) { +TEST_P(PlanTest, Basic) { google::protobuf::LinkMessageReflection(); - const RecursiveTestCase& test_case = GetParam(); + const TestCase& test_case = GetParam(); ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, ParseTestCase(test_case)); cel::RuntimeOptions options; options.container = "cel.expr.conformance.proto3"; google::protobuf::Arena arena; - // disabled. - options.max_recursion_depth = 0; CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); ASSERT_OK(SetupBuilder(builder)); @@ -275,9 +177,6 @@ TEST_P(RecursivePlanTest, Disabled) { builder.CreateExpression(&parsed_expr.expr(), &parsed_expr.source_info())); - EXPECT_THAT(dynamic_cast(plan.get()), - IsNull()); - Activation activation; ASSERT_OK(SetupActivation(activation, &arena)); @@ -287,8 +186,8 @@ TEST_P(RecursivePlanTest, Disabled) { } INSTANTIATE_TEST_SUITE_P( - RecursivePlanTest, RecursivePlanTest, - testing::ValuesIn(std::vector{ + PlanTest, PlanTest, + testing::ValuesIn(std::vector{ {"constant", "'abc'", test::IsCelString("abc")}, {"call", "1 + 2", test::IsCelInt64(3)}, {"nested_call", "1 + 1 + 1 + 1", test::IsCelInt64(4)}, @@ -545,7 +444,7 @@ INSTANTIATE_TEST_SUITE_P( } })pb"}}), - [](const testing::TestParamInfo& info) -> std::string { + [](const testing::TestParamInfo& info) -> std::string { return info.param.test_name; }); diff --git a/eval/compiler/constant_folding.cc b/eval/compiler/constant_folding.cc index 118fc94c5..3d308b81f 100644 --- a/eval/compiler/constant_folding.cc +++ b/eval/compiler/constant_folding.cc @@ -19,7 +19,6 @@ #include #include -#include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -31,7 +30,6 @@ #include "common/value.h" #include "eval/compiler/flat_expr_builder_extensions.h" #include "eval/compiler/resolver.h" -#include "eval/eval/const_value_step.h" #include "eval/eval/evaluator_core.h" #include "internal/status_macros.h" #include "runtime/activation.h" @@ -49,12 +47,11 @@ using ::cel::builtin::kAnd; using ::cel::builtin::kOr; using ::cel::builtin::kTernary; using ::cel::runtime_internal::ConvertConstant; -using ::google::api::expr::runtime::CreateConstValueDirectStep; -using ::google::api::expr::runtime::CreateConstValueStep; using ::google::api::expr::runtime::EvaluationListener; using ::google::api::expr::runtime::ExecutionFrame; using ::google::api::expr::runtime::ExecutionPath; using ::google::api::expr::runtime::ExecutionPathView; +using ::google::api::expr::runtime::ExpressionStep; using ::google::api::expr::runtime::FlatExpressionEvaluatorState; using ::google::api::expr::runtime::PlannerContext; using ::google::api::expr::runtime::ProgramOptimizer; @@ -78,8 +75,9 @@ class ConstantFoldingExtension : public ProgramOptimizer { const TypeProvider& type_provider) : shared_arena_(std::move(shared_arena)), shared_message_factory_(std::move(shared_message_factory)), - state_(kDefaultStackLimit, kComprehensionSlotCount, type_provider, - descriptor_pool, message_factory, arena) {} + state_(kDefaultStackLimit, kComprehensionSlotCount, + /*iterator_stack_size=*/0, type_provider, descriptor_pool, + message_factory, arena) {} absl::Status OnPreVisit(google::api::expr::runtime::PlannerContext& context, const Expr& node) override; @@ -96,7 +94,7 @@ class ConstantFoldingExtension : public ProgramOptimizer { static constexpr size_t kComprehensionSlotCount = 0; absl_nullable std::shared_ptr shared_arena_; - ABSL_ATTRIBUTE_UNUSED + [[maybe_unused]] absl_nullable std::shared_ptr shared_message_factory_; Activation empty_; FlatExpressionEvaluatorState state_; @@ -192,6 +190,14 @@ absl::Status ConstantFoldingExtension::OnPostVisit(PlannerContext& context, is_const_.pop_back(); if (is_const == IsConst::kNonConst) { + if (context.options().enable_subexpression_inlining && + node.has_ident_expr()) { + ExecutionPathView subplan = context.GetSubplan(node); + if (subplan.size() == 1 && + google::api::expr::runtime::IsConstant(subplan[0])) { + return absl::OkStatus(); + } + } // update parent if (!is_const_.empty()) { is_const_.back() = IsConst::kNonConst; @@ -203,18 +209,18 @@ absl::Status ConstantFoldingExtension::OnPostVisit(PlannerContext& context, // This subexpression is already optimized out or suppressed. return absl::OkStatus(); } - // copy string to managed handle if backed by the original program. + // Ensure that string constants are copied to the constant folding arena Value value; if (node.has_const_expr()) { CEL_ASSIGN_OR_RETURN(value, ConvertConstant(node.const_expr(), state_.arena())); } else { - ExecutionFrame frame(subplan, empty_, context.options(), state_); state_.Reset(); // Update stack size to accommodate sub expression. // This only results in a vector resize if the new maxsize is greater than // the current capacity. state_.value_stack().SetMaxSize(subplan.size()); + ExecutionFrame frame(subplan, empty_, context.options(), state_); auto result = frame.Evaluate(); // If this would be a runtime error, then don't adjust the program plan, but @@ -229,22 +235,9 @@ absl::Status ConstantFoldingExtension::OnPostVisit(PlannerContext& context, } } - // If recursive planning enabled (recursion limit unbounded or at least 1), - // use a recursive (direct) step for the folded constant. - // - // Constant folding is applied leaf to root based on the program plan so far, - // so the planner will have an opportunity to validate that the recursion - // limit is being followed when visiting parent nodes in the AST. - if (context.options().max_recursion_depth != 0) { - return context.ReplaceSubplan( - node, CreateConstValueDirectStep(std::move(value), node.id()), 1); - } - // Otherwise make a stack machine plan. ExecutionPath new_plan; - CEL_ASSIGN_OR_RETURN( - new_plan.emplace_back(), - CreateConstValueStep(std::move(value), node.id(), false)); + new_plan.push_back(ExpressionStep::MakeConstant(value, node.id())); return context.ReplaceSubplan(node, std::move(new_plan)); } diff --git a/eval/compiler/constant_folding_test.cc b/eval/compiler/constant_folding_test.cc index d1c0c31e0..c612e7c1f 100644 --- a/eval/compiler/constant_folding_test.cc +++ b/eval/compiler/constant_folding_test.cc @@ -29,7 +29,6 @@ #include "common/value.h" #include "eval/compiler/flat_expr_builder_extensions.h" #include "eval/compiler/resolver.h" -#include "eval/eval/const_value_step.h" #include "eval/eval/create_list_step.h" #include "eval/eval/create_map_step.h" #include "eval/eval/evaluator_core.h" @@ -58,10 +57,10 @@ using ::cel::runtime_internal::IssueCollector; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::expr::ParsedExpr; using ::google::api::expr::parser::Parse; -using ::google::api::expr::runtime::CreateConstValueStep; using ::google::api::expr::runtime::CreateCreateListStep; using ::google::api::expr::runtime::CreateCreateStructStepForMap; using ::google::api::expr::runtime::ExecutionPath; +using ::google::api::expr::runtime::ExpressionStep; using ::google::api::expr::runtime::PlannerContext; using ::google::api::expr::runtime::ProgramBuilder; using ::google::api::expr::runtime::ProgramOptimizer; @@ -115,26 +114,25 @@ TEST_F(UpdatedConstantFoldingTest, SkipsTernary) { program_builder.EnterSubexpression(&call); // condition program_builder.EnterSubexpression(&condition); - ASSERT_OK_AND_ASSIGN(auto step, - CreateConstValueStep(cel::BoolValue(true), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::BoolValue(true), condition.id())); program_builder.ExitSubexpression(&condition); // true program_builder.EnterSubexpression(&true_branch); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::BoolValue(true), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::BoolValue(true), true_branch.id())); program_builder.ExitSubexpression(&true_branch); // false program_builder.EnterSubexpression(&false_branch); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::BoolValue(true), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::BoolValue(true), false_branch.id())); program_builder.ExitSubexpression(&false_branch); // ternary. - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::NullValue(), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::NullValue(), call.id())); program_builder.ExitSubexpression(&call); std::shared_ptr arena; @@ -179,21 +177,20 @@ TEST_F(UpdatedConstantFoldingTest, SkipsOr) { // left program_builder.EnterSubexpression(&left_condition); - ASSERT_OK_AND_ASSIGN(auto step, - CreateConstValueStep(cel::BoolValue(false), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::BoolValue(false), left_condition.id())); program_builder.ExitSubexpression(&left_condition); // right program_builder.EnterSubexpression(&right_condition); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::BoolValue(true), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::BoolValue(true), right_condition.id())); program_builder.ExitSubexpression(&right_condition); // op // Just a placeholder. - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::NullValue(), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::NullValue(), call.id())); program_builder.ExitSubexpression(&call); std::shared_ptr arena; @@ -235,21 +232,20 @@ TEST_F(UpdatedConstantFoldingTest, SkipsAnd) { // left program_builder.EnterSubexpression(&left_condition); - ASSERT_OK_AND_ASSIGN(auto step, - CreateConstValueStep(cel::BoolValue(true), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::BoolValue(true), left_condition.id())); program_builder.ExitSubexpression(&left_condition); // right program_builder.EnterSubexpression(&right_condition); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::BoolValue(false), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeConstant(cel::BoolValue(false), + right_condition.id())); program_builder.ExitSubexpression(&right_condition); // op // Just a placeholder. - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::NullValue(), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::NullValue(), call.id())); program_builder.ExitSubexpression(&call); std::shared_ptr arena; @@ -291,19 +287,19 @@ TEST_F(UpdatedConstantFoldingTest, CreatesList) { // elem one program_builder.EnterSubexpression(&elem_one); - ASSERT_OK_AND_ASSIGN(auto step, CreateConstValueStep(cel::IntValue(1L), 1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(1L), elem_one.id())); program_builder.ExitSubexpression(&elem_one); // elem two program_builder.EnterSubexpression(&elem_two); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::IntValue(2L), 2)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(2L), elem_two.id())); program_builder.ExitSubexpression(&elem_two); // createlist - ASSERT_OK_AND_ASSIGN(step, CreateCreateListStep(create_list.list_expr(), 3)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(CreateCreateListStep( + create_list.list_expr().elements().size(), {}, create_list.id())); program_builder.ExitSubexpression(&create_list); std::shared_ptr arena; @@ -349,37 +345,37 @@ TEST_F(UpdatedConstantFoldingTest, CreatesLargeList) { // 0 ASSERT_TRUE(program_builder.EnterSubexpression(&elem0) != nullptr); - ASSERT_OK_AND_ASSIGN(auto step, CreateConstValueStep(cel::IntValue(1L), 1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(1L), elem0.id())); program_builder.ExitSubexpression(&elem0); // 1 ASSERT_TRUE(program_builder.EnterSubexpression(&elem1)); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::IntValue(2L), 2)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(2L), elem1.id())); program_builder.ExitSubexpression(&elem1); // 2 ASSERT_TRUE(program_builder.EnterSubexpression(&elem2) != nullptr); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::IntValue(3L), 3)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(3L), elem2.id())); program_builder.ExitSubexpression(&elem2); // 3 ASSERT_TRUE(program_builder.EnterSubexpression(&elem3) != nullptr); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::IntValue(4L), 4)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(4L), elem3.id())); program_builder.ExitSubexpression(&elem3); // 4 ASSERT_TRUE(program_builder.EnterSubexpression(&elem4) != nullptr); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::IntValue(5L), 5)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(5L), elem4.id())); program_builder.ExitSubexpression(&elem4); // createlist - ASSERT_OK_AND_ASSIGN(step, CreateCreateListStep(create_list.list_expr(), 6)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(CreateCreateListStep( + create_list.list_expr().elements().size(), {}, create_list.id())); program_builder.ExitSubexpression(&create_list); std::shared_ptr arena; @@ -426,21 +422,22 @@ TEST_F(UpdatedConstantFoldingTest, CreatesMap) { // key program_builder.EnterSubexpression(&key); - ASSERT_OK_AND_ASSIGN(auto step, CreateConstValueStep(cel::IntValue(1L), 1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(1L), key.id())); program_builder.ExitSubexpression(&key); // value program_builder.EnterSubexpression(&value); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::IntValue(2L), 2)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(2L), value.id())); program_builder.ExitSubexpression(&value); // create map ASSERT_OK_AND_ASSIGN( - step, CreateCreateStructStepForMap(create_map.map_expr().entries().size(), - {}, 3)); - program_builder.AddStep(std::move(step)); + auto step_map, + CreateCreateStructStepForMap(create_map.map_expr().entries().size(), {})); + program_builder.AddStep( + ExpressionStep::MakeGenericStep(std::move(step_map), create_map.id())); program_builder.ExitSubexpression(&create_map); std::shared_ptr arena; @@ -481,22 +478,22 @@ TEST_F(UpdatedConstantFoldingTest, CreatesInvalidMap) { // key program_builder.EnterSubexpression(&key); - ASSERT_OK_AND_ASSIGN(auto step, - CreateConstValueStep(cel::DoubleValue(1.0), 1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::DoubleValue(1.0), key.id())); program_builder.ExitSubexpression(&key); // value program_builder.EnterSubexpression(&value); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::IntValue(2L), 2)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(2L), value.id())); program_builder.ExitSubexpression(&value); // create map ASSERT_OK_AND_ASSIGN( - step, CreateCreateStructStepForMap(create_map.map_expr().entries().size(), - {}, 3)); - program_builder.AddStep(std::move(step)); + auto step_invalid_map, + CreateCreateStructStepForMap(create_map.map_expr().entries().size(), {})); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + std::move(step_invalid_map), create_map.id())); program_builder.ExitSubexpression(&create_map); std::shared_ptr arena; @@ -536,21 +533,20 @@ TEST_F(UpdatedConstantFoldingTest, ErrorsOnUnexpectedOrder) { program_builder.EnterSubexpression(&call); // left program_builder.EnterSubexpression(&left_condition); - ASSERT_OK_AND_ASSIGN(auto step, - CreateConstValueStep(cel::BoolValue(true), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::BoolValue(true), left_condition.id())); program_builder.ExitSubexpression(&left_condition); // right program_builder.EnterSubexpression(&right_condition); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::BoolValue(false), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeConstant(cel::BoolValue(false), + right_condition.id())); program_builder.ExitSubexpression(&right_condition); // op // Just a placeholder. - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::NullValue(), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::NullValue(), call.id())); program_builder.ExitSubexpression(&call); std::shared_ptr arena; @@ -568,6 +564,55 @@ TEST_F(UpdatedConstantFoldingTest, ErrorsOnUnexpectedOrder) { StatusIs(absl::StatusCode::kInternal)); } +TEST_F(UpdatedConstantFoldingTest, FoldsListWithInlinedConstantIdent) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr list_ast, + ParseFromCel("[a, 2]")); + const Expr& create_list = list_ast->root_expr(); + const Expr& elem_a = create_list.list_expr().elements()[0].expr(); + const Expr& elem_two = create_list.list_expr().elements()[1].expr(); + + ProgramBuilder program_builder; + program_builder.EnterSubexpression(&create_list); + // Simulate `a` having been inlined as a constant step by FlatExprVisitor. + program_builder.EnterSubexpression(&elem_a); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(10), elem_a.id())); + program_builder.ExitSubexpression(&elem_a); + + program_builder.EnterSubexpression(&elem_two); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(2), elem_two.id())); + program_builder.ExitSubexpression(&elem_two); + + program_builder.AddStep(CreateCreateListStep( + create_list.list_expr().elements().size(), {}, create_list.id())); + program_builder.ExitSubexpression(&create_list); + + options_.enable_subexpression_inlining = true; + std::shared_ptr arena; + PlannerContext context(env_, resolver_, options_, + type_registry_.GetComposedTypeProvider(), + issue_collector_, program_builder, arena); + + ProgramOptimizerFactory constant_folder_factory = + CreateConstantFoldingOptimizer(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr constant_folder, + constant_folder_factory(context, *list_ast)); + + ASSERT_THAT(constant_folder->OnPreVisit(context, create_list), IsOk()); + ASSERT_THAT(constant_folder->OnPreVisit(context, elem_a), IsOk()); + ASSERT_THAT(constant_folder->OnPostVisit(context, elem_a), IsOk()); + ASSERT_THAT(constant_folder->OnPreVisit(context, elem_two), IsOk()); + ASSERT_THAT(constant_folder->OnPostVisit(context, elem_two), IsOk()); + ASSERT_THAT(constant_folder->OnPostVisit(context, create_list), IsOk()); + + ExecutionPath path = std::move(program_builder).FlattenMain(); + EXPECT_THAT(path, SizeIs(1)); + cel::Value folded_list; + ASSERT_TRUE(google::api::expr::runtime::GetIfConstant(path[0], folded_list)); + EXPECT_TRUE(folded_list.IsList()); +} + } // namespace } // namespace cel::runtime_internal diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index e38c912c0..3626937e9 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -30,8 +31,8 @@ #include #include "absl/algorithm/container.h" -#include "absl/base/attributes.h" -#include "absl/base/optimization.h" +#include "absl/base/nullability.h" +#include "absl/cleanup/cleanup.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/container/node_hash_map.h" @@ -45,13 +46,11 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/strings/strip.h" -#include "absl/types/optional.h" #include "absl/types/span.h" #include "absl/types/variant.h" #include "base/ast.h" #include "base/builtins.h" #include "base/type_provider.h" -#include "common/allocator.h" #include "common/ast.h" #include "common/ast_traverse.h" #include "common/ast_visitor.h" @@ -64,24 +63,18 @@ #include "eval/compiler/flat_expr_builder_extensions.h" #include "eval/compiler/resolver.h" #include "eval/eval/comprehension_step.h" -#include "eval/eval/const_value_step.h" #include "eval/eval/container_access_step.h" #include "eval/eval/create_list_step.h" #include "eval/eval/create_map_step.h" #include "eval/eval/create_struct_step.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/equality_steps.h" #include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" #include "eval/eval/function_step.h" #include "eval/eval/ident_step.h" -#include "eval/eval/jump_step.h" -#include "eval/eval/lazy_init_step.h" -#include "eval/eval/logic_step.h" #include "eval/eval/optional_or_step.h" #include "eval/eval/select_step.h" #include "eval/eval/shadowable_value_step.h" #include "eval/eval/ternary_step.h" -#include "eval/eval/trace_step.h" #include "internal/status_macros.h" #include "runtime/internal/convert_constant.h" #include "runtime/internal/issue_collector.h" @@ -97,7 +90,6 @@ namespace { using ::cel::Ast; using ::cel::AstTraverse; using ::cel::RuntimeIssue; -using ::cel::StringValue; using ::cel::Value; using ::cel::runtime_internal::ConvertConstant; using ::cel::runtime_internal::GetLegacyRuntimeTypeProvider; @@ -111,17 +103,15 @@ constexpr absl::string_view kBlock = "cel.@block"; // Forward declare to resolve circular dependency for short_circuiting visitors. class FlatExprVisitor; -// Error code for failed recursive program building. Generally indicates an -// optimization doesn't support recursive programs. -absl::Status FailedRecursivePlanning() { - return absl::InternalError( - "failed to build recursive program. check for unsupported optimizations"); -} - // Helper for bookkeeping variables mapped to indexes. class IndexManager { public: - IndexManager() : next_free_slot_(0), max_slot_count_(0) {} + IndexManager() + : next_free_slot_(0), + max_slot_count_(0), + iter_stack_count_(0), + max_iter_stack_count_(0), + lazy_iter_stack_count_(0) {} size_t ReserveSlots(size_t n) { size_t result = next_free_slot_; @@ -137,11 +127,30 @@ class IndexManager { return next_free_slot_; } + void ReserveIterStack() { + ++iter_stack_count_; + max_iter_stack_count_ = std::max(max_iter_stack_count_, iter_stack_count_); + } + void ReleaseIterStack() { --iter_stack_count_; } + void AddLazyIterStack() { ++lazy_iter_stack_count_; } + size_t max_slot_count() const { return max_slot_count_; } + // Returns the maximum number of iterator stack slots required to evaluate the + // expression by tracking the maximum comprehension nesting in the main + // expression and dedicating a slot for any comprehensions evaluated lazily. + // This is not as tight as possible, but is a correct upper bound estimate and + // is tight for small expressions. + size_t IterStackRequirement() const { + return max_iter_stack_count_ + lazy_iter_stack_count_; + } + private: size_t next_free_slot_; size_t max_slot_count_; + size_t iter_stack_count_; + size_t max_iter_stack_count_; + size_t lazy_iter_stack_count_; }; // Helper for computing jump offsets. @@ -156,13 +165,6 @@ struct ProgramStepIndex { // A convenience wrapper for offset-calculating logic. class Jump { public: - // Default constructor for empty jump. - // - // Users must check that jump is non-empty before calling member functions. - explicit Jump() : self_index_{-1, nullptr}, jump_step_(nullptr) {} - Jump(ProgramStepIndex self_index, JumpStepBase* jump_step) - : self_index_(self_index), jump_step_(jump_step) {} - static absl::StatusOr CalculateOffset(ProgramStepIndex base, ProgramStepIndex target) { if (target.subexpression != base.subexpression) { @@ -172,21 +174,19 @@ class Jump { } int offset = base.subexpression->CalculateOffset(base.index, target.index); - return offset; - } - absl::Status set_target(ProgramStepIndex target) { - CEL_ASSIGN_OR_RETURN(int offset, CalculateOffset(self_index_, target)); + // The offset may be packed into a 31-bit int field. Technically, the offset + // could be larger for a well-formed expression with more than 2**30 steps, + // but such a case is unlikely to exist at this time. + constexpr int kMaxJumpOffset = (1 << 30) - 1; + constexpr int kMinJumpOffset = -(1 << 30); - jump_step_->set_jump_offset(offset); - return absl::OkStatus(); + if (offset < kMinJumpOffset || offset > kMaxJumpOffset) { + return absl::InternalError("Jump offset exceeds supported range: " + + absl::StrCat(offset)); + } + return offset; } - - bool exists() { return jump_step_ != nullptr; } - - private: - ProgramStepIndex self_index_; - JumpStepBase* jump_step_; }; class CondVisitor { @@ -198,14 +198,7 @@ class CondVisitor { virtual void PostVisitTarget(const cel::Expr* expr) {} }; -enum class BinaryCond { - kAnd = 0, - kOr, - kOptionalOr, - kOptionalOrValue, -}; - -// Visitor managing the "&&" and "||" operatiions. +// Visitor managing the "&&" and "||" (boolean logic) operations. // Implements short-circuiting if enabled. // // With short-circuiting enabled, generates a program like: @@ -218,21 +211,43 @@ enum class BinaryCond { // | i + 3 | BooleanOperator | Op(arg1, arg2) | // | i + 4 | | arg1 | Op(arg1, arg2) | // +-------------+------------------------+------------------------+ -class BinaryCondVisitor : public CondVisitor { +class LogicalCondVisitor : public CondVisitor { public: - explicit BinaryCondVisitor(FlatExprVisitor* visitor, BinaryCond cond, - bool short_circuiting) - : visitor_(visitor), cond_(cond), short_circuiting_(short_circuiting) {} + explicit LogicalCondVisitor(FlatExprVisitor* visitor, bool is_or, + bool short_circuiting) + : visitor_(visitor), is_or_(is_or), short_circuiting_(short_circuiting) {} void PreVisit(const cel::Expr* expr) override; void PostVisitArg(int arg_num, const cel::Expr* expr) override; void PostVisit(const cel::Expr* expr) override; + + private: + FlatExprVisitor* visitor_; + const bool is_or_; + std::vector jump_steps_; + bool short_circuiting_; +}; + +// Visitor managing optional "or" and "orValue" operations. +// Implements short-circuiting if enabled. +class OptionalOrCondVisitor : public CondVisitor { + public: + explicit OptionalOrCondVisitor(FlatExprVisitor* visitor, bool is_or_value, + bool short_circuiting) + : visitor_(visitor), + is_or_value_(is_or_value), + short_circuiting_(short_circuiting) {} + + void PreVisit(const cel::Expr* expr) override; + void PostVisitArg(int arg_num, const cel::Expr* expr) override {} void PostVisitTarget(const cel::Expr* expr) override; + void PostVisit(const cel::Expr* expr) override; private: FlatExprVisitor* visitor_; - const BinaryCond cond_; - Jump jump_step_; + const bool is_or_value_; + OptionalHasValueJumpStep* jump_step_ = nullptr; + ProgramStepIndex jump_step_pos_{-1, nullptr}; bool short_circuiting_; }; @@ -246,9 +261,8 @@ class TernaryCondVisitor : public CondVisitor { private: FlatExprVisitor* visitor_; - Jump jump_to_second_; - Jump error_jump_; - Jump jump_after_first_; + std::optional cond_jump_pos_; + std::optional jump_after_first_pos_; }; class ExhaustiveTernaryCondVisitor : public CondVisitor { @@ -293,12 +307,15 @@ size_t SizeHint(const cel::Expr& expr) { // macro implementation. It is not exhaustive, so it is unsafe to use with // custom comprehensions outside of the standard macros or hand crafted ASTs. bool IsOptimizableListAppend(const cel::ComprehensionExpr* comprehension, - bool enable_comprehension_list_append) { + bool enable_comprehension_list_append, + bool short_circuiting = true) { if (!enable_comprehension_list_append) { return false; } absl::string_view accu_var = comprehension->accu_var(); - if (accu_var.empty() || + if (accu_var.empty() || !absl::StartsWith(accu_var, "@") || + !comprehension->has_result() || + !comprehension->result().has_ident_expr() || comprehension->result().ident_expr().name() != accu_var) { return false; } @@ -319,18 +336,22 @@ bool IsOptimizableListAppend(const cel::ComprehensionExpr* comprehension, if (call_expr->function() == cel::builtin::kTernary && call_expr->args().size() == 3) { - if (!call_expr->args()[1].has_call_expr()) { + if (!short_circuiting || !call_expr->args()[1].has_call_expr() || + !call_expr->args()[2].has_ident_expr() || + call_expr->args()[2].ident_expr().name() != accu_var) { return false; } call_expr = &(call_expr->args()[1].call_expr()); } - return call_expr->function() == cel::builtin::kAdd && + return !call_expr->has_target() && + call_expr->function() == cel::builtin::kAdd && call_expr->args().size() == 2 && call_expr->args()[0].has_ident_expr() && call_expr->args()[0].ident_expr().name() == accu_var && call_expr->args()[1].has_list_expr() && - call_expr->args()[1].list_expr().elements().size() == 1; + call_expr->args()[1].list_expr().elements().size() == 1 && + !call_expr->args()[1].list_expr().elements()[0].optional(); } // Assuming `IsOptimizableListAppend()` return true, return a pointer to the @@ -364,7 +385,8 @@ const cel::Expr* GetOptimizableListAppendOperand( // map transformations. It is not exhaustive, so it is unsafe to use with custom // comprehensions outside of the standard macros or hand crafted ASTs. bool IsOptimizableMapInsert(const cel::ComprehensionExpr* comprehension, - bool enable_comprehension_mutable_map) { + bool enable_comprehension_mutable_map, + bool short_circuiting = true) { if (!enable_comprehension_mutable_map) { return false; } @@ -372,12 +394,14 @@ bool IsOptimizableMapInsert(const cel::ComprehensionExpr* comprehension, return false; } absl::string_view accu_var = comprehension->accu_var(); - if (accu_var.empty() || !comprehension->has_result() || + if (accu_var.empty() || !absl::StartsWith(accu_var, "@") || + !comprehension->has_result() || !comprehension->result().has_ident_expr() || comprehension->result().ident_expr().name() != accu_var) { return false; } - if (!comprehension->accu_init().has_map_expr()) { + if (!comprehension->accu_init().has_map_expr() || + !comprehension->accu_init().map_expr().entries().empty()) { return false; } if (!comprehension->loop_step().has_call_expr()) { @@ -387,12 +411,15 @@ bool IsOptimizableMapInsert(const cel::ComprehensionExpr* comprehension, if (call_expr->function() == cel::builtin::kTernary && call_expr->args().size() == 3) { - if (!call_expr->args()[1].has_call_expr()) { + if (!short_circuiting || !call_expr->args()[1].has_call_expr() || + !call_expr->args()[2].has_ident_expr() || + call_expr->args()[2].ident_expr().name() != accu_var) { return false; } call_expr = &(call_expr->args()[1].call_expr()); } - return call_expr->function() == "cel.@mapInsert" && + return !call_expr->has_target() && + call_expr->function() == "cel.@mapInsert" && (call_expr->args().size() == 2 || call_expr->args().size() == 3) && call_expr->args()[0].has_ident_expr() && call_expr->args()[0].ident_expr().name() == accu_var; @@ -414,13 +441,10 @@ bool IsBlock(const cel::CallExpr* call) { return call->function() == kBlock; } // Visitor for Comprehension expressions. class ComprehensionVisitor { public: - explicit ComprehensionVisitor(FlatExprVisitor* visitor, bool short_circuiting, - bool is_trivial, size_t iter_slot, - size_t iter2_slot, size_t accu_slot) + explicit ComprehensionVisitor(FlatExprVisitor* visitor, bool is_trivial, + size_t iter_slot, size_t iter2_slot, + size_t accu_slot) : visitor_(visitor), - next_step_(nullptr), - cond_step_(nullptr), - short_circuiting_(short_circuiting), is_trivial_(is_trivial), accu_init_extracted_(false), iter_slot_(iter_slot), @@ -448,14 +472,14 @@ class ComprehensionVisitor { absl::Status PostVisitArgDefault(cel::ComprehensionArg arg_num, const cel::Expr* comprehension_expr); + ComprehensionInitStepInfo* absl_nullable GetInitStep(); + ComprehensionCondStepInfo* absl_nullable GetCondStep(); + ComprehensionNextStepInfo* absl_nullable GetNextStep(); + FlatExprVisitor* visitor_; - ComprehensionInitStep* init_step_; - ComprehensionNextStep* next_step_; - ComprehensionCondStep* cond_step_; - ProgramStepIndex init_step_pos_; - ProgramStepIndex next_step_pos_; - ProgramStepIndex cond_step_pos_; - bool short_circuiting_; + std::optional init_step_pos_; + std::optional next_step_pos_; + std::optional cond_step_pos_; bool is_trivial_; bool accu_init_extracted_; size_t iter_slot_; @@ -463,17 +487,6 @@ class ComprehensionVisitor { size_t accu_slot_; }; -absl::flat_hash_set MakeOptionalIndicesSet( - const cel::ListExpr& create_list_expr) { - absl::flat_hash_set optional_indices; - for (size_t i = 0; i < create_list_expr.elements().size(); ++i) { - if (create_list_expr.elements()[i].optional()) { - optional_indices.insert(static_cast(i)); - } - } - return optional_indices; -} - absl::flat_hash_set MakeOptionalIndicesSet( const cel::StructExpr& create_struct_expr) { absl::flat_hash_set optional_indices; @@ -496,6 +509,17 @@ absl::flat_hash_set MakeOptionalIndicesSet( return optional_indices; } +absl::flat_hash_set MakeOptionalIndicesSet( + const cel::ListExpr& list_expr) { + absl::flat_hash_set optional_indices; + for (size_t i = 0; i < list_expr.elements().size(); ++i) { + if (list_expr.elements()[i].optional()) { + optional_indices.insert(i); + } + } + return optional_indices; +} + class FlatExprVisitor : public cel::AstVisitor { public: enum class CallHandlerResult { @@ -514,7 +538,7 @@ class FlatExprVisitor : public cel::AstVisitor { FlatExprVisitor( const Resolver& resolver, const cel::RuntimeOptions& options, std::vector> program_optimizers, - const absl::flat_hash_map& reference_map, + const absl::flat_hash_map& type_map, const cel::TypeProvider& type_provider, IssueCollector& issue_collector, ProgramBuilder& program_builder, PlannerContext& extension_context, bool enable_optional_types) @@ -524,6 +548,7 @@ class FlatExprVisitor : public cel::AstVisitor { resolved_select_expr_(nullptr), options_(options), program_optimizers_(std::move(program_optimizers)), + type_map_(type_map), issue_collector_(issue_collector), program_builder_(program_builder), extension_context_(extension_context), @@ -586,12 +611,6 @@ class FlatExprVisitor : public cel::AstVisitor { } } - void SetMaxRecursionDepth(int max_recursion_depth) { - max_recursion_depth_ = max_recursion_depth; - } - - bool PlanRecursiveProgram() const { return max_recursion_depth_ > 0; } - void PreVisitExpr(const cel::Expr& expr) override { ValidateOrError(!absl::holds_alternative(expr.kind()), "Invalid empty expression"); @@ -603,6 +622,10 @@ class FlatExprVisitor : public cel::AstVisitor { resume_from_suppressed_branch_ = &expr; } + if (extension_context_.resolve_types()) { + MaybeResolveType(expr); + } + if (block_.has_value()) { BlockInfo& block = *block_; if (block.in && block.bindings_set.contains(&expr)) { @@ -622,7 +645,7 @@ class FlatExprVisitor : public cel::AstVisitor { program_optimizers_) { absl::Status status = optimizer->OnPreVisit(extension_context_, expr); if (!status.ok()) { - SetProgressStatusError(status); + SetProgressStatusIfError(status); } } } @@ -639,25 +662,17 @@ class FlatExprVisitor : public cel::AstVisitor { program_optimizers_) { absl::Status status = optimizer->OnPostVisit(extension_context_, expr); if (!status.ok()) { - SetProgressStatusError(status); + SetProgressStatusIfError(status); return; } } - auto* subexpression = program_builder_.current(); - if (subexpression != nullptr && options_.enable_recursive_tracing && - subexpression->IsRecursive()) { - auto program = subexpression->ExtractRecursiveProgram(); - subexpression->set_recursive_program( - std::make_unique(std::move(program.step)), program.depth); - } - program_builder_.ExitSubexpression(&expr); if (!comprehension_stack_.empty() && comprehension_stack_.back().is_optimizable_bind && (&comprehension_stack_.back().comprehension->accu_init() == &expr)) { - SetProgressStatusError( + SetProgressStatusIfError( MaybeExtractSubexpression(&expr, comprehension_stack_.back())); } @@ -666,7 +681,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (block.current_binding == &expr) { int index = program_builder_.ExtractSubexpression(&expr); if (index == -1) { - SetProgressStatusError( + SetProgressStatusIfError( absl::InvalidArgumentError("failed to extract subexpression")); return; } @@ -683,22 +698,15 @@ class FlatExprVisitor : public cel::AstVisitor { } absl::StatusOr converted_value = - ConvertConstant(const_expr, cel::NewDeleteAllocator()); + ConvertConstant(const_expr, extension_context_.MutableArena()); if (!converted_value.ok()) { - SetProgressStatusError(converted_value.status()); + SetProgressStatusIfError(converted_value.status()); return; } - if (options_.max_recursion_depth > 0 || options_.max_recursion_depth < 0) { - SetRecursiveStep(CreateConstValueDirectStep( - std::move(converted_value).value(), expr.id()), - 1); - return; - } - - AddStep( - CreateConstValueStep(std::move(converted_value).value(), expr.id())); + AddStep(ExpressionStep::MakeConstant(std::move(converted_value).value(), + expr.id())); } struct SlotLookupResult { @@ -722,13 +730,13 @@ class FlatExprVisitor : public cel::AstVisitor { if (absl::ConsumePrefix(&index_suffix, "@index")) { size_t index; if (!absl::SimpleAtoi(index_suffix, &index)) { - SetProgressStatusError( + SetProgressStatusIfError( issue_collector_.AddIssue(RuntimeIssue::CreateError( absl::InvalidArgumentError("bad @index")))); return {-1, -1}; } if (index >= block.size) { - SetProgressStatusError( + SetProgressStatusIfError( issue_collector_.AddIssue(RuntimeIssue::CreateError( absl::InvalidArgumentError(absl::StrCat( "invalid @index greater than number of bindings: ", @@ -736,7 +744,7 @@ class FlatExprVisitor : public cel::AstVisitor { return {-1, -1}; } if (index >= block.current_index) { - SetProgressStatusError( + SetProgressStatusIfError( issue_collector_.AddIssue(RuntimeIssue::CreateError( absl::InvalidArgumentError(absl::StrCat( "@index references current or future binding: ", index, @@ -754,7 +762,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (record.iter_var_in_scope && record.comprehension->iter_var() == path) { if (record.is_optimizable_bind) { - SetProgressStatusError(issue_collector_.AddIssue( + SetProgressStatusIfError(issue_collector_.AddIssue( RuntimeIssue::CreateWarning(absl::InvalidArgumentError( "Unexpected iter_var access in trivial comprehension")))); return {-1, -1}; @@ -781,7 +789,7 @@ class FlatExprVisitor : public cel::AstVisitor { // If we see a CSE generated comprehension variable that was not // resolvable through the normal comprehension scope resolution, reject it // now rather than surfacing errors at activation time. - SetProgressStatusError( + SetProgressStatusIfError( issue_collector_.AddIssue(RuntimeIssue::CreateError( absl::InvalidArgumentError("out of scope reference to CSE " "generated comprehension variable")))); @@ -808,39 +816,33 @@ class FlatExprVisitor : public cel::AstVisitor { SlotLookupResult slot = LookupSlot(path); if (slot.subexpression >= 0) { - auto* subexpression = + const ProgramBuilder::Subexpression* subexpression = program_builder_.GetExtractedSubexpression(slot.subexpression); if (subexpression == nullptr) { - SetProgressStatusError( + SetProgressStatusIfError( absl::InternalError("bad subexpression reference")); return; } - if (subexpression->IsRecursive()) { - const auto& program = subexpression->recursive_program(); - SetRecursiveStep( - CreateDirectLazyInitStep(slot.slot, program.step.get(), expr.id()), - program.depth + 1); - } else { - // Off by one since mainline expression will be index 0. - AddStep( - CreateLazyInitStep(slot.slot, slot.subexpression + 1, expr.id())); + if (options_.enable_subexpression_inlining) { + if (std::optional inlined_step = + subexpression->MaybeInline(expr.id()); + inlined_step.has_value()) { + AddStep(*std::move(inlined_step)); + return; + } } + // Off by one since mainline expression will be index 0. + AddStep(ExpressionStep::MakeLazyInitStep( + slot.slot, slot.subexpression + 1, expr.id())); return; } else if (slot.slot >= 0) { - if (options_.max_recursion_depth != 0) { - SetRecursiveStep( - CreateDirectSlotIdentStep(ident_expr.name(), slot.slot, expr.id()), - 1); - } else { - AddStep( - CreateIdentStepForSlot(ident_expr.name(), slot.slot, expr.id())); - } + AddStep(ExpressionStep::MakeReadSlotStep(slot.slot, expr.id())); return; } // Attempt to resolve a select expression as a namespaced identifier for an // enum or type constant value. - absl::optional const_value; + std::optional const_value; int64_t select_root_id = -1; std::string path_candidate; @@ -878,24 +880,17 @@ class FlatExprVisitor : public cel::AstVisitor { if (const_value) { // If the path starts with a dot, strip it. absl::string_view name = absl::StripPrefix(path, "."); - if (options_.max_recursion_depth != 0) { - SetRecursiveStep( - CreateDirectShadowableValueStep( - name, std::move(const_value).value(), select_root_id), - 1); - return; - } - AddStep(CreateShadowableValueStep(name, std::move(const_value).value(), - select_root_id)); + AddStep(CreateShadowableValueStep(name, std::move(const_value).value()), + select_root_id); return; } - absl::string_view ident_name = absl::StripPrefix(ident_expr.name(), "."); - if (options_.max_recursion_depth != 0) { - SetRecursiveStep(CreateDirectIdentStep(ident_name, expr.id()), 1); - } else { - AddStep(CreateIdentStep(ident_name, expr.id())); + if (PlanningSuppressed()) { + return; } + absl::string_view ident_name = absl::StripPrefix(ident_expr.name(), "."); + size_t ident_index = AddOrUpdateIdentifier(ident_name); + AddStep(ExpressionStep::MakeIdentifierStep(ident_index, expr.id())); } void PreVisitSelect(const cel::Expr& expr, @@ -962,27 +957,39 @@ class FlatExprVisitor : public cel::AstVisitor { return; } - if (auto depth = RecursionEligible(); depth.has_value()) { - auto deps = ExtractRecursiveDependencies(); - if (deps.size() != 1) { - SetProgressStatusError(absl::InternalError( - "unexpected number of dependencies for select operation.")); - return; + std::string field = select_expr.field(); + std::optional struct_type; + std::optional field_type; + if (options_.enable_typed_field_access) { + std::optional operand_type = + extension_context_.GetResolvedType(select_expr.operand().id()); + if (operand_type.has_value() && operand_type->IsStruct()) { + struct_type = operand_type->GetStruct(); + if (struct_type.has_value()) { + auto field_lookup = + extension_context_.type_reflector().FindStructTypeFieldByName( + *struct_type, select_expr.field()); + // Swallow error to fallback to duck typing behavior. + if (field_lookup.ok() && field_lookup->has_value()) { + field_type = *std::move(field_lookup); + } + } } - StringValue field = cel::StringValue(select_expr.field()); - - SetRecursiveStep( - CreateDirectSelectStep(std::move(deps[0]), std::move(field), - select_expr.test_only(), expr.id(), - options_.enable_empty_wrapper_null_unboxing, - enable_optional_types_), - *depth + 1); - return; } - AddStep(CreateSelectStep(select_expr, expr.id(), + if (field_type.has_value()) { + AddStep( + CreateTypedSelectStep(std::move(field), *struct_type, + *std::move(field_type), select_expr.test_only(), + options_.enable_empty_wrapper_null_unboxing, + enable_optional_types_), + expr.id(), /*stack_delta=*/0); + return; + } + AddStep(CreateSelectStep(std::move(field), select_expr.test_only(), options_.enable_empty_wrapper_null_unboxing, - enable_optional_types_)); + enable_optional_types_), + expr.id(), /*stack_delta=*/0); } // Call node handler group. @@ -997,11 +1004,11 @@ class FlatExprVisitor : public cel::AstVisitor { std::unique_ptr cond_visitor; if (call_expr.function() == cel::builtin::kAnd) { - cond_visitor = std::make_unique( - this, BinaryCond::kAnd, options_.short_circuiting); + cond_visitor = std::make_unique( + this, /*is_or=*/false, options_.short_circuiting); } else if (call_expr.function() == cel::builtin::kOr) { - cond_visitor = std::make_unique( - this, BinaryCond::kOr, options_.short_circuiting); + cond_visitor = std::make_unique( + this, /*is_or=*/true, options_.short_circuiting); } else if (call_expr.function() == cel::builtin::kTernary) { if (options_.short_circuiting) { cond_visitor = std::make_unique(this); @@ -1011,18 +1018,18 @@ class FlatExprVisitor : public cel::AstVisitor { } else if (enable_optional_types_ && call_expr.function() == kOptionalOrFn && call_expr.has_target() && call_expr.args().size() == 1) { - cond_visitor = std::make_unique( - this, BinaryCond::kOptionalOr, options_.short_circuiting); + cond_visitor = std::make_unique( + this, /*is_or_value=*/false, options_.short_circuiting); } else if (enable_optional_types_ && call_expr.function() == kOptionalOrValueFn && call_expr.has_target() && call_expr.args().size() == 1) { - cond_visitor = std::make_unique( - this, BinaryCond::kOptionalOrValue, options_.short_circuiting); + cond_visitor = std::make_unique( + this, /*is_or_value=*/true, options_.short_circuiting); } else if (IsBlock(&call_expr)) { // cel.@block if (block_.has_value()) { // There can only be one for now. - SetProgressStatusError( + SetProgressStatusIfError( absl::InvalidArgumentError("multiple cel.@block are not allowed")); return; } @@ -1030,32 +1037,28 @@ class FlatExprVisitor : public cel::AstVisitor { BlockInfo& block = *block_; block.in = true; if (call_expr.args().empty()) { - SetProgressStatusError(absl::InvalidArgumentError( + SetProgressStatusIfError(absl::InvalidArgumentError( "malformed cel.@block: missing list of bound expressions")); return; } if (call_expr.args().size() != 2) { - SetProgressStatusError(absl::InvalidArgumentError( + SetProgressStatusIfError(absl::InvalidArgumentError( "malformed cel.@block: missing bound expression")); return; } if (!call_expr.args()[0].has_list_expr()) { - SetProgressStatusError( + SetProgressStatusIfError( absl::InvalidArgumentError("malformed cel.@block: first argument " "is not a list of bound expressions")); return; } const auto& list_expr = call_expr.args().front().list_expr(); block.size = list_expr.elements().size(); - if (block.size == 0) { - SetProgressStatusError(absl::InvalidArgumentError( - "malformed cel.@block: list of bound expressions is empty")); - return; - } + block.bindings_set.reserve(block.size); for (const auto& list_expr_element : list_expr.elements()) { if (list_expr_element.optional()) { - SetProgressStatusError( + SetProgressStatusIfError( absl::InvalidArgumentError("malformed cel.@block: list of bound " "expressions contains an optional")); return; @@ -1078,189 +1081,6 @@ class FlatExprVisitor : public cel::AstVisitor { } } - // Returns the maximum recursion depth of the current program if it is - // eligible for recursion, or nullopt if it is not. - absl::optional RecursionEligible() { - if (!PlanRecursiveProgram() || program_builder_.current() == nullptr) { - return absl::nullopt; - } - return program_builder_.current()->RecursiveDependencyDepth(); - } - - std::vector> - ExtractRecursiveDependencies() { - // Must check recursion eligibility before calling. - ABSL_DCHECK(program_builder_.current() != nullptr); - - return program_builder_.current()->ExtractRecursiveDependencies(); - } - - void MakeTernaryRecursive(const cel::Expr* expr) { - if (expr->call_expr().args().size() != 3) { - SetProgressStatusError(absl::InvalidArgumentError( - "unexpected number of args for builtin ternary")); - return; - } - - const cel::Expr* condition_expr = &expr->call_expr().args()[0]; - const cel::Expr* left_expr = &expr->call_expr().args()[1]; - const cel::Expr* right_expr = &expr->call_expr().args()[2]; - - auto* condition_plan = program_builder_.GetSubexpression(condition_expr); - auto* left_plan = program_builder_.GetSubexpression(left_expr); - auto* right_plan = program_builder_.GetSubexpression(right_expr); - - if (condition_plan == nullptr || !condition_plan->IsRecursive() || - left_plan == nullptr || !left_plan->IsRecursive() || - right_plan == nullptr || !right_plan->IsRecursive()) { - SetProgressStatusError(FailedRecursivePlanning()); - return; - } - - int max_depth = std::max({0, condition_plan->recursive_program().depth, - left_plan->recursive_program().depth, - right_plan->recursive_program().depth}); - - SetRecursiveStep( - CreateDirectTernaryStep(condition_plan->ExtractRecursiveProgram().step, - left_plan->ExtractRecursiveProgram().step, - right_plan->ExtractRecursiveProgram().step, - expr->id(), options_.short_circuiting), - max_depth + 1); - } - - void MakeShortcircuitRecursive(const cel::Expr* expr, bool is_or) { - if (expr->call_expr().args().size() != 2) { - SetProgressStatusError(absl::InvalidArgumentError( - "unexpected number of args for builtin boolean operator &&/||")); - return; - } - const cel::Expr* left_expr = &expr->call_expr().args()[0]; - const cel::Expr* right_expr = &expr->call_expr().args()[1]; - - auto* left_plan = program_builder_.GetSubexpression(left_expr); - auto* right_plan = program_builder_.GetSubexpression(right_expr); - - if (left_plan == nullptr || !left_plan->IsRecursive() || - right_plan == nullptr || !right_plan->IsRecursive()) { - SetProgressStatusError(FailedRecursivePlanning()); - return; - } - - int max_depth = std::max({0, left_plan->recursive_program().depth, - right_plan->recursive_program().depth}); - - if (is_or) { - SetRecursiveStep( - CreateDirectOrStep(left_plan->ExtractRecursiveProgram().step, - right_plan->ExtractRecursiveProgram().step, - expr->id(), options_.short_circuiting), - max_depth + 1); - } else { - SetRecursiveStep( - CreateDirectAndStep(left_plan->ExtractRecursiveProgram().step, - right_plan->ExtractRecursiveProgram().step, - expr->id(), options_.short_circuiting), - max_depth + 1); - } - } - - void MakeOptionalShortcircuit(const cel::Expr* expr, bool is_or_value) { - if (!expr->call_expr().has_target() || - expr->call_expr().args().size() != 1) { - SetProgressStatusError(absl::InvalidArgumentError( - "unexpected number of args for optional.or{Value}")); - return; - } - const cel::Expr* left_expr = &expr->call_expr().target(); - const cel::Expr* right_expr = &expr->call_expr().args()[0]; - - auto* left_plan = program_builder_.GetSubexpression(left_expr); - auto* right_plan = program_builder_.GetSubexpression(right_expr); - - if (left_plan == nullptr || !left_plan->IsRecursive() || - right_plan == nullptr || !right_plan->IsRecursive()) { - SetProgressStatusError(FailedRecursivePlanning()); - return; - } - int max_depth = std::max({0, left_plan->recursive_program().depth, - right_plan->recursive_program().depth}); - - SetRecursiveStep(CreateDirectOptionalOrStep( - expr->id(), left_plan->ExtractRecursiveProgram().step, - right_plan->ExtractRecursiveProgram().step, - is_or_value, options_.short_circuiting), - max_depth + 1); - } - - void MaybeMakeBindRecursive(const cel::Expr* expr, - const cel::ComprehensionExpr* comprehension, - size_t accu_slot) { - if (!PlanRecursiveProgram()) { - return; - } - - auto* result_plan = - program_builder_.GetSubexpression(&comprehension->result()); - - if (result_plan == nullptr || !result_plan->IsRecursive()) { - SetProgressStatusError(FailedRecursivePlanning()); - return; - } - - int result_depth = result_plan->recursive_program().depth; - - auto program = result_plan->ExtractRecursiveProgram(); - SetRecursiveStep( - CreateDirectBindStep(accu_slot, std::move(program.step), expr->id()), - result_depth + 1); - } - - void MaybeMakeComprehensionRecursive( - const cel::Expr* expr, const cel::ComprehensionExpr* comprehension, - size_t iter_slot, size_t iter2_slot, size_t accu_slot) { - if (!PlanRecursiveProgram()) { - return; - } - - auto* accu_plan = - program_builder_.GetSubexpression(&comprehension->accu_init()); - auto* range_plan = - program_builder_.GetSubexpression(&comprehension->iter_range()); - auto* loop_plan = - program_builder_.GetSubexpression(&comprehension->loop_step()); - auto* condition_plan = - program_builder_.GetSubexpression(&comprehension->loop_condition()); - auto* result_plan = - program_builder_.GetSubexpression(&comprehension->result()); - if (accu_plan == nullptr || !accu_plan->IsRecursive() || - range_plan == nullptr || !range_plan->IsRecursive() || - loop_plan == nullptr || !loop_plan->IsRecursive() || - condition_plan == nullptr || !condition_plan->IsRecursive() || - result_plan == nullptr || !result_plan->IsRecursive()) { - SetProgressStatusError(FailedRecursivePlanning()); - return; - } - - int max_depth = 0; - max_depth = std::max(max_depth, accu_plan->recursive_program().depth); - max_depth = std::max(max_depth, range_plan->recursive_program().depth); - max_depth = std::max(max_depth, loop_plan->recursive_program().depth); - max_depth = std::max(max_depth, condition_plan->recursive_program().depth); - max_depth = std::max(max_depth, result_plan->recursive_program().depth); - - auto step = CreateDirectComprehensionStep( - iter_slot, iter2_slot, accu_slot, - range_plan->ExtractRecursiveProgram().step, - accu_plan->ExtractRecursiveProgram().step, - loop_plan->ExtractRecursiveProgram().step, - condition_plan->ExtractRecursiveProgram().step, - result_plan->ExtractRecursiveProgram().step, options_.short_circuiting, - expr->id()); - - SetRecursiveStep(std::move(step), max_depth + 1); - } - // Invoked after all child nodes are processed. void PostVisitCall(const cel::Expr& expr, const cel::CallExpr& call_expr) override { @@ -1346,38 +1166,55 @@ class FlatExprVisitor : public cel::AstVisitor { slot_count = 0; } } + // If this is in the scope of an optimized bind accu-init, account the slots // to the outermost bind-init scope. // // The init expression is effectively inlined at the first usage in the // critical path (which is unknown at plan time), so the used slots need to // be dedicated for the entire scope of that bind. + bool in_bind_init = false; for (ComprehensionStackRecord& record : comprehension_stack_) { if (record.in_accu_init && record.is_optimizable_bind) { record.slot_count += slot_count; slot_count = 0; + in_bind_init = true; break; } // If no bind init subexpression, account normally. } + // Iterator frames are releasable if not in a bind (no iterator push), and + // not in a lazy subexpression (which may evaluate in different implicit + // scopes). + const bool is_stack_releasable = + !is_bind && !in_bind_init && (!block_.has_value() || !block_->in); + + if (is_stack_releasable) { + index_manager_.ReserveIterStack(); + } else if (!is_bind) { + index_manager_.AddLazyIterStack(); + } + comprehension_stack_.push_back( {&expr, &comprehension, iter_slot, iter2_slot, accu_slot, slot_count, /*subexpression=*/-1, /*.is_optimizable_list_append=*/ IsOptimizableListAppend(&comprehension, - options_.enable_comprehension_list_append), + options_.enable_comprehension_list_append, + options_.short_circuiting), /*.is_optimizable_map_insert=*/ IsOptimizableMapInsert(&comprehension, - options_.enable_comprehension_mutable_map), + options_.enable_comprehension_mutable_map, + options_.short_circuiting), /*.is_optimizable_bind=*/is_bind, + /*.is_stack_releasable=*/is_stack_releasable, /*.iter_var_in_scope=*/false, /*.iter_var2_in_scope=*/false, /*.accu_var_in_scope=*/false, /*.in_accu_init=*/false, - std::make_unique(this, options_.short_circuiting, - is_bind, iter_slot, iter2_slot, - accu_slot)}); + std::make_unique(this, is_bind, iter_slot, + iter2_slot, accu_slot)}); comprehension_stack_.back().visitor->PreVisit(&expr); } @@ -1398,6 +1235,10 @@ class FlatExprVisitor : public cel::AstVisitor { record.visitor->PostVisit(&expr); index_manager_.ReleaseSlots(record.slot_count); + if (record.is_stack_releasable) { + ABSL_DCHECK(!record.is_optimizable_bind); + index_manager_.ReleaseIterStack(); + } comprehension_stack_.pop_back(); } @@ -1466,7 +1307,7 @@ class FlatExprVisitor : public cel::AstVisitor { return; } - SetProgressStatusError(comprehension_stack_.back().visitor->PostVisitArg( + SetProgressStatusIfError(comprehension_stack_.back().visitor->PostVisitArg( comprehension_arg, comprehension_stack_.back().expr)); } @@ -1512,32 +1353,27 @@ class FlatExprVisitor : public cel::AstVisitor { comprehension_stack_.back(); if (comprehension.is_optimizable_list_append) { if (&(comprehension.comprehension->accu_init()) == &expr) { - if (PlanRecursiveProgram()) { - SetRecursiveStep(CreateDirectMutableListStep(expr.id()), 1); - return; - } - AddStep(CreateMutableListStep(expr.id())); + AddStep(ExpressionStep::MakeNewMutableListStep(expr.id())); return; } if (GetOptimizableListAppendOperand(comprehension.comprehension) == &expr) { + // Avoid wrapping the appended element in a create list step. return; } } } - if (absl::optional depth = RecursionEligible(); depth.has_value()) { - auto deps = ExtractRecursiveDependencies(); - if (deps.size() != list_expr.elements().size()) { - SetProgressStatusError(absl::InternalError( - "Unexpected number of plan elements for CreateList expr")); + absl::flat_hash_set optional_indices = + MakeOptionalIndicesSet(list_expr); + for (size_t index : optional_indices) { + if (!ValidateOrError(index < list_expr.elements().size(), + "Optional index out of range: ", index, + ", list size: ", list_expr.elements().size())) { return; } - auto step = CreateDirectListStep( - std::move(deps), MakeOptionalIndicesSet(list_expr), expr.id()); - SetRecursiveStep(std::move(step), *depth + 1); - return; } - AddStep(CreateCreateListStep(list_expr, expr.id())); + AddStep(CreateCreateListStep(list_expr.elements().size(), + std::move(optional_indices), expr.id())); } // CreateStruct node handler. @@ -1551,7 +1387,7 @@ class FlatExprVisitor : public cel::AstVisitor { auto status_or_resolved_fields = ResolveCreateStructFields(struct_expr, expr.id()); if (!status_or_resolved_fields.ok()) { - SetProgressStatusError(status_or_resolved_fields.status()); + SetProgressStatusIfError(status_or_resolved_fields.status()); return; } std::string resolved_name = @@ -1559,23 +1395,14 @@ class FlatExprVisitor : public cel::AstVisitor { std::vector fields = std::move(status_or_resolved_fields.value().second); - if (auto depth = RecursionEligible(); depth.has_value()) { - auto deps = ExtractRecursiveDependencies(); - if (deps.size() != struct_expr.fields().size()) { - SetProgressStatusError(absl::InternalError( - "Unexpected number of plan elements for CreateStruct expr")); - return; - } - auto step = CreateDirectCreateStructStep( - std::move(resolved_name), std::move(fields), std::move(deps), - MakeOptionalIndicesSet(struct_expr), expr.id()); - SetRecursiveStep(std::move(step), *depth + 1); - return; - } - + size_t num_fields = fields.size(); + int64_t stack_delta = + num_fields <= static_cast(std::numeric_limits::max()) + ? 1 - static_cast(num_fields) + : std::numeric_limits::max(); AddStep(CreateCreateStructStep(std::move(resolved_name), std::move(fields), - MakeOptionalIndicesSet(struct_expr), - expr.id())); + MakeOptionalIndicesSet(struct_expr)), + expr.id(), stack_delta); } void PostVisitMap(const cel::Expr& expr, @@ -1590,31 +1417,21 @@ class FlatExprVisitor : public cel::AstVisitor { comprehension_stack_.back(); if (comprehension.is_optimizable_map_insert) { if (&(comprehension.comprehension->accu_init()) == &expr) { - if (PlanRecursiveProgram()) { - SetRecursiveStep(CreateDirectMutableMapStep(expr.id()), 1); - return; - } - AddStep(CreateMutableMapStep(expr.id())); + AddStep(CreateMutableMapStep(), expr.id()); return; } } } - if (auto depth = RecursionEligible(); depth.has_value()) { - auto deps = ExtractRecursiveDependencies(); - if (deps.size() != 2 * map_expr.entries().size()) { - SetProgressStatusError(absl::InternalError( - "Unexpected number of plan elements for CreateStruct expr")); - return; - } - auto step = CreateDirectCreateMapStep( - std::move(deps), MakeOptionalIndicesSet(map_expr), expr.id()); - SetRecursiveStep(std::move(step), *depth + 1); - return; - } - AddStep(CreateCreateStructStepForMap(map_expr.entries().size(), - MakeOptionalIndicesSet(map_expr), - expr.id())); + size_t num_entries = map_expr.entries().size(); + int64_t stack_delta = + num_entries <= + static_cast(std::numeric_limits::max() / 2) + ? 1 - 2 * static_cast(num_entries) + : std::numeric_limits::max(); + AddStep(CreateCreateStructStepForMap(num_entries, + MakeOptionalIndicesSet(map_expr)), + expr.id(), stack_delta); } absl::Status progress_status() const { return progress_status_; } @@ -1639,16 +1456,10 @@ class FlatExprVisitor : public cel::AstVisitor { auto lazy_overloads = resolver_.FindLazyOverloads( function, call_expr->has_target(), num_args, expr->id()); if (!lazy_overloads.empty()) { - if (auto depth = RecursionEligible(); depth.has_value()) { - auto args = program_builder_.current()->ExtractRecursiveDependencies(); - SetRecursiveStep(CreateDirectLazyFunctionStep( - expr->id(), *call_expr, std::move(args), - std::move(lazy_overloads)), - *depth + 1); - return; - } - AddStep(CreateFunctionStep(*call_expr, expr->id(), - std::move(lazy_overloads))); + AddStep(ExpressionStep::MakeLazyFunctionStep( + CreateLazyFunctionStep(*call_expr, expr->id(), + std::move(lazy_overloads)), + expr->id())); return; } @@ -1665,24 +1476,14 @@ class FlatExprVisitor : public cel::AstVisitor { "No overloads provided for FunctionStep creation"), RuntimeIssue::ErrorCode::kNoMatchingOverload)); if (!status.ok()) { - SetProgressStatusError(status); + SetProgressStatusIfError(status); return; } } - if (auto recursion_depth = RecursionEligible(); - recursion_depth.has_value()) { - // Nonnull while active -- nullptr indicates logic error elsewhere in the - // builder. - ABSL_DCHECK(program_builder_.current() != nullptr); - auto args = program_builder_.current()->ExtractRecursiveDependencies(); - SetRecursiveStep( - CreateDirectFunctionStep(expr->id(), *call_expr, std::move(args), - std::move(overloads)), - *recursion_depth + 1); - return; - } - AddStep(CreateFunctionStep(*call_expr, expr->id(), std::move(overloads))); + AddStep(ExpressionStep::MakeEagerFunctionStep( + CreateFunctionStep(*call_expr, expr->id(), std::move(overloads)), + expr->id())); } // Add a step to the program, taking ownership. If successful, returns the @@ -1691,43 +1492,38 @@ class FlatExprVisitor : public cel::AstVisitor { // Note: the pointer is only guaranteed to stay valid until the parent // subexpression is finalized. Optimizers may modify the program plan which // may free the step at that point. - ExpressionStep* AddStep( - absl::StatusOr> step) { - if (step.ok()) { - return AddStep(*std::move(step)); - } else { - SetProgressStatusError(step.status()); + template + std::enable_if_t, T*> AddStep( + std::unique_ptr step, int64_t expr_id = -1, int64_t stack_delta = 1) { + if (progress_status_.ok() && !PlanningSuppressed()) { + T* ptr = step.get(); + program_builder_.AddStep(ExpressionStep::MakeGenericStep( + std::move(step), expr_id, stack_delta)); + return ptr; } return nullptr; } template - std::enable_if_t, T*> AddStep( - std::unique_ptr step) { - if (progress_status_.ok() && !PlanningSuppressed()) { - return static_cast(program_builder_.AddStep(std::move(step))); + std::enable_if_t, T*> AddStep( + absl::StatusOr> step, int64_t expr_id = -1, + int64_t stack_delta = 1) { + if (step.ok()) { + return AddStep(*std::move(step), expr_id, stack_delta); + } else { + SetProgressStatusIfError(step.status()); } return nullptr; } - void SetRecursiveStep(std::unique_ptr step, int depth) { - if (!progress_status_.ok() || PlanningSuppressed()) { - return; - } - if (program_builder_.current() == nullptr) { - SetProgressStatusError(absl::InternalError( - "CEL AST traversal out of order in flat_expr_builder.")); - return; - } - program_builder_.current()->set_recursive_program(std::move(step), depth); - if (depth > max_recursion_depth_) { - SetProgressStatusError(absl::InvalidArgumentError( - absl::StrCat("Maximum recursion depth of ", - options_.max_recursion_depth, " exceeded"))); + ExpressionStep* AddStep(ExpressionStep step) { + if (progress_status_.ok() && !PlanningSuppressed()) { + return program_builder_.AddStep(std::move(step)); } + return nullptr; } - void SetProgressStatusError(const absl::Status& status) { + void SetProgressStatusIfError(const absl::Status& status) { if (progress_status_.ok() && !status.ok()) { progress_status_ = status; } @@ -1756,6 +1552,19 @@ class FlatExprVisitor : public cel::AstVisitor { size_t slot_count() const { return index_manager_.max_slot_count(); } + size_t iter_stack_count() const { + return index_manager_.IterStackRequirement(); + } + + IdentifierTable BuildIdentifierTable(size_t& total_slot_count) && { + IdentifierTable table; + table.reserve(identifiers_.size()); + for (std::string& name : identifiers_) { + table.push_back(Identifier{std::move(name), total_slot_count++}); + } + return table; + } + void AddOptimizer(std::unique_ptr optimizer) { program_optimizers_.push_back(std::move(optimizer)); } @@ -1769,7 +1578,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (valid_expression) { return true; } - SetProgressStatusError(absl::InvalidArgumentError( + SetProgressStatusIfError(absl::InvalidArgumentError( absl::StrCat(error_message, message_parts...))); return false; } @@ -1787,6 +1596,7 @@ class FlatExprVisitor : public cel::AstVisitor { bool is_optimizable_list_append; bool is_optimizable_map_insert; bool is_optimizable_bind; + bool is_stack_releasable; bool iter_var_in_scope; bool iter_var2_in_scope; bool accu_var_in_scope; @@ -1855,7 +1665,7 @@ class FlatExprVisitor : public cel::AstVisitor { int64_t expr_id) { absl::string_view ast_name = create_struct_expr.name(); - absl::optional> type; + std::optional> type; CEL_ASSIGN_OR_RETURN(type, resolver_.FindType(ast_name, expr_id)); if (!type.has_value()) { @@ -1904,6 +1714,10 @@ class FlatExprVisitor : public cel::AstVisitor { CallHandlerResult HandleHeterogeneousEqualityIn(const cel::Expr& expr, const cel::CallExpr& call); + void MaybeResolveType(const cel::Expr& expr); + + size_t AddOrUpdateIdentifier(absl::string_view name); + const Resolver& resolver_; const cel::TypeProvider& type_provider_; absl::Status progress_status_; @@ -1925,15 +1739,17 @@ class FlatExprVisitor : public cel::AstVisitor { absl::flat_hash_set suppressed_branches_; const cel::Expr* resume_from_suppressed_branch_ = nullptr; std::vector> program_optimizers_; + const absl::flat_hash_map& type_map_; IssueCollector& issue_collector_; ProgramBuilder& program_builder_; PlannerContext& extension_context_; IndexManager index_manager_; + std::vector identifiers_; + absl::flat_hash_map identifier_map_; bool enable_optional_types_; - absl::optional block_; - int max_recursion_depth_ = 0; + std::optional block_; }; FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleIndex( @@ -1948,21 +1764,8 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleIndex( return CallHandlerResult::kIntercepted; } - if (auto depth = RecursionEligible(); depth.has_value()) { - auto args = ExtractRecursiveDependencies(); - if (args.size() != 2) { - SetProgressStatusError(absl::InvalidArgumentError( - "unexpected number of args for builtin index operator")); - return CallHandlerResult::kIntercepted; - } - SetRecursiveStep( - CreateDirectContainerAccessStep(std::move(args[0]), std::move(args[1]), - enable_optional_types_, expr.id()), - *depth + 1); - return CallHandlerResult::kIntercepted; - } - AddStep( - CreateContainerAccessStep(call_expr, expr.id(), enable_optional_types_)); + AddStep(CreateContainerAccessStep(call_expr, enable_optional_types_), + expr.id(), /*stack_delta=*/-1); return CallHandlerResult::kIntercepted; } @@ -1975,18 +1778,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleNot( return CallHandlerResult::kIntercepted; } - if (auto depth = RecursionEligible(); depth.has_value()) { - auto args = ExtractRecursiveDependencies(); - if (args.size() != 1) { - SetProgressStatusError(absl::InvalidArgumentError( - "unexpected number of args for builtin not operator")); - return CallHandlerResult::kIntercepted; - } - SetRecursiveStep(CreateDirectNotStep(std::move(args[0]), expr.id()), - *depth + 1); - return CallHandlerResult::kIntercepted; - } - AddStep(CreateNotStep(expr.id())); + AddStep(ExpressionStep::MakeBooleanNotStep(expr.id())); return CallHandlerResult::kIntercepted; } @@ -1998,20 +1790,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleNotStrictlyFalse( return CallHandlerResult::kIntercepted; } - if (auto depth = RecursionEligible(); depth.has_value()) { - auto args = ExtractRecursiveDependencies(); - if (args.size() != 1) { - SetProgressStatusError( - absl::InvalidArgumentError("unexpected number of args for builtin " - "@not_strictly_false operator")); - return CallHandlerResult::kIntercepted; - } - SetRecursiveStep( - CreateDirectNotStrictlyFalseStep(std::move(args[0]), expr.id()), - *depth + 1); - return CallHandlerResult::kIntercepted; - } - AddStep(CreateNotStrictlyFalseStep(expr.id())); + AddStep(ExpressionStep::MakeNotStrictlyFalseStep(expr.id())); return CallHandlerResult::kIntercepted; } @@ -2020,7 +1799,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleBlock( ABSL_DCHECK(call_expr.function() == kBlock); if (!block_.has_value() || block_->expr != &expr || call_expr.args().size() != 2 || call_expr.has_target()) { - SetProgressStatusError( + SetProgressStatusIfError( absl::InvalidArgumentError("unexpected call to internal cel.@block")); return CallHandlerResult::kIntercepted; } @@ -2029,30 +1808,11 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleBlock( block.in = false; index_manager().ReleaseSlots(block.slot_count); - // Check if eligible for recursion and update the plan if so. - // - // The first argument to @block is the list of initializers. These don't - // generate a plan in the main program (they are tracked separately to support - // lazy evaluation) so we only need to extract the second argument -- the body - // of the block that uses the initializers. - ProgramBuilder::Subexpression* body_subexpression = - program_builder_.GetSubexpression(&call_expr.args()[1]); - - if (options_.max_recursion_depth != 0 && body_subexpression != nullptr && - body_subexpression->IsRecursive() && - (options_.max_recursion_depth < 0 || - body_subexpression->recursive_program().depth < - options_.max_recursion_depth)) { - auto recursive_program = body_subexpression->ExtractRecursiveProgram(); - SetRecursiveStep( - CreateDirectBlockStep(block.index, block.slot_count, - std::move(recursive_program.step), expr.id()), - recursive_program.depth + 1); - return CallHandlerResult::kIntercepted; - } - // Otherwise, iterative plan. - AddStep(CreateClearSlotsStep(block.index, block.slot_count, expr.id())); + if (block.slot_count > 0) { + AddStep(ExpressionStep::MakeClearSlotsStep(block.index, block.slot_count, + expr.id())); + } return CallHandlerResult::kIntercepted; } @@ -2070,21 +1830,20 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleListAppend( const cel::ComprehensionExpr* comprehension = comprehension_stack_.back().comprehension; const cel::Expr& loop_step = comprehension->loop_step(); + // Macro loop_step for a map() will contain a list concat operation: // accu_var + [elem] - if (&loop_step == &expr) { - AddResolvedFunctionStep(&call_expr, &expr, - cel::builtin::kRuntimeListAppend); - return CallHandlerResult::kIntercepted; - } + const bool is_map_loop_step = &loop_step == &expr; + // Macro loop_step for a filter() will contain a ternary: // filter ? accu_var + [elem] : accu_var - if (loop_step.has_call_expr() && + const bool is_filter_loop_step = + loop_step.has_call_expr() && loop_step.call_expr().function() == cel::builtin::kTernary && loop_step.call_expr().args().size() == 3 && - &(loop_step.call_expr().args()[1]) == &expr) { - AddResolvedFunctionStep(&call_expr, &expr, - cel::builtin::kRuntimeListAppend); + &(loop_step.call_expr().args()[1]) == &expr; + if (is_map_loop_step || is_filter_loop_step) { + AddStep(ExpressionStep::MakeMutableListAppendStep(expr.id())); return CallHandlerResult::kIntercepted; } } @@ -2100,20 +1859,8 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleHeterogeneousEquality( return CallHandlerResult::kIntercepted; } - if (auto depth = RecursionEligible(); depth.has_value()) { - auto args = ExtractRecursiveDependencies(); - if (args.size() != 2) { - SetProgressStatusError(absl::InvalidArgumentError( - "unexpected number of args for builtin equality operator")); - return CallHandlerResult::kIntercepted; - } - SetRecursiveStep( - CreateDirectEqualityStep(std::move(args[0]), std::move(args[1]), - inequality, expr.id()), - *depth + 1); - return CallHandlerResult::kIntercepted; - } - AddStep(CreateEqualityStep(inequality, expr.id())); + AddStep(inequality ? ExpressionStep::MakeFastNotEqualStep(expr.id()) + : ExpressionStep::MakeFastEqualStep(expr.id())); return CallHandlerResult::kIntercepted; } @@ -2125,149 +1872,112 @@ FlatExprVisitor::HandleHeterogeneousEqualityIn(const cel::Expr& expr, return CallHandlerResult::kIntercepted; } - if (auto depth = RecursionEligible(); depth.has_value()) { - auto args = ExtractRecursiveDependencies(); - if (args.size() != 2) { - SetProgressStatusError(absl::InvalidArgumentError( - "unexpected number of args for builtin 'in' operator")); - return CallHandlerResult::kIntercepted; - } - SetRecursiveStep( - CreateDirectInStep(std::move(args[0]), std::move(args[1]), expr.id()), - *depth + 1); - return CallHandlerResult::kIntercepted; - } - - AddStep(CreateInStep(expr.id())); + AddStep(ExpressionStep::MakeFastInStep(expr.id())); return CallHandlerResult::kIntercepted; } -void BinaryCondVisitor::PreVisit(const cel::Expr* expr) { - switch (cond_) { - case BinaryCond::kAnd: - ABSL_FALLTHROUGH_INTENDED; - case BinaryCond::kOr: - visitor_->ValidateOrError( - !expr->call_expr().has_target() && - expr->call_expr().args().size() == 2, - "Invalid argument count for a binary function call."); - break; - case BinaryCond::kOptionalOr: - ABSL_FALLTHROUGH_INTENDED; - case BinaryCond::kOptionalOrValue: - visitor_->ValidateOrError(expr->call_expr().has_target() && - expr->call_expr().args().size() == 1, - "Invalid argument count for or/orValue call."); - break; +void FlatExprVisitor::MaybeResolveType(const cel::Expr& expr) { + // Try to resolve the type from the type map, but don't fail if it's not + // there. This permits cases where the runtime type is compatible but not + // the same as the type checked type. + auto it = type_map_.find(expr.id()); + if (it == type_map_.end()) { + return; } + extension_context_.TryResolveType(expr.id(), it->second); } -void BinaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - return; +size_t FlatExprVisitor::AddOrUpdateIdentifier(absl::string_view name) { + auto it = identifier_map_.find(name); + if (it == identifier_map_.end()) { + size_t index = identifiers_.size(); + it = identifier_map_.emplace(name, index).first; + identifiers_.push_back(std::string(name)); } - if (short_circuiting_ && arg_num == 0 && - (cond_ == BinaryCond::kAnd || cond_ == BinaryCond::kOr)) { - // If first branch evaluation result is enough to determine output, - // jump over the second branch and provide result of the first argument as - // final output. - // Retain a pointer to the jump step so we can update the target after - // planning the second argument. - std::unique_ptr jump_step; - switch (cond_) { - case BinaryCond::kAnd: - jump_step = CreateCondJumpStep(false, true, {}, expr->id()); - break; - case BinaryCond::kOr: - jump_step = CreateCondJumpStep(true, true, {}, expr->id()); - break; - default: - ABSL_UNREACHABLE(); + return it->second; +} + +void LogicalCondVisitor::PreVisit(const cel::Expr* expr) { + visitor_->ValidateOrError( + !expr->call_expr().has_target() && expr->call_expr().args().size() >= 2, + "Invalid argument count for a binary function call."); +} + +void LogicalCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { + const int last_arg_index = expr->call_expr().args().size() - 1; + const size_t num_args = expr->call_expr().args().size(); + if (arg_num == last_arg_index) { + if (is_or_) { + visitor_->AddStep( + ExpressionStep::MakeBooleanOrStep(num_args, expr->id())); + } else { + visitor_->AddStep( + ExpressionStep::MakeBooleanAndStep(num_args, expr->id())); + } + if (short_circuiting_ && !jump_steps_.empty()) { + ProgramStepIndex target = visitor_->GetCurrentIndex(); + for (const ProgramStepIndex& jump_pos : jump_steps_) { + absl::StatusOr offset = Jump::CalculateOffset(jump_pos, target); + if (!offset.ok()) { + visitor_->SetProgressStatusIfError(offset.status()); + continue; + } + ExpressionStep* step = + jump_pos.subexpression->GetIfExpressionStep(jump_pos.index); + BoolJumpStepInfo* jump_info = GetIfBoolJumpStep(step); + if (jump_info != nullptr) { + jump_info->set = true; + jump_info->offset = *offset; + } + } } + } + if (short_circuiting_ && arg_num < last_arg_index) { ProgramStepIndex index = visitor_->GetCurrentIndex(); - if (JumpStepBase* jump_step_ptr = visitor_->AddStep(std::move(jump_step)); - jump_step_ptr) { - jump_step_ = Jump(index, jump_step_ptr); - } + visitor_->AddStep(is_or_ ? ExpressionStep::MakeBooleanOrJumpStep( + /*arg_count=*/arg_num + 1) + : ExpressionStep::MakeBooleanAndJumpStep( + /*arg_count=*/arg_num + 1)); + jump_steps_.push_back(index); } } -void BinaryCondVisitor::PostVisitTarget(const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { +void LogicalCondVisitor::PostVisit(const cel::Expr* expr) { +} + +void OptionalOrCondVisitor::PreVisit(const cel::Expr* expr) { + visitor_->ValidateOrError( + expr->call_expr().has_target() && expr->call_expr().args().size() == 1, + "Invalid argument count for or/orValue call."); +} + +void OptionalOrCondVisitor::PostVisitTarget(const cel::Expr* expr) { + if (!short_circuiting_) { return; } - if (short_circuiting_ && (cond_ == BinaryCond::kOptionalOr || - cond_ == BinaryCond::kOptionalOrValue)) { - // If first branch evaluation result is enough to determine output, - // jump over the second branch and provide result of the first argument as - // final output. - // Retain a pointer to the jump step so we can update the target after - // planning the second argument. - std::unique_ptr jump_step; - switch (cond_) { - case BinaryCond::kOptionalOr: - jump_step = CreateOptionalHasValueJumpStep(false, expr->id()); - break; - case BinaryCond::kOptionalOrValue: - jump_step = CreateOptionalHasValueJumpStep(true, expr->id()); - break; - default: - ABSL_UNREACHABLE(); - } - ProgramStepIndex index = visitor_->GetCurrentIndex(); - if (JumpStepBase* jump_step_ptr = visitor_->AddStep(std::move(jump_step)); - jump_step_ptr) { - jump_step_ = Jump(index, jump_step_ptr); - } - } + + // Keep a pointer to the jump step so we can update the target after + // planning the second argument. + jump_step_pos_ = visitor_->GetCurrentIndex(); + jump_step_ = visitor_->AddStep(CreateOptionalHasValueJumpStep(is_or_value_)); } -void BinaryCondVisitor::PostVisit(const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - switch (cond_) { - case BinaryCond::kAnd: - visitor_->MakeShortcircuitRecursive(expr, /*is_or=*/false); - break; - case BinaryCond::kOr: - visitor_->MakeShortcircuitRecursive(expr, /*is_or=*/true); - break; - case BinaryCond::kOptionalOr: - visitor_->MakeOptionalShortcircuit(expr, - /*is_or_value=*/false); - break; - case BinaryCond::kOptionalOrValue: - visitor_->MakeOptionalShortcircuit(expr, - /*is_or_value=*/true); - break; - default: - ABSL_UNREACHABLE(); - } +void OptionalOrCondVisitor::PostVisit(const cel::Expr* expr) { + visitor_->AddStep(CreateOptionalOrStep(is_or_value_), expr->id()); + if (!short_circuiting_) { return; } - switch (cond_) { - case BinaryCond::kAnd: - visitor_->AddStep(CreateAndStep(expr->id())); - break; - case BinaryCond::kOr: - visitor_->AddStep(CreateOrStep(expr->id())); - break; - case BinaryCond::kOptionalOr: - visitor_->AddStep( - CreateOptionalOrStep(/*is_or_value=*/false, expr->id())); - break; - case BinaryCond::kOptionalOrValue: - visitor_->AddStep(CreateOptionalOrStep(/*is_or_value=*/true, expr->id())); - break; - default: - ABSL_UNREACHABLE(); - } - if (short_circuiting_) { - // If short-circuiting is enabled, point the conditional jump past the - // boolean operator step. - visitor_->SetProgressStatusError( - jump_step_.set_target(visitor_->GetCurrentIndex())); + ABSL_DCHECK(jump_step_ != nullptr) + << "OptionalOrCondVisitor::PostVisit: jump_step_ is null"; + absl::StatusOr offset = + Jump::CalculateOffset(jump_step_pos_, visitor_->GetCurrentIndex()); + if (!offset.ok()) { + visitor_->SetProgressStatusIfError(offset.status()); + return; } + + jump_step_->set_jump_offset(*offset); } void TernaryCondVisitor::PreVisit(const cel::Expr* expr) { @@ -2277,9 +1987,6 @@ void TernaryCondVisitor::PreVisit(const cel::Expr* expr) { } void TernaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - return; - } // Ternary operator "_?_:_" requires a special handing. // In contrary to regular function call, its execution affects the control // flow of the overall CEL expression. @@ -2293,39 +2000,35 @@ void TernaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { // condition argument for ternary operator if (arg_num == 0) { - // Jump in case of error or non-bool - ProgramStepIndex error_jump_pos = visitor_->GetCurrentIndex(); - auto* error_jump = - visitor_->AddStep(CreateBoolCheckJumpStep({}, expr->id())); - if (error_jump) { - error_jump_ = Jump(error_jump_pos, error_jump); - } - - // Jump to the second branch of execution - // Value is to be removed from the stack. - ProgramStepIndex cond_jump_pos = visitor_->GetCurrentIndex(); - auto* jump_to_second = - visitor_->AddStep(CreateCondJumpStep(false, false, {}, expr->id())); - if (jump_to_second) { - jump_to_second_ = - Jump(cond_jump_pos, static_cast(jump_to_second)); - } + // Jump in case of error or non-bool, or jump to the second branch of + // execution if false. + cond_jump_pos_ = visitor_->GetCurrentIndex(); + visitor_->AddStep(ExpressionStep::MakeTernaryJumpStep()); } else if (arg_num == 1) { // Jump after the first and over the second branch of execution. - // Value is to be removed from the stack. - ProgramStepIndex jump_pos = visitor_->GetCurrentIndex(); - auto* jump_after_first = visitor_->AddStep(CreateJumpStep({}, expr->id())); - if (!jump_after_first) { + jump_after_first_pos_ = visitor_->GetCurrentIndex(); + visitor_->AddStep(ExpressionStep::MakeFixedJumpStep()); + + ExpressionStep* cond_step = + cond_jump_pos_.has_value() + ? cond_jump_pos_->subexpression->GetIfExpressionStep( + cond_jump_pos_->index) + : nullptr; + TernaryJumpStepInfo* ternary_info = GetIfTernaryJumpStep(cond_step); + if (!visitor_->ValidateOrError( + ternary_info != nullptr, + "Error configuring ternary operator: jump_to_second_ is null")) { return; } - jump_after_first_ = Jump(jump_pos, jump_after_first); - if (visitor_->ValidateOrError( - jump_to_second_.exists(), - "Error configuring ternary operator: jump_to_second_ is null")) { - visitor_->SetProgressStatusError( - jump_to_second_.set_target(visitor_->GetCurrentIndex())); + absl::StatusOr offset = + Jump::CalculateOffset(*cond_jump_pos_, visitor_->GetCurrentIndex()); + if (!offset.ok()) { + visitor_->SetProgressStatusIfError(offset.status()); + return; } + + ternary_info->jump_to_second_offset = *offset; } // Code executed after traversing the final branch of execution // (arg_num == 2) is placed in PostVisitCall, to make this method less @@ -2333,22 +2036,42 @@ void TernaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { } void TernaryCondVisitor::PostVisit(const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - visitor_->MakeTernaryRecursive(expr); - return; - } // Determine and set jump offset in jump instruction. + ExpressionStep* cond_step = + cond_jump_pos_.has_value() + ? cond_jump_pos_->subexpression->GetIfExpressionStep( + cond_jump_pos_->index) + : nullptr; + TernaryJumpStepInfo* ternary_info = GetIfTernaryJumpStep(cond_step); if (visitor_->ValidateOrError( - error_jump_.exists(), + ternary_info != nullptr, "Error configuring ternary operator: error_jump_ is null")) { - visitor_->SetProgressStatusError( - error_jump_.set_target(visitor_->GetCurrentIndex())); + absl::StatusOr offset = + Jump::CalculateOffset(*cond_jump_pos_, visitor_->GetCurrentIndex()); + if (!offset.ok()) { + visitor_->SetProgressStatusIfError(offset.status()); + } else { + ternary_info->set = true; + ternary_info->error_offset = *offset; + } } + ExpressionStep* jump_after_first_step = + jump_after_first_pos_.has_value() + ? jump_after_first_pos_->subexpression->GetIfExpressionStep( + jump_after_first_pos_->index) + : nullptr; + FixedJumpStepInfo* fixed_info = GetIfFixedJumpStep(jump_after_first_step); if (visitor_->ValidateOrError( - jump_after_first_.exists(), + fixed_info != nullptr, "Error configuring ternary operator: jump_after_first_ is null")) { - visitor_->SetProgressStatusError( - jump_after_first_.set_target(visitor_->GetCurrentIndex())); + absl::StatusOr offset = Jump::CalculateOffset( + *jump_after_first_pos_, visitor_->GetCurrentIndex()); + if (!offset.ok()) { + visitor_->SetProgressStatusIfError(offset.status()); + } else { + fixed_info->set = true; + fixed_info->offset = *offset; + } } } @@ -2359,11 +2082,7 @@ void ExhaustiveTernaryCondVisitor::PreVisit(const cel::Expr* expr) { } void ExhaustiveTernaryCondVisitor::PostVisit(const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - visitor_->MakeTernaryRecursive(expr); - return; - } - visitor_->AddStep(CreateTernaryStep(expr->id())); + visitor_->AddStep(CreateTernaryStep(), expr->id(), /*stack_delta=*/-2); } void ComprehensionVisitor::PreVisit(const cel::Expr* expr) { @@ -2374,77 +2093,146 @@ void ComprehensionVisitor::PreVisit(const cel::Expr* expr) { } } +ComprehensionInitStepInfo* absl_nullable ComprehensionVisitor::GetInitStep() { + if (!init_step_pos_) { + return nullptr; + } + ExpressionStep* step = + init_step_pos_->subexpression->GetIfExpressionStep(init_step_pos_->index); + if (!step) { + return nullptr; + } + return GetIfComprehensionInitStep(*step); +} + +ComprehensionCondStepInfo* absl_nullable ComprehensionVisitor::GetCondStep() { + if (!cond_step_pos_) { + return nullptr; + } + ExpressionStep* step = + cond_step_pos_->subexpression->GetIfExpressionStep(cond_step_pos_->index); + if (!step) { + return nullptr; + } + return GetIfComprehensionCondStep(*step); +} + +ComprehensionNextStepInfo* absl_nullable ComprehensionVisitor::GetNextStep() { + if (!next_step_pos_) { + return nullptr; + } + ExpressionStep* step = + next_step_pos_->subexpression->GetIfExpressionStep(next_step_pos_->index); + if (!step) { + return nullptr; + } + return GetIfComprehensionNextStep(*step); +} + absl::Status ComprehensionVisitor::PostVisitArgDefault( cel::ComprehensionArg arg_num, const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - return absl::OkStatus(); - } switch (arg_num) { case cel::ITER_RANGE: { init_step_pos_ = visitor_->GetCurrentIndex(); - init_step_ = visitor_->AddStep( - std::make_unique(expr->id())); + if (iter_slot_ != iter2_slot_) { + ABSL_DCHECK_EQ(iter2_slot_, iter_slot_ + 1); + ABSL_DCHECK_EQ(accu_slot_, iter2_slot_ + 1); + visitor_->AddStep(ExpressionStep::MakeComprehensionInitStep( + static_cast(iter_slot_), ComprehensionVersion::kV2)); + } else { + ABSL_DCHECK_EQ(accu_slot_, iter_slot_ + 1); + visitor_->AddStep(ExpressionStep::MakeComprehensionInitStep( + static_cast(iter_slot_), ComprehensionVersion::kV1)); + } break; } case cel::ACCU_INIT: { next_step_pos_ = visitor_->GetCurrentIndex(); - next_step_ = visitor_->AddStep(std::make_unique( - iter_slot_, iter2_slot_, accu_slot_, expr->id())); + if (iter_slot_ != iter2_slot_) { + visitor_->AddStep(ExpressionStep::MakeComprehensionNext2Step()); + } else { + visitor_->AddStep(ExpressionStep::MakeComprehensionNextStep()); + } break; } case cel::LOOP_CONDITION: { cond_step_pos_ = visitor_->GetCurrentIndex(); - cond_step_ = visitor_->AddStep(std::make_unique( - iter_slot_, iter2_slot_, accu_slot_, short_circuiting_, expr->id())); + if (iter_slot_ != iter2_slot_) { + visitor_->AddStep(ExpressionStep::MakeComprehensionCond2Step()); + } else { + visitor_->AddStep(ExpressionStep::MakeComprehensionCondStep()); + } break; } case cel::LOOP_STEP: { ProgramStepIndex index = visitor_->GetCurrentIndex(); - auto* jump_to_next = visitor_->AddStep(CreateJumpStep({}, expr->id())); - if (!jump_to_next) { + ExpressionStep* jump_step = + visitor_->AddStep(ExpressionStep::MakeFixedJumpStep()); + FixedJumpStepInfo* fixed_info = GetIfFixedJumpStep(jump_step); + if (fixed_info == nullptr) { + // either an error occurred earlier or planning is suppressed. break; } - Jump jump_helper(index, jump_to_next); - visitor_->SetProgressStatusError(jump_helper.set_target(next_step_pos_)); + CEL_ASSIGN_OR_RETURN(int offset, + Jump::CalculateOffset(index, *next_step_pos_)); + fixed_info->set = true; + fixed_info->offset = offset; // Set offsets jumping to the result step. - if (cond_step_) { - CEL_ASSIGN_OR_RETURN( - int jump_from_cond, - Jump::CalculateOffset(cond_step_pos_, visitor_->GetCurrentIndex())); - cond_step_->set_jump_offset(jump_from_cond); + if (auto* cond_step = GetCondStep(); cond_step != nullptr) { + CEL_ASSIGN_OR_RETURN(int jump_from_cond, + Jump::CalculateOffset( + *cond_step_pos_, visitor_->GetCurrentIndex())); + cond_step->set_jump_offset(jump_from_cond); + } else { + return absl::InvalidArgumentError("Comprehension is malformed"); } - if (next_step_) { - CEL_ASSIGN_OR_RETURN( - int jump_from_next, - Jump::CalculateOffset(next_step_pos_, visitor_->GetCurrentIndex())); + if (auto* next_step = GetNextStep(); next_step != nullptr) { + CEL_ASSIGN_OR_RETURN(int jump_from_next, + Jump::CalculateOffset( + *next_step_pos_, visitor_->GetCurrentIndex())); - next_step_->set_jump_offset(jump_from_next); + next_step->set_jump_offset(jump_from_next); + } else { + return absl::InvalidArgumentError("Comprehension is malformed"); } break; } case cel::RESULT: { - if (!init_step_ || !next_step_ || !cond_step_) { + if (!init_step_pos_ || !next_step_pos_ || !cond_step_pos_) { // Encountered an error earlier. Can't determine where to jump. break; } - visitor_->AddStep(CreateComprehensionFinishStep(accu_slot_, expr->id())); + visitor_->AddStep( + ExpressionStep::MakeComprehensionFinishStep(accu_slot_, expr->id())); // Set offsets jumping past the result step in case of errors. - CEL_ASSIGN_OR_RETURN( - int jump_from_init, - Jump::CalculateOffset(init_step_pos_, visitor_->GetCurrentIndex())); - init_step_->set_error_jump_offset(jump_from_init); - - CEL_ASSIGN_OR_RETURN( - int jump_from_next, - Jump::CalculateOffset(next_step_pos_, visitor_->GetCurrentIndex())); - next_step_->set_error_jump_offset(jump_from_next); - - CEL_ASSIGN_OR_RETURN( - int jump_from_cond, - Jump::CalculateOffset(cond_step_pos_, visitor_->GetCurrentIndex())); - cond_step_->set_error_jump_offset(jump_from_cond); + if (auto* init_step = GetInitStep(); init_step != nullptr) { + CEL_ASSIGN_OR_RETURN(int jump_from_init, + Jump::CalculateOffset( + *init_step_pos_, visitor_->GetCurrentIndex())); + init_step->set_error_jump_offset(jump_from_init); + } else { + return absl::InvalidArgumentError("Comprehension is malformed"); + } + + if (auto* next_step = GetNextStep(); next_step != nullptr) { + CEL_ASSIGN_OR_RETURN(int jump_from_next, + Jump::CalculateOffset( + *next_step_pos_, visitor_->GetCurrentIndex())); + next_step->set_error_jump_offset(jump_from_next); + } else { + return absl::InvalidArgumentError("Comprehension is malformed"); + } + + if (auto* cond_step = GetCondStep(); cond_step != nullptr) { + CEL_ASSIGN_OR_RETURN(int jump_from_cond, + Jump::CalculateOffset( + *cond_step_pos_, visitor_->GetCurrentIndex())); + cond_step->set_error_jump_offset(jump_from_cond); + } else { + return absl::InvalidArgumentError("Comprehension is malformed"); + } break; } } @@ -2453,16 +2241,13 @@ absl::Status ComprehensionVisitor::PostVisitArgDefault( void ComprehensionVisitor::PostVisitArgTrivial(cel::ComprehensionArg arg_num, const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - return; - } switch (arg_num) { case cel::ITER_RANGE: { break; } case cel::ACCU_INIT: { if (!accu_init_extracted_) { - visitor_->AddStep(CreateAssignSlotAndPopStep(accu_slot_)); + visitor_->AddStep(ExpressionStep::MakeAssignSlotAndPopStep(accu_slot_)); } break; } @@ -2473,20 +2258,14 @@ void ComprehensionVisitor::PostVisitArgTrivial(cel::ComprehensionArg arg_num, break; } case cel::RESULT: { - visitor_->AddStep(CreateClearSlotStep(accu_slot_, expr->id())); + visitor_->AddStep( + ExpressionStep::MakeClearSlotStep(accu_slot_, expr->id())); break; } } } void ComprehensionVisitor::PostVisit(const cel::Expr* expr) { - if (is_trivial_) { - visitor_->MaybeMakeBindRecursive(expr, &expr->comprehension_expr(), - accu_slot_); - return; - } - visitor_->MaybeMakeComprehensionRecursive( - expr, &expr->comprehension_expr(), iter_slot_, iter2_slot_, accu_slot_); } // Flattens the expression table into the end of the mainline expression vector @@ -2513,13 +2292,64 @@ std::vector FlattenExpressionTable( return subexpression_indexes; } +std::optional CheckedDeltaAdd(int64_t current, + std::optional delta) { + if (!delta.has_value()) { + return std::nullopt; + } + if (*delta > 0 && current > std::numeric_limits::max() - *delta) { + return std::nullopt; + } + if (*delta < 0 && current < std::numeric_limits::min() - *delta) { + return std::nullopt; + } + current += *delta; + if (current < 0) { + return std::nullopt; + } + return current; +} + +// Conservative estimate of the maximum value stack size needed for the given +// subexpressions. +// +// If overflow occurs, returns fallback_size, which is the total number of +// steps in the program. +size_t EstimateMaxStackSize(absl::Span subexpressions, + size_t fallback_size) { + size_t total_max_stack = 0; + for (ExecutionPathView path : subexpressions) { + int64_t current = 0; + int64_t max_depth = 0; + for (const ExpressionStep& step : path) { + std::optional next = CheckedDeltaAdd(current, step.StackDelta()); + if (!next.has_value()) { + return fallback_size; + } + current = *next; + max_depth = std::max(max_depth, current); + } + if (static_cast(max_depth) > + std::numeric_limits::max() - total_max_stack) { + return fallback_size; + } + total_max_stack += static_cast(max_depth); + } + return total_max_stack; +} + absl::Status CheckAstExtensions( - const std::vector& extensions) { + const std::vector& extensions, + bool enable_select_optimization) { for (const cel::ExtensionSpec& extension : extensions) { if (extension.id() == "cel_block" && extension.version().major() == 1) { // cel_block v1 is always supported. continue; } + if (extension.id() == "select_optimization" && + extension.version().major() == 1 && enable_select_optimization) { + continue; + } // TODO(uncreated-issue/89): Add support for json field names. return absl::InvalidArgumentError(absl::StrCat( @@ -2542,6 +2372,12 @@ absl::StatusOr FlatExprBuilder::CreateExpressionImpl( ? RuntimeIssue::Severity::kWarning : RuntimeIssue::Severity::kError; IssueCollector issue_collector(max_severity); + auto record_issues = absl::MakeCleanup([issues, &issue_collector] { + if (issues == nullptr) { + return; + } + *issues = issue_collector.ExtractIssues(); + }); absl::StatusOr> runtime_extensions = ExtractAndValidateRuntimeExtensions(*ast); @@ -2549,12 +2385,13 @@ absl::StatusOr FlatExprBuilder::CreateExpressionImpl( if (!runtime_extensions.ok()) { CEL_RETURN_IF_ERROR(issue_collector.AddIssue( RuntimeIssue::CreateError(runtime_extensions.status()))); - } - - auto status = CheckAstExtensions(*runtime_extensions); - if (!status.ok()) { - CEL_RETURN_IF_ERROR( - issue_collector.AddIssue(RuntimeIssue::CreateError(status))); + } else { + absl::Status status = + CheckAstExtensions(*runtime_extensions, enable_select_optimization_); + if (!status.ok()) { + CEL_RETURN_IF_ERROR( + issue_collector.AddIssue(RuntimeIssue::CreateError(status))); + } } Resolver resolver(container_, function_registry_, type_registry_, @@ -2582,17 +2419,10 @@ absl::StatusOr FlatExprBuilder::CreateExpressionImpl( // These objects are expected to remain scoped to one build call -- references // to them shouldn't be persisted in any part of the result expression. FlatExprVisitor visitor(resolver, options_, std::move(optimizers), - ast->reference_map(), GetTypeProvider(), - issue_collector, program_builder, extension_context, + ast->type_map(), GetTypeProvider(), issue_collector, + program_builder, extension_context, enable_optional_types_); - if (options_.max_recursion_depth == -1 || options_.max_recursion_depth > 0) { - int depth_limit = options_.max_recursion_depth == -1 - ? std::numeric_limits::max() - : options_.max_recursion_depth; - visitor.SetMaxRecursionDepth(depth_limit); - } - cel::TraversalOptions opts; opts.use_comprehension_callbacks = true; AstTraverse(ast->root_expr(), visitor, opts); @@ -2601,22 +2431,46 @@ absl::StatusOr FlatExprBuilder::CreateExpressionImpl( return visitor.progress_status(); } - if (issues != nullptr) { - (*issues) = issue_collector.ExtractIssues(); + size_t total_slot_count = visitor.slot_count(); + size_t iterator_stack_size = visitor.iter_stack_count(); + IdentifierTable identifiers = + std::move(visitor).BuildIdentifierTable(total_slot_count); + + if (total_slot_count > std::numeric_limits::max() || + program_builder.ExtractedSubexpressionCount() > + std::numeric_limits::max()) { + // Impractical to trigger (we'd run out of memory first), but assuming this + // allows us to pack references to slots and subexpressions. + return absl::InternalError( + "Expression too large to be executed, exceeds uint32_t limits."); } ExecutionPath execution_path; std::vector subexpressions = FlattenExpressionTable(program_builder, execution_path); + size_t value_stack_size = + EstimateMaxStackSize(subexpressions, execution_path.size()); + + // Iterate over execution path and update identifier steps with resolved slot. + // TODO(b/572083203): Should invert this so the local slots are stored + // after the global ones. + for (ExpressionStep& step : execution_path) { + if (IdentStepInfo* ident_step = GetIfIdentStep(step); + ident_step != nullptr) { + ident_step->slot_index = static_cast( + identifiers[ident_step->ident_index].slot_index); + } + } return FlatExpression(std::move(execution_path), std::move(subexpressions), - visitor.slot_count(), GetTypeProvider(), options_, - std::move(arena)); + total_slot_count, iterator_stack_size, + GetTypeProvider(), options_, std::move(arena), + std::move(identifiers), value_stack_size); } const cel::TypeProvider& FlatExprBuilder::GetTypeProvider() const { return use_legacy_type_provider_ ? static_cast( - *GetLegacyRuntimeTypeProvider(type_registry_)) + GetLegacyRuntimeTypeProvider(type_registry_)) : GetRuntimeTypeProvider(type_registry_); } diff --git a/eval/compiler/flat_expr_builder.h b/eval/compiler/flat_expr_builder.h index aa4d0b4e5..fc6066883 100644 --- a/eval/compiler/flat_expr_builder.h +++ b/eval/compiler/flat_expr_builder.h @@ -81,6 +81,10 @@ class FlatExprBuilder { bool optional_types_enabled() const { return enable_optional_types_; } + // Called when select optimization is enabled to accept ASTs tagged with the + // `select_optimization` runtime extension. + void enable_select_optimization() { enable_select_optimization_ = true; } + private: const cel::TypeProvider& GetTypeProvider() const; @@ -90,6 +94,7 @@ class FlatExprBuilder { cel::RuntimeOptions options_; std::string container_; bool enable_optional_types_ = false; + bool enable_select_optimization_ = false; // TODO(uncreated-issue/45): evaluate whether we should use a shared_ptr here to // allow built expressions to keep the registries alive. const cel::FunctionRegistry& function_registry_; diff --git a/eval/compiler/flat_expr_builder_comprehensions_test.cc b/eval/compiler/flat_expr_builder_comprehensions_test.cc index 9d46d8dd8..df60a456d 100644 --- a/eval/compiler/flat_expr_builder_comprehensions_test.cc +++ b/eval/compiler/flat_expr_builder_comprehensions_test.cc @@ -14,15 +14,22 @@ * limitations under the License. */ +#include +#include #include #include "cel/expr/syntax.pb.h" #include "google/protobuf/field_mask.pb.h" #include "absl/status/status.h" #include "absl/strings/string_view.h" +#include "base/ast.h" +#include "common/source.h" +#include "common/value.h" +#include "common/value_testing.h" #include "eval/compiler/cel_expression_builder_flat_impl.h" #include "eval/compiler/comprehension_vulnerability_check.h" #include "eval/compiler/flat_expr_builder.h" +#include "eval/eval/evaluator_core.h" #include "eval/public/activation.h" #include "eval/public/builtin_func_registrar.h" #include "eval/public/cel_attribute.h" @@ -32,10 +39,15 @@ #include "eval/public/containers/container_backed_list_impl.h" #include "eval/public/testing/matchers.h" #include "eval/testutil/test_message.pb.h" +#include "extensions/bindings_ext.h" #include "internal/testing.h" #include "parser/parser.h" +#include "parser/parser_interface.h" +#include "runtime/activation.h" +#include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" #include "runtime/runtime_options.h" +#include "testutil/test_macros.h" #include "google/protobuf/arena.h" #include "google/protobuf/text_format.h" @@ -43,30 +55,25 @@ namespace google::api::expr::runtime { namespace { +using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::expr::CheckedExpr; using ::cel::expr::ParsedExpr; using ::testing::HasSubstr; -class CelExpressionBuilderFlatImplComprehensionsTest - : public testing::TestWithParam { +class CelExpressionBuilderFlatImplComprehensionsTest : public testing::Test { public: CelExpressionBuilderFlatImplComprehensionsTest() = default; - bool enable_recursive_planning() { return GetParam(); } - cel::RuntimeOptions GetRuntimeOptions() { cel::RuntimeOptions options; - if (enable_recursive_planning()) { - options.max_recursion_depth = -1; - } options.enable_comprehension_list_append = true; return options; } }; -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, NestedComp) { +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, NestedComp) { cel::RuntimeOptions options = GetRuntimeOptions(); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); @@ -84,7 +91,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, NestedComp) { EXPECT_THAT(*result.ListOrDie(), testing::SizeIs(2)); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, MapComp) { +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, MapComp) { cel::RuntimeOptions options = GetRuntimeOptions(); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); @@ -105,7 +112,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, MapComp) { test::EqualsCelValue(CelValue::CreateInt64(4))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, ExistsOneTrue) { +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ExistsOneTrue) { cel::RuntimeOptions options = GetRuntimeOptions(); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); @@ -122,7 +129,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, ExistsOneTrue) { EXPECT_THAT(result, test::IsCelBool(true)); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, ExistsOneFalse) { +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ExistsOneFalse) { cel::RuntimeOptions options = GetRuntimeOptions(); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); @@ -139,7 +146,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, ExistsOneFalse) { EXPECT_THAT(result, test::IsCelBool(false)); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, ListCompWithUnknowns) { +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ListCompWithUnknowns) { cel::RuntimeOptions options = GetRuntimeOptions(); options.unknown_processing = UnknownProcessingOptions::kAttributeAndFunction; CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); @@ -152,9 +159,11 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, ListCompWithUnknowns) { &parsed_expr.source_info())); Activation activation; - activation.set_unknown_attribute_patterns({CelAttributePattern{ - "items", - {CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1))}}}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern{ + "items", + {CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1))}}}), + IsOk()); ContainerBackedListImpl list_impl = ContainerBackedListImpl({ CelValue::CreateInt64(1), // element items[1] is marked unknown, so the computation should produce @@ -176,7 +185,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, ListCompWithUnknowns) { testing::Eq(1)); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, InvalidComprehensionWithRewrite) { CheckedExpr expr; // The rewrite step which occurs when an identifier gets a more qualified name @@ -213,7 +222,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("Invalid empty expression")))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ComprehensionWithConcatVulernability) { CheckedExpr expr; // The comprehension loop step performs an unsafe concatenation of the @@ -267,7 +276,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("memory exhaustion vulnerability"))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ComprehensionWithListVulernability) { CheckedExpr expr; // The comprehension @@ -311,7 +320,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("memory exhaustion vulnerability"))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ComprehensionWithStructVulernability) { CheckedExpr expr; // The comprehension loop step builds a deeply nested struct which expands @@ -368,7 +377,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("memory exhaustion vulnerability"))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ComprehensionWithNestedComprehensionResultVulernability) { CheckedExpr expr; // The nested comprehension performs an unsafe concatenation on the parent @@ -436,7 +445,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("memory exhaustion vulnerability"))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ComprehensionWithNestedComprehensionLoopStepVulernability) { CheckedExpr expr; // The nested comprehension performs an unsafe concatenation on the parent @@ -483,7 +492,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("memory exhaustion vulnerability"))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ComprehensionWithNestedComprehensionLoopStepVulernabilityResult) { CheckedExpr expr; // The nested comprehension performs an unsafe concatenation on the parent @@ -535,7 +544,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("memory exhaustion vulnerability"))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ComprehensionWithNestedComprehensionLoopStepIterRangeVulnerability) { CheckedExpr expr; // The nested comprehension unsafely modifies the parent accumulator @@ -582,7 +591,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("memory exhaustion vulnerability"))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, InvalidBindComprehension) { ParsedExpr expr; // Trivial comprehensions (such as cel.bind), are optimized by skipping the @@ -626,12 +635,118 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("Unexpected iter_var access in trivial comprehension"))); } -INSTANTIATE_TEST_SUITE_P(TestSuite, - CelExpressionBuilderFlatImplComprehensionsTest, - testing::Bool(), - [](const testing::TestParamInfo& info) { - return info.param ? "recursive" : "default"; - }); +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, + BindInsideComprehensionWithComprehensionInInitIterStackCount) { + std::unique_ptr parser_builder = cel::NewParserBuilder(); + ASSERT_THAT( + parser_builder->AddLibrary(cel::extensions::BindingsParserLibrary()), + IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr parser, + parser_builder->Build()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr source, + parser->PrepareSource( + "[[1, 2], [3, 4]].map(l, cel.bind(filtered, l.filter(x, x > 1), " + "size(filtered) > 0 ? filtered[0] : 0)) == [2, 3]")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, parser->Parse(*source)); + + cel::RuntimeOptions options = GetRuntimeOptions(); + std::shared_ptr env = + NewTestingRuntimeEnv(); + CelExpressionBuilderFlatImpl builder(env, options); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + + ASSERT_OK_AND_ASSIGN( + FlatExpression flat_expr, + builder.flat_expr_builder().CreateExpressionImpl(std::move(ast), + /*issues=*/nullptr)); + EXPECT_EQ(flat_expr.iterator_stack_size(), 2); + + cel::Activation activation; + google::protobuf::Arena arena; + FlatExpressionEvaluatorState state = flat_expr.MakeEvaluatorState( + env->descriptor_pool.get(), env->MutableMessageFactory(), &arena); + ASSERT_OK_AND_ASSIGN( + cel::Value result, + flat_expr.EvaluateWithCallback(activation, /*embedder_context=*/nullptr, + EvaluationListener(), state)); + EXPECT_THAT(result, cel::test::BoolValueIs(true)); +} + +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, + BlockWithComprehensionInBindingAndBodyIterStackCount) { + std::unique_ptr parser_builder = cel::NewParserBuilder(); + ASSERT_THAT(parser_builder->AddLibrary(cel::test::TestMacrosParserLibrary()), + IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr parser, + parser_builder->Build()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr source, + parser->PrepareSource( + "cel.block([[1, 2, 3].filter(x, x > 1), [1, 2, 3].filter(x, x < 3)], " + "[1, 2].map(y, y in cel.index(0) || y in cel.index(1))) == [true, " + "true]")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, parser->Parse(*source)); + + cel::RuntimeOptions options = GetRuntimeOptions(); + std::shared_ptr env = + NewTestingRuntimeEnv(); + CelExpressionBuilderFlatImpl builder(env, options); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + + ASSERT_OK_AND_ASSIGN( + FlatExpression flat_expr, + builder.flat_expr_builder().CreateExpressionImpl(std::move(ast), + /*issues=*/nullptr)); + EXPECT_EQ(flat_expr.iterator_stack_size(), 3); + + cel::Activation activation; + google::protobuf::Arena arena; + FlatExpressionEvaluatorState state = flat_expr.MakeEvaluatorState( + env->descriptor_pool.get(), env->MutableMessageFactory(), &arena); + ASSERT_OK_AND_ASSIGN( + cel::Value result, + flat_expr.EvaluateWithCallback(activation, /*embedder_context=*/nullptr, + EvaluationListener(), state)); + EXPECT_THAT(result, cel::test::BoolValueIs(true)); +} + +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, + NonNestedComprehensionsIterStackCount) { + std::unique_ptr parser_builder = cel::NewParserBuilder(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr parser, + parser_builder->Build()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr source, + parser->PrepareSource( + "[1, 2].map(x, x + 1) + [3, 4].filter(y, y > 3) == [2, 3, 4]")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, parser->Parse(*source)); + + cel::RuntimeOptions options = GetRuntimeOptions(); + std::shared_ptr env = + NewTestingRuntimeEnv(); + CelExpressionBuilderFlatImpl builder(env, options); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + + ASSERT_OK_AND_ASSIGN( + FlatExpression flat_expr, + builder.flat_expr_builder().CreateExpressionImpl(std::move(ast), + /*issues=*/nullptr)); + EXPECT_EQ(flat_expr.iterator_stack_size(), 1); + + cel::Activation activation; + google::protobuf::Arena arena; + FlatExpressionEvaluatorState state = flat_expr.MakeEvaluatorState( + env->descriptor_pool.get(), env->MutableMessageFactory(), &arena); + ASSERT_OK_AND_ASSIGN( + cel::Value result, + flat_expr.EvaluateWithCallback(activation, /*embedder_context=*/nullptr, + EvaluationListener(), state)); + EXPECT_THAT(result, cel::test::BoolValueIs(true)); +} } // namespace diff --git a/eval/compiler/flat_expr_builder_extensions.cc b/eval/compiler/flat_expr_builder_extensions.cc index 463b48425..bf8f6f74c 100644 --- a/eval/compiler/flat_expr_builder_extensions.cc +++ b/eval/compiler/flat_expr_builder_extensions.cc @@ -13,10 +13,11 @@ // limitations under the License. #include "eval/compiler/flat_expr_builder_extensions.h" -#include #include +#include #include #include +#include #include #include @@ -27,11 +28,14 @@ #include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/types/optional.h" #include "absl/types/variant.h" +#include "common/ast.h" #include "common/expr.h" -#include "eval/eval/direct_expression_step.h" +#include "common/type.h" +#include "common/type_spec_resolver.h" +#include "common/value.h" #include "eval/eval/evaluator_core.h" +#include "eval/eval/ident_step.h" namespace google::api::expr::runtime { @@ -39,29 +43,35 @@ namespace { using Subexpression = google::api::expr::runtime::ProgramBuilder::Subexpression; -// Remap a recursive program to its parent if the parent is a transparent -// wrapper. -void MaybeReassignChildRecursiveProgram(Subexpression* parent) { - if (parent->IsFlattened() || parent->IsRecursive()) { - return; - } - if (parent->elements().size() != 1) { - return; - } - auto* child_alternative = - absl::get_if(&parent->elements()[0]); - if (child_alternative == nullptr) { - return; - } - - auto& child_subexpression = *child_alternative; - if (!child_subexpression->IsRecursive()) { - return; +const ExpressionStep* absl_nullable TryGetSingleStep( + const Subexpression* absl_nonnull subexpr) { + while (subexpr != nullptr) { + if (subexpr->IsFlattened()) { + const ExecutionPath& elements = subexpr->flattened_elements(); + if (elements.size() != 1) { + return nullptr; + } + return &elements[0]; + } + const auto& elements = subexpr->elements(); + if (elements.size() != 1) { + return nullptr; + } + const auto& element = elements[0]; + if (const ExpressionStep* step = absl::get_if(&element); + step != nullptr) { + return step; + } + // This might happen if an AST node is interpreted as a no-op, but we still + // create a subexpression for it. + if (Subexpression* const* child = absl::get_if(&element); + child != nullptr) { + subexpr = *child; + continue; + } + return nullptr; } - - auto child_program = child_subexpression->ExtractRecursiveProgram(); - parent->set_recursive_program(std::move(child_program.step), - child_program.depth); + return nullptr; } } // namespace @@ -72,8 +82,6 @@ Subexpression::Subexpression(const cel::Expr* self, ProgramBuilder* owner) size_t Subexpression::ComputeSize() const { if (IsFlattened()) { return flattened_elements().size(); - } else if (IsRecursive()) { - return 1; } std::vector to_expand{this}; size_t size = 0; @@ -83,9 +91,6 @@ size_t Subexpression::ComputeSize() const { if (expr->IsFlattened()) { size += expr->flattened_elements().size(); continue; - } else if (expr->IsRecursive()) { - size += 1; - continue; } for (const auto& elem : expr->elements()) { if (auto* child = absl::get_if(&elem); child != nullptr) { @@ -98,45 +103,6 @@ size_t Subexpression::ComputeSize() const { return size; } -absl::optional Subexpression::RecursiveDependencyDepth() const { - auto* tree = absl::get_if(&program_); - int depth = 0; - if (tree == nullptr) { - return absl::nullopt; - } - for (const auto& element : *tree) { - auto* subexpression = absl::get_if(&element); - if (subexpression == nullptr) { - return absl::nullopt; - } - if (!(*subexpression)->IsRecursive()) { - return absl::nullopt; - } - depth = std::max(depth, (*subexpression)->recursive_program().depth); - } - return depth; -} - -std::vector> -Subexpression::ExtractRecursiveDependencies() const { - auto* tree = absl::get_if(&program_); - std::vector> dependencies; - if (tree == nullptr) { - return {}; - } - for (const auto& element : *tree) { - auto* subexpression = absl::get_if(&element); - if (subexpression == nullptr) { - return {}; - } - if (!(*subexpression)->IsRecursive()) { - return {}; - } - dependencies.push_back((*subexpression)->ExtractRecursiveProgram().step); - } - return dependencies; -} - Subexpression* absl_nullable Subexpression::ExtractChild(Subexpression* child) { ABSL_DCHECK(child != nullptr); if (IsFlattened()) { @@ -161,7 +127,6 @@ Subexpression* absl_nullable Subexpression::ExtractChild(Subexpression* child) { // target step. int Subexpression::CalculateOffset(int base, int target) const { ABSL_DCHECK(!IsFlattened()); - ABSL_DCHECK(!IsRecursive()); int sign = 1; int start = base + 1; @@ -205,7 +170,7 @@ void Subexpression::Flatten() { return; } - std::vector> flat; + ExecutionPath flat; std::vector flatten_stack; @@ -220,11 +185,6 @@ void Subexpression::Flatten() { absl::c_move(elements, std::back_inserter(flat)); elements.clear(); continue; - } else if (subexpr->IsRecursive()) { - flat.push_back(std::make_unique( - std::move(subexpr->ExtractRecursiveProgram().step), - subexpr->self_->id())); - continue; } auto& elements = subexpr->elements(); size_t size = elements.size(); @@ -237,8 +197,7 @@ void Subexpression::Flatten() { flatten_stack.push_back({subexpr, i + 1}); flatten_stack.push_back({*child, 0}); break; - } else if (auto* step = - absl::get_if>(&element); + } else if (auto* step = absl::get_if(&element); step != nullptr) { flat.push_back(std::move(*step)); } else { @@ -252,15 +211,7 @@ void Subexpression::Flatten() { program_ = std::move(flat); } -Subexpression::RecursiveProgram Subexpression::ExtractRecursiveProgram() { - ABSL_DCHECK(IsRecursive()); - auto result = std::move(absl::get(program_)); - program_.emplace>(); - return result; -} - -bool Subexpression::ExtractTo( - std::vector>& out) { +bool Subexpression::ExtractTo(ExecutionPath& out) { if (!IsFlattened()) { return false; } @@ -272,9 +223,34 @@ bool Subexpression::ExtractTo( return true; } -std::vector> -ProgramBuilder::FlattenSubexpression(Subexpression* expr) { - std::vector> out; +bool Subexpression::IsInlinable() const { + const ExpressionStep* step = TryGetSingleStep(this); + if (step == nullptr) { + return false; + } + return IsConstant(*step) || GetIfIdentStep(*step) != nullptr; +} + +std::optional Subexpression::MaybeInline( + int64_t expr_id) const { + const ExpressionStep* step = TryGetSingleStep(this); + if (step == nullptr) { + return std::nullopt; + } + cel::Value value; + if (GetIfConstant(*step, value)) { + return ExpressionStep::MakeConstant(value, expr_id); + } + if (const IdentStepInfo* ident = GetIfIdentStep(*step); ident != nullptr) { + return ExpressionStep::MakeIdentifierStep(ident->ident_index, + ident->slot_index, expr_id); + } + return std::nullopt; +} + +ExecutionPath ProgramBuilder::FlattenSubexpression( + Subexpression* absl_nonnull expr) { + ExecutionPath out; if (!expr) { return out; @@ -329,8 +305,6 @@ Subexpression* absl_nullable ProgramBuilder::ExitSubexpression( ABSL_DCHECK(expr == current_->self_); ABSL_DCHECK(GetSubexpression(expr) == current_); - MaybeReassignChildRecursiveProgram(current_); - Subexpression* result = GetSubexpression(current_->parent_); ABSL_DCHECK(result != nullptr || current_ == root_); current_ = result; @@ -347,13 +321,16 @@ Subexpression* absl_nullable ProgramBuilder::GetSubexpression( return it->second.get(); } -ExpressionStep* absl_nullable ProgramBuilder::AddStep( - std::unique_ptr step) { +ExpressionStep* absl_nullable ProgramBuilder::AddStep(ExpressionStep step) { if (current_ == nullptr) { return nullptr; } - auto* step_ptr = step.get(); - return current_->AddStep(std::move(step)) ? step_ptr : nullptr; + if (current_->IsFlattened()) { + current_->flattened_elements().push_back(std::move(step)); + return ¤t_->flattened_elements().back(); + } + auto& elem = current_->elements().emplace_back(std::move(step)); + return absl::get_if(&elem); } int ProgramBuilder::ExtractSubexpression(const cel::Expr* expr) { @@ -390,8 +367,15 @@ Subexpression* absl_nullable ProgramBuilder::MakeSubexpression( return it->second.get(); } -bool PlannerContext::IsSubplanInspectable(const cel::Expr& node) const { - return program_builder_.GetSubexpression(&node) != nullptr; +bool PlannerContext::NodeHasInspectablePlan(const cel::Expr& node) const { + const Subexpression* subexpression = program_builder_.GetSubexpression(&node); + if (subexpression == nullptr) { + return false; + } + if (subexpression->IsFlattened()) { + return !subexpression->flattened_elements().empty(); + } + return !subexpression->elements().empty(); } ExecutionPathView PlannerContext::GetSubplan(const cel::Expr& node) { @@ -437,6 +421,21 @@ absl::Status PlannerContext::ReplaceSubplan(const cel::Expr& node, return absl::OkStatus(); } +std::optional PlannerContext::TryResolveType( + int64_t expr_id, const cel::TypeSpec& ast_type) { + auto it = resolved_types_.find(expr_id); + if (it != resolved_types_.end()) { + return it->second; + } + absl::StatusOr type = + cel::ConvertTypeSpecToType(ast_type, type_reflector_, MutableArena()); + if (!type.ok()) { + return std::nullopt; + } + resolved_types_.insert(it, {expr_id, *type}); + return *type; +} + void ProgramBuilder::Reset() { root_ = nullptr; current_ = nullptr; @@ -444,21 +443,8 @@ void ProgramBuilder::Reset() { subprogram_map_.clear(); } -absl::Status PlannerContext::ReplaceSubplan( - const cel::Expr& node, std::unique_ptr step, - int depth) { - auto* subexpression = program_builder_.GetSubexpression(&node); - if (subexpression == nullptr) { - return absl::InternalError( - "attempted to update program step for untracked expr node"); - } - - subexpression->set_recursive_program(std::move(step), depth); - return absl::OkStatus(); -} - -absl::Status PlannerContext::AddSubplanStep( - const cel::Expr& node, std::unique_ptr step) { +absl::Status PlannerContext::AddSubplanStep(const cel::Expr& node, + ExpressionStep step) { auto* subexpression = program_builder_.GetSubexpression(&node); if (subexpression == nullptr) { diff --git a/eval/compiler/flat_expr_builder_extensions.h b/eval/compiler/flat_expr_builder_extensions.h index 21e37b2a8..14d847dca 100644 --- a/eval/compiler/flat_expr_builder_extensions.h +++ b/eval/compiler/flat_expr_builder_extensions.h @@ -23,8 +23,11 @@ #define THIRD_PARTY_CEL_CPP_EVAL_COMPILER_FLAT_EXPR_BUILDER_EXTENSIONS_H_ #include +#include #include +#include #include +#include #include #include "absl/base/attributes.h" @@ -34,18 +37,15 @@ #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/types/optional.h" #include "absl/types/variant.h" #include "base/ast.h" #include "base/type_provider.h" #include "common/expr.h" -#include "common/native_type.h" +#include "common/type.h" #include "common/type_reflector.h" #include "eval/compiler/resolver.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/trace_step.h" -#include "internal/casts.h" +#include "eval/eval/expression_step_logic.h" #include "runtime/internal/issue_collector.h" #include "runtime/internal/runtime_env.h" #include "runtime/runtime_options.h" @@ -84,18 +84,12 @@ class ProgramBuilder { // Must be tied to a ProgramBuilder to coordinate relationships. class Subexpression { private: - using Element = absl::variant, - Subexpression* absl_nonnull>; + using Element = absl::variant; using TreePlan = std::vector; - using FlattenedPlan = std::vector>; + using FlattenedPlan = ExecutionPath; public: - struct RecursiveProgram { - std::unique_ptr step; - int depth; - }; - ~Subexpression() = default; // Not copyable or movable. @@ -105,11 +99,7 @@ class ProgramBuilder { Subexpression& operator=(Subexpression&&) = delete; // Add a program step at the current end of the subexpression. - bool AddStep(std::unique_ptr step) { - if (IsRecursive()) { - return false; - } - + bool AddStep(ExpressionStep step) { if (IsFlattened()) { flattened_elements().push_back(std::move(step)); return true; @@ -125,6 +115,13 @@ class ProgramBuilder { elements().push_back(expr); } + ExpressionStep* absl_nullable GetIfExpressionStep(size_t index) { + if (index >= elements().size()) { + return nullptr; + } + return std::get_if(&elements()[index]); + } + // Accessor for elements (either simple steps or subexpressions). // // Value is undefined if in the expression has already been flattened. @@ -141,38 +138,16 @@ class ProgramBuilder { // Accessor for program steps. // // Value is undefined if in the expression has not yet been flattened. - std::vector>& flattened_elements() { + ExecutionPath& flattened_elements() { ABSL_DCHECK(IsFlattened()); return absl::get(program_); } - const std::vector>& - flattened_elements() const { + const ExecutionPath& flattened_elements() const { ABSL_DCHECK(IsFlattened()); return absl::get(program_); } - void set_recursive_program(std::unique_ptr step, - int depth) { - program_ = RecursiveProgram{std::move(step), depth}; - } - - const RecursiveProgram& recursive_program() const { - ABSL_DCHECK(IsRecursive()); - return absl::get(program_); - } - - absl::optional RecursiveDependencyDepth() const; - - std::vector> - ExtractRecursiveDependencies() const; - - RecursiveProgram ExtractRecursiveProgram(); - - bool IsRecursive() const { - return absl::holds_alternative(program_); - } - // Compute the current number of program steps in this subexpression and // its dependencies. size_t ComputeSize() const; @@ -202,7 +177,15 @@ class ProgramBuilder { // ownership of the given steps. // // Returns false if the subexpression is not currently flattened. - bool ExtractTo(std::vector>& out); + bool ExtractTo(ExecutionPath& out); + + // Returns true if this subexpression is trivial (reduces to a single + // constant or identifier step) and can be inlined at reference sites. + bool IsInlinable() const; + + // Creates an inlined ExpressionStep for this subexpression with the + // given expr_id, or returns std::nullopt if !IsInlinable(). + std::optional MaybeInline(int64_t expr_id) const; private: Subexpression(const cel::Expr* self, ProgramBuilder* owner); @@ -213,7 +196,7 @@ class ProgramBuilder { // // This adds complexity, but supports swapping to a flat representation as // needed. - absl::variant program_; + absl::variant program_; const cel::Expr* self_; const cel::Expr* absl_nullable parent_; @@ -234,6 +217,11 @@ class ProgramBuilder { // programs table to starting state. std::vector FlattenSubexpressions(); + // Return the current number of subexpressions in the program builder. + size_t ExtractedSubexpressionCount() const { + return extracted_subexpressions_.size(); + } + // Returns the current subexpression where steps and new subexpressions are // added. // @@ -276,6 +264,15 @@ class ProgramBuilder { return extracted_subexpressions_[index]; } + const Subexpression* absl_nullable GetExtractedSubexpression( + size_t index) const { + if (index >= extracted_subexpressions_.size()) { + return nullptr; + } + + return extracted_subexpressions_[index]; + } + // Return index to the extracted subexpression. // // Returns -1 if the subexpression is not found. @@ -284,16 +281,14 @@ class ProgramBuilder { // Add a program step to the current subexpression. // If successful, returns the step pointer. // - // Note: If successful, the pointer should remain valid until the parent - // expression is finalized. Optimizers may modify the program plan which may - // free the step at that point. - ExpressionStep* absl_nullable AddStep(std::unique_ptr step); + // Note: If successful, the pointer should remain valid until a further call + // to AddStep, AddSubexpression, or Flatten. + ExpressionStep* absl_nullable AddStep(ExpressionStep step); void Reset(); private: - static std::vector> - FlattenSubexpression(Subexpression* absl_nonnull expr); + static ExecutionPath FlattenSubexpression(Subexpression* absl_nonnull expr); Subexpression* absl_nullable MakeSubexpression(const cel::Expr* expr); @@ -303,31 +298,6 @@ class ProgramBuilder { SubprogramMap subprogram_map_; }; -// Attempt to downcast a specific type of recursive step. -template -const Subclass* TryDowncastDirectStep(const DirectExpressionStep* step) { - if (step == nullptr) { - return nullptr; - } - - auto type_id = step->GetNativeTypeId(); - if (type_id == cel::NativeTypeId::For()) { - const auto* trace_step = cel::internal::down_cast(step); - auto deps = trace_step->GetDependencies(); - if (!deps.has_value() || deps->size() != 1) { - return nullptr; - } - step = deps->at(0); - type_id = step->GetNativeTypeId(); - } - - if (type_id == cel::NativeTypeId::For()) { - return cel::internal::down_cast(step); - } - - return nullptr; -} - // Class representing FlatExpr internals exposed to extensions. class PlannerContext { public: @@ -346,15 +316,16 @@ class PlannerContext { issue_collector_(issue_collector), program_builder_(program_builder), arena_(arena), + message_factory_(std::move(message_factory)), explicit_arena_(arena_ != nullptr), - message_factory_(std::move(message_factory)) {} + resolve_types_(options_.enable_typed_field_access) {} ProgramBuilder& program_builder() { return program_builder_; } // Returns true if the subplan is inspectable. // // If false, the node is not mapped to a subexpression in the program builder. - bool IsSubplanInspectable(const cel::Expr& node) const; + bool NodeHasInspectablePlan(const cel::Expr& node) const; // Return a view to the current subplan representing node. // @@ -376,17 +347,14 @@ class PlannerContext { // expr->program mapping for any descendants. absl::Status ReplaceSubplan(const cel::Expr& node, ExecutionPath path); - // Replace the subplan associated with node with a new recursive subplan. - // - // This operation clears any existing plan to which removes the - // expr->program mapping for any descendants. - absl::Status ReplaceSubplan(const cel::Expr& node, - std::unique_ptr step, - int depth); - // Extend the current subplan with the given expression step. + absl::Status AddSubplanStep(const cel::Expr& node, ExpressionStep step); absl::Status AddSubplanStep(const cel::Expr& node, - std::unique_ptr step); + std::unique_ptr step, + int64_t expr_id = -1, int64_t stack_delta = 1) { + return AddSubplanStep(node, ExpressionStep::MakeGenericStep( + std::move(step), expr_id, stack_delta)); + } const Resolver& resolver() const { return resolver_; } const cel::TypeReflector& type_reflector() const { return type_reflector_; } @@ -410,6 +378,33 @@ class PlannerContext { return arena_.get(); } + // Whether the planner should resolve types in the AST to concrete cel::Type + // values. + bool resolve_types() const { return resolve_types_; } + + void set_resolve_types() { resolve_types_ = true; } + + // Returns the resolved type if available. + // + // This can only be used as a hint since we can't guarantee that the AST was + // type checked or can be fully trusted (e.g. post-processing broke IDs). + // + // This is populated by FlatExprBuilder on the PreVisit pass for the Expr. + std::optional GetResolvedType(int64_t expr_id) const { + auto it = resolved_types_.find(expr_id); + if (it != resolved_types_.end()) { + return it->second; + } + return std::nullopt; + } + + std::optional TryResolveType(int64_t expr_id, + const cel::TypeSpec& ast_type); + + void SetResolvedType(int64_t expr_id, cel::Type type) { + resolved_types_[expr_id] = std::move(type); + } + // Returns `true` if a message factory was explicitly provided during // planning. bool HasExplicitMessageFactory() const { return message_factory_ != nullptr; } @@ -427,8 +422,10 @@ class PlannerContext { cel::runtime_internal::IssueCollector& issue_collector_; ProgramBuilder& program_builder_; std::shared_ptr& arena_; - const bool explicit_arena_; const std::shared_ptr message_factory_; + const bool explicit_arena_; + bool resolve_types_; + absl::flat_hash_map resolved_types_; }; // Interface for Ast Transforms. diff --git a/eval/compiler/flat_expr_builder_extensions_test.cc b/eval/compiler/flat_expr_builder_extensions_test.cc index 45913e61b..661bd8953 100644 --- a/eval/compiler/flat_expr_builder_extensions_test.cc +++ b/eval/compiler/flat_expr_builder_extensions_test.cc @@ -14,22 +14,21 @@ #include "eval/compiler/flat_expr_builder_extensions.h" #include +#include #include -#include #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "common/expr.h" -#include "common/native_type.h" +#include "common/type.h" #include "common/value.h" #include "eval/compiler/resolver.h" -#include "eval/eval/const_value_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" #include "eval/eval/function_step.h" -#include "internal/status_macros.h" +#include "eval/eval/ident_step.h" #include "internal/testing.h" #include "runtime/function_registry.h" #include "runtime/internal/issue_collector.h" @@ -53,7 +52,15 @@ using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::runtime_internal::RuntimeEnv; using ::testing::ElementsAre; using ::testing::IsEmpty; -using ::testing::Optional; + +class TestStepLogic : public ExpressionStepLogic { + public: + void Evaluate(ExecutionFrame* frame) const override {} +}; + +std::unique_ptr MakeTestStepLogic() { + return std::make_unique(); +} using Subexpression = ProgramBuilder::Subexpression; @@ -78,13 +85,13 @@ class PlannerContextTest : public testing::Test { MATCHER_P(UniquePtrHolds, ptr, "") { const auto& got = arg; - return ptr == got.get(); + return got.IsGenericStep() && ptr == got.GetGenericStep(); } struct SimpleTreeSteps { - const ExpressionStep* a; - const ExpressionStep* b; - const ExpressionStep* c; + const ExpressionStepLogic* a; + const ExpressionStepLogic* b; + const ExpressionStepLogic* c; }; // simulate a program of: @@ -94,20 +101,23 @@ struct SimpleTreeSteps { absl::StatusOr InitSimpleTree( const Expr& a, const Expr& b, const Expr& c, ProgramBuilder& program_builder) { - CEL_ASSIGN_OR_RETURN(auto a_step, CreateConstValueStep(cel::NullValue(), -1)); - CEL_ASSIGN_OR_RETURN(auto b_step, CreateConstValueStep(cel::NullValue(), -1)); - CEL_ASSIGN_OR_RETURN(auto c_step, CreateConstValueStep(cel::NullValue(), -1)); + auto a_step = MakeTestStepLogic(); + auto b_step = MakeTestStepLogic(); + auto c_step = MakeTestStepLogic(); SimpleTreeSteps result{a_step.get(), b_step.get(), c_step.get()}; program_builder.EnterSubexpression(&a); program_builder.EnterSubexpression(&b); - program_builder.AddStep(std::move(b_step)); + program_builder.AddStep( + ExpressionStep::MakeGenericStep(std::move(b_step), -1)); program_builder.ExitSubexpression(&b); program_builder.EnterSubexpression(&c); - program_builder.AddStep(std::move(c_step)); + program_builder.AddStep( + ExpressionStep::MakeGenericStep(std::move(c_step), -1)); program_builder.ExitSubexpression(&c); - program_builder.AddStep(std::move(a_step)); + program_builder.AddStep( + ExpressionStep::MakeGenericStep(std::move(a_step), -1)); program_builder.ExitSubexpression(&a); return result; @@ -136,7 +146,7 @@ TEST_F(PlannerContextTest, GetPlan) { UniquePtrHolds(step_ptrs.a))); Expr d; - EXPECT_FALSE(context.IsSubplanInspectable(d)); + EXPECT_FALSE(context.NodeHasInspectablePlan(d)); EXPECT_THAT(context.GetSubplan(d), IsEmpty()); } @@ -160,10 +170,9 @@ TEST_F(PlannerContextTest, ReplacePlan) { ExecutionPath new_a; - ASSERT_OK_AND_ASSIGN(auto new_a_step, - CreateConstValueStep(cel::NullValue(), -1)); - const ExpressionStep* new_a_step_ptr = new_a_step.get(); - new_a.push_back(std::move(new_a_step)); + auto new_a_step = MakeTestStepLogic(); + const ExpressionStepLogic* new_a_step_ptr = new_a_step.get(); + new_a.push_back(ExpressionStep::MakeGenericStep(std::move(new_a_step), -1)); ASSERT_THAT(context.ReplaceSubplan(a, std::move(new_a)), IsOk()); @@ -186,8 +195,8 @@ TEST_F(PlannerContextTest, ExtractPlan) { type_registry_.GetComposedTypeProvider(), issue_collector_, program_builder, arena); - EXPECT_TRUE(context.IsSubplanInspectable(a)); - EXPECT_TRUE(context.IsSubplanInspectable(b)); + EXPECT_TRUE(context.NodeHasInspectablePlan(a)); + EXPECT_TRUE(context.NodeHasInspectablePlan(b)); ASSERT_OK_AND_ASSIGN(ExecutionPath extracted, context.ExtractSubplan(b)); @@ -226,7 +235,7 @@ TEST_F(PlannerContextTest, ReplacePlanUpdatesParent) { type_registry_.GetComposedTypeProvider(), issue_collector_, program_builder, arena); - EXPECT_TRUE(context.IsSubplanInspectable(a)); + EXPECT_TRUE(context.NodeHasInspectablePlan(a)); ASSERT_THAT(context.ReplaceSubplan(c, {}), IsOk()); @@ -251,14 +260,12 @@ TEST_F(PlannerContextTest, ReplacePlanUpdatesSibling) { ExecutionPath new_b; - ASSERT_OK_AND_ASSIGN(auto b1_step, - CreateConstValueStep(cel::NullValue(), -1)); - const ExpressionStep* b1_step_ptr = b1_step.get(); - new_b.push_back(std::move(b1_step)); - ASSERT_OK_AND_ASSIGN(auto b2_step, - CreateConstValueStep(cel::NullValue(), -1)); - const ExpressionStep* b2_step_ptr = b2_step.get(); - new_b.push_back(std::move(b2_step)); + auto b1_step = MakeTestStepLogic(); + const ExpressionStepLogic* b1_step_ptr = b1_step.get(); + new_b.push_back(ExpressionStep::MakeGenericStep(std::move(b1_step), -1)); + auto b2_step = MakeTestStepLogic(); + const ExpressionStepLogic* b2_step_ptr = b2_step.get(); + new_b.push_back(ExpressionStep::MakeGenericStep(std::move(b2_step), -1)); ASSERT_THAT(context.ReplaceSubplan(b, std::move(new_b)), IsOk()); @@ -302,10 +309,9 @@ TEST_F(PlannerContextTest, AddSubplanStep) { ASSERT_OK_AND_ASSIGN(auto plan_steps, InitSimpleTree(a, b, c, program_builder)); - ASSERT_OK_AND_ASSIGN(auto b2_step, - CreateConstValueStep(cel::NullValue(), -1)); + auto b2_step = MakeTestStepLogic(); - const ExpressionStep* b2_step_ptr = b2_step.get(); + const ExpressionStepLogic* b2_step_ptr = b2_step.get(); std::shared_ptr arena; PlannerContext context(env_, resolver_, options_, @@ -332,8 +338,7 @@ TEST_F(PlannerContextTest, AddSubplanStepFailsOnUnknownNode) { ASSERT_THAT(InitSimpleTree(a, b, c, program_builder).status(), IsOk()); - ASSERT_OK_AND_ASSIGN(auto b2_step, - CreateConstValueStep(cel::NullValue(), -1)); + auto b2_step = MakeTestStepLogic(); std::shared_ptr arena; PlannerContext context(env_, resolver_, options_, @@ -346,6 +351,23 @@ TEST_F(PlannerContextTest, AddSubplanStepFailsOnUnknownNode) { StatusIs(absl::StatusCode::kInternal)); } +TEST_F(PlannerContextTest, ResolvedTypes) { + ProgramBuilder program_builder; + std::shared_ptr arena; + options_.enable_typed_field_access = false; + PlannerContext context(env_, resolver_, options_, + type_registry_.GetComposedTypeProvider(), + issue_collector_, program_builder, arena); + + EXPECT_FALSE(context.resolve_types()); + context.set_resolve_types(); + EXPECT_TRUE(context.resolve_types()); + + EXPECT_EQ(context.GetResolvedType(1), std::nullopt); + context.SetResolvedType(1, cel::IntType()); + EXPECT_THAT(context.GetResolvedType(1), testing::Optional(cel::IntType())); +} + class ProgramBuilderTest : public testing::Test { public: ProgramBuilderTest() : type_registry_(), function_registry_() {} @@ -480,8 +502,9 @@ TEST_F(ProgramBuilderTest, ExtractWorks) { program_builder.EnterSubexpression(&b); program_builder.ExitSubexpression(&b); - ASSERT_OK_AND_ASSIGN(auto a_step, CreateConstValueStep(cel::NullValue(), -1)); - program_builder.AddStep(std::move(a_step)); + auto a_step = MakeTestStepLogic(); + program_builder.AddStep( + ExpressionStep::MakeGenericStep(std::move(a_step), -1)); program_builder.EnterSubexpression(&c); program_builder.ExitSubexpression(&c); program_builder.ExitSubexpression(&a); @@ -518,53 +541,99 @@ TEST_F(ProgramBuilderTest, ExtractToRequiresFlatten) { UniquePtrHolds(step_ptrs.a))); } -TEST_F(ProgramBuilderTest, Recursive) { +TEST_F(ProgramBuilderTest, IsInlinableAndMaybeInlineConstant) { Expr a; Expr b; - Expr c; - ProgramBuilder program_builder; program_builder.EnterSubexpression(&a); program_builder.EnterSubexpression(&b); - program_builder.current()->set_recursive_program( - CreateConstValueDirectStep(cel::NullValue()), 1); + program_builder.AddStep(ExpressionStep::MakeConstant(cel::IntValue(42), 10)); program_builder.ExitSubexpression(&b); - program_builder.EnterSubexpression(&c); - program_builder.current()->set_recursive_program( - CreateConstValueDirectStep(cel::NullValue()), 1); - program_builder.ExitSubexpression(&c); + program_builder.ExitSubexpression(&a); + + Subexpression* subexpr_a = program_builder.GetSubexpression(&a); + Subexpression* subexpr_b = program_builder.GetSubexpression(&b); + ASSERT_TRUE(subexpr_a != nullptr); + ASSERT_TRUE(subexpr_b != nullptr); - ASSERT_FALSE(program_builder.current()->IsFlattened()); - ASSERT_FALSE(program_builder.current()->IsRecursive()); - ASSERT_TRUE(program_builder.GetSubexpression(&b)->IsRecursive()); - ASSERT_TRUE(program_builder.GetSubexpression(&c)->IsRecursive()); + // Unflattened direct constant step and nested single-child subexpression. + EXPECT_TRUE(subexpr_b->IsInlinable()); + EXPECT_TRUE(subexpr_a->IsInlinable()); - EXPECT_EQ(program_builder.GetSubexpression(&b)->recursive_program().depth, 1); - EXPECT_EQ(program_builder.GetSubexpression(&c)->recursive_program().depth, 1); + std::optional inlined_b = subexpr_b->MaybeInline(99); + ASSERT_TRUE(inlined_b.has_value()); + EXPECT_EQ(inlined_b->id(), 99); + cel::Value val; + ASSERT_TRUE(GetIfConstant(*inlined_b, val)); + EXPECT_EQ(val.GetInt().NativeValue(), 42); - cel::CallExpr call_expr; - call_expr.set_function("_==_"); - call_expr.mutable_args().emplace_back(); - call_expr.mutable_args().emplace_back(); + // Flattened constant step remains inlinable. + subexpr_a->Flatten(); + EXPECT_TRUE(subexpr_a->IsInlinable()); + std::optional inlined_a = subexpr_a->MaybeInline(100); + ASSERT_TRUE(inlined_a.has_value()); + EXPECT_EQ(inlined_a->id(), 100); + ASSERT_TRUE(GetIfConstant(*inlined_a, val)); + EXPECT_EQ(val.GetInt().NativeValue(), 42); +} - auto max_depth = program_builder.current()->RecursiveDependencyDepth(); +TEST_F(ProgramBuilderTest, IsInlinableAndMaybeInlineIdentifier) { + Expr a; + ProgramBuilder program_builder; - EXPECT_THAT(max_depth, Optional(1)); + program_builder.EnterSubexpression(&a); + program_builder.AddStep(ExpressionStep::MakeIdentifierStep(3, 7, 10)); + program_builder.ExitSubexpression(&a); - auto deps = program_builder.current()->ExtractRecursiveDependencies(); + Subexpression* subexpr_a = program_builder.GetSubexpression(&a); + ASSERT_TRUE(subexpr_a != nullptr); - program_builder.current()->set_recursive_program( - CreateDirectFunctionStep(-1, call_expr, std::move(deps), {}), - *max_depth + 1); + EXPECT_TRUE(subexpr_a->IsInlinable()); + std::optional inlined = subexpr_a->MaybeInline(55); + ASSERT_TRUE(inlined.has_value()); + EXPECT_EQ(inlined->id(), 55); + const IdentStepInfo* ident_info = GetIfIdentStep(*inlined); + ASSERT_TRUE(ident_info != nullptr); + EXPECT_EQ(ident_info->ident_index, 3); + EXPECT_EQ(ident_info->slot_index, 7); - program_builder.ExitSubexpression(&a); + subexpr_a->Flatten(); + EXPECT_TRUE(subexpr_a->IsInlinable()); + std::optional inlined_flat = subexpr_a->MaybeInline(56); + ASSERT_TRUE(inlined_flat.has_value()); + EXPECT_EQ(inlined_flat->id(), 56); + ident_info = GetIfIdentStep(*inlined_flat); + ASSERT_TRUE(ident_info != nullptr); + EXPECT_EQ(ident_info->ident_index, 3); + EXPECT_EQ(ident_info->slot_index, 7); +} + +TEST_F(ProgramBuilderTest, NotInlinableWhenNonTrivial) { + Expr a; + Expr b; + Expr c; + ProgramBuilder program_builder; - auto path = program_builder.FlattenMain(); + ASSERT_OK_AND_ASSIGN(SimpleTreeSteps step_ptrs, + InitSimpleTree(a, b, c, program_builder)); + (void)step_ptrs; - ASSERT_THAT(path, testing::SizeIs(1)); - EXPECT_TRUE(path[0]->GetNativeTypeId() == - cel::NativeTypeId::For()); + Subexpression* subexpr_a = program_builder.GetSubexpression(&a); + Subexpression* subexpr_b = program_builder.GetSubexpression(&b); + ASSERT_TRUE(subexpr_a != nullptr); + ASSERT_TRUE(subexpr_b != nullptr); + + // Single generic step is not inlinable. + EXPECT_FALSE(subexpr_b->IsInlinable()); + EXPECT_EQ(subexpr_b->MaybeInline(1), std::nullopt); + + // Multi-step subexpression is not inlinable (unflattened and flattened). + EXPECT_FALSE(subexpr_a->IsInlinable()); + EXPECT_EQ(subexpr_a->MaybeInline(1), std::nullopt); + subexpr_a->Flatten(); + EXPECT_FALSE(subexpr_a->IsInlinable()); + EXPECT_EQ(subexpr_a->MaybeInline(1), std::nullopt); } } // namespace diff --git a/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc b/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc index afe7c5f9f..9081b4813 100644 --- a/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc +++ b/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc @@ -1,122 +1,86 @@ // A collection of tests that confirm that short-circuit and non-short-circuit // produce expressions with the same outputs. #include +#include +#include +#include "absl/log/absl_check.h" #include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "absl/strings/substitute.h" -#include "base/builtins.h" #include "eval/compiler/cel_expression_builder_flat_impl.h" #include "eval/public/activation.h" +#include "eval/public/builtin_func_registrar.h" #include "eval/public/cel_attribute.h" #include "eval/public/cel_expression.h" #include "eval/public/cel_value.h" #include "eval/public/unknown_attribute_set.h" #include "eval/public/unknown_set.h" #include "internal/testing.h" +#include "parser/options.h" +#include "parser/parser.h" #include "runtime/internal/runtime_env_testing.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" -#include "google/protobuf/text_format.h" namespace google::api::expr::runtime { namespace { +using ::absl_testing::IsOk; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::expr::Expr; +using ::google::api::expr::parser::Parse; using ::testing::Eq; using ::testing::SizeIs; -constexpr char kTwoLogicalOp[] = R"cel( -id: 1 -call_expr { - function: "$0" - args { - id: 2 - ident_expr { - name: "var1", - } - } - args { - id: 3 - call_expr { - function: "$0" - args { - id: 4 - ident_expr { - name: "var2" - } - } - args { - id: 5 - ident_expr { - name: "var3" - } - } - } - } -} -)cel"; - -constexpr char kTernaryExpr[] = R"cel( -id: 1 -call_expr { - function: "_?_:_" - args { - id: 2 - ident_expr { - name: "cond" - } - } - args { - id: 3 - ident_expr { - name: "arg1" - } - } - args { - id: 4 - ident_expr { - name: "arg2" - } - } -})cel"; - void BuildAndEval(CelExpressionBuilder* builder, const Expr& expr, const Activation& activation, google::protobuf::Arena* arena, CelValue* result) { - ASSERT_OK_AND_ASSIGN(auto expression, + ASSERT_OK_AND_ASSIGN(std::unique_ptr expression, builder->CreateExpression(&expr, nullptr)); - auto value = expression->Evaluate(activation, arena); - ASSERT_OK(value); + ASSERT_OK_AND_ASSIGN(CelValue value, expression->Evaluate(activation, arena)); - *result = *value; + *result = value; } -class ShortCircuitingTest : public testing::TestWithParam { +class ShortCircuitingTest + : public testing::TestWithParam> { public: + bool short_circuiting() const { return std::get<0>(GetParam()); } + bool enable_variadic() const { return std::get<1>(GetParam()); } + std::unique_ptr GetBuilder( - bool enable_unknowns = false) { + bool enable_unknowns = false, + bool enable_comprehension_list_append = false) { cel::RuntimeOptions options; - options.short_circuiting = GetParam(); + options.short_circuiting = short_circuiting(); + options.enable_comprehension_list_append = enable_comprehension_list_append; if (enable_unknowns) { options.unknown_processing = cel::UnknownProcessingOptions::kAttributeAndFunction; } auto result = std::make_unique( NewTestingRuntimeEnv(), options); + ABSL_CHECK_OK(RegisterBuiltinFunctions(result->GetRegistry())); return result; } + + Expr ParseExpr(absl::string_view expression) { + cel::ParserOptions options; + options.enable_variadic_logical_operators = enable_variadic(); + auto parsed_expr = Parse(expression, "", options); + ABSL_CHECK_OK(parsed_expr.status()); + return parsed_expr->expr(); + } }; TEST_P(ShortCircuitingTest, BasicAnd) { - Expr expr; + Expr expr = ParseExpr("var1 && var2 && var3"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( - absl::Substitute(kTwoLogicalOp, ::cel::builtin::kAnd), &expr)); auto builder = GetBuilder(); activation.InsertValue("var1", CelValue::CreateBool(true)); @@ -140,11 +104,9 @@ TEST_P(ShortCircuitingTest, BasicAnd) { } TEST_P(ShortCircuitingTest, BasicOr) { - Expr expr; + Expr expr = ParseExpr("var1 || var2 || var3"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( - absl::Substitute(kTwoLogicalOp, ::cel::builtin::kOr), &expr)); auto builder = GetBuilder(); activation.InsertValue("var1", CelValue::CreateBool(false)); @@ -168,11 +130,9 @@ TEST_P(ShortCircuitingTest, BasicOr) { } TEST_P(ShortCircuitingTest, ErrorAnd) { - Expr expr; + Expr expr = ParseExpr("var1 && var2 && var3"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( - absl::Substitute(kTwoLogicalOp, ::cel::builtin::kAnd), &expr)); auto builder = GetBuilder(); absl::Status error = absl::InternalError("error"); @@ -198,11 +158,9 @@ TEST_P(ShortCircuitingTest, ErrorAnd) { } TEST_P(ShortCircuitingTest, ErrorOr) { - Expr expr; + Expr expr = ParseExpr("var1 || var2 || var3"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( - absl::Substitute(kTwoLogicalOp, ::cel::builtin::kOr), &expr)); auto builder = GetBuilder(); absl::Status error = absl::InternalError("error"); @@ -228,15 +186,15 @@ TEST_P(ShortCircuitingTest, ErrorOr) { } TEST_P(ShortCircuitingTest, UnknownAnd) { - Expr expr; + Expr expr = ParseExpr("var1 && var2 && var3"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( - absl::Substitute(kTwoLogicalOp, ::cel::builtin::kAnd), &expr)); auto builder = GetBuilder(/* enable_unknowns=*/true); absl::Status error = absl::InternalError("error"); - activation.set_unknown_attribute_patterns({CelAttributePattern("var1", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern("var1", {})}), + IsOk()); activation.InsertValue("var2", CelValue::CreateError(&error)); activation.InsertValue("var3", CelValue::CreateBool(false)); @@ -260,15 +218,15 @@ TEST_P(ShortCircuitingTest, UnknownAnd) { } TEST_P(ShortCircuitingTest, UnknownOr) { - Expr expr; + Expr expr = ParseExpr("var1 || var2 || var3"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( - absl::Substitute(kTwoLogicalOp, ::cel::builtin::kOr), &expr)); auto builder = GetBuilder(/* enable_unknowns=*/true); absl::Status error = absl::InternalError("error"); - activation.set_unknown_attribute_patterns({CelAttributePattern("var1", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern("var1", {})}), + IsOk()); activation.InsertValue("var2", CelValue::CreateError(&error)); activation.InsertValue("var3", CelValue::CreateBool(true)); @@ -292,10 +250,9 @@ TEST_P(ShortCircuitingTest, UnknownOr) { } TEST_P(ShortCircuitingTest, BasicTernary) { - Expr expr; + Expr expr = ParseExpr("cond ? arg1 : arg2"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(kTernaryExpr, &expr)); auto builder = GetBuilder(); activation.InsertValue("cond", CelValue::CreateBool(true)); @@ -319,10 +276,9 @@ TEST_P(ShortCircuitingTest, BasicTernary) { } TEST_P(ShortCircuitingTest, TernaryErrorHandling) { - Expr expr; + Expr expr = ParseExpr("cond ? arg1 : arg2"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(kTernaryExpr, &expr)); auto builder = GetBuilder(); absl::Status error1 = absl::InternalError("error1"); @@ -349,10 +305,9 @@ TEST_P(ShortCircuitingTest, TernaryErrorHandling) { } TEST_P(ShortCircuitingTest, TernaryUnknownCondHandling) { - Expr expr; + Expr expr = ParseExpr("cond ? arg1 : arg2"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(kTernaryExpr, &expr)); auto builder = GetBuilder(/*enable_unknowns=*/true); absl::Status error = absl::InternalError("error1"); @@ -361,7 +316,9 @@ TEST_P(ShortCircuitingTest, TernaryUnknownCondHandling) { activation.InsertValue("arg1", CelValue::CreateError(&error)); activation.InsertValue("arg2", CelValue::CreateInt64(-1)); - activation.set_unknown_attribute_patterns({CelAttributePattern("cond", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern("cond", {})}), + IsOk()); CelValue result; ASSERT_NO_FATAL_FAILURE( @@ -373,9 +330,11 @@ TEST_P(ShortCircuitingTest, TernaryUnknownCondHandling) { EXPECT_THAT(attrs.begin()->variable_name(), Eq("cond")); // Unknown branches are discarded if condition is unknown - activation.set_unknown_attribute_patterns({CelAttributePattern("cond", {}), - CelAttributePattern("arg1", {}), - CelAttributePattern("arg2", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern("cond", {}), + CelAttributePattern("arg1", {}), + CelAttributePattern("arg2", {})}), + IsOk()); ASSERT_NO_FATAL_FAILURE( BuildAndEval(builder.get(), expr, activation, &arena, &result)); @@ -386,10 +345,9 @@ TEST_P(ShortCircuitingTest, TernaryUnknownCondHandling) { } TEST_P(ShortCircuitingTest, TernaryUnknownArgsHandling) { - Expr expr; + Expr expr = ParseExpr("cond ? arg1 : arg2"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(kTernaryExpr, &expr)); auto builder = GetBuilder(/*enable_unknowns=*/true); absl::Status error = absl::InternalError("error1"); @@ -399,7 +357,9 @@ TEST_P(ShortCircuitingTest, TernaryUnknownArgsHandling) { activation.InsertValue("arg2", CelValue::CreateInt64(-1)); // Unknown arg is discarded if condition chooses other branch. - activation.set_unknown_attribute_patterns({CelAttributePattern("arg1", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern("arg1", {})}), + IsOk()); CelValue result; @@ -409,8 +369,10 @@ TEST_P(ShortCircuitingTest, TernaryUnknownArgsHandling) { EXPECT_EQ(result.Int64OrDie(), -1); // Branches won't merge if both are unknown. - activation.set_unknown_attribute_patterns( - {CelAttributePattern("arg1", {}), CelAttributePattern("arg2", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns( + {CelAttributePattern("arg1", {}), CelAttributePattern("arg2", {})}), + IsOk()); ASSERT_NO_FATAL_FAILURE( BuildAndEval(builder.get(), expr, activation, &arena, &result)); @@ -421,10 +383,9 @@ TEST_P(ShortCircuitingTest, TernaryUnknownArgsHandling) { } TEST_P(ShortCircuitingTest, TernaryUnknownAndErrorHandling) { - Expr expr; + Expr expr = ParseExpr("cond ? arg1 : arg2"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(kTernaryExpr, &expr)); auto builder = GetBuilder(/*enable_unknowns=*/true); absl::Status error = absl::InternalError("error1"); @@ -434,8 +395,10 @@ TEST_P(ShortCircuitingTest, TernaryUnknownAndErrorHandling) { activation.InsertValue("arg2", CelValue::CreateInt64(-1)); // Error cond discards args - activation.set_unknown_attribute_patterns( - {CelAttributePattern("arg1", {}), CelAttributePattern("arg2", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns( + {CelAttributePattern("arg1", {}), CelAttributePattern("arg2", {})}), + IsOk()); CelValue result; @@ -445,7 +408,9 @@ TEST_P(ShortCircuitingTest, TernaryUnknownAndErrorHandling) { EXPECT_EQ(*result.ErrorOrDie(), error); // Error arg discarded if condition unknown - activation.set_unknown_attribute_patterns({CelAttributePattern("cond", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern("cond", {})}), + IsOk()); ASSERT_TRUE(activation.RemoveValueEntry("arg1")); activation.InsertValue("arg1", CelValue::CreateError(&error)); @@ -457,16 +422,79 @@ TEST_P(ShortCircuitingTest, TernaryUnknownAndErrorHandling) { EXPECT_EQ(attrs.begin()->variable_name(), "cond"); } -const char* TestName(testing::TestParamInfo info) { - if (info.param) { - return "short_circuit_enabled"; - } else { - return "short_circuit_disabled"; +TEST_P(ShortCircuitingTest, FilterComprehension) { + Expr expr = ParseExpr("[1, 2, 3, 4].filter(x, x > 2) == [3, 4]"); + Expr empty_expr = ParseExpr("[1, 2, 3].filter(x, false) == []"); + Activation activation; + google::protobuf::Arena arena; + + for (bool enable_list_append : {false, true}) { + std::unique_ptr builder = + GetBuilder(/*enable_unknowns=*/false, enable_list_append); + + CelValue result; + ASSERT_NO_FATAL_FAILURE( + BuildAndEval(builder.get(), expr, activation, &arena, &result)); + ASSERT_TRUE(result.IsBool()) << result.DebugString(); + EXPECT_TRUE(result.BoolOrDie()); + + ASSERT_NO_FATAL_FAILURE( + BuildAndEval(builder.get(), empty_expr, activation, &arena, &result)); + ASSERT_TRUE(result.IsBool()) << result.DebugString(); + EXPECT_TRUE(result.BoolOrDie()); + } +} + +TEST_P(ShortCircuitingTest, MapWithFilterComprehension) { + Expr expr = ParseExpr("[1, 2, 3, 4].map(x, x % 2 == 1, x * 10) == [10, 30]"); + Expr empty_expr = ParseExpr("[1, 2, 3].map(x, false, x * 10) == []"); + Activation activation; + google::protobuf::Arena arena; + + for (bool enable_list_append : {false, true}) { + std::unique_ptr builder = + GetBuilder(/*enable_unknowns=*/false, enable_list_append); + + CelValue result; + ASSERT_NO_FATAL_FAILURE( + BuildAndEval(builder.get(), expr, activation, &arena, &result)); + ASSERT_TRUE(result.IsBool()) << result.DebugString(); + EXPECT_TRUE(result.BoolOrDie()); + + ASSERT_NO_FATAL_FAILURE( + BuildAndEval(builder.get(), empty_expr, activation, &arena, &result)); + ASSERT_TRUE(result.IsBool()) << result.DebugString(); + EXPECT_TRUE(result.BoolOrDie()); } } +TEST_P(ShortCircuitingTest, MapComprehension) { + Expr expr = ParseExpr("[1, 2, 3].map(x, x * 10) == [10, 20, 30]"); + Activation activation; + google::protobuf::Arena arena; + + for (bool enable_list_append : {false, true}) { + std::unique_ptr builder = + GetBuilder(/*enable_unknowns=*/false, enable_list_append); + + CelValue result; + ASSERT_NO_FATAL_FAILURE( + BuildAndEval(builder.get(), expr, activation, &arena, &result)); + ASSERT_TRUE(result.IsBool()) << result.DebugString(); + EXPECT_TRUE(result.BoolOrDie()); + } +} + +std::string TestName(testing::TestParamInfo> info) { + return absl::StrCat( + std::get<0>(info.param) ? "short_circuit_enabled" + : "short_circuit_disabled", + "_", std::get<1>(info.param) ? "variadic_enabled" : "variadic_disabled"); +} + INSTANTIATE_TEST_SUITE_P(Test, ShortCircuitingTest, - testing::Values(false, true), &TestName); + testing::Combine(testing::Bool(), testing::Bool()), + &TestName); } // namespace diff --git a/eval/compiler/flat_expr_builder_test.cc b/eval/compiler/flat_expr_builder_test.cc index 5fc20f01e..a15a2e4f1 100644 --- a/eval/compiler/flat_expr_builder_test.cc +++ b/eval/compiler/flat_expr_builder_test.cc @@ -16,16 +16,16 @@ #include #include +#include #include +#include #include #include #include #include "cel/expr/checked.pb.h" #include "cel/expr/syntax.pb.h" -#include "google/protobuf/field_mask.pb.h" #include "google/protobuf/descriptor.pb.h" -#include "absl/base/nullability.h" #include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" @@ -36,16 +36,30 @@ #include "absl/time/time.h" #include "absl/types/span.h" #include "base/builtins.h" +#include "checker/validation_result.h" +#include "common/ast.h" +#include "common/decl.h" +#include "common/expr.h" #include "common/function_descriptor.h" #include "common/kind.h" +#include "common/type.h" #include "common/value.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" #include "eval/compiler/cel_expression_builder_flat_impl.h" #include "eval/compiler/constant_folding.h" +#include "eval/compiler/flat_expr_builder_extensions.h" #include "eval/compiler/qualified_reference_resolver.h" +#include "eval/eval/cel_expression_flat_impl.h" +#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" +#include "eval/eval/ident_step.h" #include "eval/public/activation.h" +#include "eval/public/ast_rewrite.h" +#include "eval/public/base_activation.h" #include "eval/public/builtin_func_registrar.h" #include "eval/public/cel_attribute.h" -#include "eval/public/cel_builtins.h" #include "eval/public/cel_expr_builder_factory.h" #include "eval/public/cel_expression.h" #include "eval/public/cel_function.h" @@ -55,15 +69,18 @@ #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_map_impl.h" #include "eval/public/portable_cel_function_adapter.h" +#include "eval/public/source_position.h" #include "eval/public/structs/cel_proto_descriptor_pool_builder.h" #include "eval/public/structs/cel_proto_wrapper.h" #include "eval/public/testing/matchers.h" #include "eval/public/unknown_attribute_set.h" #include "eval/public/unknown_set.h" #include "eval/testutil/test_message.pb.h" +#include "extensions/protobuf/ast_converters.h" #include "internal/proto_matchers.h" #include "internal/status_macros.h" #include "internal/testing.h" +#include "parser/options.h" #include "parser/parser.h" #include "runtime/function.h" #include "runtime/function_adapter.h" @@ -191,7 +208,7 @@ TEST(FlatExprBuilderTest, RuntimeExtensionsError) { auto* ext = source_info.add_extensions(); ext->set_id("ext1"); ext->add_affected_components( - cel::expr::SourceInfo_Extension_Component_COMPONENT_RUNTIME); + cel::expr::SourceInfo::Extension::COMPONENT_RUNTIME); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); EXPECT_THAT(builder.CreateExpression(&expr, &source_info).status(), @@ -199,6 +216,51 @@ TEST(FlatExprBuilderTest, RuntimeExtensionsError) { HasSubstr("unsupported CEL extension: ext1"))); } +TEST(FlatExprBuilderTest, SelectOptimizationRuntimeExtension) { + Expr expr; + expr.mutable_const_expr()->set_bool_value(true); + auto create = [&](CelExpressionBuilder& builder, absl::string_view id, + int major, int minor) { + SourceInfo source_info; + auto* ext = source_info.add_extensions(); + ext->set_id(id); + ext->mutable_version()->set_major(major); + ext->mutable_version()->set_minor(minor); + ext->add_affected_components(SourceInfo::Extension::COMPONENT_RUNTIME); + return builder.CreateExpression(&expr, &source_info).status(); + }; + + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); + // Other program optimizers don't imply select optimization support. + builder.flat_expr_builder().AddProgramOptimizer( + [](auto&, const auto&) { return nullptr; }); + EXPECT_THAT( + create(builder, "select_optimization", 1, 0), + StatusIs( + absl::StatusCode::kInvalidArgument, + HasSubstr("unsupported CEL extension: select_optimization@1.0"))); + + builder.flat_expr_builder().enable_select_optimization(); + EXPECT_THAT(create(builder, "select_optimization", 1, 0), IsOk()); + // Minor versions are informational (see SourceInfo.Extension.Version). + EXPECT_THAT(create(builder, "select_optimization", 1, 1), IsOk()); + EXPECT_THAT(create(builder, "select_optimization", 2, 0), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(create(builder, "ext1", 1, 0), + StatusIs(absl::StatusCode::kInvalidArgument)); + + // The legacy API also plans the 3-arg form under default fail_on_warnings. + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed, + parser::Parse("f(x, [[1, 'field', 5]], 0)")); + parsed.mutable_expr()->mutable_call_expr()->set_function("cel.@attribute"); + expr = parsed.expr(); + InterpreterOptions options; + options.enable_select_optimization = true; + EXPECT_THAT( + create(*CreateCelExpressionBuilder(options), "select_optimization", 1, 0), + IsOk()); +} + TEST(FlatExprBuilderTest, ConstValueUnset) { Expr expr; SourceInfo source_info; @@ -255,7 +317,7 @@ TEST(FlatExprBuilderTest, BinaryCallTooManyArguments) { CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); auto* call = expr.mutable_call_expr(); - call->set_function(builtin::kAnd); + call->set_function(cel::builtin::kAnd); call->mutable_target()->mutable_const_expr()->set_string_value("random"); call->add_args()->mutable_const_expr()->set_bool_value(false); call->add_args()->mutable_const_expr()->set_bool_value(true); @@ -269,7 +331,7 @@ TEST(FlatExprBuilderTest, TernaryCallTooManyArguments) { Expr expr; SourceInfo source_info; auto* call = expr.mutable_call_expr(); - call->set_function(builtin::kTernary); + call->set_function(cel::builtin::kTernary); call->mutable_target()->mutable_const_expr()->set_string_value("random"); call->add_args()->mutable_const_expr()->set_bool_value(false); call->add_args()->mutable_const_expr()->set_int64_value(1); @@ -469,7 +531,7 @@ TEST(FlatExprBuilderTest, IdentExprUnsetName) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"(ident_expr {})", &expr); + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"(ident_expr {})", &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -482,10 +544,10 @@ TEST(FlatExprBuilderTest, SelectExprUnsetField) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"(select_expr{ + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"(select_expr{ operand{ ident_expr {name: 'var'} } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -498,11 +560,11 @@ TEST(FlatExprBuilderTest, SelectExprUnsetOperand) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"(select_expr{ + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"(select_expr{ field: 'field' operand { id: 1 } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -515,7 +577,8 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetAccuVar) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"(comprehension_expr{})", &expr); + ASSERT_TRUE( + google::protobuf::TextFormat::ParseFromString(R"(comprehension_expr{})", &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); EXPECT_THAT(builder.CreateExpression(&expr, &source_info).status(), @@ -527,10 +590,10 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetIterVar) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( comprehension_expr{accu_var: "a"} )", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); EXPECT_THAT(builder.CreateExpression(&expr, &source_info).status(), @@ -542,12 +605,12 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetAccuInit) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( comprehension_expr{ accu_var: "a" iter_var: "b"} )", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); EXPECT_THAT(builder.CreateExpression(&expr, &source_info).status(), @@ -559,7 +622,7 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetLoopCondition) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( comprehension_expr{ accu_var: 'a' iter_var: 'b' @@ -567,7 +630,7 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetLoopCondition) { const_expr {bool_value: true} }} )", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); EXPECT_THAT(builder.CreateExpression(&expr, &source_info).status(), @@ -579,7 +642,7 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetLoopStep) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( comprehension_expr{ accu_var: 'a' iter_var: 'b' @@ -590,7 +653,7 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetLoopStep) { const_expr {bool_value: true} }} )", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); EXPECT_THAT(builder.CreateExpression(&expr, &source_info).status(), @@ -602,7 +665,7 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetResult) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( comprehension_expr{ accu_var: 'a' iter_var: 'b' @@ -616,7 +679,7 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetResult) { const_expr {bool_value: false} }} )", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); EXPECT_THAT(builder.CreateExpression(&expr, &source_info).status(), @@ -628,7 +691,7 @@ TEST(FlatExprBuilderTest, MapComprehension) { Expr expr; SourceInfo source_info; // {1: "", 2: ""}.all(x, x > 0) - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( comprehension_expr { iter_var: "k" accu_var: "accu" @@ -665,7 +728,7 @@ TEST(FlatExprBuilderTest, MapComprehension) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -683,7 +746,7 @@ TEST(FlatExprBuilderTest, InvalidContainer) { Expr expr; SourceInfo source_info; // foo && bar - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( call_expr { function: "_&&_" args { @@ -697,7 +760,7 @@ TEST(FlatExprBuilderTest, InvalidContainer) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -909,7 +972,7 @@ TEST(FlatExprBuilderTest, ParsedNamespacedFunctionSupportDisabled) { TEST(FlatExprBuilderTest, BasicCheckedExprSupport) { CheckedExpr expr; // foo && bar - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( expr { id: 1 call_expr { @@ -928,7 +991,7 @@ TEST(FlatExprBuilderTest, BasicCheckedExprSupport) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -946,7 +1009,7 @@ TEST(FlatExprBuilderTest, BasicCheckedExprSupport) { TEST(FlatExprBuilderTest, CheckedExprWithReferenceMap) { CheckedExpr expr; // `foo.var1` && `bar.var2` - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( reference_map { key: 2 value { @@ -988,7 +1051,7 @@ TEST(FlatExprBuilderTest, CheckedExprWithReferenceMap) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); builder.flat_expr_builder().AddAstTransform( @@ -1008,7 +1071,7 @@ TEST(FlatExprBuilderTest, CheckedExprWithReferenceMap) { TEST(FlatExprBuilderTest, CheckedExprWithReferenceMapFunction) { CheckedExpr expr; // ext.and(var1, bar.var2) - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( reference_map { key: 1 value { @@ -1057,7 +1120,7 @@ TEST(FlatExprBuilderTest, CheckedExprWithReferenceMapFunction) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); builder.flat_expr_builder().AddAstTransform( @@ -1082,7 +1145,7 @@ TEST(FlatExprBuilderTest, CheckedExprWithReferenceMapFunction) { TEST(FlatExprBuilderTest, CheckedExprActivationMissesReferences) { CheckedExpr expr; // && . - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( reference_map { key: 2 value { @@ -1125,7 +1188,7 @@ TEST(FlatExprBuilderTest, CheckedExprActivationMissesReferences) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); builder.flat_expr_builder().AddAstTransform( @@ -1149,8 +1212,8 @@ TEST(FlatExprBuilderTest, CheckedExprActivationMissesReferences) { std::vector> map_pairs{ {CelValue::CreateStringView("var2"), CelValue::CreateBool(false)}}; - std::unique_ptr map_value = - *CreateContainerBackedMap(absl::MakeSpan(map_pairs)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr map_value, + CreateContainerBackedMap(absl::MakeSpan(map_pairs))); activation.InsertValue("bar", CelValue::CreateMap(map_value.get())); ASSERT_OK_AND_ASSIGN(result, cel_expr->Evaluate(activation, &arena)); ASSERT_TRUE(result.IsBool()); @@ -1160,7 +1223,7 @@ TEST(FlatExprBuilderTest, CheckedExprActivationMissesReferences) { TEST(FlatExprBuilderTest, CheckedExprWithReferenceMapAndConstantFolding) { CheckedExpr expr; // {`var1`: 'hello'} - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( reference_map { key: 3 value { @@ -1190,7 +1253,7 @@ TEST(FlatExprBuilderTest, CheckedExprWithReferenceMapAndConstantFolding) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); builder.flat_expr_builder().AddAstTransform( @@ -1213,7 +1276,7 @@ TEST(FlatExprBuilderTest, ComprehensionWorksForError) { Expr expr; SourceInfo source_info; // {}[0].all(x, x) should evaluate OK but return an error value - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( id: 4 comprehension_expr { iter_var: "x" @@ -1278,7 +1341,7 @@ TEST(FlatExprBuilderTest, ComprehensionWorksForError) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -1295,7 +1358,7 @@ TEST(FlatExprBuilderTest, ComprehensionWorksForNonContainer) { Expr expr; SourceInfo source_info; // 0.all(x, x) should evaluate OK but return an error value. - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( id: 4 comprehension_expr { iter_var: "x" @@ -1349,7 +1412,7 @@ TEST(FlatExprBuilderTest, ComprehensionWorksForNonContainer) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -1721,7 +1784,7 @@ TEST(FlatExprBuilderTest, NameCollisionWithComprehensionVarLeadingDot) { TEST(FlatExprBuilderTest, MapFieldPresence) { Expr expr; SourceInfo source_info; - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( id: 1, select_expr{ operand { @@ -1731,7 +1794,7 @@ TEST(FlatExprBuilderTest, MapFieldPresence) { field: "string_int32_map" test_only: true })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_OK_AND_ASSIGN(auto cel_expr, @@ -1765,7 +1828,7 @@ TEST(FlatExprBuilderTest, MapFieldPresence) { TEST(FlatExprBuilderTest, RepeatedFieldPresence) { Expr expr; SourceInfo source_info; - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( id: 1, select_expr{ operand { @@ -1775,7 +1838,7 @@ TEST(FlatExprBuilderTest, RepeatedFieldPresence) { field: "int32_list" test_only: true })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_OK_AND_ASSIGN(auto cel_expr, @@ -1811,7 +1874,7 @@ absl::Status RunTernaryExpression(CelValue selector, CelValue value1, Expr expr; SourceInfo source_info; auto call_expr = expr.mutable_call_expr(); - call_expr->set_function(builtin::kTernary); + call_expr->set_function(cel::builtin::kTernary); auto arg0 = call_expr->add_args(); arg0->mutable_ident_expr()->set_name("selector"); @@ -1840,7 +1903,7 @@ TEST(FlatExprBuilderTest, Ternary) { Expr expr; SourceInfo source_info; auto call_expr = expr.mutable_call_expr(); - call_expr->set_function(builtin::kTernary); + call_expr->set_function(cel::builtin::kTernary); auto arg0 = call_expr->add_args(); arg0->mutable_ident_expr()->set_name("selector"); @@ -2820,6 +2883,7 @@ TEST(FlatExprBuilderTest, BlockNotListOfBoundExpressions) { TEST(FlatExprBuilderTest, BlockEmptyListOfBoundExpressions) { ParsedExpr parsed_expr; + // Allowed, but degenerate case. ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( R"pb( expr: { @@ -2835,10 +2899,8 @@ TEST(FlatExprBuilderTest, BlockEmptyListOfBoundExpressions) { CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); EXPECT_THAT( builder.CreateExpression(&parsed_expr.expr(), &parsed_expr.source_info()), - StatusIs( - absl::StatusCode::kInvalidArgument, - HasSubstr( - "malformed cel.@block: list of bound expressions is empty"))); + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("invalid @index greater than number of bindings:"))); } TEST(FlatExprBuilderTest, BlockOptionalListOfBoundExpressions) { @@ -2901,6 +2963,887 @@ TEST(FlatExprBuilderTest, BlockNested) { HasSubstr("multiple cel.@block are not allowed"))); } +struct VariadicLogicalEvalTestCase { + std::string label; + std::string expr; + std::string a_val; + std::string b_val; + std::string c_val; + std::string expected_type; // "bool", "error", "unknown" + bool expected_bool = false; +}; + +class FlatExprBuilderVariadicLogicalTest + : public testing::TestWithParam {}; + +TEST_P(FlatExprBuilderVariadicLogicalTest, Evaluate) { + const auto& test_case = GetParam(); + parser::ParserOptions parser_options; + parser_options.enable_variadic_logical_operators = true; + ASSERT_OK_AND_ASSIGN( + ParsedExpr parsed_expr, + parser::Parse(test_case.expr, test_case.label, parser_options)); + + cel::RuntimeOptions options; + options.unknown_processing = + cel::UnknownProcessingOptions::kAttributeAndFunction; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + ASSERT_OK_AND_ASSIGN(auto cel_expr, + builder.CreateExpression(&parsed_expr.expr(), + &parsed_expr.source_info())); + + Activation activation; + google::protobuf::Arena arena; + std::vector unknown_patterns; + + // Set up variables: + auto insert_value = [&](absl::string_view name, const std::string& val) { + if (val == "true") { + activation.InsertValue(name, CelValue::CreateBool(true)); + } else if (val == "false") { + activation.InsertValue(name, CelValue::CreateBool(false)); + } else if (val == "error") { + activation.InsertValue(name, CreateErrorValue(&arena, "test error")); + } else if (val == "unknown1" || val == "unknown2") { + activation.InsertValue(name, CelValue::CreateBool(true)); + unknown_patterns.push_back(CreateCelAttributePattern(name, {})); + } + }; + + insert_value("a", test_case.a_val); + insert_value("b", test_case.b_val); + insert_value("c", test_case.c_val); + + if (!unknown_patterns.empty()) { + ASSERT_THAT( + activation.SetUnknownAttributePatterns(std::move(unknown_patterns)), + IsOk()); + } + + ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr->Evaluate(activation, &arena)); + + if (test_case.expected_type == "bool") { + ASSERT_TRUE(result.IsBool()) << result.DebugString(); + EXPECT_EQ(result.BoolOrDie(), test_case.expected_bool); + } else if (test_case.expected_type == "error") { + EXPECT_TRUE(result.IsError()) << result.DebugString(); + } else if (test_case.expected_type == "unknown") { + EXPECT_TRUE(result.IsUnknownSet()) << result.DebugString(); + } +} + +INSTANTIATE_TEST_SUITE_P( + FlatExprBuilderVariadicLogicalTest, FlatExprBuilderVariadicLogicalTest, + testing::Values( + VariadicLogicalEvalTestCase{"AND_AllTrue", "a && b && c", "true", + "true", "true", "bool", true}, + VariadicLogicalEvalTestCase{"AND_ShortCircuitFalse", "a && b && c", + "true", "false", "unset", "bool", false}, + VariadicLogicalEvalTestCase{"AND_ShortCircuitFirstFalse", "a && b && c", + "false", "unset", "unset", "bool", false}, + VariadicLogicalEvalTestCase{"OR_AllFalse", "a || b || c", "false", + "false", "false", "bool", false}, + VariadicLogicalEvalTestCase{"OR_ShortCircuitTrue", "a || b || c", + "false", "true", "unset", "bool", true}, + VariadicLogicalEvalTestCase{"OR_ShortCircuitFirstTrue", "a || b || c", + "true", "unset", "unset", "bool", true}, + VariadicLogicalEvalTestCase{"AND_Error", "a && b && c", "true", "error", + "true", "error"}, + VariadicLogicalEvalTestCase{"AND_ShortCircuitBeforeError", + "a && b && c", "false", "error", "unset", + "bool", false}, + VariadicLogicalEvalTestCase{"OR_Error", "a || b || c", "false", "error", + "false", "error"}, + VariadicLogicalEvalTestCase{"OR_ShortCircuitBeforeError", "a || b || c", + "true", "error", "unset", "bool", true}, + VariadicLogicalEvalTestCase{"AND_Unknown", "a && b && c", "true", + "unknown1", "true", "unknown"}, + VariadicLogicalEvalTestCase{"AND_ShortCircuitBeforeUnknown", + "a && b && c", "false", "unknown1", "unset", + "bool", false}, + VariadicLogicalEvalTestCase{"OR_Unknown", "a || b || c", "false", + "unknown1", "false", "unknown"}, + VariadicLogicalEvalTestCase{"OR_ShortCircuitBeforeUnknown", + "a || b || c", "true", "unknown1", "unset", + "bool", true}, + VariadicLogicalEvalTestCase{"AND_UnknownAggregation", "a && b && c", + "unknown1", "unknown2", "true", "unknown"}, + VariadicLogicalEvalTestCase{"OR_UnknownAggregation", "a || b || c", + "unknown1", "unknown2", "false", "unknown"}, + VariadicLogicalEvalTestCase{"Exists_True", "[a, b, c].exists(x, x)", + "false", "false", "true", "bool", true}, + VariadicLogicalEvalTestCase{"Exists_Unknown", "[a, b, c].exists(x, x)", + "false", "unknown1", "false", "unknown"}, + VariadicLogicalEvalTestCase{"All_False", "[a, b, c].all(x, x)", "true", + "true", "false", "bool", false}, + VariadicLogicalEvalTestCase{"All_Unknown", "[a, b, c].all(x, x)", + "true", "unknown1", "true", "unknown"})); + +void ReplaceResultAccumulatorWithLegacyName(Expr* expr) { + class LegacyAccumulatorRewriter : public AstRewriterBase { + public: + bool PostVisitRewrite(Expr* expr, const SourcePosition*) override { + if (expr->has_ident_expr() && expr->ident_expr().name() == "@result") { + expr->mutable_ident_expr()->set_name("__result__"); + return true; + } + if (expr->has_comprehension_expr() && + expr->comprehension_expr().accu_var() == "@result") { + expr->mutable_comprehension_expr()->set_accu_var("__result__"); + return true; + } + return false; + } + } rewriter; + AstRewrite(expr, /*source_info=*/nullptr, &rewriter); +} + +TEST(FlatExprBuilderTest, LegacyAccumulatorReferenceDoesNotMutateInPlace) { + cel::RuntimeOptions options; + options.enable_comprehension_list_append = true; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + + for (absl::string_view expr_str : { + "[1, 2].map(x, (__result__ + [x]).size()) == [1, 2]", + "[1, 2, 3].filter(x, (__result__ + [x]).size() > 1) == []", + "[1, 2, 3].filter(x, (__result__ + [x]).size() == 1) == [1]", + }) { + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(expr_str)); + ReplaceResultAccumulatorWithLegacyName(parsed_expr.mutable_expr()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_expr, + builder.CreateExpression(&parsed_expr.expr(), + &parsed_expr.source_info())); + + Activation activation; + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(CelValue result, + cel_expr->Evaluate(activation, &arena)); + EXPECT_THAT(result, test::IsCelBool(true)) << expr_str; + } +} + +TEST(FlatExprBuilderTest, + ComprehensionTernaryNonIdentFalseBranchSkipsMutableListAppend) { + // Hand-crafted comprehension where the false branch of the ternary loop_step + // is not the accumulator variable: + // accu_var = "__result__", accu_init = [] + // loop_step = x > 1 ? (__result__ + [x]) : [0] + // For iter_range = [1, 2, 3]: + // x = 1 -> false branch -> __result__ becomes [0] + // x = 2 -> true branch -> __result__ becomes [0, 2] + // x = 3 -> true branch -> __result__ becomes [0, 2, 3] + ASSERT_OK_AND_ASSIGN( + ParsedExpr parsed_expr, + parser::Parse("[1, 2, 3].filter(x, x > 1) == [0, 2, 3]")); + Expr* filter_expr = + parsed_expr.mutable_expr()->mutable_call_expr()->mutable_args(0); + ASSERT_TRUE(filter_expr->has_comprehension_expr()); + Expr* false_branch = filter_expr->mutable_comprehension_expr() + ->mutable_loop_step() + ->mutable_call_expr() + ->mutable_args(2); + false_branch->Clear(); + false_branch->mutable_list_expr() + ->add_elements() + ->mutable_const_expr() + ->set_int64_value(0); + + cel::RuntimeOptions options; + options.enable_comprehension_list_append = true; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_expr, + builder.CreateExpression(&parsed_expr.expr(), + &parsed_expr.source_info())); + + Activation activation; + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr->Evaluate(activation, &arena)); + EXPECT_THAT(result, test::IsCelBool(true)); +} + +TEST(FlatExprBuilderTest, StackSizeEstimationBuiltinsAndGenericSteps) { + cel::RuntimeOptions options; + options.short_circuiting = false; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + + { + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed, parser::Parse("1 + 2 + 3 + 4")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + cel::extensions::CreateAstFromParsedExpr(parsed)); + ASSERT_OK_AND_ASSIGN(FlatExpression expr, + builder.flat_expr_builder().CreateExpressionImpl( + std::move(ast), nullptr)); + EXPECT_EQ(expr.path().size(), 7); + EXPECT_EQ(expr.value_stack_size(), 2); + } + + { + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed, + parser::Parse("{'a': 1, 'b': 2}['a']")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + cel::extensions::CreateAstFromParsedExpr(parsed)); + ASSERT_OK_AND_ASSIGN(FlatExpression expr, + builder.flat_expr_builder().CreateExpressionImpl( + std::move(ast), nullptr)); + // 'a', 1, 'b', 2, CreateMap(2), 'a', ContainerAccess + EXPECT_EQ(expr.path().size(), 7); + EXPECT_EQ(expr.value_stack_size(), 4); + } + + { + ASSERT_OK_AND_ASSIGN( + ParsedExpr parsed, + parser::Parse("google.api.expr.runtime.TestMessage{int64_value: 1, " + "string_value: 'a'}.int64_value")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + cel::extensions::CreateAstFromParsedExpr(parsed)); + ASSERT_OK_AND_ASSIGN(FlatExpression expr, + builder.flat_expr_builder().CreateExpressionImpl( + std::move(ast), nullptr)); + // 1, 'a', CreateStruct(2), Select + EXPECT_EQ(expr.path().size(), 4); + EXPECT_EQ(expr.value_stack_size(), 2); + } + + { + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed, parser::Parse("true ? 1 : 2")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + cel::extensions::CreateAstFromParsedExpr(parsed)); + ASSERT_OK_AND_ASSIGN(FlatExpression expr, + builder.flat_expr_builder().CreateExpressionImpl( + std::move(ast), nullptr)); + // true, 1, 2, TernaryStep + EXPECT_EQ(expr.path().size(), 4); + EXPECT_EQ(expr.value_stack_size(), 3); + } +} + +TEST(FlatExprBuilderTest, StackSizeEstimationShortCircuiting) { + cel::RuntimeOptions options; + options.short_circuiting = true; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + + { + // Conditionals over-count in linear stack estimation because TernaryJump + // and FixedJump have a worst-case delta of 0 and both branches are walked + // sequentially without a merge step. + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed, + parser::Parse("(c1 ? 1 : 2) + (c2 ? 3 : 4)")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + cel::extensions::CreateAstFromParsedExpr(parsed)); + ASSERT_OK_AND_ASSIGN(FlatExpression expr, + builder.flat_expr_builder().CreateExpressionImpl( + std::move(ast), nullptr)); + // c1, TernaryJump, 1, FixedJump, 2, c2, TernaryJump, 3, FixedJump, 4, + + EXPECT_EQ(expr.path().size(), 11); + EXPECT_EQ(expr.value_stack_size(), 6); + } + + { + // Comprehension: [1, 2, 3].all(x, x > 0) + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed, + parser::Parse("[1, 2, 3].all(x, x > 0)")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + cel::extensions::CreateAstFromParsedExpr(parsed)); + ASSERT_OK_AND_ASSIGN(FlatExpression expr, + builder.flat_expr_builder().CreateExpressionImpl( + std::move(ast), nullptr)); + EXPECT_EQ(expr.path().size(), 19); + EXPECT_EQ(expr.value_stack_size(), 5); + } + + { + // Mixed logical operators: (a && b) || (c && d) + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed, + parser::Parse("(a && b) || (c && d)")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + cel::extensions::CreateAstFromParsedExpr(parsed)); + ASSERT_OK_AND_ASSIGN(FlatExpression expr, + builder.flat_expr_builder().CreateExpressionImpl( + std::move(ast), nullptr)); + // a, AndJump, b, And(2), OrJump, c, AndJump, d, And(2), Or(2) + EXPECT_EQ(expr.path().size(), 10); + EXPECT_EQ(expr.value_stack_size(), 3); + } +} + +TEST(FlatExprBuilderTest, StackSizeEstimationWithLazySubexpressions) { + ParsedExpr parsed_expr; + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( + R"pb( + expr: { + call_expr: { + function: "cel.@block" + args { + list_expr: { + elements { + call_expr: { + function: "_+_" + args { const_expr: { int64_value: 1 } } + args { const_expr: { int64_value: 2 } } + } + } + elements { + call_expr: { + function: "_+_" + args { const_expr: { int64_value: 3 } } + args { const_expr: { int64_value: 4 } } + } + } + } + } + args { + call_expr: { + function: "_+_" + args { ident_expr: { name: "@index0" } } + args { ident_expr: { name: "@index1" } } + } + } + } + } + )pb", + &parsed_expr)); + + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + cel::extensions::CreateAstFromParsedExpr(parsed_expr)); + ASSERT_OK_AND_ASSIGN(FlatExpression expr, + builder.flat_expr_builder().CreateExpressionImpl( + std::move(ast), nullptr)); + // Main program: LazyInit(@index0), LazyInit(@index1), +, ClearSlots -> max 2 + // Subexpr 1: 1, 2, + -> max 2 + // Subexpr 2: 3, 4, + -> max 2 + // Total max stack: 2 + 2 + 2 = 6 (while total steps = 10) + EXPECT_EQ(expr.path().size(), 10); + EXPECT_EQ(expr.value_stack_size(), 6); +} + +class NoOpStepWithCustomDelta : public ExpressionStepLogic { + public: + void Evaluate(ExecutionFrame* frame) const override {} +}; + +class CustomDeltaInjectorOptimizer : public ProgramOptimizer { + public: + explicit CustomDeltaInjectorOptimizer(int64_t stack_delta) + : stack_delta_(stack_delta) {} + + absl::Status OnPreVisit(PlannerContext& context, + const cel::Expr& node) override { + return absl::OkStatus(); + } + + absl::Status OnPostVisit(PlannerContext& context, + const cel::Expr& node) override { + if (node.id() == 1) { + return context.AddSubplanStep( + node, std::make_unique(), -1, stack_delta_); + } + return absl::OkStatus(); + } + + private: + int64_t stack_delta_; +}; + +TEST(FlatExprBuilderTest, StackSizeEstimationOverflowAndUnderflowFallback) { + for (int64_t bad_delta : + {static_cast(std::numeric_limits::max()), + static_cast(-10)}) { + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + builder.flat_expr_builder().AddProgramOptimizer( + [bad_delta](PlannerContext&, const cel::Ast&) + -> absl::StatusOr> { + return std::make_unique(bad_delta); + }); + + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed, parser::Parse("1 + 2 + 3")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + cel::extensions::CreateAstFromParsedExpr(parsed)); + ASSERT_OK_AND_ASSIGN(FlatExpression expr, + builder.flat_expr_builder().CreateExpressionImpl( + std::move(ast), nullptr)); + EXPECT_EQ(expr.value_stack_size(), expr.path().size()); + } +} + +TEST(FlatExprBuilderTest, ListAppendComprehensionWithTargetFailsPlanning) { + ParsedExpr parsed_expr; + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( + R"pb( + expr { + comprehension_expr { + iter_var: "x" + iter_range { + list_expr { + elements { const_expr { int64_value: 1 } } + elements { const_expr { int64_value: 2 } } + elements { const_expr { int64_value: 3 } } + elements { const_expr { int64_value: 4 } } + } + } + accu_var: "__result__" + accu_init { list_expr {} } + loop_condition { const_expr { bool_value: true } } + loop_step { + call_expr { + target { const_expr { int64_value: 0 } } + function: "_+_" + args { ident_expr { name: "__result__" } } + args { + list_expr { elements { const_expr { int64_value: 1 } } } + } + } + } + result { ident_expr { name: "__result__" } } + } + } + )pb", + &parsed_expr)); + + cel::RuntimeOptions options; + options.enable_comprehension_list_append = true; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + + EXPECT_THAT( + builder.CreateExpression(&parsed_expr.expr(), &parsed_expr.source_info()), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +TEST(FlatExprBuilderTest, ProgramOptimizerResolvesTypesWhenRequested) { + std::shared_ptr env = + NewTestingRuntimeEnv(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler_builder, + cel::NewCompilerBuilder(env->descriptor_pool)); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT(compiler_builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("x", cel::IntType())), + IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + compiler_builder->Build()); + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler->Compile("x == 1")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + validation_result.ReleaseAst()); + + class TypeRecordingOptimizer : public ProgramOptimizer { + public: + explicit TypeRecordingOptimizer( + absl::flat_hash_map& pre_visit_types, + absl::flat_hash_map& post_visit_types) + : pre_visit_types_(pre_visit_types), + post_visit_types_(post_visit_types) {} + + absl::Status OnPreVisit(PlannerContext& context, + const cel::Expr& node) override { + if (std::optional type = context.GetResolvedType(node.id()); + type.has_value()) { + pre_visit_types_[node.id()] = *type; + } + return absl::OkStatus(); + } + + absl::Status OnPostVisit(PlannerContext& context, + const cel::Expr& node) override { + if (std::optional type = context.GetResolvedType(node.id()); + type.has_value()) { + post_visit_types_[node.id()] = *type; + } + return absl::OkStatus(); + } + + private: + absl::flat_hash_map& pre_visit_types_; + absl::flat_hash_map& post_visit_types_; + }; + + cel::RuntimeOptions options; + options.enable_typed_field_access = false; + CelExpressionBuilderFlatImpl builder(env, options); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + + absl::flat_hash_map pre_visit_types; + absl::flat_hash_map post_visit_types; + builder.flat_expr_builder().AddProgramOptimizer( + [&](PlannerContext& context, const cel::Ast&) + -> absl::StatusOr> { + context.set_resolve_types(); + return std::make_unique(pre_visit_types, + post_visit_types); + }); + + ASSERT_OK_AND_ASSIGN(FlatExpression flat_expr, + builder.flat_expr_builder().CreateExpressionImpl( + std::move(ast), /*issues=*/nullptr)); + EXPECT_THAT(pre_visit_types, + testing::UnorderedElementsAre(testing::Pair(1, cel::IntType()), + testing::Pair(2, cel::BoolType()), + testing::Pair(3, cel::IntType()))); + EXPECT_THAT(post_visit_types, + testing::UnorderedElementsAre(testing::Pair(1, cel::IntType()), + testing::Pair(2, cel::BoolType()), + testing::Pair(3, cel::IntType()))); +} + +class CountingActivation : public BaseActivation { + public: + void InsertInt(absl::string_view name, int64_t val) { + values_[name] = CelValue::CreateInt64(val); + } + + void InsertBool(absl::string_view name, bool val) { + values_[name] = CelValue::CreateBool(val); + } + + std::vector FindFunctionOverloads( + absl::string_view) const override { + return {}; + } + + std::optional FindValue(absl::string_view name, + google::protobuf::Arena*) const override { + lookup_counts_[name]++; + auto it = values_.find(name); + if (it != values_.end()) { + return it->second; + } + return std::nullopt; + } + + int LookupCount(absl::string_view name) const { + auto it = lookup_counts_.find(name); + return it != lookup_counts_.end() ? it->second : 0; + } + + private: + absl::flat_hash_map values_; + mutable absl::flat_hash_map lookup_counts_; +}; + +TEST(FlatExprBuilderTest, IdentifierTableRepeatedRootIdentifierReceivesSlot) { + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse("x + y + x")); + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_expr, + builder.CreateExpression(&parsed_expr.expr(), + &parsed_expr.source_info())); + + const FlatExpression& flat_expr = + static_cast(cel_expr.get()) + ->flat_expression(); + ASSERT_THAT(flat_expr.identifiers(), SizeIs(2)); + EXPECT_EQ(flat_expr.identifiers()[0].name, "x"); + EXPECT_EQ(flat_expr.identifiers()[0].slot_index, 0); + EXPECT_EQ(flat_expr.identifiers()[1].name, "y"); + EXPECT_EQ(flat_expr.identifiers()[1].slot_index, 1); + EXPECT_EQ(flat_expr.comprehension_slots_size(), 2); + EXPECT_EQ(flat_expr.iterator_stack_size(), 0); + + CountingActivation activation; + activation.InsertInt("x", 10); + activation.InsertInt("y", 20); + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr->Evaluate(activation, &arena)); + EXPECT_THAT(result, test::IsCelInt64(40)); + EXPECT_EQ(activation.LookupCount("x"), 1); + EXPECT_EQ(activation.LookupCount("y"), 1); +} + +TEST(FlatExprBuilderTest, + IdentifierTableInComprehensionReceivesSlotAboveComprehensionSlots) { + ASSERT_OK_AND_ASSIGN( + ParsedExpr parsed_expr, + parser::Parse("[1, 2, 3].all(i, i < limit) && single_var")); + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_expr, + builder.CreateExpression(&parsed_expr.expr(), + &parsed_expr.source_info())); + + const FlatExpression& flat_expr = + static_cast(cel_expr.get()) + ->flat_expression(); + // Local comprehension variables (i, @result) are not in the root identifier + // table; only "limit" and "single_var" are present. + ASSERT_THAT(flat_expr.identifiers(), SizeIs(2)); + EXPECT_EQ(flat_expr.identifiers()[0].name, "limit"); + // Comprehension uses 2 slots (accu_var at 0, iter_var at 1), so "limit" gets + // slot 2 and "single_var" gets slot 3. + EXPECT_EQ(flat_expr.identifiers()[0].slot_index, 2); + EXPECT_EQ(flat_expr.identifiers()[1].name, "single_var"); + EXPECT_EQ(flat_expr.identifiers()[1].slot_index, 3); + EXPECT_EQ(flat_expr.comprehension_slots_size(), 4); + EXPECT_EQ(flat_expr.iterator_stack_size(), 1); + + CountingActivation activation; + activation.InsertInt("limit", 5); + activation.InsertBool("single_var", true); + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr->Evaluate(activation, &arena)); + EXPECT_THAT(result, test::IsCelBool(true)); + EXPECT_EQ(activation.LookupCount("limit"), 1); + EXPECT_EQ(activation.LookupCount("single_var"), 1); + EXPECT_EQ(activation.LookupCount("i"), 0); +} + +// Returns a parsed expression containing a `cel.@block` call with trivial +// subexpressions. +// +// Rough policy equivalent: +// +// variables: +// - name: a +// expression: "10" +// - name: b +// expression: "x" +// rule: (a + b) + b +// +// Which lowers to: +// cel.@block([10, x], (@index0 + @index1) + @index1) +ParsedExpr MakeTrivialBlockParsedExpr() { + ParsedExpr parsed_expr; + ABSL_CHECK(google::protobuf::TextFormat::ParseFromString( + R"pb( + expr: { + id: 1 + call_expr: { + function: "cel.@block" + args { + id: 2 + list_expr: { + elements { + id: 3 + const_expr: { int64_value: 10 } + } + elements { + id: 4 + ident_expr: { name: "x" } + } + } + } + args { + id: 5 + call_expr: { + function: "_+_" + args { + id: 6 + call_expr: { + function: "_+_" + args { + id: 7 + ident_expr: { name: "@index0" } + } + args { + id: 8 + ident_expr: { name: "@index1" } + } + } + } + args { + id: 9 + ident_expr: { name: "@index1" } + } + } + } + } + } + )pb", + &parsed_expr)); + return parsed_expr; +} + +TEST(FlatExprBuilderTest, BlockInlinesTrivialSubexpressionsEnabled) { + ParsedExpr parsed_expr = MakeTrivialBlockParsedExpr(); + + cel::RuntimeOptions options; + options.enable_subexpression_inlining = true; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), + absl_testing::IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_expr, + builder.CreateExpression(&parsed_expr.expr(), + &parsed_expr.source_info())); + + const FlatExpression& flat_expr = + static_cast(cel_expr.get()) + ->flat_expression(); + ASSERT_FALSE(flat_expr.subexpressions().empty()); + ExecutionPathView main_path = flat_expr.subexpressions()[0]; + ASSERT_GE(main_path.size(), 3); + + cel::Value const_val; + EXPECT_TRUE(GetIfConstant(main_path[0], const_val)); + EXPECT_EQ(main_path[0].id(), 7); + EXPECT_EQ(const_val.GetInt().NativeValue(), 10); + const IdentStepInfo* ident1 = GetIfIdentStep(main_path[1]); + ASSERT_NE(ident1, nullptr); + EXPECT_EQ(main_path[1].id(), 8); + + CountingActivation activation; + activation.InsertInt("x", 5); + google::protobuf::Arena arena; + std::vector traced_ids; + ASSERT_OK_AND_ASSIGN(CelValue result, + cel_expr->Trace(activation, &arena, + [&](int64_t id, const CelValue&, + google::protobuf::Arena*) -> absl::Status { + traced_ids.push_back(id); + return absl::OkStatus(); + })); + EXPECT_THAT(result, test::IsCelInt64(20)); + EXPECT_EQ(activation.LookupCount("x"), 1); + // We don't guarantee trace specifics here, just documenting that it + // is deterministic and changed from the non-inlined version. + EXPECT_THAT(traced_ids, testing::ElementsAre(7, 8, 6, 9, 5, 1)); +} + +TEST(FlatExprBuilderTest, BlockInlinesTrivialSubexpressionsDisabled) { + ParsedExpr parsed_expr = MakeTrivialBlockParsedExpr(); + + cel::RuntimeOptions options; + options.enable_subexpression_inlining = false; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), + absl_testing::IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_expr, + builder.CreateExpression(&parsed_expr.expr(), + &parsed_expr.source_info())); + + const FlatExpression& flat_expr = + static_cast(cel_expr.get()) + ->flat_expression(); + ASSERT_FALSE(flat_expr.subexpressions().empty()); + ExecutionPathView main_path = flat_expr.subexpressions()[0]; + EXPECT_EQ(main_path.size(), 6); + EXPECT_EQ(main_path[0].kind(), ExpressionStepKind::kLazyInit); + + CountingActivation activation; + activation.InsertInt("x", 5); + google::protobuf::Arena arena; + std::vector traced_ids; + ASSERT_OK_AND_ASSIGN(CelValue result, + cel_expr->Trace(activation, &arena, + [&](int64_t id, const CelValue&, + google::protobuf::Arena*) -> absl::Status { + traced_ids.push_back(id); + return absl::OkStatus(); + })); + EXPECT_THAT(result, test::IsCelInt64(20)); + EXPECT_EQ(activation.LookupCount("x"), 1); + // Here trace reports the init step on first access. + EXPECT_THAT(traced_ids, testing::ElementsAre(3, 4, 6, 9, 5, 1)); +} + +TEST(FlatExprBuilderTest, BindInlinesConstantAndFoldsMainExpression) { + ParsedExpr parsed_expr; + // Equivalent to: cel.bind(a, 1 + 2, (a + 3) + x) + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( + R"pb( + expr { + id: 1 + comprehension_expr { + iter_var: "#unused" + iter_range { + id: 2 + list_expr {} + } + accu_var: "a" + accu_init { + id: 3 + call_expr { + function: "_+_" + args { + id: 4 + const_expr { int64_value: 1 } + } + args { + id: 5 + const_expr { int64_value: 2 } + } + } + } + loop_condition { + id: 6 + const_expr { bool_value: false } + } + loop_step { + id: 7 + ident_expr { name: "a" } + } + result { + id: 8 + call_expr { + function: "_+_" + args { + id: 9 + call_expr { + function: "_+_" + args { + id: 10 + ident_expr { name: "a" } + } + args { + id: 11 + const_expr { int64_value: 3 } + } + } + } + args { + id: 12 + ident_expr { name: "x" } + } + } + } + } + } + )pb", + &parsed_expr)); + + InterpreterOptions options; + google::protobuf::Arena arena; + options.constant_folding = true; + options.constant_arena = &arena; + options.enable_subexpression_inlining = true; + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr builder, + CreateConstantFoldingConformanceTestExprBuilder(options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_expr, + builder->CreateExpression(&parsed_expr.expr(), + &parsed_expr.source_info())); + + const FlatExpression& flat_expr = + static_cast(cel_expr.get()) + ->flat_expression(); + ASSERT_FALSE(flat_expr.subexpressions().empty()); + ExecutionPathView main_path = flat_expr.subexpressions()[0]; + // Main expression consists of: + // 0: Constant(6) (folded from `a + 3` after `a` was inlined as Constant(3)) + // 1: Identifier(x) + // 2: _+_ + // 3: ClearSlot + ASSERT_THAT(main_path, SizeIs(4)); + cel::Value folded_val; + ASSERT_TRUE(GetIfConstant(main_path[0], folded_val)); + EXPECT_EQ(folded_val.GetInt().NativeValue(), 6); + EXPECT_EQ(main_path[0].id(), 9); + + Activation activation; + activation.InsertValue("x", CelValue::CreateInt64(10)); + ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr->Evaluate(activation, &arena)); + EXPECT_THAT(result, test::IsCelInt64(16)); +} } // namespace } // namespace google::api::expr::runtime diff --git a/eval/compiler/instrumentation.cc b/eval/compiler/instrumentation.cc index 3e37bdb45..7f522c739 100644 --- a/eval/compiler/instrumentation.cc +++ b/eval/compiler/instrumentation.cc @@ -37,14 +37,17 @@ class InstrumentStep : public ExpressionStepBase { expr_id_(expr_id), instrumentation_(std::move(instrumentation)) {} - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { if (!frame->value_stack().HasEnough(1)) { - return absl::InternalError("stack underflow in instrument step."); + frame->Abort(absl::InternalError("stack underflow in instrument step.")); + return; } - return instrumentation_(expr_id_, frame->value_stack().Peek()); - - return absl::OkStatus(); + if (absl::Status status = + instrumentation_(expr_id_, frame->value_stack().Peek()); + !status.ok()) { + frame->Abort(std::move(status)); + } } private: @@ -69,7 +72,8 @@ class InstrumentOptimizer : public ProgramOptimizer { } return context.AddSubplanStep( - node, std::make_unique(node.id(), instrumentation_)); + node, std::make_unique(node.id(), instrumentation_), + node.id()); } private: diff --git a/eval/compiler/instrumentation_test.cc b/eval/compiler/instrumentation_test.cc index 630f398d1..cfd1c4fdc 100644 --- a/eval/compiler/instrumentation_test.cc +++ b/eval/compiler/instrumentation_test.cc @@ -105,6 +105,7 @@ TEST_F(InstrumentationTest, Basic) { ASSERT_OK_AND_ASSIGN(auto value, plan.EvaluateWithCallback( activation, /*embedder_context=*/nullptr, EvaluationListener(), state)); + static_cast(value); // AST for the test expression: // + <4> @@ -150,6 +151,7 @@ TEST_F(InstrumentationTest, BasicWithConstFolding) { ASSERT_OK_AND_ASSIGN(auto value, plan.EvaluateWithCallback( activation, /*embedder_context=*/nullptr, EvaluationListener(), state)); + static_cast(value); // AST for the test expression: // + <4> @@ -190,6 +192,7 @@ TEST_F(InstrumentationTest, AndShortCircuit) { ASSERT_OK_AND_ASSIGN(auto value, plan.EvaluateWithCallback( activation, /*embedder_context=*/nullptr, EvaluationListener(), state)); + static_cast(value); EXPECT_THAT(expr_ids, ElementsAre(1, 2, 3)); @@ -198,6 +201,7 @@ TEST_F(InstrumentationTest, AndShortCircuit) { ASSERT_OK_AND_ASSIGN( value, plan.EvaluateWithCallback(activation, /*embedder_context=*/nullptr, EvaluationListener(), state)); + static_cast(value); EXPECT_THAT(expr_ids, ElementsAre(1, 2, 3, 1, 3)); } @@ -232,6 +236,7 @@ TEST_F(InstrumentationTest, OrShortCircuit) { ASSERT_OK_AND_ASSIGN(auto value, plan.EvaluateWithCallback( activation, /*embedder_context=*/nullptr, EvaluationListener(), state)); + static_cast(value); EXPECT_THAT(expr_ids, ElementsAre(1, 2, 3)); expr_ids.clear(); @@ -240,6 +245,7 @@ TEST_F(InstrumentationTest, OrShortCircuit) { ASSERT_OK_AND_ASSIGN( value, plan.EvaluateWithCallback(activation, /*embedder_context=*/nullptr, EvaluationListener(), state)); + static_cast(value); EXPECT_THAT(expr_ids, ElementsAre(1, 3)); } @@ -275,6 +281,7 @@ TEST_F(InstrumentationTest, Ternary) { ASSERT_OK_AND_ASSIGN(auto value, plan.EvaluateWithCallback( activation, /*embedder_context=*/nullptr, EvaluationListener(), state)); + static_cast(value); // AST // ?:() <2> @@ -288,6 +295,7 @@ TEST_F(InstrumentationTest, Ternary) { ASSERT_OK_AND_ASSIGN( value, plan.EvaluateWithCallback(activation, /*embedder_context=*/nullptr, EvaluationListener(), state)); + static_cast(value); EXPECT_THAT(expr_ids, ElementsAre(1, 4, 2)); expr_ids.clear(); diff --git a/eval/compiler/qualified_reference_resolver.cc b/eval/compiler/qualified_reference_resolver.cc index 67f86ebb6..a46034298 100644 --- a/eval/compiler/qualified_reference_resolver.cc +++ b/eval/compiler/qualified_reference_resolver.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -26,7 +27,6 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "base/ast.h" #include "base/builtins.h" #include "common/ast.h" @@ -81,9 +81,9 @@ bool OverloadExists(const Resolver& resolver, absl::string_view name, // Return the qualified name of the most qualified matching overload, or // nullopt if no matches are found. -absl::optional BestOverloadMatch(const Resolver& resolver, - absl::string_view base_name, - int argument_count) { +std::optional BestOverloadMatch(const Resolver& resolver, + absl::string_view base_name, + int argument_count) { if (IsSpecialFunction(base_name)) { return std::string(base_name); } @@ -99,7 +99,7 @@ absl::optional BestOverloadMatch(const Resolver& resolver, return *name; } } - return absl::nullopt; + return std::nullopt; } // Rewriter visitor for resolving references. @@ -262,27 +262,27 @@ class ReferenceResolver : public cel::AstRewriterBase { // Convert a select expr sub tree into a namespace name if possible. // If any operand of the top element is a not a select or an ident node, // return nullopt. - absl::optional ToNamespace(const Expr& expr) { - absl::optional maybe_parent_namespace; + std::optional ToNamespace(const Expr& expr) { + std::optional maybe_parent_namespace; if (rewritten_reference_.find(expr.id()) != rewritten_reference_.end()) { // The target expr matches a reference (resolved to an ident decl). // This should not be treated as a function qualifier. - return absl::nullopt; + return std::nullopt; } if (expr.has_ident_expr()) { return expr.ident_expr().name(); } else if (expr.has_select_expr()) { if (expr.select_expr().test_only()) { - return absl::nullopt; + return std::nullopt; } maybe_parent_namespace = ToNamespace(expr.select_expr().operand()); if (!maybe_parent_namespace.has_value()) { - return absl::nullopt; + return std::nullopt; } return absl::StrCat(*maybe_parent_namespace, ".", expr.select_expr().field()); } else { - return absl::nullopt; + return std::nullopt; } } diff --git a/eval/compiler/regex_precompilation_optimization.cc b/eval/compiler/regex_precompilation_optimization.cc index b94cae383..f4275ebd3 100644 --- a/eval/compiler/regex_precompilation_optimization.cc +++ b/eval/compiler/regex_precompilation_optimization.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -26,19 +27,14 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "base/builtins.h" #include "common/ast.h" -#include "common/casting.h" #include "common/expr.h" #include "common/native_type.h" #include "common/value.h" #include "eval/compiler/flat_expr_builder_extensions.h" -#include "eval/eval/compiler_constant_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/regex_match_step.h" -#include "internal/casts.h" #include "internal/re2_options.h" #include "internal/status_macros.h" #include "re2/re2.h" @@ -48,14 +44,10 @@ namespace { using ::cel::Ast; using ::cel::CallExpr; -using ::cel::Cast; using ::cel::Expr; -using ::cel::InstanceOf; using ::cel::NativeTypeId; using ::cel::Reference; -using ::cel::StringValue; using ::cel::Value; -using ::cel::internal::down_cast; using ReferenceMap = absl::flat_hash_map; @@ -145,7 +137,7 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { // Try to check if the regex is valid, whether or not we can actually update // the plan. - absl::optional pattern = + std::optional pattern = GetConstantString(context, subexpression, node, pattern_expr); if (!pattern.has_value()) { return absl::OkStatus(); @@ -168,7 +160,7 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { } private: - absl::optional GetConstantString( + std::optional GetConstantString( PlannerContext& context, ProgramBuilder::Subexpression* absl_nullable subexpression, const Expr& call_expr, const Expr& re_expr) const { @@ -178,35 +170,23 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { if (subexpression == nullptr || subexpression->IsFlattened()) { // Already modified, can't recover the input pattern. - return absl::nullopt; + return std::nullopt; } - absl::optional constant; - if (subexpression->IsRecursive()) { - const auto& program = subexpression->recursive_program(); - auto deps = program.step->GetDependencies(); - if (deps.has_value() && deps->size() == 2) { - const auto* re_plan = - TryDowncastDirectStep(deps->at(1)); - if (re_plan != nullptr) { - constant = re_plan->value(); - } - } - } else { + std::optional constant; // otherwise stack-machine program. ExecutionPathView re_plan = context.GetSubplan(re_expr); - if (re_plan.size() == 1 && - re_plan[0]->GetNativeTypeId() == - NativeTypeId::For()) { - constant = - down_cast(re_plan[0].get())->value(); + if (re_plan.size() == 1) { + cel::Value val; + if (GetIfConstant(re_plan[0], val)) { + constant = std::move(val); + } } - } - if (constant.has_value() && InstanceOf(*constant)) { - return Cast(*constant).ToString(); + if (constant.has_value() && constant->IsString()) { + return constant->GetString().ToString(); } - return absl::nullopt; + return std::nullopt; } absl::Status RewritePlan( @@ -214,33 +194,10 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { ProgramBuilder::Subexpression* absl_nonnull subexpression, const Expr& call, const Expr& subject, std::shared_ptr regex_program) { - if (subexpression->IsRecursive()) { - return RewriteRecursivePlan(subexpression, call, subject, - std::move(regex_program)); - } return RewriteStackMachinePlan(context, call, subject, std::move(regex_program)); } - absl::Status RewriteRecursivePlan( - ProgramBuilder::Subexpression* absl_nonnull subexpression, - const Expr& call, const Expr& subject, - std::shared_ptr regex_program) { - auto program = subexpression->ExtractRecursiveProgram(); - auto deps = program.step->ExtractDependencies(); - if (!deps.has_value() || deps->size() != 2) { - // Possibly already const-folded, put the plan back. - subexpression->set_recursive_program(std::move(program.step), - program.depth); - return absl::OkStatus(); - } - subexpression->set_recursive_program( - CreateDirectRegexMatchStep(call.id(), std::move(deps->at(0)), - std::move(regex_program)), - program.depth); - return absl::OkStatus(); - } - absl::Status RewriteStackMachinePlan( PlannerContext& context, const Expr& call, const Expr& subject, std::shared_ptr regex_program) { @@ -251,9 +208,10 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { CEL_ASSIGN_OR_RETURN(ExecutionPath new_plan, context.ExtractSubplan(subject)); - CEL_ASSIGN_OR_RETURN( - new_plan.emplace_back(), - CreateRegexMatchStep(std::move(regex_program), call.id())); + CEL_ASSIGN_OR_RETURN(auto step, + CreateRegexMatchStep(std::move(regex_program))); + new_plan.push_back( + ExpressionStep::MakeGenericStep(std::move(step), call.id())); return context.ReplaceSubplan(call, std::move(new_plan)); } diff --git a/eval/compiler/resolver.cc b/eval/compiler/resolver.cc index 4e3fa3841..3acc39165 100644 --- a/eval/compiler/resolver.cc +++ b/eval/compiler/resolver.cc @@ -102,8 +102,8 @@ absl::Span Resolver::GetPrefixesFor( return namespace_prefixes_; } -absl::optional Resolver::FindConstant(absl::string_view name, - int64_t expr_id) const { +std::optional Resolver::FindConstant(absl::string_view name, + int64_t expr_id) const { auto prefixes = GetPrefixesFor(name); for (const auto& prefix : prefixes) { std::string qualified_name = absl::StrCat(prefix, name); @@ -128,7 +128,7 @@ absl::optional Resolver::FindConstant(absl::string_view name, return TypeValue(**type_value); } } - return absl::nullopt; + return std::nullopt; } std::vector Resolver::FindOverloads( @@ -196,8 +196,8 @@ std::vector Resolver::FindLazyOverloads( auto prefixes = GetPrefixesFor(name); for (const auto& prefix : prefixes) { std::string qualified_name = absl::StrCat(prefix, name); - funcs = function_registry_.FindLazyOverloadsByArity(name, receiver_style, - arity); + funcs = function_registry_.FindLazyOverloadsByArity(qualified_name, + receiver_style, arity); if (!funcs.empty()) { return funcs; } @@ -205,7 +205,7 @@ std::vector Resolver::FindLazyOverloads( return funcs; } -absl::StatusOr>> +absl::StatusOr>> Resolver::FindType(absl::string_view name, int64_t expr_id) const { auto prefixes = GetPrefixesFor(name); for (auto& prefix : prefixes) { @@ -216,7 +216,7 @@ Resolver::FindType(absl::string_view name, int64_t expr_id) const { return std::make_pair(std::move(qualified_name), std::move(*maybe_type)); } } - return absl::nullopt; + return std::nullopt; } } // namespace google::api::expr::runtime diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 44c7ded79..1e133ca3a 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -35,30 +35,64 @@ package_group( cc_library( name = "evaluator_core", srcs = [ + "bitset_steps.cc", + "create_list_step.cc", + "equality_steps.cc", "evaluator_core.cc", + "function_step.cc", ], hdrs = [ + "bitset_steps.h", + "comprehension_step.h", + "create_list_step.h", + "equality_steps.h", "evaluator_core.h", + "function_step.h", + "ident_step.h", + "lazy_init_step.h", ], deps = [ - ":attribute_utility", + ":attribute_trail", ":comprehension_slots", ":evaluator_stack", + ":expression_step_logic", ":iterator_stack", + "//base:attribute_matcher", + "//base:attributes", + "//base:builtins", "//base:data", - "//common:native_type", + "//common:expr", + "//common:function_descriptor", + "//common:kind", + "//common:optional_ref", "//common:value", + "//common:value_kind", + "//common/internal:attribute_trail", + "//common/internal:unknowns", + "//eval/internal:errors", + "//internal:number", + "//internal:status_macros", "//runtime", "//runtime:activation_interface", + "//runtime:function", + "//runtime:function_overload_reference", + "//runtime:function_provider", + "//runtime:function_registry", "//runtime:runtime_options", - "//runtime/internal:activation_attribute_matcher_access", + "//runtime/internal:errors", + "//runtime/standard:equality_functions", + "@com_google_absl//absl/base", + "@com_google_absl//absl/base:config", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_protobuf//:protobuf", ], @@ -73,11 +107,7 @@ cc_library( "cel_expression_flat_impl.h", ], deps = [ - ":attribute_trail", - ":comprehension_slots", - ":direct_expression_step", ":evaluator_core", - "//common:native_type", "//common:value", "//eval/internal:adapter_activation_impl", "//eval/internal:interop", @@ -88,10 +118,9 @@ cc_library( "//internal:status_macros", "//runtime/internal:runtime_env", "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/memory", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", "@com_google_protobuf//:protobuf", ], ) @@ -102,14 +131,13 @@ cc_library( "comprehension_slots.h", ], deps = [ - ":attribute_trail", "//common:value", + "//common/internal:attribute_trail", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:fixed_array", "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/types:optional", ], ) @@ -121,7 +149,6 @@ cc_test( deps = [ ":attribute_trail", ":comprehension_slots", - "//base:attributes", "//base:data", "//common:memory", "//common:value", @@ -138,8 +165,8 @@ cc_library( "evaluator_stack.h", ], deps = [ - ":attribute_trail", "//common:value", + "//common/internal:attribute_trail", "//internal:align", "//internal:new", "@com_google_absl//absl/base:core_headers", @@ -148,7 +175,6 @@ cc_library( "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/meta:type_traits", - "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", ], ) @@ -159,6 +185,7 @@ cc_test( "evaluator_stack_test.cc", ], deps = [ + ":attribute_trail", ":evaluator_stack", "//base:attributes", "//common:value", @@ -167,25 +194,19 @@ cc_test( ) cc_library( - name = "expression_step_base", + name = "expression_step_logic", hdrs = [ - "expression_step_base.h", + "expression_step_logic.h", ], - deps = [":evaluator_core"], + deps = ["//common:native_type"], ) cc_library( - name = "const_value_step", + name = "expression_step_base", hdrs = [ - "const_value_step.h", - ], - deps = [ - ":compiler_constant_step", - ":direct_expression_step", - ":evaluator_core", - "//common:value", - "@com_google_absl//absl/status:statusor", + "expression_step_base.h", ], + deps = [":expression_step_logic"], ) cc_library( @@ -198,19 +219,16 @@ cc_library( ], deps = [ ":attribute_trail", - ":attribute_utility", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", + ":expression_step_logic", "//base:attributes", - "//common:casting", "//common:expr", "//common:kind", "//common:value", "//common:value_kind", "//eval/internal:errors", "//internal:number", - "//internal:status_macros", "//runtime/internal:errors", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", @@ -226,12 +244,10 @@ cc_library( srcs = ["regex_match_step.cc"], hdrs = ["regex_match_step.h"], deps = [ - ":attribute_trail", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", + ":expression_step_logic", "//common:value", - "//internal:status_macros", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:cord", @@ -242,19 +258,12 @@ cc_library( cc_library( name = "ident_step", - srcs = [ - "ident_step.cc", - ], - hdrs = [ - "ident_step.h", - ], deps = [ - ":attribute_trail", - ":comprehension_slots", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", - "//common:expr", + ":expression_step_logic", + "//base:attribute_matcher", + "//base:attributes", "//common:value", "//eval/internal:errors", "//internal:status_macros", @@ -267,37 +276,8 @@ cc_library( cc_library( name = "function_step", - srcs = [ - "function_step.cc", - ], - hdrs = [ - "function_step.h", - ], deps = [ - ":attribute_trail", - ":direct_expression_step", ":evaluator_core", - ":expression_step_base", - "//common:casting", - "//common:expr", - "//common:function_descriptor", - "//common:kind", - "//common:value", - "//common:value_kind", - "//eval/internal:errors", - "//internal:status_macros", - "//runtime:activation_interface", - "//runtime:function", - "//runtime:function_overload_reference", - "//runtime:function_provider", - "//runtime:function_registry", - "//runtime/internal:errors", - "@com_google_absl//absl/container:inlined_vector", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", ], ) @@ -311,18 +291,18 @@ cc_library( ], deps = [ ":attribute_trail", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", - "//common:expr", + ":expression_step_logic", + "//base:attributes", + "//common:legacy_value", + "//common:type", "//common:value", "//common:value_kind", - "//eval/internal:errors", "//internal:status_macros", "//runtime:runtime_options", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -340,19 +320,14 @@ cc_library( "create_list_step.h", ], deps = [ - ":attribute_trail", - ":attribute_utility", - ":direct_expression_step", ":evaluator_core", - ":expression_step_base", - "//common:casting", - "//common:expr", + "//common:optional_ref", "//common:value", - "//internal:status_macros", + "//common/internal:attribute_trail", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", ], ) @@ -365,12 +340,11 @@ cc_library( "create_struct_step.h", ], deps = [ - ":attribute_trail", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", - "//common:casting", + ":expression_step_logic", "//common:value", + "//common/internal:attribute_trail", "//internal:status_macros", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/status", @@ -389,104 +363,44 @@ cc_library( "create_map_step.h", ], deps = [ - ":attribute_trail", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", - "//common:casting", + ":expression_step_logic", "//common:value", + "//common/internal:attribute_trail", "//internal:status_macros", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", ], ) cc_library( - name = "jump_step", - srcs = [ - "jump_step.cc", - ], - hdrs = [ - "jump_step.h", - ], + name = "equality_steps", deps = [ ":evaluator_core", - ":expression_step_base", - "//common:value", - "//eval/internal:errors", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/types:optional", - "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", ], ) cc_library( - name = "logic_step", - srcs = [ - "logic_step.cc", - ], - hdrs = [ - "logic_step.h", - ], + name = "bitset_steps", deps = [ - ":attribute_trail", - ":direct_expression_step", ":evaluator_core", - ":expression_step_base", - "//base:builtins", - "//common:casting", - "//common:value", - "//common:value_kind", - "//eval/internal:errors", - "//internal:status_macros", - "//runtime/internal:errors", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", - ], -) - -cc_library( - name = "equality_steps", - srcs = [ - "equality_steps.cc", - ], - hdrs = [ - "equality_steps.h", - ], - deps = [ - ":attribute_trail", - ":direct_expression_step", - ":evaluator_core", - ":expression_step_base", - "//base:builtins", - "//common:value", - "//common:value_kind", - "//internal:number", - "//internal:status_macros", - "//runtime/internal:errors", - "//runtime/standard:equality_functions", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", ], ) cc_test( - name = "equality_steps_test", + name = "bitset_steps_test", srcs = [ - "equality_steps_test.cc", + "bitset_steps_test.cc", ], deps = [ ":attribute_trail", - ":direct_expression_step", - ":equality_steps", ":evaluator_core", + ":expression_step_logic", "//base:attributes", + "//common:native_type", "//common:value", "//common:value_kind", "//common:value_testing", @@ -504,32 +418,30 @@ cc_test( ], ) -cc_library( - name = "comprehension_step", +cc_test( + name = "equality_steps_test", srcs = [ - "comprehension_step.cc", - ], - hdrs = [ - "comprehension_step.h", + "equality_steps_test.cc", ], deps = [ - ":attribute_trail", - ":comprehension_slots", - ":direct_expression_step", ":evaluator_core", - ":expression_step_base", + ":expression_step_logic", "//base:attributes", - "//common:casting", + "//common:unknown", "//common:value", "//common:value_kind", - "//eval/internal:errors", - "//internal:status_macros", - "@com_google_absl//absl/base", - "@com_google_absl//absl/base:core_headers", + "//common:value_testing", + "//common/internal:attribute_trail", + "//internal:testing", + "//internal:testing_descriptor_pool", + "//internal:testing_message_factory", + "//runtime:activation", + "//runtime:runtime_options", + "//runtime/internal:runtime_type_provider", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", + "@com_google_protobuf//:protobuf", ], ) @@ -540,34 +452,19 @@ cc_test( "comprehension_step_test.cc", ], deps = [ - ":attribute_trail", ":cel_expression_flat_impl", - ":comprehension_slots", - ":comprehension_step", - ":const_value_step", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", - ":ident_step", "//base:data", "//common:expr", - "//common:value", - "//common:value_testing", "//eval/public:activation", "//eval/public:cel_attribute", "//eval/public:cel_value", "//eval/public/structs:cel_proto_wrapper", - "//internal:status_macros", "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", "//runtime:runtime_options", "//runtime/internal:runtime_env_testing", - "//runtime/internal:runtime_type_provider", - "@com_google_absl//absl/memory", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", @@ -584,6 +481,7 @@ cc_test( deps = [ ":cel_expression_flat_impl", ":evaluator_core", + ":expression_step_logic", "//base:data", "//common:value", "//eval/compiler:cel_expression_builder_flat_impl", @@ -595,9 +493,12 @@ cc_test( "//internal:testing_descriptor_pool", "//internal:testing_message_factory", "//runtime:activation", + "//runtime:function_overload_reference", + "//runtime:function_registry", "//runtime:runtime_options", "//runtime/internal:runtime_env_testing", "//runtime/internal:runtime_type_provider", + "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/status", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", @@ -613,9 +514,8 @@ cc_test( deps = [ ":cel_expression_flat_impl", ":container_access_step", - ":direct_expression_step", ":evaluator_core", - ":ident_step", + ":expression_step_logic", "//base:builtins", "//base:data", "//common:ast", @@ -633,10 +533,13 @@ cc_test( "//eval/public/testing:matchers", "//internal:testing", "//parser", + "//runtime:runtime_options", "//runtime/internal:runtime_env", "//runtime/internal:runtime_env_testing", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:struct_cc_proto", @@ -658,7 +561,6 @@ cc_test( "//internal:testing", "//parser", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", @@ -672,25 +574,22 @@ cc_test( "ident_step_test.cc", ], deps = [ - ":attribute_trail", ":cel_expression_flat_impl", ":evaluator_core", - ":ident_step", "//base:data", - "//common:casting", "//common:memory", - "//common:value", "//eval/public:activation", + "//eval/public:base_activation", "//eval/public:cel_attribute", + "//eval/public:cel_function", "//eval/public:cel_value", "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", "//runtime:runtime_options", + "//runtime/internal:runtime_env", "//runtime/internal:runtime_env_testing", - "//runtime/internal:runtime_type_provider", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", "@com_google_protobuf//:protobuf", ], ) @@ -703,12 +602,8 @@ cc_test( ], deps = [ ":cel_expression_flat_impl", - ":const_value_step", - ":direct_expression_step", ":evaluator_core", - ":function_step", - ":ident_step", - "//base:builtins", + "//base:attributes", "//base:data", "//common:constant", "//common:expr", @@ -726,10 +621,7 @@ cc_test( "//eval/public/testing:matchers", "//eval/testutil:test_message_cc_proto", "//internal:testing", - "//runtime:function_overload_reference", - "//runtime:function_registry", "//runtime:runtime_options", - "//runtime:standard_functions", "//runtime/internal:runtime_env_testing", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", @@ -747,37 +639,20 @@ cc_test( "logic_step_test.cc", ], deps = [ - ":attribute_trail", ":cel_expression_flat_impl", - ":const_value_step", - ":direct_expression_step", ":evaluator_core", - ":ident_step", - ":logic_step", - "//base:attributes", "//base:data", - "//common:casting", "//common:expr", - "//common:unknown", - "//common:value", "//eval/public:activation", - "//eval/public:cel_attribute", "//eval/public:cel_value", - "//eval/public:unknown_attribute_set", - "//eval/public:unknown_set", "//internal:status_macros", "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", "//runtime:runtime_options", "//runtime/internal:runtime_env", "//runtime/internal:runtime_env_testing", - "//runtime/internal:runtime_type_provider", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", - "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", "@com_google_protobuf//:protobuf", ], @@ -790,48 +665,40 @@ cc_test( "select_step_test.cc", ], deps = [ - ":attribute_trail", ":cel_expression_flat_impl", - ":const_value_step", ":evaluator_core", - ":ident_step", ":select_step", "//base:attributes", "//base:data", - "//common:casting", "//common:expr", "//common:legacy_value", + "//common:type", "//common:value", - "//common:value_testing", "//eval/public:activation", "//eval/public:cel_attribute", "//eval/public:cel_value", + "//eval/public:unknown_attribute_set", + "//eval/public:unknown_set", "//eval/public/containers:container_backed_map_impl", "//eval/public/structs:cel_proto_wrapper", - "//eval/public/structs:legacy_type_adapter", "//eval/public/structs:trivial_legacy_type_info", - "//eval/public/testing:matchers", "//eval/testutil:test_extensions_cc_proto", "//eval/testutil:test_message_cc_proto", - "//extensions/protobuf:value", "//internal:proto_matchers", "//internal:status_macros", "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", "//runtime:runtime_options", "//runtime/internal:runtime_env", "//runtime/internal:runtime_env_testing", - "//runtime/internal:runtime_type_provider", "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", + "@com_google_protobuf//:protobuf", "@com_google_protobuf//:wrappers_cc_proto", ], ) @@ -843,19 +710,12 @@ cc_test( "create_list_step_test.cc", ], deps = [ - ":attribute_trail", ":cel_expression_flat_impl", - ":const_value_step", - ":create_list_step", - ":direct_expression_step", ":evaluator_core", - ":ident_step", "//base:attributes", "//base:data", - "//common:casting", "//common:expr", "//common:value", - "//common:value_testing", "//eval/internal:interop", "//eval/public:activation", "//eval/public:cel_attribute", @@ -864,13 +724,9 @@ cc_test( "//eval/public/testing:matchers", "//internal:status_macros", "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", "//runtime:runtime_options", "//runtime/internal:runtime_env", "//runtime/internal:runtime_env_testing", - "//runtime/internal:runtime_type_provider", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", @@ -889,11 +745,8 @@ cc_test( deps = [ ":cel_expression_flat_impl", ":create_struct_step", - ":direct_expression_step", ":evaluator_core", - ":ident_step", "//base:data", - "//common:expr", "//eval/public:activation", "//eval/public:cel_type_registry", "//eval/public:cel_value", @@ -928,9 +781,7 @@ cc_test( deps = [ ":cel_expression_flat_impl", ":create_map_step", - ":direct_expression_step", ":evaluator_core", - ":ident_step", "//base:data", "//common:expr", "//eval/public:activation", @@ -954,74 +805,15 @@ cc_test( cc_library( name = "attribute_trail", - srcs = ["attribute_trail.cc"], hdrs = ["attribute_trail.h"], deps = [ + "//base:attribute_matcher", "//base:attributes", - "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/utility", - ], -) - -cc_test( - name = "attribute_trail_test", - size = "small", - srcs = [ - "attribute_trail_test.cc", - ], - deps = [ - ":attribute_trail", - "//eval/public:cel_attribute", - "//eval/public:cel_value", - "//internal:testing", - "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", - ], -) - -cc_library( - name = "attribute_utility", - srcs = ["attribute_utility.cc"], - hdrs = ["attribute_utility.h"], - deps = [ - ":attribute_trail", - "//base:attributes", - "//base:function_result", - "//base:function_result_set", - "//base/internal:unknown_set", - "//common:casting", - "//common:function_descriptor", - "//common:unknown", - "//common:value", - "//eval/internal:errors", - "//internal:status_macros", - "//runtime/internal:attribute_matcher", - "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", - ], -) - -cc_test( - name = "attribute_utility_test", - size = "small", - srcs = [ - "attribute_utility_test.cc", - ], - deps = [ - ":attribute_trail", - ":attribute_utility", - "//base:attributes", - "//common:unknown", "//common:value", - "//eval/public:cel_attribute", - "//eval/public:cel_value", - "//eval/public:unknown_attribute_set", - "//eval/public:unknown_set", - "//internal:testing", - "//runtime/internal:attribute_matcher", - "@com_google_absl//absl/types:span", - "@com_google_protobuf//:protobuf", + "//common/internal:attribute_trail", + "//common/internal:unknowns", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/strings:string_view", ], ) @@ -1034,16 +826,13 @@ cc_library( "ternary_step.h", ], deps = [ - ":attribute_trail", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", + ":expression_step_logic", "//base:builtins", "//common:value", "//eval/internal:errors", - "//internal:status_macros", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", ], ) @@ -1054,31 +843,16 @@ cc_test( "ternary_step_test.cc", ], deps = [ - ":attribute_trail", ":cel_expression_flat_impl", - ":const_value_step", - ":direct_expression_step", ":evaluator_core", - ":ident_step", ":ternary_step", - "//base:attributes", "//base:data", - "//common:casting", - "//common:expr", - "//common:value", "//eval/public:activation", "//eval/public:cel_value", - "//eval/public:unknown_attribute_set", - "//eval/public:unknown_set", - "//internal:status_macros", "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", "//runtime:runtime_options", "//runtime/internal:runtime_env", "//runtime/internal:runtime_env_testing", - "//runtime/internal:runtime_type_provider", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_protobuf//:protobuf", @@ -1090,13 +864,10 @@ cc_library( srcs = ["shadowable_value_step.cc"], hdrs = ["shadowable_value_step.h"], deps = [ - ":attribute_trail", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", + ":expression_step_logic", "//common:value", - "//internal:status_macros", - "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:string_view", @@ -1123,67 +894,17 @@ cc_test( "//runtime/internal:runtime_env_testing", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status:statusor", - ], -) - -cc_library( - name = "compiler_constant_step", - srcs = ["compiler_constant_step.cc"], - hdrs = ["compiler_constant_step.h"], - deps = [ - ":attribute_trail", - ":direct_expression_step", - ":evaluator_core", - ":expression_step_base", - "//common:native_type", - "//common:value", - "@com_google_absl//absl/status", - ], -) - -cc_test( - name = "compiler_constant_step_test", - srcs = ["compiler_constant_step_test.cc"], - deps = [ - ":compiler_constant_step", - ":evaluator_core", - "//common:native_type", - "//common:value", - "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", - "//runtime:runtime_options", - "//runtime/internal:runtime_type_provider", + "@com_google_absl//absl/strings:string_view", "@com_google_protobuf//:protobuf", ], ) -cc_library( - name = "lazy_init_step", - srcs = ["lazy_init_step.cc"], - hdrs = ["lazy_init_step.h"], - deps = [ - ":attribute_trail", - ":comprehension_slots", - ":direct_expression_step", - ":evaluator_core", - ":expression_step_base", - "//common:value", - "//internal:status_macros", - "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/status", - "@com_google_cel_spec//proto/cel/expr:value_cc_proto", - ], -) - cc_test( name = "lazy_init_step_test", srcs = ["lazy_init_step_test.cc"], deps = [ - ":const_value_step", + ":comprehension_slots", ":evaluator_core", - ":lazy_init_step", "//base:data", "//common:value", "//internal:testing", @@ -1196,80 +917,20 @@ cc_test( ], ) -cc_library( - name = "direct_expression_step", - srcs = ["direct_expression_step.cc"], - hdrs = ["direct_expression_step.h"], - deps = [ - ":attribute_trail", - ":evaluator_core", - "//common:native_type", - "//common:value", - "//internal:status_macros", - "@com_google_absl//absl/status", - "@com_google_absl//absl/types:optional", - ], -) - -cc_library( - name = "trace_step", - hdrs = ["trace_step.h"], - deps = [ - ":attribute_trail", - ":direct_expression_step", - ":evaluator_core", - "//common:native_type", - "//common:value", - "//internal:status_macros", - "@com_google_absl//absl/status", - "@com_google_absl//absl/types:optional", - ], -) - cc_library( name = "optional_or_step", srcs = ["optional_or_step.cc"], hdrs = ["optional_or_step.h"], deps = [ ":attribute_trail", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", - ":jump_step", - "//common:casting", + ":expression_step_logic", "//common:value", - "//internal:status_macros", "//runtime/internal:errors", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", - ], -) - -cc_test( - name = "optional_or_step_test", - srcs = ["optional_or_step_test.cc"], - deps = [ - ":attribute_trail", - ":const_value_step", - ":direct_expression_step", - ":evaluator_core", - ":optional_or_step", - "//common:casting", - "//common:value", - "//common:value_kind", - "//common:value_testing", - "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", - "//runtime:runtime_options", - "//runtime/internal:errors", - "//runtime/internal:runtime_type_provider", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/status", "@com_google_protobuf//:protobuf", ], ) @@ -1280,6 +941,7 @@ cc_library( deps = [ "//common:value", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:inlined_vector", "@com_google_absl//absl/log:absl_check", ], ) diff --git a/eval/eval/attribute_trail.cc b/eval/eval/attribute_trail.cc deleted file mode 100644 index 6b5db896e..000000000 --- a/eval/eval/attribute_trail.cc +++ /dev/null @@ -1,28 +0,0 @@ -#include "eval/eval/attribute_trail.h" - -#include -#include -#include -#include -#include - -#include "base/attribute.h" - -namespace google::api::expr::runtime { - -// Creates AttributeTrail with attribute path incremented by "qualifier". -AttributeTrail AttributeTrail::Step(cel::AttributeQualifier qualifier) const { - // Cannot continue void trail - if (empty()) return AttributeTrail(); - - std::vector qualifiers; - qualifiers.reserve(attribute_->qualifier_path().size() + 1); - std::copy_n(attribute_->qualifier_path().begin(), - attribute_->qualifier_path().size(), - std::back_inserter(qualifiers)); - qualifiers.push_back(std::move(qualifier)); - return AttributeTrail(cel::Attribute(std::string(attribute_->variable_name()), - std::move(qualifiers))); -} - -} // namespace google::api::expr::runtime diff --git a/eval/eval/attribute_trail.h b/eval/eval/attribute_trail.h index 576d0be34..bd76571cf 100644 --- a/eval/eval/attribute_trail.h +++ b/eval/eval/attribute_trail.h @@ -1,63 +1,15 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_ATTRIBUTE_TRAIL_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_ATTRIBUTE_TRAIL_H_ -#include -#include - -#include "absl/types/optional.h" -#include "absl/utility/utility.h" -#include "base/attribute.h" +#include "base/attribute_matcher.h" +#include "common/internal/attribute_trail.h" namespace google::api::expr::runtime { -// AttributeTrail reflects current attribute path. -// It is functionally similar to cel::Attribute, yet intended to have better -// complexity on attribute path increment operations. -// TODO(issues/41) Current AttributeTrail implementation is equivalent to -// cel::Attribute - improve it. -// Intended to be used in conjunction with cel::Value, describing the attribute -// value originated from. -// Empty AttributeTrail denotes object with attribute path not defined -// or supported. -class AttributeTrail { - public: - AttributeTrail() : attribute_(absl::nullopt) {} - - explicit AttributeTrail(std::string variable_name) - : attribute_(absl::in_place, std::move(variable_name)) {} - - explicit AttributeTrail(cel::Attribute attribute) - : attribute_(std::move(attribute)) {} - - // NOLINTNEXTLINE(google-explicit-constructor) - AttributeTrail(absl::nullopt_t) : AttributeTrail() {} - - AttributeTrail(const AttributeTrail&) = default; - AttributeTrail& operator=(const AttributeTrail&) = default; - AttributeTrail(AttributeTrail&&) = default; - AttributeTrail& operator=(AttributeTrail&&) = default; - - AttributeTrail& operator=(absl::nullopt_t) { - attribute_.reset(); - return *this; - } - - // Creates AttributeTrail with attribute path incremented by "qualifier". - AttributeTrail Step(cel::AttributeQualifier qualifier) const; - - // Creates AttributeTrail with attribute path incremented by "qualifier". - AttributeTrail Step(const std::string* qualifier) const { - return Step(cel::AttributeQualifier::OfString(*qualifier)); - } - - // Returns CelAttribute that corresponds to content of AttributeTrail. - const cel::Attribute& attribute() const { return attribute_.value(); } - - bool empty() const { return !attribute_.has_value(); } - - private: - absl::optional attribute_; -}; +using cel::AttributeMatch; +using cel::common_internal::AttributeLeadMatch; +using cel::common_internal::AttributeTrail; +using cel::common_internal::UnknownAccumulator; } // namespace google::api::expr::runtime diff --git a/eval/eval/attribute_trail_test.cc b/eval/eval/attribute_trail_test.cc deleted file mode 100644 index 3143b9ed4..000000000 --- a/eval/eval/attribute_trail_test.cc +++ /dev/null @@ -1,31 +0,0 @@ -#include "eval/eval/attribute_trail.h" - -#include - -#include "cel/expr/syntax.pb.h" -#include "eval/public/cel_attribute.h" -#include "eval/public/cel_value.h" -#include "internal/testing.h" - -namespace google::api::expr::runtime { - -// Attribute Trail behavior -TEST(AttributeTrailTest, AttributeTrailEmptyStep) { - std::string step = "step"; - CelValue step_value = CelValue::CreateString(&step); - AttributeTrail trail; - ASSERT_TRUE(trail.Step(&step).empty()); - ASSERT_TRUE(trail.Step(CreateCelAttributeQualifier(step_value)).empty()); -} - -TEST(AttributeTrailTest, AttributeTrailStep) { - std::string step = "step"; - CelValue step_value = CelValue::CreateString(&step); - - AttributeTrail trail = AttributeTrail("ident").Step(&step); - - ASSERT_EQ(trail.attribute(), - CelAttribute("ident", {CreateCelAttributeQualifier(step_value)})); -} - -} // namespace google::api::expr::runtime diff --git a/eval/eval/attribute_utility.cc b/eval/eval/attribute_utility.cc deleted file mode 100644 index 117516caf..000000000 --- a/eval/eval/attribute_utility.cc +++ /dev/null @@ -1,260 +0,0 @@ -#include "eval/eval/attribute_utility.h" - -#include -#include -#include - -#include "absl/status/statusor.h" -#include "absl/types/optional.h" -#include "absl/types/span.h" -#include "base/attribute.h" -#include "base/attribute_set.h" -#include "base/function_result.h" -#include "base/function_result_set.h" -#include "base/internal/unknown_set.h" -#include "common/casting.h" -#include "common/function_descriptor.h" -#include "common/unknown.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/internal/errors.h" -#include "internal/status_macros.h" -#include "runtime/internal/attribute_matcher.h" - -namespace google::api::expr::runtime { - -using ::cel::Attribute; -using ::cel::AttributePattern; -using ::cel::AttributeSet; -using ::cel::Cast; -using ::cel::ErrorValue; -using ::cel::FunctionResult; -using ::cel::FunctionResultSet; -using ::cel::InstanceOf; -using ::cel::UnknownValue; -using ::cel::Value; -using ::cel::base_internal::UnknownSet; -using ::cel::runtime_internal::AttributeMatcher; - -using Accumulator = AttributeUtility::Accumulator; -using MatchResult = AttributeMatcher::MatchResult; - -DefaultAttributeMatcher::DefaultAttributeMatcher( - absl::Span unknown_patterns, - absl::Span missing_patterns) - : unknown_patterns_(unknown_patterns), - missing_patterns_(missing_patterns) {} - -DefaultAttributeMatcher::DefaultAttributeMatcher() = default; - -AttributeMatcher::MatchResult MatchAgainstPatterns( - absl::Span patterns, const Attribute& attr) { - MatchResult result = MatchResult::NONE; - for (const auto& pattern : patterns) { - auto current_match = pattern.IsMatch(attr); - if (current_match == cel::AttributePattern::MatchType::FULL) { - return MatchResult::FULL; - } - if (current_match == cel::AttributePattern::MatchType::PARTIAL) { - result = MatchResult::PARTIAL; - } - } - return result; -} - -DefaultAttributeMatcher::MatchResult DefaultAttributeMatcher::CheckForUnknown( - const Attribute& attr) const { - return MatchAgainstPatterns(unknown_patterns_, attr); -} - -DefaultAttributeMatcher::MatchResult DefaultAttributeMatcher::CheckForMissing( - const Attribute& attr) const { - return MatchAgainstPatterns(missing_patterns_, attr); -} - -bool AttributeUtility::CheckForMissingAttribute( - const AttributeTrail& trail) const { - if (trail.empty()) { - return false; - } - // Missing attributes are only treated as errors if the attribute exactly - // matches (so no guard against passing partial state to a function as with - // unknowns). This was initially a design oversight, but is difficult to - // change now. - return matcher_->CheckForMissing(trail.attribute()) == - AttributeMatcher::MatchResult::FULL; -} - -// Checks whether particular corresponds to any patterns that define unknowns. -bool AttributeUtility::CheckForUnknown(const AttributeTrail& trail, - bool use_partial) const { - if (trail.empty()) { - return false; - } - MatchResult result = matcher_->CheckForUnknown(trail.attribute()); - - if (result == MatchResult::FULL || - (use_partial && result == MatchResult::PARTIAL)) { - return true; - } - return false; -} - -// Creates merged UnknownAttributeSet. -// Scans over the args collection, merges any UnknownSets found in -// it together with initial_set (if initial_set is not null). -// Returns pointer to merged set or nullptr, if there were no sets to merge. -absl::optional AttributeUtility::MergeUnknowns( - absl::Span args) const { - // Empty unknown value may be used as a sentinel in some tests so need to - // distinguish unset (nullopt) and empty(engaged empty value). - absl::optional result_set; - - for (const auto& value : args) { - if (!value->Is()) continue; - if (!result_set.has_value()) { - result_set.emplace(); - } - const auto& current_set = value.GetUnknown(); - - cel::base_internal::UnknownSetAccess::Add( - *result_set, UnknownSet(current_set.attribute_set(), - current_set.function_result_set())); - } - - if (!result_set.has_value()) { - return absl::nullopt; - } - - return UnknownValue(cel::Unknown(result_set->unknown_attributes(), - result_set->unknown_function_results())); -} - -UnknownValue AttributeUtility::MergeUnknownValues( - const UnknownValue& left, const UnknownValue& right) const { - // Empty unknown value may be used as a sentinel in some tests so need to - // distinguish unset (nullopt) and empty(engaged empty value). - AttributeSet attributes; - FunctionResultSet function_results; - attributes.Add(left.attribute_set()); - function_results.Add(left.function_result_set()); - attributes.Add(right.attribute_set()); - function_results.Add(right.function_result_set()); - - return UnknownValue( - cel::Unknown(std::move(attributes), std::move(function_results))); -} - -// Creates merged UnknownAttributeSet. -// Scans over the args collection, determines if there matches to unknown -// patterns, merges attributes together with those from initial_set -// (if initial_set is not null). -// Returns pointer to merged set or nullptr, if there were no sets to merge. -AttributeSet AttributeUtility::CheckForUnknowns( - absl::Span args, bool use_partial) const { - AttributeSet attribute_set; - - for (const auto& trail : args) { - if (CheckForUnknown(trail, use_partial)) { - attribute_set.Add(trail.attribute()); - } - } - - return attribute_set; -} - -// Creates merged UnknownAttributeSet. -// Merges together attributes from UnknownAttributeSets found in the args -// collection, attributes from attr that match unknown pattern -// patterns, and attributes from initial_set -// (if initial_set is not null). -// Returns pointer to merged set or nullptr, if there were no sets to merge. -absl::optional AttributeUtility::IdentifyAndMergeUnknowns( - absl::Span args, absl::Span attrs, - bool use_partial) const { - absl::optional result_set; - - // Identify new unknowns by attribute patterns. - cel::AttributeSet attr_set = CheckForUnknowns(attrs, use_partial); - if (!attr_set.empty()) { - result_set.emplace(std::move(attr_set)); - } - - // merge down existing unknown sets - absl::optional arg_unknowns = MergeUnknowns(args); - - if (!result_set.has_value()) { - // No new unknowns so no need to check for presence of existing unknowns -- - // just forward. - return arg_unknowns; - } - - if (arg_unknowns.has_value()) { - cel::base_internal::UnknownSetAccess::Add( - *result_set, UnknownSet((*arg_unknowns).attribute_set(), - (*arg_unknowns).function_result_set())); - } - - return UnknownValue(cel::Unknown(result_set->unknown_attributes(), - result_set->unknown_function_results())); -} - -UnknownValue AttributeUtility::CreateUnknownSet(cel::Attribute attr) const { - return UnknownValue(cel::Unknown(AttributeSet({std::move(attr)}))); -} - -absl::StatusOr AttributeUtility::CreateMissingAttributeError( - const cel::Attribute& attr) const { - CEL_ASSIGN_OR_RETURN(std::string message, attr.AsString()); - return cel::ErrorValue( - cel::runtime_internal::CreateMissingAttributeError(message)); -} - -UnknownValue AttributeUtility::CreateUnknownSet( - const cel::FunctionDescriptor& fn_descriptor, int64_t expr_id, - absl::Span args) const { - return UnknownValue( - cel::Unknown(FunctionResultSet(FunctionResult(fn_descriptor, expr_id)))); -} - -void AttributeUtility::Add(Accumulator& a, const cel::UnknownValue& v) const { - a.attribute_set_.Add(v.attribute_set()); - a.function_result_set_.Add(v.function_result_set()); -} - -void AttributeUtility::Add(Accumulator& a, const AttributeTrail& attr) const { - a.attribute_set_.Add(attr.attribute()); -} - -void Accumulator::Add(const UnknownValue& value) { - unknown_present_ = true; - parent_.Add(*this, value); -} - -void Accumulator::Add(const AttributeTrail& attr) { parent_.Add(*this, attr); } - -void Accumulator::MaybeAdd(const Value& v) { - if (v.IsUnknown()) { - Add(v.GetUnknown()); - } -} - -void Accumulator::MaybeAdd(const Value& v, const AttributeTrail& attr) { - if (v.IsUnknown()) { - Add(v.GetUnknown()); - } else if (parent_.CheckForUnknown(attr, /*use_partial=*/true)) { - Add(attr); - } -} - -bool Accumulator::IsEmpty() const { - return !unknown_present_ && attribute_set_.empty() && - function_result_set_.empty(); -} - -cel::UnknownValue Accumulator::Build() && { - return cel::UnknownValue( - cel::Unknown(std::move(attribute_set_), std::move(function_result_set_))); -} - -} // namespace google::api::expr::runtime diff --git a/eval/eval/attribute_utility.h b/eval/eval/attribute_utility.h deleted file mode 100644 index 94a5158f0..000000000 --- a/eval/eval/attribute_utility.h +++ /dev/null @@ -1,179 +0,0 @@ -#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_UNKNOWNS_UTILITY_H_ -#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_UNKNOWNS_UTILITY_H_ - -#include - -#include "absl/base/nullability.h" -#include "absl/status/statusor.h" -#include "absl/types/optional.h" -#include "absl/types/span.h" -#include "base/attribute.h" -#include "base/attribute_set.h" -#include "base/function_result_set.h" -#include "common/function_descriptor.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "runtime/internal/attribute_matcher.h" - -namespace google::api::expr::runtime { - -// Default implementation of the attribute matcher. -// Scans the attribute trail against a list of unknown or missing patterns. -class DefaultAttributeMatcher : public cel::runtime_internal::AttributeMatcher { - private: - using MatchResult = cel::runtime_internal::AttributeMatcher::MatchResult; - - public: - DefaultAttributeMatcher( - absl::Span unknown_patterns, - absl::Span missing_patterns); - - DefaultAttributeMatcher(); - - MatchResult CheckForUnknown(const cel::Attribute& attr) const override; - MatchResult CheckForMissing(const cel::Attribute& attr) const override; - - private: - absl::Span unknown_patterns_; - absl::Span missing_patterns_; -}; - -// Helper class for handling unknowns and missing attribute logic. Provides -// helpers for merging unknown sets from arguments on the stack and for -// identifying unknown/missing attributes based on the patterns for a given -// Evaluation. -// Neither moveable nor copyable. -class AttributeUtility { - public: - class Accumulator { - public: - Accumulator(const Accumulator&) = delete; - Accumulator& operator=(const Accumulator&) = delete; - Accumulator(Accumulator&&) = delete; - Accumulator& operator=(Accumulator&&) = delete; - - // Add to the accumulated unknown attributes and functions. - void Add(const cel::UnknownValue& v); - void Add(const AttributeTrail& attr); - - // Add to the accumulated set of unknowns if value is UnknownValue. - void MaybeAdd(const cel::Value& v); - - // Add to the accumulated set of unknowns if value is UnknownValue or - // the attribute trail is (partially) unknown. This version prefers - // preserving an already present unknown value over a new one matching the - // attribute trail. - // - // Uses partial matching (a pattern matches the attribute or any - // sub-attribute). - void MaybeAdd(const cel::Value& v, const AttributeTrail& attr); - - bool IsEmpty() const; - - cel::UnknownValue Build() &&; - - private: - explicit Accumulator(const AttributeUtility& parent) - : parent_(parent), unknown_present_(false) {} - - friend class AttributeUtility; - const AttributeUtility& parent_; - - cel::AttributeSet attribute_set_; - cel::FunctionResultSet function_result_set_; - - // Some tests will use an empty unknown set as a sentinel. - // Preserve forwarding behavior. - bool unknown_present_; - }; - - AttributeUtility(absl::Span unknown_patterns, - absl::Span missing_patterns) - : default_matcher_(unknown_patterns, missing_patterns), - matcher_(&default_matcher_) {} - - explicit AttributeUtility( - const cel::runtime_internal::AttributeMatcher* absl_nonnull matcher) - : matcher_(matcher) {} - - AttributeUtility(const AttributeUtility&) = delete; - AttributeUtility& operator=(const AttributeUtility&) = delete; - AttributeUtility(AttributeUtility&&) = delete; - AttributeUtility& operator=(AttributeUtility&&) = delete; - - // Checks whether particular corresponds to any patterns that define missing - // attribute. - bool CheckForMissingAttribute(const AttributeTrail& trail) const; - - // Checks whether trail corresponds to any patterns that define unknowns. - bool CheckForUnknown(const AttributeTrail& trail, bool use_partial) const; - - // Checks whether trail corresponds to any patterns that identify - // unknowns. Only matches exactly (exact attribute match for self or parent). - bool CheckForUnknownExact(const AttributeTrail& trail) const { - return CheckForUnknown(trail, false); - } - - // Checks whether trail corresponds to any patterns that define unknowns. - // Matches if a parent or any descendant (select or index of) the attribute. - bool CheckForUnknownPartial(const AttributeTrail& trail) const { - return CheckForUnknown(trail, true); - } - - // Creates merged UnknownAttributeSet. - // Scans over the args collection, determines if there matches to unknown - // patterns and returns the (possibly empty) collection. - cel::AttributeSet CheckForUnknowns(absl::Span args, - bool use_partial) const; - - // Creates merged UnknownValue. - // Scans over the args collection, merges any UnknownValues found. - // Returns the merged UnknownValue or nullopt if not found. - absl::optional MergeUnknowns( - absl::Span args) const; - - // Creates a merged UnknownValue from two unknown values. - cel::UnknownValue MergeUnknownValues(const cel::UnknownValue& left, - const cel::UnknownValue& right) const; - - // Creates merged UnknownValue. - // Merges together UnknownValues found in the args - // along with attributes from attr that match the configured unknown patterns - // Returns returns the merged UnknownValue if available or nullopt. - absl::optional IdentifyAndMergeUnknowns( - absl::Span args, absl::Span attrs, - bool use_partial) const; - - // Create an initial UnknownSet from a single attribute. - cel::UnknownValue CreateUnknownSet(cel::Attribute attr) const; - - // Factory function for missing attribute errors. - absl::StatusOr CreateMissingAttributeError( - const cel::Attribute& attr) const; - - // Create an initial UnknownSet from a single missing function call. - cel::UnknownValue CreateUnknownSet( - const cel::FunctionDescriptor& fn_descriptor, int64_t expr_id, - absl::Span args) const; - - Accumulator CreateAccumulator() const ABSL_ATTRIBUTE_LIFETIME_BOUND { - return Accumulator(*this); - } - - void set_matcher( - const cel::runtime_internal::AttributeMatcher* absl_nonnull matcher) { - matcher_ = matcher; - } - - private: - // Workaround friend visibility. - void Add(Accumulator& a, const cel::UnknownValue& v) const; - void Add(Accumulator& a, const AttributeTrail& attr) const; - - DefaultAttributeMatcher default_matcher_; - const cel::runtime_internal::AttributeMatcher* absl_nonnull matcher_; -}; - -} // namespace google::api::expr::runtime - -#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_UNKNOWNS_UTILITY_H_ diff --git a/eval/eval/attribute_utility_test.cc b/eval/eval/attribute_utility_test.cc deleted file mode 100644 index f3dbc0d06..000000000 --- a/eval/eval/attribute_utility_test.cc +++ /dev/null @@ -1,216 +0,0 @@ -#include "eval/eval/attribute_utility.h" - -#include -#include - -#include "absl/types/span.h" -#include "base/attribute.h" -#include "base/attribute_set.h" -#include "common/unknown.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/public/cel_attribute.h" -#include "eval/public/cel_value.h" -#include "eval/public/unknown_attribute_set.h" -#include "eval/public/unknown_set.h" -#include "internal/testing.h" -#include "runtime/internal/attribute_matcher.h" -#include "google/protobuf/arena.h" - -namespace google::api::expr::runtime { - -using ::cel::AttributeSet; - -using ::cel::UnknownValue; -using ::cel::Value; -using ::testing::Eq; -using ::testing::SizeIs; -using ::testing::UnorderedPointwise; - -class AttributeUtilityTest : public ::testing::Test { - public: - AttributeUtilityTest() = default; - - protected: - google::protobuf::Arena arena_; -}; - -absl::Span NoPatterns() { return {}; } - -TEST_F(AttributeUtilityTest, UnknownsUtilityCheckUnknowns) { - std::vector unknown_patterns = { - CelAttributePattern("unknown0", {CreateCelAttributeQualifierPattern( - CelValue::CreateInt64(1))}), - CelAttributePattern("unknown0", {CreateCelAttributeQualifierPattern( - CelValue::CreateInt64(2))}), - CelAttributePattern("unknown1", {}), - CelAttributePattern("unknown2", {}), - }; - - std::vector missing_attribute_patterns; - - AttributeUtility utility(unknown_patterns, missing_attribute_patterns); - // no match for void trail - ASSERT_FALSE(utility.CheckForUnknown(AttributeTrail(), true)); - ASSERT_FALSE(utility.CheckForUnknown(AttributeTrail(), false)); - - AttributeTrail unknown_trail0("unknown0"); - - { ASSERT_FALSE(utility.CheckForUnknown(unknown_trail0, false)); } - - { ASSERT_TRUE(utility.CheckForUnknown(unknown_trail0, true)); } - - { - ASSERT_TRUE(utility.CheckForUnknown( - unknown_trail0.Step( - CreateCelAttributeQualifier(CelValue::CreateInt64(1))), - false)); - } - - { - ASSERT_TRUE(utility.CheckForUnknown( - unknown_trail0.Step( - CreateCelAttributeQualifier(CelValue::CreateInt64(1))), - true)); - } -} - -TEST_F(AttributeUtilityTest, UnknownsUtilityMergeUnknownsFromValues) { - std::vector unknown_patterns; - - std::vector missing_attribute_patterns; - - CelAttribute attribute0("unknown0", {}); - CelAttribute attribute1("unknown1", {}); - - AttributeUtility utility(unknown_patterns, missing_attribute_patterns); - - UnknownValue unknown_set0 = - cel::UnknownValue(cel::Unknown(AttributeSet({attribute0}))); - UnknownValue unknown_set1 = - cel::UnknownValue(cel::Unknown(AttributeSet({attribute1}))); - - std::vector values = { - unknown_set0, - unknown_set1, - cel::BoolValue(true), - cel::IntValue(1), - }; - - absl::optional unknown_set = utility.MergeUnknowns(values); - ASSERT_TRUE(unknown_set.has_value()); - EXPECT_THAT((*unknown_set).attribute_set(), - UnorderedPointwise( - Eq(), std::vector{attribute0, attribute1})); -} - -TEST_F(AttributeUtilityTest, UnknownsUtilityCheckForUnknownsFromAttributes) { - std::vector unknown_patterns = { - CelAttributePattern("unknown0", - {CelAttributeQualifierPattern::CreateWildcard()}), - }; - - std::vector missing_attribute_patterns; - - AttributeTrail trail0("unknown0"); - AttributeTrail trail1("unknown1"); - - CelAttribute attribute1("unknown1", {}); - UnknownSet unknown_set1(UnknownAttributeSet({attribute1})); - - AttributeUtility utility(unknown_patterns, missing_attribute_patterns); - - UnknownSet unknown_attr_set(utility.CheckForUnknowns( - { - AttributeTrail(), // To make sure we handle empty trail gracefully. - trail0.Step(CreateCelAttributeQualifier(CelValue::CreateInt64(1))), - trail0.Step(CreateCelAttributeQualifier(CelValue::CreateInt64(2))), - }, - false)); - - UnknownSet unknown_set(unknown_set1, unknown_attr_set); - - ASSERT_THAT(unknown_set.unknown_attributes(), SizeIs(3)); -} - -TEST_F(AttributeUtilityTest, UnknownsUtilityCheckForMissingAttributes) { - std::vector unknown_patterns; - - std::vector missing_attribute_patterns; - - AttributeTrail trail("destination"); - trail = - trail.Step(CreateCelAttributeQualifier(CelValue::CreateStringView("ip"))); - - AttributeUtility utility0(unknown_patterns, missing_attribute_patterns); - EXPECT_FALSE(utility0.CheckForMissingAttribute(trail)); - - missing_attribute_patterns.push_back(CelAttributePattern( - "destination", - {CreateCelAttributeQualifierPattern(CelValue::CreateStringView("ip"))})); - - AttributeUtility utility1(unknown_patterns, missing_attribute_patterns); - EXPECT_TRUE(utility1.CheckForMissingAttribute(trail)); -} - -TEST_F(AttributeUtilityTest, CreateUnknownSet) { - AttributeTrail trail("destination"); - trail = - trail.Step(CreateCelAttributeQualifier(CelValue::CreateStringView("ip"))); - - std::vector empty_patterns; - AttributeUtility utility(empty_patterns, empty_patterns); - - UnknownValue set = utility.CreateUnknownSet(trail.attribute()); - ASSERT_THAT(set.attribute_set(), SizeIs(1)); - ASSERT_OK_AND_ASSIGN(auto elem, set.attribute_set().begin()->AsString()); - EXPECT_EQ(elem, "destination.ip"); -} - -class FakeMatcher : public cel::runtime_internal::AttributeMatcher { - private: - using MatchResult = cel::runtime_internal::AttributeMatcher::MatchResult; - - public: - MatchResult CheckForUnknown(const cel::Attribute& attr) const override { - std::string attr_str = attr.AsString().value_or(""); - if (attr_str == "device.foo") { - return MatchResult::FULL; - } else if (attr_str == "device") { - return MatchResult::PARTIAL; - } - return MatchResult::NONE; - } - - MatchResult CheckForMissing(const cel::Attribute& attr) const override { - std::string attr_str = attr.AsString().value_or(""); - - if (attr_str == "device2.foo") { - return MatchResult::FULL; - } else if (attr_str == "device2") { - return MatchResult::PARTIAL; - } - return MatchResult::NONE; - } -}; - -TEST_F(AttributeUtilityTest, CustomMatcher) { - AttributeTrail trail("device"); - - AttributeUtility utility(NoPatterns(), NoPatterns()); - FakeMatcher matcher; - utility.set_matcher(&matcher); - EXPECT_TRUE(utility.CheckForUnknownPartial(trail)); - EXPECT_FALSE(utility.CheckForUnknownExact(trail)); - - trail = trail.Step(cel::AttributeQualifier::OfString("foo")); - EXPECT_TRUE(utility.CheckForUnknownExact(trail)); - EXPECT_TRUE(utility.CheckForUnknownPartial(trail)); - - trail = AttributeTrail("device2"); - EXPECT_FALSE(utility.CheckForMissingAttribute(trail)); - trail = trail.Step(cel::AttributeQualifier::OfString("foo")); - EXPECT_TRUE(utility.CheckForMissingAttribute(trail)); -} - -} // namespace google::api::expr::runtime diff --git a/eval/eval/bitset_steps.cc b/eval/eval/bitset_steps.cc new file mode 100644 index 000000000..001ed8a22 --- /dev/null +++ b/eval/eval/bitset_steps.cc @@ -0,0 +1,233 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "eval/eval/bitset_steps.h" + +#include +#include +#include + +#include "absl/base/optimization.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "base/builtins.h" +#include "common/internal/attribute_trail.h" +#include "common/value.h" +#include "common/value_kind.h" +#include "eval/eval/evaluator_core.h" +#include "internal/number.h" +#include "internal/status_macros.h" +#include "runtime/internal/errors.h" + +namespace google::api::expr::runtime { + +namespace { + +using ::cel::BoolValue; +using ::cel::ListValue; +using ::cel::Value; +using ::cel::ValueKind; +using ::cel::common_internal::AttributeTrail; +using ::cel::internal::Number; + +enum class BitSetOp { + kAndIsNonZero, + kAndEqual, + kEqual, +}; + +} // namespace + +uint64_t ExtractBit(const Value& value) { + switch (value.kind()) { + case ValueKind::kInt: { + int64_t v = value.GetInt().NativeValue(); + if (v >= 0 && v < 64) { + return uint64_t{1} << v; + } + return 0; + } + case ValueKind::kUint: { + uint64_t v = value.GetUint().NativeValue(); + if (v < 64) { + return uint64_t{1} << v; + } + return 0; + } + case ValueKind::kDouble: { + Number number = Number::FromDouble(value.GetDouble().NativeValue()); + if (number.LosslessConvertibleToUint()) { + uint64_t v = number.AsUint(); + if (v < 64) { + return uint64_t{1} << v; + } + } + return 0; + } + default: + return 0; + } +} + +namespace { + +absl::StatusOr EvaluateBitAndIsNonZero(ExecutionFrameBase& frame, + const ListValue& list, + uint64_t bitmask) { + if (bitmask == 0) { + return BoolValue(false); + } + bool intersects = false; + CEL_RETURN_IF_ERROR(list.ForEach( + [&](size_t /*index*/, const Value& element) -> absl::StatusOr { + uint64_t bit = ExtractBit(element); + if ((bit & bitmask) != 0) { + intersects = true; + return false; + } + return true; + }, + frame.descriptor_pool(), frame.message_factory(), frame.arena())); + return BoolValue(intersects); +} + +absl::StatusOr EvaluateBitAndEqual(ExecutionFrameBase& frame, + const ListValue& list, + uint64_t bitmask) { + if (bitmask == 0) { + return BoolValue(true); + } + uint64_t remaining = bitmask; + CEL_RETURN_IF_ERROR(list.ForEach( + [&](size_t /*index*/, const Value& element) -> absl::StatusOr { + remaining &= ~ExtractBit(element); + return remaining != 0; + }, + frame.descriptor_pool(), frame.message_factory(), frame.arena())); + return BoolValue(remaining == 0); +} + +absl::StatusOr EvaluateBitEqual(ExecutionFrameBase& frame, + const ListValue& list, + uint64_t bitmask) { + uint64_t list_mask = 0; + bool has_unexpected = false; + CEL_RETURN_IF_ERROR(list.ForEach( + [&](size_t /*index*/, const Value& element) -> absl::StatusOr { + uint64_t bit = ExtractBit(element); + if ((bit & bitmask) == 0) { + has_unexpected = true; + return false; + } + list_mask |= bit; + return true; + }, + frame.descriptor_pool(), frame.message_factory(), frame.arena())); + return BoolValue(!has_unexpected && list_mask == bitmask); +} + +absl::StatusOr EvaluateBitSetOp(ExecutionFrameBase& frame, + const Value& arg, + const AttributeTrail& attr, + uint64_t bitmask, BitSetOp op) { + if (arg.IsError()) { + return arg; + } + + if (frame.unknown_processing_enabled()) { + if (auto unknown = cel::common_internal::PartiallyIdentifyAndMergeUnknowns( + absl::MakeConstSpan(&arg, 1), absl::MakeConstSpan(&attr, 1), + frame.unknown_tree()); + unknown.has_value()) { + return *unknown; + } + } + + if (!arg.IsList()) { + return cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(cel::builtin::kIn), + frame.arena()); + } + + const ListValue& list = arg.GetList(); + + switch (op) { + case BitSetOp::kAndIsNonZero: + return EvaluateBitAndIsNonZero(frame, list, bitmask); + case BitSetOp::kAndEqual: + return EvaluateBitAndEqual(frame, list, bitmask); + case BitSetOp::kEqual: + return EvaluateBitEqual(frame, list, bitmask); + } + ABSL_UNREACHABLE(); +} + +void EvaluateBitSetStep(uint64_t bitmask, BitSetOp op, ExecutionFrame& frame) { + if (!frame.value_stack().HasEnough(1)) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + const Value& arg = frame.value_stack().Peek(); + const AttributeTrail& attr = frame.value_stack().PeekAttribute(); + + absl::StatusOr result = + EvaluateBitSetOp(frame, arg, attr, bitmask, op); + if (!result.ok()) { + frame.Abort(std::move(result).status()); + return; + } + frame.value_stack().PopAndPush(*std::move(result)); +} + +} // namespace + +void EvaluateBitIsSetStep(uint64_t bitmask, ExecutionFrame& frame) { + if (!frame.value_stack().HasEnough(1)) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + const Value& arg = frame.value_stack().Peek(); + if (arg.IsError()) { + return; + } + + if (frame.unknown_processing_enabled()) { + if (auto unknown = cel::common_internal::PartiallyIdentifyAndMergeUnknowns( + frame.value_stack().GetSpan(1), + frame.value_stack().GetAttributeSpan(1), frame.unknown_tree()); + unknown.has_value()) { + frame.value_stack().PopAndPush(1, *unknown); + return; + } + } + + uint64_t bit = ExtractBit(arg); + frame.value_stack().PopAndPush(BoolValue((bit & bitmask) != 0)); +} + +void EvaluateBitAndIsNonZeroStep(uint64_t bitmask, ExecutionFrame& frame) { + EvaluateBitSetStep(bitmask, BitSetOp::kAndIsNonZero, frame); +} + +void EvaluateBitAndEqualStep(uint64_t bitmask, ExecutionFrame& frame) { + EvaluateBitSetStep(bitmask, BitSetOp::kAndEqual, frame); +} + +void EvaluateBitEqualStep(uint64_t bitmask, ExecutionFrame& frame) { + EvaluateBitSetStep(bitmask, BitSetOp::kEqual, frame); +} + +} // namespace google::api::expr::runtime diff --git a/eval/eval/bitset_steps.h b/eval/eval/bitset_steps.h new file mode 100644 index 000000000..553ca42b5 --- /dev/null +++ b/eval/eval/bitset_steps.h @@ -0,0 +1,43 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_BITSET_STEPS_H_ +#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_BITSET_STEPS_H_ + +#include + +#include "common/value.h" + +namespace google::api::expr::runtime { + +class ExecutionFrame; + +// Extracts a single-bit bitmask (`1ULL << v`) for a `cel::Value` equal to an +// integer `v` in `[0, 63]` under heterogeneous equality. +// +// Return values: +// - `1ULL << v` (non-zero) if `value` is `int`, `uint`, or `double` with an +// exact integer value `v` in `[0, 63]`. +// - `0` otherwise (if `value` is outside `[0, 63]`, is not an exact integer, +// or is not a numeric value). +uint64_t ExtractBit(const cel::Value& value); + +void EvaluateBitIsSetStep(uint64_t bitmask, ExecutionFrame& frame); +void EvaluateBitAndIsNonZeroStep(uint64_t bitmask, ExecutionFrame& frame); +void EvaluateBitAndEqualStep(uint64_t bitmask, ExecutionFrame& frame); +void EvaluateBitEqualStep(uint64_t bitmask, ExecutionFrame& frame); + +} // namespace google::api::expr::runtime + +#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_BITSET_STEPS_H_ diff --git a/eval/eval/bitset_steps_test.cc b/eval/eval/bitset_steps_test.cc new file mode 100644 index 000000000..520227f0a --- /dev/null +++ b/eval/eval/bitset_steps_test.cc @@ -0,0 +1,555 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "eval/eval/bitset_steps.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "base/attribute.h" +#include "common/native_type.h" +#include "common/value.h" +#include "common/value_kind.h" +#include "common/value_testing.h" +#include "eval/eval/attribute_trail.h" +#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "internal/testing_message_factory.h" +#include "runtime/activation.h" +#include "runtime/internal/runtime_type_provider.h" +#include "runtime/runtime_options.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" + +namespace google::api::expr::runtime { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::StatusIs; +using ::cel::Attribute; +using ::cel::DoubleValue; +using ::cel::ErrorValue; +using ::cel::IntValue; +using ::cel::ListValueBuilderPtr; +using ::cel::StringValue; +using ::cel::UintValue; +using ::cel::UnknownValue; +using ::cel::Value; +using ::cel::ValueKind; +using ::cel::test::BoolValueIs; +using ::cel::test::ErrorValueIs; +using ::cel::test::ValueKindIs; +using ::testing::HasSubstr; + +AttributeTrail MakeTrailForVariable(absl::string_view variable, + const cel::Activation& activation) { + AttributeTrail trail; + cel::Value dummy; + cel::common_internal::UnknownTree unknown_tree; + static_cast(cel::common_internal::AttributeLeadMatch( + activation.GetUnknownAttributeMatcher(), + activation.GetKnownAttributeMatcher(), + activation.GetMissingAttributeMatcher(), variable, dummy, trail, + &unknown_tree)); + return trail; +} + +class ValueStep : public ExpressionStepLogic { + public: + ValueStep(Value value, AttributeTrail attr) + : ExpressionStepLogic(), + value_(std::move(value)), + attr_(std::move(attr)) {} + explicit ValueStep(Value value) + : ExpressionStepLogic(), value_(std::move(value)), attr_() {} + + void Evaluate(ExecutionFrame* frame) const override { + frame->value_stack().Push(value_, attr_); + } + + private: + Value value_; + AttributeTrail attr_; +}; + +Value MakeList(google::protobuf::Arena* absl_nonnull arena, + const std::vector& elements) { + ListValueBuilderPtr builder = cel::NewListValueBuilder(arena); + for (const Value& elem : elements) { + ABSL_CHECK_OK(builder->Add(elem)); + } + return std::move(*builder).Build(); +} + +enum class OutputType { kBoolTrue, kBoolFalse, kError, kUnknown }; + +struct BitIsSetTestCase { + Value input; + uint64_t bitmask; + OutputType expected_result; +}; + +class BitIsSetTest : public ::testing::TestWithParam {}; + +TEST_P(BitIsSetTest, Evaluate) { + const BitIsSetTestCase& test_case = GetParam(); + cel::Activation activation; + google::protobuf::Arena arena; + cel::RuntimeOptions opts; + opts.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; + cel::runtime_internal::RuntimeTypeProvider type_provider( + cel::internal::GetTestingDescriptorPool()); + + FlatExpressionEvaluatorState state( + /*value_stack_size=*/5, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, + cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena); + + ExecutionPath steps; + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(test_case.input))); + steps.push_back(ExpressionStep::MakeBitIsSetStep(test_case.bitmask)); + + ExecutionFrame frame(steps, activation, opts, state); + + ASSERT_OK_AND_ASSIGN(Value result, frame.Evaluate()); + + switch (test_case.expected_result) { + case OutputType::kBoolTrue: + EXPECT_THAT(result, BoolValueIs(true)); + break; + case OutputType::kBoolFalse: + EXPECT_THAT(result, BoolValueIs(false)); + break; + case OutputType::kError: + EXPECT_THAT(result, ValueKindIs(ValueKind::kError)); + break; + case OutputType::kUnknown: + EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); + break; + } +} + +INSTANTIATE_TEST_SUITE_P( + BitIsSetTest, BitIsSetTest, + testing::Values( + // IntValue boundaries and basics + BitIsSetTestCase{IntValue(0), 0b101, OutputType::kBoolTrue}, + BitIsSetTestCase{IntValue(1), 0b101, OutputType::kBoolFalse}, + BitIsSetTestCase{IntValue(2), 0b101, OutputType::kBoolTrue}, + BitIsSetTestCase{IntValue(63), uint64_t{1} << 63, + OutputType::kBoolTrue}, + BitIsSetTestCase{IntValue(62), uint64_t{1} << 63, + OutputType::kBoolFalse}, + BitIsSetTestCase{IntValue(-1), ~uint64_t{0}, OutputType::kBoolFalse}, + BitIsSetTestCase{IntValue(std::numeric_limits::min()), + ~uint64_t{0}, OutputType::kBoolFalse}, + BitIsSetTestCase{IntValue(64), ~uint64_t{0}, OutputType::kBoolFalse}, + BitIsSetTestCase{IntValue(std::numeric_limits::max()), + ~uint64_t{0}, OutputType::kBoolFalse}, + BitIsSetTestCase{IntValue(0), 0, OutputType::kBoolFalse}, + // UintValue boundaries and basics + BitIsSetTestCase{UintValue(0), 0b101, OutputType::kBoolTrue}, + BitIsSetTestCase{UintValue(1), 0b101, OutputType::kBoolFalse}, + BitIsSetTestCase{UintValue(63), uint64_t{1} << 63, + OutputType::kBoolTrue}, + BitIsSetTestCase{UintValue(64), ~uint64_t{0}, OutputType::kBoolFalse}, + BitIsSetTestCase{UintValue(std::numeric_limits::max()), + ~uint64_t{0}, OutputType::kBoolFalse}, + // DoubleValue (heterogeneous equality is always enabled for bitset + // steps) + BitIsSetTestCase{DoubleValue(0.0), 0b101, OutputType::kBoolTrue}, + BitIsSetTestCase{DoubleValue(2.0), 0b101, OutputType::kBoolTrue}, + BitIsSetTestCase{DoubleValue(1.0), 0b101, OutputType::kBoolFalse}, + BitIsSetTestCase{DoubleValue(63.0), uint64_t{1} << 63, + OutputType::kBoolTrue}, + BitIsSetTestCase{DoubleValue(64.0), ~uint64_t{0}, + OutputType::kBoolFalse}, + BitIsSetTestCase{DoubleValue(-1.0), ~uint64_t{0}, + OutputType::kBoolFalse}, + BitIsSetTestCase{DoubleValue(2.5), ~uint64_t{0}, + OutputType::kBoolFalse}, + BitIsSetTestCase{DoubleValue(NAN), ~uint64_t{0}, + OutputType::kBoolFalse}, + BitIsSetTestCase{DoubleValue(INFINITY), ~uint64_t{0}, + OutputType::kBoolFalse}, + // Non-numeric and Unknown + BitIsSetTestCase{StringValue(), 0b101, OutputType::kBoolFalse}, + BitIsSetTestCase{UnknownValue(), 0b101, OutputType::kUnknown})); + +TEST(BitIsSetStandaloneTest, PropagatesErrorAndPartialUnknown) { + cel::Activation activation; + google::protobuf::Arena arena; + cel::RuntimeOptions opts; + opts.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; + cel::runtime_internal::RuntimeTypeProvider type_provider( + cel::internal::GetTestingDescriptorPool()); + + FlatExpressionEvaluatorState state( + /*value_stack_size=*/5, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, + cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena); + + ASSERT_THAT(activation.SetUnknownPatterns({cel::AttributePattern( + "foo", {cel::AttributeQualifierPattern::OfString("bar")})}), + IsOk()); + + { + ExecutionPath steps; + steps.push_back(ExpressionStep::MakeGenericStep(std::make_unique( + IntValue(0), MakeTrailForVariable("foo", activation)))); + steps.push_back(ExpressionStep::MakeBitIsSetStep(0b1)); + ExecutionFrame frame(steps, activation, opts, state); + ASSERT_OK_AND_ASSIGN(Value result, frame.Evaluate()); + EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); + } + + { + ExecutionPath steps; + steps.push_back(ExpressionStep::MakeGenericStep(std::make_unique( + ErrorValue::From(absl::CancelledError("custom_err"), &arena), + MakeTrailForVariable("foo", activation)))); + steps.push_back(ExpressionStep::MakeBitIsSetStep(0b1)); + ExecutionFrame frame(steps, activation, opts, state); + ASSERT_OK_AND_ASSIGN(Value result, frame.Evaluate()); + EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kCancelled, + "custom_err"))); + } +} + +TEST(BitIsSetStandaloneTest, StackUnderflow) { + cel::Activation activation; + google::protobuf::Arena arena; + cel::RuntimeOptions opts; + cel::runtime_internal::RuntimeTypeProvider type_provider( + cel::internal::GetTestingDescriptorPool()); + + FlatExpressionEvaluatorState state( + /*value_stack_size=*/5, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, + cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena); + + ExecutionPath steps; + steps.push_back(ExpressionStep::MakeBitIsSetStep(0b1)); + ExecutionFrame frame(steps, activation, opts, state); + EXPECT_THAT(frame.Evaluate(), + StatusIs(absl::StatusCode::kInternal, HasSubstr("underflow"))); +} + +enum class BitSetStepKind { kAndIsNonZero, kAndEqual, kEqual }; + +ExpressionStep MakeBitSetStepForKind(BitSetStepKind kind, uint64_t bitmask) { + switch (kind) { + case BitSetStepKind::kAndIsNonZero: + return ExpressionStep::MakeBitAndIsNonZeroStep(bitmask); + case BitSetStepKind::kAndEqual: + return ExpressionStep::MakeBitAndEqualStep(bitmask); + case BitSetStepKind::kEqual: + return ExpressionStep::MakeBitEqualStep(bitmask); + } +} + +struct BitSetTestCase { + BitSetStepKind kind; + std::vector list_elements; + uint64_t bitmask; + OutputType expected_result; +}; + +class BitSetTest : public ::testing::TestWithParam {}; + +TEST_P(BitSetTest, Evaluate) { + const BitSetTestCase& test_case = GetParam(); + cel::Activation activation; + google::protobuf::Arena arena; + cel::RuntimeOptions opts; + opts.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; + cel::runtime_internal::RuntimeTypeProvider type_provider( + cel::internal::GetTestingDescriptorPool()); + + FlatExpressionEvaluatorState state( + /*value_stack_size=*/5, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, + cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena); + + ExecutionPath steps; + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(MakeList(&arena, test_case.list_elements)))); + steps.push_back(MakeBitSetStepForKind(test_case.kind, test_case.bitmask)); + + ExecutionFrame frame(steps, activation, opts, state); + + ASSERT_OK_AND_ASSIGN(Value result, frame.Evaluate()); + + switch (test_case.expected_result) { + case OutputType::kBoolTrue: + EXPECT_THAT(result, BoolValueIs(true)); + break; + case OutputType::kBoolFalse: + EXPECT_THAT(result, BoolValueIs(false)); + break; + case OutputType::kError: + EXPECT_THAT(result, ValueKindIs(ValueKind::kError)); + break; + case OutputType::kUnknown: + EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); + break; + } +} + +INSTANTIATE_TEST_SUITE_P( + BitSetTest, BitSetTest, + testing::Values( + // kBitAndIsNonZero (Any / overlap) + BitSetTestCase{ + BitSetStepKind::kAndIsNonZero, {}, 0b101, OutputType::kBoolFalse}, + BitSetTestCase{BitSetStepKind::kAndIsNonZero, + {IntValue(1), IntValue(3)}, + 0b101, + OutputType::kBoolFalse}, + BitSetTestCase{BitSetStepKind::kAndIsNonZero, + {IntValue(1), UintValue(2)}, + 0b101, + OutputType::kBoolTrue}, + BitSetTestCase{BitSetStepKind::kAndIsNonZero, + {UintValue(63)}, + uint64_t{1} << 63, + OutputType::kBoolTrue}, + BitSetTestCase{BitSetStepKind::kAndIsNonZero, + {IntValue(-5), IntValue(100), DoubleValue(0.0)}, + 0b101, + OutputType::kBoolTrue}, + // kBitAndEqual (All / contains all bits of mask) + BitSetTestCase{BitSetStepKind::kAndEqual, {}, 0, OutputType::kBoolTrue}, + BitSetTestCase{ + BitSetStepKind::kAndEqual, {}, 0b101, OutputType::kBoolFalse}, + BitSetTestCase{BitSetStepKind::kAndEqual, + {IntValue(0)}, + 0b101, + OutputType::kBoolFalse}, + BitSetTestCase{BitSetStepKind::kAndEqual, + {IntValue(0), UintValue(2)}, + 0b101, + OutputType::kBoolTrue}, + BitSetTestCase{BitSetStepKind::kAndEqual, + {IntValue(0), DoubleValue(2.0), IntValue(5), + IntValue(100), IntValue(-1)}, + 0b101, + OutputType::kBoolTrue}, + // kBitEqual (Equivalent / exact set match) + BitSetTestCase{BitSetStepKind::kEqual, {}, 0, OutputType::kBoolTrue}, + BitSetTestCase{ + BitSetStepKind::kEqual, {}, 0b101, OutputType::kBoolFalse}, + BitSetTestCase{BitSetStepKind::kEqual, + {IntValue(0)}, + 0b101, + OutputType::kBoolFalse}, + BitSetTestCase{BitSetStepKind::kEqual, + {IntValue(0), UintValue(2)}, + 0b101, + OutputType::kBoolTrue}, + BitSetTestCase{BitSetStepKind::kEqual, + {IntValue(0), UintValue(2), DoubleValue(0.0)}, + 0b101, + OutputType::kBoolTrue}, + BitSetTestCase{BitSetStepKind::kEqual, + {IntValue(0), IntValue(1), UintValue(2)}, + 0b101, + OutputType::kBoolFalse}, + BitSetTestCase{BitSetStepKind::kEqual, + {IntValue(0), UintValue(2), IntValue(-1)}, + 0b101, + OutputType::kBoolFalse}, + BitSetTestCase{BitSetStepKind::kEqual, + {IntValue(0), UintValue(2), IntValue(64)}, + 0b101, + OutputType::kBoolFalse}, + BitSetTestCase{BitSetStepKind::kEqual, + {IntValue(0), UintValue(2), UintValue(64)}, + 0b101, + OutputType::kBoolFalse}, + BitSetTestCase{BitSetStepKind::kEqual, + {IntValue(0), UintValue(2), DoubleValue(1.5)}, + 0b101, + OutputType::kBoolFalse}, + BitSetTestCase{BitSetStepKind::kEqual, + {IntValue(0), UintValue(2), DoubleValue(NAN)}, + 0b101, + OutputType::kBoolFalse}, + // Non-numeric element inside list (heterogeneous equality treats as not + // equal to any integer in [0, 63]) + BitSetTestCase{BitSetStepKind::kAndIsNonZero, + {StringValue()}, + 0b101, + OutputType::kBoolFalse}, + BitSetTestCase{BitSetStepKind::kAndIsNonZero, + {StringValue(), IntValue(2)}, + 0b101, + OutputType::kBoolTrue}, + BitSetTestCase{BitSetStepKind::kAndEqual, + {IntValue(0), IntValue(2), StringValue()}, + 0b101, + OutputType::kBoolTrue}, + BitSetTestCase{BitSetStepKind::kAndEqual, + {IntValue(0), StringValue()}, + 0b101, + OutputType::kBoolFalse}, + BitSetTestCase{BitSetStepKind::kEqual, + {IntValue(0), IntValue(2), StringValue()}, + 0b101, + OutputType::kBoolFalse})); + +TEST(BitSetStandaloneTest, HandlesNonListAndErrorsAndUnknowns) { + cel::Activation activation; + google::protobuf::Arena arena; + cel::RuntimeOptions opts; + opts.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; + cel::runtime_internal::RuntimeTypeProvider type_provider( + cel::internal::GetTestingDescriptorPool()); + + FlatExpressionEvaluatorState state( + /*value_stack_size=*/5, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, + cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena); + + ASSERT_THAT(activation.SetUnknownPatterns({cel::AttributePattern( + "foo", {cel::AttributeQualifierPattern::OfString("bar")})}), + IsOk()); + + for (BitSetStepKind kind : + {BitSetStepKind::kAndIsNonZero, BitSetStepKind::kAndEqual, + BitSetStepKind::kEqual}) { + // Non-list top-of-stack operand returns NoMatchingOverloadError. + { + ExecutionPath steps; + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(IntValue(1)))); + steps.push_back(MakeBitSetStepForKind(kind, 0b1)); + ExecutionFrame frame(steps, activation, opts, state); + ASSERT_OK_AND_ASSIGN(Value result, frame.Evaluate()); + EXPECT_THAT(result, ErrorValueIs(absl_testing::StatusIs( + absl::StatusCode::kUnknown, + HasSubstr("No matching overload")))); + } + + // Partial unknown attribute on list returns UnknownValue. + { + ExecutionPath steps; + steps.push_back( + ExpressionStep::MakeGenericStep(std::make_unique( + MakeList(&arena, {IntValue(0)}), + MakeTrailForVariable("foo", activation)))); + steps.push_back(MakeBitSetStepForKind(kind, 0b1)); + ExecutionFrame frame(steps, activation, opts, state); + ASSERT_OK_AND_ASSIGN(Value result, frame.Evaluate()); + EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); + } + + // Top-level ErrorValue on stack is propagated. + { + ExecutionPath steps; + steps.push_back( + ExpressionStep::MakeGenericStep(std::make_unique( + ErrorValue::From(absl::AbortedError("top_err"), &arena)))); + steps.push_back(MakeBitSetStepForKind(kind, 0b1)); + ExecutionFrame frame(steps, activation, opts, state); + ASSERT_OK_AND_ASSIGN(Value result, frame.Evaluate()); + EXPECT_THAT(result, ErrorValueIs( + StatusIs(absl::StatusCode::kAborted, "top_err"))); + } + + // List implementation returning an error status aborts evaluation. + { + class UnsupportedList final : public cel::CustomListValueInterface { + private: + std::string DebugString() const override { return "unsupported"; } + absl::Status ConvertToJsonArray( + const google::protobuf::DescriptorPool* absl_nonnull, + google::protobuf::MessageFactory* absl_nonnull, + google::protobuf::Message* absl_nonnull) const override { + return absl::UnimplementedError("unimplemented"); + } + size_t Size() const override { return 1; } + absl::Status Get(size_t, const google::protobuf::DescriptorPool* absl_nonnull, + google::protobuf::MessageFactory* absl_nonnull, + google::protobuf::Arena* absl_nonnull, + Value* absl_nonnull) const override { + return absl::UnimplementedError("unimplemented"); + } + cel::CustomListValue Clone( + google::protobuf::Arena* absl_nonnull arena) const override { + return cel::CustomListValue(this, arena); + } + cel::NativeTypeId GetNativeTypeId() const override { + return cel::NativeTypeId::For(); + } + }; + + UnsupportedList unsupported_list; + ExecutionPath steps; + steps.push_back( + ExpressionStep::MakeGenericStep(std::make_unique( + cel::CustomListValue(&unsupported_list, &arena)))); + steps.push_back(MakeBitSetStepForKind(kind, 0b1)); + ExecutionFrame frame(steps, activation, opts, state); + EXPECT_THAT(frame.Evaluate(), + StatusIs(absl::StatusCode::kUnimplemented, "unimplemented")); + } + } +} + +TEST(BitSetStandaloneTest, StackUnderflow) { + cel::Activation activation; + google::protobuf::Arena arena; + cel::RuntimeOptions opts; + cel::runtime_internal::RuntimeTypeProvider type_provider( + cel::internal::GetTestingDescriptorPool()); + + FlatExpressionEvaluatorState state( + /*value_stack_size=*/5, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, + cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena); + + for (BitSetStepKind kind : + {BitSetStepKind::kAndIsNonZero, BitSetStepKind::kAndEqual, + BitSetStepKind::kEqual}) { + ExecutionPath steps; + steps.push_back(MakeBitSetStepForKind(kind, 0b1)); + ExecutionFrame frame(steps, activation, opts, state); + EXPECT_THAT(frame.Evaluate(), + StatusIs(absl::StatusCode::kInternal, HasSubstr("underflow"))); + } +} + +} // namespace +} // namespace google::api::expr::runtime diff --git a/eval/eval/cel_expression_flat_impl.cc b/eval/eval/cel_expression_flat_impl.cc index 9e35b41ad..56b4d32a2 100644 --- a/eval/eval/cel_expression_flat_impl.cc +++ b/eval/eval/cel_expression_flat_impl.cc @@ -16,18 +16,12 @@ #include #include -#include #include "absl/base/nullability.h" -#include "absl/memory/memory.h" +#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/strings/str_cat.h" -#include "common/native_type.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/comprehension_slots.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/internal/adapter_activation_impl.h" #include "eval/internal/interop.h" @@ -36,7 +30,6 @@ #include "eval/public/cel_value.h" #include "internal/casts.h" #include "internal/status_macros.h" -#include "runtime/internal/runtime_env.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -45,7 +38,6 @@ namespace google::api::expr::runtime { namespace { using ::cel::Value; -using ::cel::runtime_internal::RuntimeEnv; EvaluationListener AdaptListener(const CelEvaluationListener& listener) { if (!listener) return nullptr; @@ -74,22 +66,39 @@ CelExpressionFlatEvaluationState::CelExpressionFlatEvaluationState( : state_(expression.MakeEvaluatorState(descriptor_pool, message_factory, arena)) {} +CelExpressionFlatEvaluationState::CelExpressionFlatEvaluationState( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + const FlatExpression& expr) + : state_(expr.MakeEvaluatorState(descriptor_pool, message_factory)) {} + absl::StatusOr CelExpressionFlatImpl::Trace( - const BaseActivation& activation, CelEvaluationState* _state, - CelEvaluationListener callback) const { - auto state = - ::cel::internal::down_cast(_state); - state->state().Reset(); + const BaseActivation& activation, google::protobuf::Arena* arena, + CelEvaluationListener callback, CelEvaluationState* state) const { + std::unique_ptr inline_state; + if (state == nullptr) { + inline_state = CreateState(); + state = inline_state.get(); + } + auto derived_state = + ::cel::internal::down_cast(state); + if (arena != nullptr) { + derived_state->Rebind(arena); + } else { + arena = derived_state->arena(); + } + ABSL_DCHECK(arena != nullptr) + << "arena must be implicitly provided when using InitializeState() or " + "explicitly provided when using CreateState()"; cel::interop_internal::AdapterActivationImpl modern_activation(activation); CEL_ASSIGN_OR_RETURN(cel::Value value, flat_expression_.EvaluateWithCallback( modern_activation, /*embedder_context=*/nullptr, - AdaptListener(callback), state->state())); + AdaptListener(callback), derived_state->state())); - return cel::interop_internal::ModernValueToLegacyValueOrDie(state->arena(), - value); + return cel::interop_internal::ModernValueToLegacyValueOrDie(arena, value); } std::unique_ptr CelExpressionFlatImpl::InitializeState( @@ -99,49 +108,10 @@ std::unique_ptr CelExpressionFlatImpl::InitializeState( flat_expression_); } -absl::StatusOr CelExpressionFlatImpl::Evaluate( - const BaseActivation& activation, CelEvaluationState* state) const { - return Trace(activation, state, CelEvaluationListener()); -} - -absl::StatusOr> -CelExpressionRecursiveImpl::Create( - absl_nonnull std::shared_ptr env, - FlatExpression flat_expr) { - if (flat_expr.path().empty() || - flat_expr.path().front()->GetNativeTypeId() != - cel::NativeTypeId::For()) { - return absl::InvalidArgumentError(absl::StrCat( - "Expected a recursive program step", flat_expr.path().size())); - } - - auto* instance = - new CelExpressionRecursiveImpl(std::move(env), std::move(flat_expr)); - - return absl::WrapUnique(instance); -} - -absl::StatusOr CelExpressionRecursiveImpl::Trace( - const BaseActivation& activation, google::protobuf::Arena* arena, - CelEvaluationListener callback) const { - cel::interop_internal::AdapterActivationImpl modern_activation(activation); - ComprehensionSlots slots(flat_expression_.comprehension_slots_size()); - ExecutionFrameBase execution_frame( - modern_activation, AdaptListener(callback), flat_expression_.options(), - flat_expression_.type_provider(), env_->descriptor_pool.get(), - env_->MutableMessageFactory(), arena, - /*embedder_context=*/nullptr, slots); - - cel::Value result; - AttributeTrail trail; - CEL_RETURN_IF_ERROR(root_->Evaluate(execution_frame, result, trail)); - - return cel::interop_internal::ModernValueToLegacyValueOrDie(arena, result); -} - -absl::StatusOr CelExpressionRecursiveImpl::Evaluate( - const BaseActivation& activation, google::protobuf::Arena* arena) const { - return Trace(activation, arena, /*callback=*/nullptr); +std::unique_ptr CelExpressionFlatImpl::CreateState() const { + return std::make_unique( + env_->descriptor_pool.get(), env_->MutableMessageFactory(), + flat_expression_); } } // namespace google::api::expr::runtime diff --git a/eval/eval/cel_expression_flat_impl.h b/eval/eval/cel_expression_flat_impl.h index 7faf6856a..5e9320332 100644 --- a/eval/eval/cel_expression_flat_impl.h +++ b/eval/eval/cel_expression_flat_impl.h @@ -20,12 +20,10 @@ #include "absl/base/nullability.h" #include "absl/status/statusor.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/public/base_activation.h" #include "eval/public/cel_expression.h" #include "eval/public/cel_value.h" -#include "internal/casts.h" #include "runtime/internal/runtime_env.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -42,9 +40,18 @@ class CelExpressionFlatEvaluationState : public CelEvaluationState { google::protobuf::MessageFactory* absl_nonnull message_factory, const FlatExpression& expr); + CelExpressionFlatEvaluationState( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + const FlatExpression& expr); + google::protobuf::Arena* arena() { return state_.arena(); } FlatExpressionEvaluatorState& state() { return state_; } + void Rebind(google::protobuf::Arena* arena) { + state_.Rebind(arena, state_.message_factory()); + } + private: FlatExpressionEvaluatorState state_; }; @@ -70,104 +77,21 @@ class CelExpressionFlatImpl : public CelExpression { std::unique_ptr InitializeState( google::protobuf::Arena* arena) const override; - absl::StatusOr Evaluate(const BaseActivation& activation, - google::protobuf::Arena* arena) const override { - return Evaluate(activation, InitializeState(arena).get()); - } - - absl::StatusOr Evaluate(const BaseActivation& activation, - CelEvaluationState* state) const override; - absl::StatusOr Trace( - const BaseActivation& activation, google::protobuf::Arena* arena, - CelEvaluationListener callback) const override { - return Trace(activation, InitializeState(arena).get(), callback); - } - - absl::StatusOr Trace(const BaseActivation& activation, - CelEvaluationState* state, - CelEvaluationListener callback) const override; - - // Exposed for inspection in tests. - const FlatExpression& flat_expression() const { return flat_expression_; } - - private: - absl_nonnull std::shared_ptr env_; - FlatExpression flat_expression_; -}; - -// Implementation of the CelExpression that evaluates a recursive representation -// of the AST. -// -// This class adapts FlatExpression to implement the CelExpression interface. -// -// Assumes that the flat expression is wrapping a simple recursive program. -class CelExpressionRecursiveImpl : public CelExpression { - private: - class EvaluationState : public CelEvaluationState { - public: - explicit EvaluationState(google::protobuf::Arena* arena) : arena_(arena) {} - google::protobuf::Arena* arena() { return arena_; } - - private: - google::protobuf::Arena* arena_; - }; - - public: - static absl::StatusOr> Create( - absl_nonnull std::shared_ptr env, - FlatExpression flat_expression); - - // Move-only - CelExpressionRecursiveImpl(const CelExpressionRecursiveImpl&) = delete; - CelExpressionRecursiveImpl& operator=(const CelExpressionRecursiveImpl&) = - delete; - CelExpressionRecursiveImpl(CelExpressionRecursiveImpl&&) = default; - CelExpressionRecursiveImpl& operator=(CelExpressionRecursiveImpl&&) = delete; - // Implement CelExpression. - std::unique_ptr InitializeState( - google::protobuf::Arena* arena) const override { - return std::make_unique(arena); - } - - absl::StatusOr Evaluate(const BaseActivation& activation, - google::protobuf::Arena* arena) const override; - - absl::StatusOr Evaluate(const BaseActivation& activation, - CelEvaluationState* state) const override { - auto* state_impl = cel::internal::down_cast(state); - return Evaluate(activation, state_impl->arena()); - } + std::unique_ptr CreateState() const override; + using CelExpression::Trace; absl::StatusOr Trace(const BaseActivation& activation, google::protobuf::Arena* arena, - CelEvaluationListener callback) const override; - - absl::StatusOr Trace( - const BaseActivation& activation, CelEvaluationState* state, - CelEvaluationListener callback) const override { - auto* state_impl = cel::internal::down_cast(state); - return Trace(activation, state_impl->arena(), callback); - } + CelEvaluationListener callback, + CelEvaluationState* state) const override; // Exposed for inspection in tests. const FlatExpression& flat_expression() const { return flat_expression_; } - const DirectExpressionStep* root() const { return root_; } - private: - explicit CelExpressionRecursiveImpl( - absl_nonnull std::shared_ptr env, - FlatExpression flat_expression) - : env_(std::move(env)), - flat_expression_(std::move(flat_expression)), - root_(cel::internal::down_cast( - flat_expression_.path()[0].get()) - ->wrapped()) {} - absl_nonnull std::shared_ptr env_; FlatExpression flat_expression_; - const DirectExpressionStep* root_; }; } // namespace google::api::expr::runtime diff --git a/eval/eval/compiler_constant_step.cc b/eval/eval/compiler_constant_step.cc deleted file mode 100644 index 44a03cecd..000000000 --- a/eval/eval/compiler_constant_step.cc +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include "eval/eval/compiler_constant_step.h" - -#include "absl/status/status.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/evaluator_core.h" - -namespace google::api::expr::runtime { - -using ::cel::Value; - -absl::Status DirectCompilerConstantStep::Evaluate( - ExecutionFrameBase& frame, Value& result, AttributeTrail& attribute) const { - result = value_; - return absl::OkStatus(); -} - -absl::Status CompilerConstantStep::Evaluate(ExecutionFrame* frame) const { - frame->value_stack().Push(value_); - - return absl::OkStatus(); -} - -} // namespace google::api::expr::runtime diff --git a/eval/eval/compiler_constant_step.h b/eval/eval/compiler_constant_step.h deleted file mode 100644 index bd514a036..000000000 --- a/eval/eval/compiler_constant_step.h +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_COMPILER_CONSTANT_STEP_H_ -#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_COMPILER_CONSTANT_STEP_H_ - -#include -#include - -#include "absl/status/status.h" -#include "common/native_type.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" - -namespace google::api::expr::runtime { - -// DirectExpressionStep implementation that simply assigns a constant value. -// -// Overrides NativeTypeId() allow the FlatExprBuilder and extensions to -// inspect the underlying value. -class DirectCompilerConstantStep : public DirectExpressionStep { - public: - DirectCompilerConstantStep(cel::Value value, int64_t expr_id) - : DirectExpressionStep(expr_id), value_(std::move(value)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& attribute) const override; - - cel::NativeTypeId GetNativeTypeId() const override { - return cel::NativeTypeId::For(); - } - - const cel::Value& value() const { return value_; } - - private: - cel::Value value_; -}; - -// ExpressionStep implementation that simply pushes a constant value on the -// stack. -// -// Overrides NativeTypeId ()o allow the FlatExprBuilder and extensions to -// inspect the underlying value. -class CompilerConstantStep : public ExpressionStepBase { - public: - CompilerConstantStep(cel::Value value, int64_t expr_id, bool comes_from_ast) - : ExpressionStepBase(expr_id, comes_from_ast), value_(std::move(value)) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override; - - cel::NativeTypeId GetNativeTypeId() const override { - return cel::NativeTypeId::For(); - } - - const cel::Value& value() const { return value_; } - - private: - cel::Value value_; -}; - -} // namespace google::api::expr::runtime - -#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_COMPILER_CONSTANT_STEP_H_ diff --git a/eval/eval/compiler_constant_step_test.cc b/eval/eval/compiler_constant_step_test.cc deleted file mode 100644 index 856ca30e0..000000000 --- a/eval/eval/compiler_constant_step_test.cc +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include "eval/eval/compiler_constant_step.h" - -#include - -#include "common/native_type.h" -#include "common/value.h" -#include "eval/eval/evaluator_core.h" -#include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" -#include "runtime/internal/runtime_type_provider.h" -#include "runtime/runtime_options.h" -#include "google/protobuf/arena.h" - -namespace google::api::expr::runtime { - -namespace { - -class CompilerConstantStepTest : public testing::Test { - public: - CompilerConstantStepTest() - : type_provider_(cel::internal::GetTestingDescriptorPool()), - state_(2, 0, type_provider_, cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_) {} - - protected: - google::protobuf::Arena arena_; - cel::runtime_internal::RuntimeTypeProvider type_provider_; - FlatExpressionEvaluatorState state_; - cel::Activation empty_activation_; - cel::RuntimeOptions options_; -}; - -TEST_F(CompilerConstantStepTest, Evaluate) { - ExecutionPath path; - path.push_back( - std::make_unique(cel::IntValue(42), -1, false)); - - ExecutionFrame frame(path, empty_activation_, options_, state_); - - ASSERT_OK_AND_ASSIGN(cel::Value result, frame.Evaluate()); - - EXPECT_EQ(result.GetInt().NativeValue(), 42); -} - -TEST_F(CompilerConstantStepTest, TypeId) { - CompilerConstantStep step(cel::IntValue(42), -1, false); - - ExpressionStep& abstract_step = step; - EXPECT_EQ(abstract_step.GetNativeTypeId(), - cel::NativeTypeId::For()); -} - -TEST_F(CompilerConstantStepTest, Value) { - CompilerConstantStep step(cel::IntValue(42), -1, false); - - EXPECT_EQ(step.value().GetInt().NativeValue(), 42); -} - -} // namespace -} // namespace google::api::expr::runtime diff --git a/eval/eval/comprehension_slots.h b/eval/eval/comprehension_slots.h index 795cca7f7..d7e71431b 100644 --- a/eval/eval/comprehension_slots.h +++ b/eval/eval/comprehension_slots.h @@ -16,6 +16,8 @@ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_COMPREHENSION_SLOTS_H_ #include +#include +#include #include #include "absl/base/attributes.h" @@ -23,9 +25,8 @@ #include "absl/base/nullability.h" #include "absl/container/fixed_array.h" #include "absl/log/absl_check.h" -#include "absl/types/optional.h" +#include "common/internal/attribute_trail.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" namespace google::api::expr::runtime { @@ -49,13 +50,14 @@ class ComprehensionSlot final { return &value_; } - const AttributeTrail& attribute() const ABSL_ATTRIBUTE_LIFETIME_BOUND { + const cel::common_internal::AttributeTrail& attribute() const + ABSL_ATTRIBUTE_LIFETIME_BOUND { ABSL_DCHECK(Has()); return attribute_; } - AttributeTrail* absl_nonnull mutable_attribute() + cel::common_internal::AttributeTrail* absl_nonnull mutable_attribute() ABSL_ATTRIBUTE_LIFETIME_BOUND { ABSL_DCHECK(Has()); @@ -64,31 +66,35 @@ class ComprehensionSlot final { bool Has() const { return has_; } - void Set() { Set(cel::NullValue(), absl::nullopt); } + void Set() { Set(cel::NullValue(), cel::common_internal::AttributeTrail()); } template void Set(V&& value) { - Set(std::forward(value), absl::nullopt); + Set(std::forward(value), cel::common_internal::AttributeTrail()); } - template - void Set(V&& value, A&& attribute) { + template + void Set(V&& value, cel::common_internal::AttributeTrail attribute) { + ABSL_DCHECK(!attribute.IsFullMatch()) + << "Full matches should not be pushed onto the value stack, they " + "should be handled directly and converted to an unknown value or " + "missing error value"; value_ = std::forward(value); - attribute_ = std::forward(attribute); + attribute_ = attribute; has_ = true; } void Clear() { if (has_) { value_ = cel::NullValue(); - attribute_ = absl::nullopt; + attribute_ = cel::common_internal::AttributeTrail(); has_ = false; } } private: cel::Value value_; - AttributeTrail attribute_; + cel::common_internal::AttributeTrail attribute_; bool has_ = false; }; @@ -110,7 +116,10 @@ class ComprehensionSlots final { return *instance; } - explicit ComprehensionSlots(size_t size) : slots_(size) {} + explicit ComprehensionSlots(size_t size) : slots_(size) { + ABSL_DCHECK_LT(size, + static_cast(std::numeric_limits::max())); + } ComprehensionSlots(const ComprehensionSlots&) = delete; ComprehensionSlots& operator=(const ComprehensionSlots&) = delete; @@ -134,12 +143,13 @@ class ComprehensionSlots final { template void Set(size_t index, V&& value) { - Set(index, std::forward(value), absl::nullopt); + Set(index, std::forward(value), cel::common_internal::AttributeTrail()); } - template - void Set(size_t index, V&& value, A&& attribute) { - Get(index)->Set(std::forward(value), std::forward(attribute)); + template + void Set(size_t index, V&& value, + cel::common_internal::AttributeTrail attribute) { + Get(index)->Set(std::forward(value), attribute); } size_t size() const { return slots_.size(); } diff --git a/eval/eval/comprehension_slots_test.cc b/eval/eval/comprehension_slots_test.cc index 5f869d7cb..d3ee67d70 100644 --- a/eval/eval/comprehension_slots_test.cc +++ b/eval/eval/comprehension_slots_test.cc @@ -14,7 +14,6 @@ #include "eval/eval/comprehension_slots.h" -#include "base/attribute.h" #include "base/type_provider.h" #include "common/memory.h" #include "common/value.h" @@ -23,9 +22,6 @@ namespace google::api::expr::runtime { -using ::cel::Attribute; - -using ::absl_testing::IsOkAndHolds; using ::cel::MemoryManagerRef; using ::cel::StringValue; using ::cel::TypeProvider; @@ -38,8 +34,7 @@ TEST(ComprehensionSlots, Basic) { ComprehensionSlots::Slot* slot0 = slots.Get(0); EXPECT_FALSE(slot0->Has()); - slots.Set(0, cel::StringValue("abcd"), - AttributeTrail(Attribute("fake_attr"))); + slots.Set(0, cel::StringValue::WrapUnsafe("abcd"), AttributeTrail()); ASSERT_TRUE(slot0->Has()); @@ -49,14 +44,10 @@ TEST(ComprehensionSlots, Basic) { })) << "value is 'abcd'"; - EXPECT_THAT(slot0->attribute().attribute().AsString(), - IsOkAndHolds("fake_attr")); - slots.ClearSlot(0); EXPECT_FALSE(slot0->Has()); - slots.Set(3, cel::StringValue("abcd"), - AttributeTrail(Attribute("fake_attr"))); + slots.Set(3, cel::StringValue::WrapUnsafe("abcd"), AttributeTrail()); auto* slot3 = slots.Get(3); diff --git a/eval/eval/comprehension_step.cc b/eval/eval/comprehension_step.cc deleted file mode 100644 index 5e741d805..000000000 --- a/eval/eval/comprehension_step.cc +++ /dev/null @@ -1,685 +0,0 @@ -#include "eval/eval/comprehension_step.h" - -#include -#include -#include -#include - -#include "absl/base/attributes.h" -#include "absl/base/casts.h" -#include "absl/base/nullability.h" -#include "absl/base/optimization.h" -#include "absl/log/absl_check.h" -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "base/attribute.h" -#include "common/casting.h" -#include "common/value.h" -#include "common/value_kind.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/comprehension_slots.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" -#include "eval/internal/errors.h" -#include "internal/status_macros.h" - -namespace google::api::expr::runtime { -namespace { - -enum class IterableKind { - kList = 1, - kMap, -}; - -using ::cel::AttributeQualifier; -using ::cel::Cast; -using ::cel::InstanceOf; -using ::cel::UnknownValue; -using ::cel::Value; -using ::cel::ValueIterator; -using ::cel::ValueIteratorPtr; -using ::cel::ValueKind; -using ::cel::runtime_internal::CreateNoMatchingOverloadError; - -AttributeQualifier AttributeQualifierFromValue(const Value& v) { - switch (v.kind()) { - case ValueKind::kString: - return AttributeQualifier::OfString(v.GetString().ToString()); - case ValueKind::kInt64: - return AttributeQualifier::OfInt(v.GetInt().NativeValue()); - case ValueKind::kUint64: - return AttributeQualifier::OfUint(v.GetUint().NativeValue()); - case ValueKind::kBool: - return AttributeQualifier::OfBool(v.GetBool().NativeValue()); - default: - // Non-matching qualifier. - return AttributeQualifier(); - } -} - -class ComprehensionFinishStep final : public ExpressionStepBase { - public: - ComprehensionFinishStep(size_t accu_slot, int64_t expr_id) - : ExpressionStepBase(expr_id), accu_slot_(accu_slot) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - frame->value_stack().SwapAndPop(2, 1); - frame->comprehension_slots().ClearSlot(accu_slot_); - frame->iterator_stack().Pop(); - return absl::OkStatus(); - } - - private: - const size_t accu_slot_; -}; - -class ComprehensionDirectStep final : public DirectExpressionStep { - public: - explicit ComprehensionDirectStep( - size_t iter_slot, size_t iter2_slot, size_t accu_slot, - std::unique_ptr range, - std::unique_ptr accu_init, - std::unique_ptr loop_step, - std::unique_ptr condition_step, - std::unique_ptr result_step, bool shortcircuiting, - int64_t expr_id) - : DirectExpressionStep(expr_id), - iter_slot_(iter_slot), - iter2_slot_(iter2_slot), - accu_slot_(accu_slot), - range_(std::move(range)), - accu_init_(std::move(accu_init)), - loop_step_(std::move(loop_step)), - condition_(std::move(condition_step)), - result_step_(std::move(result_step)), - shortcircuiting_(shortcircuiting) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& trail) const override { - return iter_slot_ == iter2_slot_ ? Evaluate1(frame, result, trail) - : Evaluate2(frame, result, trail); - } - - private: - absl::Status Evaluate1(ExecutionFrameBase& frame, Value& result, - AttributeTrail& trail) const; - - absl::StatusOr Evaluate1Unknown( - ExecutionFrameBase& frame, IterableKind range_iter_kind, - const AttributeTrail& range_iter_attr, - ValueIterator* absl_nonnull range_iter, - ComprehensionSlots::Slot* absl_nonnull accu_slot, - ComprehensionSlots::Slot* absl_nonnull iter_slot, Value& result, - AttributeTrail& trail) const; - - absl::StatusOr Evaluate1Known( - ExecutionFrameBase& frame, ValueIterator* absl_nonnull range_iter, - ComprehensionSlots::Slot* absl_nonnull accu_slot, - ComprehensionSlots::Slot* absl_nonnull iter_slot, Value& result, - AttributeTrail& trail) const; - - absl::Status Evaluate2(ExecutionFrameBase& frame, Value& result, - AttributeTrail& trail) const; - - const size_t iter_slot_; - const size_t iter2_slot_; - const size_t accu_slot_; - const std::unique_ptr range_; - const std::unique_ptr accu_init_; - const std::unique_ptr loop_step_; - const std::unique_ptr condition_; - const std::unique_ptr result_step_; - const bool shortcircuiting_; -}; - -absl::Status ComprehensionDirectStep::Evaluate1(ExecutionFrameBase& frame, - Value& result, - AttributeTrail& trail) const { - Value range; - AttributeTrail range_attr; - CEL_RETURN_IF_ERROR(range_->Evaluate(frame, range, range_attr)); - - if (frame.unknown_processing_enabled() && range.IsMap()) { - if (frame.attribute_utility().CheckForUnknownPartial(range_attr)) { - result = - frame.attribute_utility().CreateUnknownSet(range_attr.attribute()); - return absl::OkStatus(); - } - } - - absl_nullability_unknown ValueIteratorPtr range_iter; - IterableKind iterable_kind; - switch (range.kind()) { - case ValueKind::kList: { - CEL_ASSIGN_OR_RETURN(range_iter, range.GetList().NewIterator()); - iterable_kind = IterableKind::kList; - } break; - case ValueKind::kMap: { - CEL_ASSIGN_OR_RETURN(range_iter, range.GetMap().NewIterator()); - iterable_kind = IterableKind::kMap; - } break; - case ValueKind::kError: - ABSL_FALLTHROUGH_INTENDED; - case ValueKind::kUnknown: - result = std::move(range); - return absl::OkStatus(); - default: - result = cel::ErrorValue(CreateNoMatchingOverloadError("")); - return absl::OkStatus(); - } - ABSL_DCHECK(range_iter != nullptr); - - ComprehensionSlots::Slot* accu_slot = - frame.comprehension_slots().Get(accu_slot_); - ABSL_DCHECK(accu_slot != nullptr); - - { - Value accu_init; - AttributeTrail accu_init_attr; - CEL_RETURN_IF_ERROR(accu_init_->Evaluate(frame, accu_init, accu_init_attr)); - accu_slot->Set(std::move(accu_init), std::move(accu_init_attr)); - } - - ComprehensionSlots::Slot* iter_slot = - frame.comprehension_slots().Get(iter_slot_); - ABSL_DCHECK(iter_slot != nullptr); - iter_slot->Set(); - - bool should_skip_result; - if (frame.unknown_processing_enabled()) { - CEL_ASSIGN_OR_RETURN( - should_skip_result, - Evaluate1Unknown(frame, iterable_kind, range_attr, range_iter.get(), - accu_slot, iter_slot, result, trail)); - } else { - CEL_ASSIGN_OR_RETURN(should_skip_result, - Evaluate1Known(frame, range_iter.get(), accu_slot, - iter_slot, result, trail)); - } - - frame.comprehension_slots().ClearSlot(iter_slot_); - if (!should_skip_result) { - CEL_RETURN_IF_ERROR(result_step_->Evaluate(frame, result, trail)); - } - frame.comprehension_slots().ClearSlot(accu_slot_); - return absl::OkStatus(); -} - -absl::StatusOr ComprehensionDirectStep::Evaluate1Unknown( - ExecutionFrameBase& frame, IterableKind range_iter_kind, - const AttributeTrail& range_iter_attr, - ValueIterator* absl_nonnull range_iter, - ComprehensionSlots::Slot* absl_nonnull accu_slot, - ComprehensionSlots::Slot* absl_nonnull iter_slot, Value& result, - AttributeTrail& trail) const { - Value condition; - AttributeTrail condition_attr; - Value key_or_value; - Value* key; - Value* value; - - switch (range_iter_kind) { - case IterableKind::kList: - key = &key_or_value; - value = iter_slot->mutable_value(); - break; - case IterableKind::kMap: - key = iter_slot->mutable_value(); - value = nullptr; - break; - default: - ABSL_UNREACHABLE(); - } - while (true) { - CEL_ASSIGN_OR_RETURN(bool ok, range_iter->Next2(frame.descriptor_pool(), - frame.message_factory(), - frame.arena(), key, value)); - if (!ok) { - break; - } - CEL_RETURN_IF_ERROR(frame.IncrementIterations()); - *iter_slot->mutable_attribute() = - range_iter_attr.Step(AttributeQualifierFromValue(*key)); - if (frame.attribute_utility().CheckForUnknownExact( - iter_slot->attribute())) { - *iter_slot->mutable_value() = frame.attribute_utility().CreateUnknownSet( - iter_slot->attribute().attribute()); - } - - // Evaluate the loop condition. - CEL_RETURN_IF_ERROR(condition_->Evaluate(frame, condition, condition_attr)); - - switch (condition.kind()) { - case ValueKind::kBool: - break; - case ValueKind::kError: - ABSL_FALLTHROUGH_INTENDED; - case ValueKind::kUnknown: - result = std::move(condition); - return true; - default: - result = - cel::ErrorValue(CreateNoMatchingOverloadError("")); - return true; - } - - if (shortcircuiting_ && !absl::implicit_cast(condition.GetBool())) { - break; - } - - // Evaluate the loop step. - CEL_RETURN_IF_ERROR(loop_step_->Evaluate(frame, *accu_slot->mutable_value(), - *accu_slot->mutable_attribute())); - } - return false; -} - -absl::StatusOr ComprehensionDirectStep::Evaluate1Known( - ExecutionFrameBase& frame, ValueIterator* absl_nonnull range_iter, - ComprehensionSlots::Slot* absl_nonnull accu_slot, - ComprehensionSlots::Slot* absl_nonnull iter_slot, Value& result, - AttributeTrail& trail) const { - Value condition; - AttributeTrail condition_attr; - - while (true) { - CEL_ASSIGN_OR_RETURN( - bool ok, - range_iter->Next1(frame.descriptor_pool(), frame.message_factory(), - frame.arena(), iter_slot->mutable_value())); - if (!ok) { - break; - } - CEL_RETURN_IF_ERROR(frame.IncrementIterations()); - - // Evaluate the loop condition. - CEL_RETURN_IF_ERROR(condition_->Evaluate(frame, condition, condition_attr)); - - switch (condition.kind()) { - case ValueKind::kBool: - break; - case ValueKind::kError: - ABSL_FALLTHROUGH_INTENDED; - case ValueKind::kUnknown: - result = std::move(condition); - return true; - default: - result = - cel::ErrorValue(CreateNoMatchingOverloadError("")); - return true; - } - - if (shortcircuiting_ && !absl::implicit_cast(condition.GetBool())) { - break; - } - - // Evaluate the loop step. - CEL_RETURN_IF_ERROR(loop_step_->Evaluate(frame, *accu_slot->mutable_value(), - *accu_slot->mutable_attribute())); - } - return false; -} - -absl::Status ComprehensionDirectStep::Evaluate2(ExecutionFrameBase& frame, - Value& result, - AttributeTrail& trail) const { - Value range; - AttributeTrail range_attr; - CEL_RETURN_IF_ERROR(range_->Evaluate(frame, range, range_attr)); - - if (frame.unknown_processing_enabled() && range.IsMap()) { - if (frame.attribute_utility().CheckForUnknownPartial(range_attr)) { - result = - frame.attribute_utility().CreateUnknownSet(range_attr.attribute()); - return absl::OkStatus(); - } - } - - absl_nullability_unknown ValueIteratorPtr range_iter; - switch (range.kind()) { - case ValueKind::kList: { - CEL_ASSIGN_OR_RETURN(range_iter, range.GetList().NewIterator()); - } break; - case ValueKind::kMap: { - CEL_ASSIGN_OR_RETURN(range_iter, range.GetMap().NewIterator()); - } break; - case ValueKind::kError: - ABSL_FALLTHROUGH_INTENDED; - case ValueKind::kUnknown: - result = std::move(range); - return absl::OkStatus(); - default: - result = cel::ErrorValue(CreateNoMatchingOverloadError("")); - return absl::OkStatus(); - } - ABSL_DCHECK(range_iter != nullptr); - - ComprehensionSlots::Slot* accu_slot = - frame.comprehension_slots().Get(accu_slot_); - ABSL_DCHECK(accu_slot != nullptr); - - { - Value accu_init; - AttributeTrail accu_init_attr; - CEL_RETURN_IF_ERROR(accu_init_->Evaluate(frame, accu_init, accu_init_attr)); - accu_slot->Set(std::move(accu_init), std::move(accu_init_attr)); - } - - ComprehensionSlots::Slot* iter_slot = - frame.comprehension_slots().Get(iter_slot_); - ABSL_DCHECK(iter_slot != nullptr); - iter_slot->Set(); - - ComprehensionSlots::Slot* iter2_slot = - frame.comprehension_slots().Get(iter2_slot_); - ABSL_DCHECK(iter2_slot != nullptr); - iter2_slot->Set(); - - Value condition; - AttributeTrail condition_attr; - bool should_skip_result = false; - - while (true) { - CEL_ASSIGN_OR_RETURN( - bool ok, - range_iter->Next2(frame.descriptor_pool(), frame.message_factory(), - frame.arena(), iter_slot->mutable_value(), - iter2_slot->mutable_value())); - if (!ok) { - break; - } - CEL_RETURN_IF_ERROR(frame.IncrementIterations()); - if (frame.unknown_processing_enabled()) { - *iter_slot->mutable_attribute() = *iter2_slot->mutable_attribute() = - range_attr.Step(AttributeQualifierFromValue(iter_slot->value())); - if (frame.attribute_utility().CheckForUnknownExact( - iter_slot->attribute())) { - *iter2_slot->mutable_value() = - frame.attribute_utility().CreateUnknownSet( - iter_slot->attribute().attribute()); - } - } - - // Evaluate the loop condition. - CEL_RETURN_IF_ERROR(condition_->Evaluate(frame, condition, condition_attr)); - - switch (condition.kind()) { - case ValueKind::kBool: - break; - case ValueKind::kError: - ABSL_FALLTHROUGH_INTENDED; - case ValueKind::kUnknown: - result = std::move(condition); - should_skip_result = true; - goto finish; - default: - result = - cel::ErrorValue(CreateNoMatchingOverloadError("")); - should_skip_result = true; - goto finish; - } - - if (shortcircuiting_ && !absl::implicit_cast(condition.GetBool())) { - break; - } - - // Evaluate the loop step. - CEL_RETURN_IF_ERROR(loop_step_->Evaluate(frame, *accu_slot->mutable_value(), - *accu_slot->mutable_attribute())); - } - -finish: - iter_slot->Clear(); - iter2_slot->Clear(); - if (!should_skip_result) { - CEL_RETURN_IF_ERROR(result_step_->Evaluate(frame, result, trail)); - } - accu_slot->Clear(); - return absl::OkStatus(); -} - -} // namespace - -absl::Status ComprehensionInitStep::Evaluate(ExecutionFrame* frame) const { - if (!frame->value_stack().HasEnough(1)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - - const Value& top = frame->value_stack().Peek(); - if (top.IsError() || top.IsUnknown()) { - return frame->JumpTo(error_jump_offset_); - } - - if (frame->enable_unknowns() && top.IsMap()) { - const AttributeTrail& top_attr = frame->value_stack().PeekAttribute(); - if (frame->attribute_utility().CheckForUnknownPartial(top_attr)) { - frame->value_stack().PopAndPush( - frame->attribute_utility().CreateUnknownSet(top_attr.attribute())); - return frame->JumpTo(error_jump_offset_); - } - } - - switch (top.kind()) { - case ValueKind::kList: { - CEL_ASSIGN_OR_RETURN(auto iterator, top.GetList().NewIterator()); - frame->iterator_stack().Push(std::move(iterator)); - } break; - case ValueKind::kMap: { - CEL_ASSIGN_OR_RETURN(auto iterator, top.GetMap().NewIterator()); - frame->iterator_stack().Push(std::move(iterator)); - } break; - default: - // Replace with an error and jump past - // ComprehensionFinishStep. - frame->value_stack().PopAndPush( - cel::ErrorValue(CreateNoMatchingOverloadError(""))); - return frame->JumpTo(error_jump_offset_); - } - - return absl::OkStatus(); -} - -absl::Status ComprehensionNextStep::Evaluate1(ExecutionFrame* frame) const { - if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - - { - Value& accu_var = frame->value_stack().Peek(); - AttributeTrail& accu_var_attr = frame->value_stack().PeekAttribute(); - frame->comprehension_slots().Set(accu_slot_, std::move(accu_var), - std::move(accu_var_attr)); - frame->value_stack().Pop(1); - } - - ComprehensionSlots::Slot* iter_slot = - frame->comprehension_slots().Get(iter_slot_); - ABSL_DCHECK(iter_slot != nullptr); - iter_slot->Set(); - - if (frame->enable_unknowns()) { - Value key_or_value; - Value* key; - Value* value; - switch (frame->value_stack().Peek().kind()) { - case ValueKind::kList: - key = &key_or_value; - value = iter_slot->mutable_value(); - break; - case ValueKind::kMap: - key = iter_slot->mutable_value(); - value = nullptr; - break; - default: - ABSL_UNREACHABLE(); - } - CEL_ASSIGN_OR_RETURN(bool ok, - frame->iterator_stack().Peek()->Next2( - frame->descriptor_pool(), frame->message_factory(), - frame->arena(), key, value)); - if (!ok) { - iter_slot->Clear(); - return frame->JumpTo(jump_offset_); - } - CEL_RETURN_IF_ERROR(frame->IncrementIterations()); - *iter_slot->mutable_attribute() = frame->value_stack().PeekAttribute().Step( - AttributeQualifierFromValue(*key)); - if (frame->attribute_utility().CheckForUnknownExact( - iter_slot->attribute())) { - *iter_slot->mutable_value() = frame->attribute_utility().CreateUnknownSet( - iter_slot->attribute().attribute()); - } - } else { - CEL_ASSIGN_OR_RETURN(bool ok, - frame->iterator_stack().Peek()->Next1( - frame->descriptor_pool(), frame->message_factory(), - frame->arena(), iter_slot->mutable_value())); - if (!ok) { - iter_slot->Clear(); - return frame->JumpTo(jump_offset_); - } - CEL_RETURN_IF_ERROR(frame->IncrementIterations()); - } - return absl::OkStatus(); -} - -absl::Status ComprehensionNextStep::Evaluate2(ExecutionFrame* frame) const { - if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - - { - Value& accu_var = frame->value_stack().Peek(); - AttributeTrail& accu_var_attr = frame->value_stack().PeekAttribute(); - frame->comprehension_slots().Set(accu_slot_, std::move(accu_var), - std::move(accu_var_attr)); - frame->value_stack().Pop(1); - } - - ComprehensionSlots::Slot* iter_slot = - frame->comprehension_slots().Get(iter_slot_); - ABSL_DCHECK(iter_slot != nullptr); - iter_slot->Set(); - - ComprehensionSlots::Slot* iter2_slot = - frame->comprehension_slots().Get(iter2_slot_); - ABSL_DCHECK(iter2_slot != nullptr); - iter2_slot->Set(); - - CEL_ASSIGN_OR_RETURN( - bool ok, - frame->iterator_stack().Peek()->Next2( - frame->descriptor_pool(), frame->message_factory(), frame->arena(), - iter_slot->mutable_value(), iter2_slot->mutable_value())); - if (!ok) { - iter_slot->Clear(); - iter2_slot->Clear(); - return frame->JumpTo(jump_offset_); - } - CEL_RETURN_IF_ERROR(frame->IncrementIterations()); - if (frame->enable_unknowns()) { - *iter_slot->mutable_attribute() = *iter2_slot->mutable_attribute() = - frame->value_stack().PeekAttribute().Step( - AttributeQualifierFromValue(iter_slot->value())); - if (frame->attribute_utility().CheckForUnknownExact( - iter2_slot->attribute())) { - *iter2_slot->mutable_value() = - frame->attribute_utility().CreateUnknownSet( - iter2_slot->attribute().attribute()); - } - } - return absl::OkStatus(); -} - -absl::Status ComprehensionCondStep::Evaluate1(ExecutionFrame* frame) const { - if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - const Value& top = frame->value_stack().Peek(); - switch (top.kind()) { - case ValueKind::kBool: - break; - case ValueKind::kError: - ABSL_FALLTHROUGH_INTENDED; - case ValueKind::kUnknown: - frame->value_stack().SwapAndPop(2, 1); - frame->comprehension_slots().ClearSlot(iter_slot_); - frame->comprehension_slots().ClearSlot(accu_slot_); - frame->iterator_stack().Pop(); - return frame->JumpTo(error_jump_offset_); - default: - frame->value_stack().PopAndPush( - 2, - cel::ErrorValue(CreateNoMatchingOverloadError(""))); - frame->comprehension_slots().ClearSlot(iter_slot_); - frame->comprehension_slots().ClearSlot(accu_slot_); - frame->iterator_stack().Pop(); - return frame->JumpTo(error_jump_offset_); - } - const bool loop_condition = absl::implicit_cast(top.GetBool()); - frame->value_stack().Pop(1); // loop_condition - if (!loop_condition && shortcircuiting_) { - return frame->JumpTo(jump_offset_); - } - return absl::OkStatus(); -} - -absl::Status ComprehensionCondStep::Evaluate2(ExecutionFrame* frame) const { - if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - const Value& top = frame->value_stack().Peek(); - switch (top.kind()) { - case ValueKind::kBool: - break; - case ValueKind::kError: - ABSL_FALLTHROUGH_INTENDED; - case ValueKind::kUnknown: - frame->value_stack().SwapAndPop(2, 1); - frame->comprehension_slots().ClearSlot(iter_slot_); - frame->comprehension_slots().ClearSlot(iter2_slot_); - frame->comprehension_slots().ClearSlot(accu_slot_); - frame->iterator_stack().Pop(); - return frame->JumpTo(error_jump_offset_); - default: - frame->value_stack().PopAndPush( - 2, - cel::ErrorValue(CreateNoMatchingOverloadError(""))); - frame->comprehension_slots().ClearSlot(iter_slot_); - frame->comprehension_slots().ClearSlot(iter2_slot_); - frame->comprehension_slots().ClearSlot(accu_slot_); - frame->iterator_stack().Pop(); - return frame->JumpTo(error_jump_offset_); - } - const bool loop_condition = absl::implicit_cast(top.GetBool()); - frame->value_stack().Pop(1); // loop_condition - if (!loop_condition && shortcircuiting_) { - return frame->JumpTo(jump_offset_); - } - return absl::OkStatus(); -} - -std::unique_ptr CreateDirectComprehensionStep( - size_t iter_slot, size_t iter2_slot, size_t accu_slot, - std::unique_ptr range, - std::unique_ptr accu_init, - std::unique_ptr loop_step, - std::unique_ptr condition_step, - std::unique_ptr result_step, bool shortcircuiting, - int64_t expr_id) { - return std::make_unique( - iter_slot, iter2_slot, accu_slot, std::move(range), std::move(accu_init), - std::move(loop_step), std::move(condition_step), std::move(result_step), - shortcircuiting, expr_id); -} - -std::unique_ptr CreateComprehensionFinishStep(size_t accu_slot, - int64_t expr_id) { - return std::make_unique(accu_slot, expr_id); -} - -} // namespace google::api::expr::runtime diff --git a/eval/eval/comprehension_step.h b/eval/eval/comprehension_step.h index 34a6afc19..9dc1c94eb 100644 --- a/eval/eval/comprehension_step.h +++ b/eval/eval/comprehension_step.h @@ -1,15 +1,10 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_COMPREHENSION_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_COMPREHENSION_STEP_H_ -#include #include #include -#include -#include "absl/status/status.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" +#include "absl/base/macros.h" namespace google::api::expr::runtime { @@ -25,95 +20,59 @@ namespace google::api::expr::runtime { // 8: 1 -> 2 // 9: ComprehensionFinishStep 2 -> 1 -class ComprehensionInitStep final : public ExpressionStepBase { - public: - explicit ComprehensionInitStep(int64_t expr_id) - : ExpressionStepBase(expr_id, /*comes_from_ast=*/false) {} - - void set_error_jump_offset(int offset) { error_jump_offset_ = offset; } - - absl::Status Evaluate(ExecutionFrame* frame) const override; - - private: - int error_jump_offset_ = std::numeric_limits::max(); +enum class ComprehensionVersion { + kV1 = 0, + kV2, }; -class ComprehensionNextStep final : public ExpressionStepBase { - public: - ComprehensionNextStep(size_t iter_slot, size_t iter2_slot, size_t accu_slot, - int64_t expr_id) - : ExpressionStepBase(expr_id, /*comes_from_ast=*/false), - iter_slot_(iter_slot), - iter2_slot_(iter2_slot), - accu_slot_(accu_slot) {} +struct ComprehensionInitStepInfo { + ABSL_DEPRECATE_AND_INLINE() + void set_error_jump_offset(int offset) { error_jump_offset = offset; } - void set_jump_offset(int offset) { jump_offset_ = offset; } - - void set_error_jump_offset(int offset) { error_jump_offset_ = offset; } - - absl::Status Evaluate(ExecutionFrame* frame) const override { - return iter_slot_ == iter2_slot_ ? Evaluate1(frame) : Evaluate2(frame); + [[nodiscard]] + uint32_t iter_slot() const { + return first_slot; } - private: - absl::Status Evaluate1(ExecutionFrame* frame) const; + [[nodiscard]] + uint32_t iter2_slot() const { + ABSL_ASSERT(v2); + return first_slot + 1; + } - absl::Status Evaluate2(ExecutionFrame* frame) const; + [[nodiscard]] + uint32_t accu_slot() const { + return first_slot + v2 + 1; + } - const size_t iter_slot_; - const size_t iter2_slot_; - const size_t accu_slot_; - int jump_offset_ = std::numeric_limits::max(); - int error_jump_offset_ = std::numeric_limits::max(); + // First slot. + uint32_t first_slot : 31; + uint32_t v2 : 1; + int32_t error_jump_offset = std::numeric_limits::max(); }; -class ComprehensionCondStep final : public ExpressionStepBase { - public: - ComprehensionCondStep(size_t iter_slot, size_t iter2_slot, size_t accu_slot, - bool shortcircuiting, int64_t expr_id) - : ExpressionStepBase(expr_id, /*comes_from_ast=*/false), - iter_slot_(iter_slot), - iter2_slot_(iter2_slot), - accu_slot_(accu_slot), - shortcircuiting_(shortcircuiting) {} - - void set_jump_offset(int offset) { jump_offset_ = offset; } +struct ComprehensionNextStepInfo { + ABSL_DEPRECATE_AND_INLINE() + void set_jump_offset(int offset) { jump_offset = offset; } - void set_error_jump_offset(int offset) { error_jump_offset_ = offset; } + ABSL_DEPRECATE_AND_INLINE() + void set_error_jump_offset(int offset) { error_jump_offset = offset; } - absl::Status Evaluate(ExecutionFrame* frame) const override { - return iter_slot_ == iter2_slot_ ? Evaluate1(frame) : Evaluate2(frame); - } + int32_t jump_offset = std::numeric_limits::max(); + int32_t error_jump_offset = std::numeric_limits::max(); +}; - private: - absl::Status Evaluate1(ExecutionFrame* frame) const; +struct ComprehensionCondStepInfo { + ABSL_DEPRECATE_AND_INLINE() + void set_jump_offset(int offset) { jump_offset = offset; } - absl::Status Evaluate2(ExecutionFrame* frame) const; + ABSL_DEPRECATE_AND_INLINE() + void set_error_jump_offset(int offset) { error_jump_offset = offset; } - const size_t iter_slot_; - const size_t iter2_slot_; - const size_t accu_slot_; - int jump_offset_ = std::numeric_limits::max(); - int error_jump_offset_ = std::numeric_limits::max(); - const bool shortcircuiting_; + int32_t jump_offset = std::numeric_limits::max(); + int32_t error_jump_offset = std::numeric_limits::max(); }; -// Creates a step for executing a comprehension. -std::unique_ptr CreateDirectComprehensionStep( - size_t iter_slot, size_t iter2_slot, size_t accu_slot, - std::unique_ptr range, - std::unique_ptr accu_init, - std::unique_ptr loop_step, - std::unique_ptr condition_step, - std::unique_ptr result_step, bool shortcircuiting, - int64_t expr_id); - -// Creates a cleanup step for the comprehension. -// Removes the comprehension context then pushes the 'result' sub expression to -// the top of the stack. -std::unique_ptr CreateComprehensionFinishStep(size_t accu_slot, - int64_t expr_id); - } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_COMPREHENSION_STEP_H_ diff --git a/eval/eval/comprehension_step_test.cc b/eval/eval/comprehension_step_test.cc index 681f8af4f..93296d7e2 100644 --- a/eval/eval/comprehension_step_test.cc +++ b/eval/eval/comprehension_step_test.cc @@ -7,19 +7,11 @@ #include "cel/expr/syntax.pb.h" #include "google/protobuf/struct.pb.h" -#include "absl/memory/memory.h" #include "absl/status/status.h" -#include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "base/type_provider.h" #include "common/expr.h" -#include "common/value.h" -#include "common/value_testing.h" -#include "eval/eval/attribute_trail.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/comprehension_slots.h" -#include "eval/eval/const_value_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" #include "eval/eval/ident_step.h" @@ -27,41 +19,23 @@ #include "eval/public/cel_attribute.h" #include "eval/public/cel_value.h" #include "eval/public/structs/cel_proto_wrapper.h" -#include "internal/status_macros.h" #include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" #include "runtime/internal/runtime_env_testing.h" -#include "runtime/internal/runtime_type_provider.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" namespace google::api::expr::runtime { namespace { -using ::absl_testing::StatusIs; -using ::cel::BoolValue; +using ::absl_testing::IsOk; using ::cel::Expr; -using ::cel::IdentExpr; -using ::cel::IntValue; using ::cel::TypeProvider; -using ::cel::Value; using ::cel::runtime_internal::NewTestingRuntimeEnv; -using ::cel::test::BoolValueIs; using ::google::protobuf::Struct; using ::google::protobuf::Arena; -using ::testing::_; using ::testing::Eq; -using ::testing::Return; using ::testing::SizeIs; -IdentExpr CreateIdent(const std::string& var) { - IdentExpr expr; - expr.set_name(var); - return expr; -} - class ListKeysStepTest : public testing::Test { public: ListKeysStepTest() = default; @@ -76,8 +50,9 @@ class ListKeysStepTest : public testing::Test { auto env = NewTestingRuntimeEnv(); return std::make_unique( env, - FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + FlatExpression(std::move(path), /*comprehension_slot_count=*/1, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, IdentifierTable{{"var", 0}})); } private: @@ -86,11 +61,10 @@ class ListKeysStepTest : public testing::Test { class GetListKeysResultStep : public ExpressionStepBase { public: - GetListKeysResultStep() : ExpressionStepBase(-1, false) {} + GetListKeysResultStep() : ExpressionStepBase() {} - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { frame->value_stack().Pop(1); - return absl::OkStatus(); } }; @@ -102,13 +76,12 @@ MATCHER_P(CelStringValue, val, "") { TEST_F(ListKeysStepTest, MapPartiallyUnknown) { ExecutionPath path; - auto result = CreateIdentStep("var", 0); - ASSERT_OK(result); - path.push_back(*std::move(result)); - ComprehensionInitStep* init_step = new ComprehensionInitStep(1); - init_step->set_error_jump_offset(1); - path.push_back(absl::WrapUnique(init_step)); - path.push_back(std::make_unique()); + path.push_back(ExpressionStep::MakeIdentifierStep(0)); + path.push_back(ExpressionStep::MakeComprehensionInitStep( + /*first_slot=*/0, ComprehensionVersion::kV2)); + GetIfComprehensionInitStep(path.back())->error_jump_offset = 1; + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); auto expression = MakeExpression(std::move(path), /*unknown_attributes=*/true); @@ -121,15 +94,17 @@ TEST_F(ListKeysStepTest, MapPartiallyUnknown) { (*value.mutable_fields())["key3"].set_number_value(3.0); activation.InsertValue("var", CelProtoWrapper::CreateMessage(&value, &arena)); - activation.set_unknown_attribute_patterns({CelAttributePattern( - "var", - {CreateCelAttributeQualifierPattern(CelValue::CreateStringView("key2")), - CreateCelAttributeQualifierPattern(CelValue::CreateStringView("foo")), - CelAttributeQualifierPattern::CreateWildcard()})}); + ASSERT_THAT(activation.SetUnknownAttributePatterns({CelAttributePattern( + "var", {CreateCelAttributeQualifierPattern( + CelValue::CreateStringView("key2")), + CreateCelAttributeQualifierPattern( + CelValue::CreateStringView("foo")), + CelAttributeQualifierPattern::CreateWildcard()})}), + IsOk()); auto eval_result = expression->Evaluate(activation, &arena); - ASSERT_OK(eval_result); + ASSERT_THAT(eval_result, IsOk()); ASSERT_TRUE(eval_result->IsUnknownSet()); const auto& attrs = eval_result->UnknownSetOrDie()->unknown_attributes(); @@ -140,13 +115,12 @@ TEST_F(ListKeysStepTest, MapPartiallyUnknown) { TEST_F(ListKeysStepTest, ErrorPassedThrough) { ExecutionPath path; - auto result = CreateIdentStep("var", 0); - ASSERT_OK(result); - path.push_back(*std::move(result)); - ComprehensionInitStep* init_step = new ComprehensionInitStep(1); - init_step->set_error_jump_offset(1); - path.push_back(absl::WrapUnique(init_step)); - path.push_back(std::make_unique()); + path.push_back(ExpressionStep::MakeIdentifierStep(0)); + path.push_back(ExpressionStep::MakeComprehensionInitStep( + /*first_slot=*/0, ComprehensionVersion::kV2)); + GetIfComprehensionInitStep(path.back())->error_jump_offset = 1; + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); auto expression = MakeExpression(std::move(path)); @@ -156,7 +130,7 @@ TEST_F(ListKeysStepTest, ErrorPassedThrough) { // Var not in activation, turns into cel error at eval time. auto eval_result = expression->Evaluate(activation, &arena); - ASSERT_OK(eval_result); + ASSERT_THAT(eval_result, IsOk()); ASSERT_TRUE(eval_result->IsError()); EXPECT_THAT(eval_result->ErrorOrDie()->message(), testing::HasSubstr("\"var\"")); @@ -165,13 +139,12 @@ TEST_F(ListKeysStepTest, ErrorPassedThrough) { TEST_F(ListKeysStepTest, UnknownSetPassedThrough) { ExecutionPath path; - auto result = CreateIdentStep("var", 0); - ASSERT_OK(result); - path.push_back(*std::move(result)); - ComprehensionInitStep* init_step = new ComprehensionInitStep(1); - init_step->set_error_jump_offset(1); - path.push_back(absl::WrapUnique(init_step)); - path.push_back(std::make_unique()); + path.push_back(ExpressionStep::MakeIdentifierStep(0)); + path.push_back(ExpressionStep::MakeComprehensionInitStep( + /*first_slot=*/0, ComprehensionVersion::kV2)); + GetIfComprehensionInitStep(path.back())->error_jump_offset = 1; + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); auto expression = MakeExpression(std::move(path), /*unknown_attributes=*/true); @@ -179,314 +152,16 @@ TEST_F(ListKeysStepTest, UnknownSetPassedThrough) { Activation activation; Arena arena; - activation.set_unknown_attribute_patterns({CelAttributePattern("var", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern("var", {})}), + IsOk()); auto eval_result = expression->Evaluate(activation, &arena); - ASSERT_OK(eval_result); + ASSERT_THAT(eval_result, IsOk()); ASSERT_TRUE(eval_result->IsUnknownSet()); EXPECT_THAT(eval_result->UnknownSetOrDie()->unknown_attributes(), SizeIs(1)); } -class MockDirectStep : public DirectExpressionStep { - public: - MockDirectStep() : DirectExpressionStep(-1) {} - - MOCK_METHOD(absl::Status, Evaluate, - (ExecutionFrameBase&, Value&, AttributeTrail&), - (const, override)); -}; - -// Test fixture for comprehensions. -// -// Comprehensions are quite involved so tests here focus on edge cases that are -// hard to exercise normally in functional-style tests for the planner. -class DirectComprehensionTest : public testing::Test { - public: - DirectComprehensionTest() - : type_provider_(cel::internal::GetTestingDescriptorPool()), slots_(2) {} - - // returns a two element list for testing [1, 2]. - absl::StatusOr MakeList() { - auto builder = cel::NewListValueBuilder(&arena_); - - CEL_RETURN_IF_ERROR(builder->Add(IntValue(1))); - CEL_RETURN_IF_ERROR(builder->Add(IntValue(2))); - return std::move(*builder).Build(); - } - - protected: - google::protobuf::Arena arena_; - cel::runtime_internal::RuntimeTypeProvider type_provider_; - ComprehensionSlots slots_; - cel::Activation empty_activation_; -}; - -TEST_F(DirectComprehensionTest, PropagateRangeNonOkStatus) { - cel::RuntimeOptions options; - - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto range_step = std::make_unique(); - MockDirectStep* mock = range_step.get(); - - ON_CALL(*mock, Evaluate(_, _, _)) - .WillByDefault(Return(absl::InternalError("test range error"))); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/std::move(range_step), - /*accu_init=*/CreateConstValueDirectStep(BoolValue(false)), - /*loop_step=*/CreateConstValueDirectStep(BoolValue(false)), - /*condition_step=*/CreateConstValueDirectStep(BoolValue(true)), - /*result_step=*/CreateDirectSlotIdentStep("__result__", 1, -1), - /*shortcircuiting=*/true, -1); - - Value result; - AttributeTrail trail; - EXPECT_THAT(compre_step->Evaluate(frame, result, trail), - StatusIs(absl::StatusCode::kInternal, "test range error")); -} - -TEST_F(DirectComprehensionTest, PropagateAccuInitNonOkStatus) { - cel::RuntimeOptions options; - - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto accu_init = std::make_unique(); - MockDirectStep* mock = accu_init.get(); - - ON_CALL(*mock, Evaluate(_, _, _)) - .WillByDefault(Return(absl::InternalError("test accu init error"))); - - ASSERT_OK_AND_ASSIGN(auto list, MakeList()); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/CreateConstValueDirectStep(std::move(list)), - /*accu_init=*/std::move(accu_init), - /*loop_step=*/CreateConstValueDirectStep(BoolValue(false)), - /*condition_step=*/CreateConstValueDirectStep(BoolValue(true)), - /*result_step=*/CreateDirectSlotIdentStep("__result__", 1, -1), - /*shortcircuiting=*/true, -1); - - Value result; - AttributeTrail trail; - EXPECT_THAT(compre_step->Evaluate(frame, result, trail), - StatusIs(absl::StatusCode::kInternal, "test accu init error")); -} - -TEST_F(DirectComprehensionTest, PropagateLoopNonOkStatus) { - cel::RuntimeOptions options; - - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto loop_step = std::make_unique(); - MockDirectStep* mock = loop_step.get(); - - ON_CALL(*mock, Evaluate(_, _, _)) - .WillByDefault(Return(absl::InternalError("test loop error"))); - - ASSERT_OK_AND_ASSIGN(auto list, MakeList()); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/CreateConstValueDirectStep(std::move(list)), - /*accu_init=*/CreateConstValueDirectStep(BoolValue(false)), - /*loop_step=*/std::move(loop_step), - /*condition_step=*/CreateConstValueDirectStep(BoolValue(true)), - /*result_step=*/CreateDirectSlotIdentStep("__result__", 1, -1), - /*shortcircuiting=*/true, -1); - - Value result; - AttributeTrail trail; - EXPECT_THAT(compre_step->Evaluate(frame, result, trail), - StatusIs(absl::StatusCode::kInternal, "test loop error")); -} - -TEST_F(DirectComprehensionTest, PropagateConditionNonOkStatus) { - cel::RuntimeOptions options; - - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto condition = std::make_unique(); - MockDirectStep* mock = condition.get(); - - ON_CALL(*mock, Evaluate(_, _, _)) - .WillByDefault(Return(absl::InternalError("test condition error"))); - - ASSERT_OK_AND_ASSIGN(auto list, MakeList()); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/CreateConstValueDirectStep(std::move(list)), - /*accu_init=*/CreateConstValueDirectStep(BoolValue(false)), - /*loop_step=*/CreateConstValueDirectStep(BoolValue(false)), - /*condition_step=*/std::move(condition), - /*result_step=*/CreateDirectSlotIdentStep("__result__", 1, -1), - /*shortcircuiting=*/true, -1); - - Value result; - AttributeTrail trail; - EXPECT_THAT(compre_step->Evaluate(frame, result, trail), - StatusIs(absl::StatusCode::kInternal, "test condition error")); -} - -TEST_F(DirectComprehensionTest, PropagateResultNonOkStatus) { - cel::RuntimeOptions options; - - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto result_step = std::make_unique(); - MockDirectStep* mock = result_step.get(); - - ON_CALL(*mock, Evaluate(_, _, _)) - .WillByDefault(Return(absl::InternalError("test result error"))); - - ASSERT_OK_AND_ASSIGN(auto list, MakeList()); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/CreateConstValueDirectStep(std::move(list)), - /*accu_init=*/CreateConstValueDirectStep(BoolValue(false)), - /*loop_step=*/CreateConstValueDirectStep(BoolValue(false)), - /*condition_step=*/CreateConstValueDirectStep(BoolValue(true)), - /*result_step=*/std::move(result_step), - /*shortcircuiting=*/true, -1); - - Value result; - AttributeTrail trail; - EXPECT_THAT(compre_step->Evaluate(frame, result, trail), - StatusIs(absl::StatusCode::kInternal, "test result error")); -} - -TEST_F(DirectComprehensionTest, Shortcircuit) { - cel::RuntimeOptions options; - - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto loop_step = std::make_unique(); - MockDirectStep* mock = loop_step.get(); - - EXPECT_CALL(*mock, Evaluate(_, _, _)) - .Times(0) - .WillRepeatedly([](ExecutionFrameBase&, Value& result, AttributeTrail&) { - result = BoolValue(false); - return absl::OkStatus(); - }); - - ASSERT_OK_AND_ASSIGN(auto list, MakeList()); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/CreateConstValueDirectStep(std::move(list)), - /*accu_init=*/CreateConstValueDirectStep(BoolValue(false)), - /*loop_step=*/std::move(loop_step), - /*condition_step=*/CreateConstValueDirectStep(BoolValue(false)), - /*result_step=*/CreateDirectSlotIdentStep("__result__", 1, -1), - /*shortcircuiting=*/true, -1); - - Value result; - AttributeTrail trail; - ASSERT_OK(compre_step->Evaluate(frame, result, trail)); - EXPECT_THAT(result, BoolValueIs(false)); -} - -TEST_F(DirectComprehensionTest, IterationLimit) { - cel::RuntimeOptions options; - options.comprehension_max_iterations = 2; - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto loop_step = std::make_unique(); - MockDirectStep* mock = loop_step.get(); - - EXPECT_CALL(*mock, Evaluate(_, _, _)) - .Times(1) - .WillRepeatedly([](ExecutionFrameBase&, Value& result, AttributeTrail&) { - result = BoolValue(false); - return absl::OkStatus(); - }); - - ASSERT_OK_AND_ASSIGN(auto list, MakeList()); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/CreateConstValueDirectStep(std::move(list)), - /*accu_init=*/CreateConstValueDirectStep(BoolValue(false)), - /*loop_step=*/std::move(loop_step), - /*condition_step=*/CreateConstValueDirectStep(BoolValue(true)), - /*result_step=*/CreateDirectSlotIdentStep("__result__", 1, -1), - /*shortcircuiting=*/true, -1); - - Value result; - AttributeTrail trail; - EXPECT_THAT(compre_step->Evaluate(frame, result, trail), - StatusIs(absl::StatusCode::kInternal)); -} - -TEST_F(DirectComprehensionTest, Exhaustive) { - cel::RuntimeOptions options; - - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto loop_step = std::make_unique(); - MockDirectStep* mock = loop_step.get(); - - EXPECT_CALL(*mock, Evaluate(_, _, _)) - .Times(2) - .WillRepeatedly([](ExecutionFrameBase&, Value& result, AttributeTrail&) { - result = BoolValue(false); - return absl::OkStatus(); - }); - - ASSERT_OK_AND_ASSIGN(auto list, MakeList()); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/CreateConstValueDirectStep(std::move(list)), - /*accu_init=*/CreateConstValueDirectStep(BoolValue(false)), - /*loop_step=*/std::move(loop_step), - /*condition_step=*/CreateConstValueDirectStep(BoolValue(false)), - /*result_step=*/CreateDirectSlotIdentStep("__result__", 1, -1), - /*shortcircuiting=*/false, -1); - - Value result; - AttributeTrail trail; - ASSERT_OK(compre_step->Evaluate(frame, result, trail)); - EXPECT_THAT(result, BoolValueIs(false)); -} - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/eval/const_value_step.h b/eval/eval/const_value_step.h deleted file mode 100644 index c3cf6a424..000000000 --- a/eval/eval/const_value_step.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_CONST_VALUE_STEP_H_ -#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_CONST_VALUE_STEP_H_ - -#include -#include -#include - -#include "absl/status/statusor.h" -#include "common/value.h" -#include "eval/eval/compiler_constant_step.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" - -namespace google::api::expr::runtime { - -// Factory method for Constant AST node expression recursive step. -inline std::unique_ptr CreateConstValueDirectStep( - cel::Value value, int64_t id = -1) { - return std::make_unique(std::move(value), id); -} - -// Factory method for Constant AST node expression step. -inline absl::StatusOr> CreateConstValueStep( - cel::Value value, int64_t expr_id, bool comes_from_ast = true) { - return std::make_unique(std::move(value), expr_id, - comes_from_ast); -} - -} // namespace google::api::expr::runtime - -#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_CONST_VALUE_STEP_H_ diff --git a/eval/eval/container_access_step.cc b/eval/eval/container_access_step.cc index fda51e34f..5e62cb7c6 100644 --- a/eval/eval/container_access_step.cc +++ b/eval/eval/container_access_step.cc @@ -2,6 +2,7 @@ #include #include +#include #include #include "absl/log/absl_check.h" @@ -11,29 +12,23 @@ #include "absl/types/optional.h" #include "absl/types/span.h" #include "base/attribute.h" -#include "common/casting.h" #include "common/expr.h" #include "common/kind.h" #include "common/value.h" #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" -#include "eval/eval/attribute_utility.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" #include "eval/internal/errors.h" #include "internal/number.h" -#include "internal/status_macros.h" #include "runtime/internal/errors.h" namespace google::api::expr::runtime { namespace { -using ::cel::AttributeQualifier; -using ::cel::Cast; using ::cel::ErrorValue; -using ::cel::InstanceOf; using ::cel::IntValue; using ::cel::ListValue; using ::cel::MapValue; @@ -55,12 +50,12 @@ absl::optional CelNumberFromValue(const Value& value) { case ValueKind::kDouble: return Number::FromDouble(value.GetDouble().NativeValue()); default: - return absl::nullopt; + return std::nullopt; } } absl::Status CheckMapKeyType(const Value& key) { - ValueKind kind = key->kind(); + ValueKind kind = key.kind(); switch (kind) { case ValueKind::kString: case ValueKind::kInt64: @@ -73,19 +68,21 @@ absl::Status CheckMapKeyType(const Value& key) { } } -AttributeQualifier AttributeQualifierFromValue(const Value& v) { - switch (v->kind()) { +cel::AttributeQualifierView AttributeQualifierViewFromValue( + const Value& v, std::string& scratch) { + switch (v.kind()) { case ValueKind::kString: - return AttributeQualifier::OfString(v.GetString().ToString()); + return cel::AttributeQualifierView::OfString( + v.GetString().ToStringView(&scratch)); case ValueKind::kInt64: - return AttributeQualifier::OfInt(v.GetInt().NativeValue()); + return cel::AttributeQualifierView::OfInt(v.GetInt().NativeValue()); case ValueKind::kUint64: - return AttributeQualifier::OfUint(v.GetUint().NativeValue()); + return cel::AttributeQualifierView::OfUint(v.GetUint().NativeValue()); case ValueKind::kBool: - return AttributeQualifier::OfBool(v.GetBool().NativeValue()); + return cel::AttributeQualifierView::OfBool(v.GetBool().NativeValue()); default: // Non-matching qualifier. - return AttributeQualifier(); + return cel::AttributeQualifierView(); } } @@ -102,7 +99,8 @@ void LookupInMap(const MapValue& cel_map, const Value& key, cel_map.Find(key, frame.descriptor_pool(), frame.message_factory(), frame.arena(), &result); if (!lookup.ok()) { - result = cel::ErrorValue(std::move(lookup).status()); + result = + cel::ErrorValue::From(std::move(lookup).status(), frame.arena()); return; } if (*lookup) { @@ -116,7 +114,8 @@ void LookupInMap(const MapValue& cel_map, const Value& key, cel_map.Find(IntValue(number->AsInt()), frame.descriptor_pool(), frame.message_factory(), frame.arena(), &result); if (!lookup.ok()) { - result = cel::ErrorValue(std::move(lookup).status()); + result = + cel::ErrorValue::From(std::move(lookup).status(), frame.arena()); return; } if (*lookup) { @@ -130,7 +129,8 @@ void LookupInMap(const MapValue& cel_map, const Value& key, cel_map.Find(UintValue(number->AsUint()), frame.descriptor_pool(), frame.message_factory(), frame.arena(), &result); if (!lookup.ok()) { - result = cel::ErrorValue(std::move(lookup).status()); + result = + cel::ErrorValue::From(std::move(lookup).status(), frame.arena()); return; } if (*lookup) { @@ -138,14 +138,15 @@ void LookupInMap(const MapValue& cel_map, const Value& key, return; } } - result = cel::ErrorValue(CreateNoSuchKeyError(key->DebugString())); + result = cel::ErrorValue::From(CreateNoSuchKeyError(key->DebugString()), + frame.arena()); return; } } absl::Status status = CheckMapKeyType(key); if (!status.ok()) { - result = cel::ErrorValue(std::move(status)); + result = cel::ErrorValue::From(std::move(status), frame.arena()); return; } @@ -153,7 +154,7 @@ void LookupInMap(const MapValue& cel_map, const Value& key, cel_map.Get(key, frame.descriptor_pool(), frame.message_factory(), frame.arena(), &result); if (!lookup.ok()) { - result = cel::ErrorValue(std::move(lookup)); + result = cel::ErrorValue::From(std::move(lookup), frame.arena()); } ABSL_DCHECK(!result.IsUnknown()); } @@ -166,26 +167,30 @@ void LookupInList(const ListValue& cel_list, const Value& key, if (number.has_value() && number->LosslessConvertibleToInt()) { maybe_idx = number->AsInt(); } - } else if (InstanceOf(key)) { + } else if (key.IsInt()) { maybe_idx = key.GetInt().NativeValue(); } if (!maybe_idx.has_value()) { - result = cel::ErrorValue(absl::UnknownError( - absl::StrCat("Index error: expected integer type, got ", - cel::KindToString(ValueKindToKind(key->kind()))))); + result = cel::ErrorValue::From( + absl::UnknownError( + absl::StrCat("Index error: expected integer type, got ", + cel::KindToString(ValueKindToKind(key->kind())))), + frame.arena()); return; } int64_t idx = *maybe_idx; auto size = cel_list.Size(); if (!size.ok()) { - result = cel::ErrorValue(size.status()); + result = cel::ErrorValue::From(size.status(), frame.arena()); return; } if (idx < 0 || idx >= *size) { - result = cel::ErrorValue(absl::UnknownError( - absl::StrCat("Index error: index=", idx, " size=", *size))); + result = + cel::ErrorValue::From(absl::UnknownError(absl::StrCat( + "Index error: index=", idx, " size=", *size)), + frame.arena()); return; } @@ -194,7 +199,7 @@ void LookupInList(const ListValue& cel_list, const Value& key, frame.arena(), &result); if (!lookup.ok()) { - result = cel::ErrorValue(std::move(lookup)); + result = cel::ErrorValue::From(std::move(lookup), frame.arena()); } ABSL_DCHECK(!result.IsUnknown()); } @@ -204,17 +209,19 @@ void LookupInContainer(const Value& container, const Value& key, // Select steps can be applied to either maps or messages switch (container.kind()) { case ValueKind::kMap: { - LookupInMap(Cast(container), key, frame, result); + LookupInMap(container.GetMap(), key, frame, result); return; } case ValueKind::kList: { - LookupInList(Cast(container), key, frame, result); + LookupInList(container.GetList(), key, frame, result); return; } default: - result = cel::ErrorValue(absl::InvalidArgumentError( - absl::StrCat("Invalid container type: '", - ValueKindToString(container->kind()), "'"))); + result = + cel::ErrorValue::From(absl::InvalidArgumentError(absl::StrCat( + "Invalid container type: '", + ValueKindToString(container->kind()), "'")), + frame.arena()); return; } } @@ -224,8 +231,7 @@ void PerformLookup(ExecutionFrameBase& frame, const Value& container, bool enable_optional_types, Value& result, AttributeTrail& trail) { if (frame.unknown_processing_enabled()) { - AttributeUtility::Accumulator unknowns = - frame.attribute_utility().CreateAccumulator(); + UnknownAccumulator unknowns(frame.unknown_tree()); unknowns.MaybeAdd(container); unknowns.MaybeAdd(key); @@ -234,20 +240,23 @@ void PerformLookup(ExecutionFrameBase& frame, const Value& container, return; } - trail = container_trail.Step(AttributeQualifierFromValue(key)); - - if (frame.attribute_utility().CheckForUnknownExact(trail)) { - result = frame.attribute_utility().CreateUnknownSet(trail.attribute()); + std::string scratch; + cel::AttributeQualifierView qualifier = + AttributeQualifierViewFromValue(key, scratch); + if (qualifier && container_trail.Match( + qualifier, result, trail, frame.unknown_tree())) { return; } } - if (InstanceOf(container)) { + if (container.IsError()) { result = container; + trail = AttributeTrail(); return; } - if (InstanceOf(key)) { + if (key.IsError()) { result = key; + trail = AttributeTrail(); return; } @@ -260,7 +269,7 @@ void PerformLookup(ExecutionFrameBase& frame, const Value& container, Value value; optional_value.Value(&value); LookupInContainer(value, key, frame, result); - if (auto error_value = cel::As(result); + if (auto error_value = result.AsError(); error_value && cel::IsNoSuchKey(*error_value)) { result = cel::OptionalValue::None(); return; @@ -276,21 +285,20 @@ void PerformLookup(ExecutionFrameBase& frame, const Value& container, // message. class ContainerAccessStep : public ExpressionStepBase { public: - ContainerAccessStep(int64_t expr_id, bool enable_optional_types) - : ExpressionStepBase(expr_id), - enable_optional_types_(enable_optional_types) {} + explicit ContainerAccessStep(bool enable_optional_types) + : ExpressionStepBase(), enable_optional_types_(enable_optional_types) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: bool enable_optional_types_; }; -absl::Status ContainerAccessStep::Evaluate(ExecutionFrame* frame) const { +void ContainerAccessStep::Evaluate(ExecutionFrame* frame) const { if (!frame->value_stack().HasEnough(kNumContainerAccessArguments)) { - return absl::Status( - absl::StatusCode::kInternal, - "Insufficient arguments supplied for ContainerAccess-type expression"); + frame->Abort(absl::InternalError( + "Insufficient arguments supplied for ContainerAccess-type expression")); + return; } Value result; @@ -303,68 +311,19 @@ absl::Status ContainerAccessStep::Evaluate(ExecutionFrame* frame) const { enable_optional_types_, result, result_trail); frame->value_stack().PopAndPush(kNumContainerAccessArguments, std::move(result), std::move(result_trail)); - - return absl::OkStatus(); -} - -class DirectContainerAccessStep : public DirectExpressionStep { - public: - DirectContainerAccessStep( - std::unique_ptr container_step, - std::unique_ptr key_step, - bool enable_optional_types, int64_t expr_id) - : DirectExpressionStep(expr_id), - container_step_(std::move(container_step)), - key_step_(std::move(key_step)), - enable_optional_types_(enable_optional_types) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& trail) const override; - - private: - std::unique_ptr container_step_; - std::unique_ptr key_step_; - bool enable_optional_types_; -}; - -absl::Status DirectContainerAccessStep::Evaluate(ExecutionFrameBase& frame, - Value& result, - AttributeTrail& trail) const { - Value container; - Value key; - AttributeTrail container_trail; - AttributeTrail key_trail; - - CEL_RETURN_IF_ERROR( - container_step_->Evaluate(frame, container, container_trail)); - CEL_RETURN_IF_ERROR(key_step_->Evaluate(frame, key, key_trail)); - - PerformLookup(frame, container, key, container_trail, enable_optional_types_, - result, trail); - - return absl::OkStatus(); } } // namespace -std::unique_ptr CreateDirectContainerAccessStep( - std::unique_ptr container_step, - std::unique_ptr key_step, bool enable_optional_types, - int64_t expr_id) { - return std::make_unique( - std::move(container_step), std::move(key_step), enable_optional_types, - expr_id); -} - // Factory method for Select - based Execution step -absl::StatusOr> CreateContainerAccessStep( - const cel::CallExpr& call, int64_t expr_id, bool enable_optional_types) { +absl::StatusOr> CreateContainerAccessStep( + const cel::CallExpr& call, bool enable_optional_types) { int arg_count = call.args().size() + (call.has_target() ? 1 : 0); if (arg_count != kNumContainerAccessArguments) { return absl::InvalidArgumentError(absl::StrCat( "Invalid argument count for index operation: ", arg_count)); } - return std::make_unique(expr_id, enable_optional_types); + return std::make_unique(enable_optional_types); } } // namespace google::api::expr::runtime diff --git a/eval/eval/container_access_step.h b/eval/eval/container_access_step.h index b7af5e895..38d4b34d4 100644 --- a/eval/eval/container_access_step.h +++ b/eval/eval/container_access_step.h @@ -1,25 +1,17 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_CONTAINER_ACCESS_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_CONTAINER_ACCESS_STEP_H_ -#include #include #include "absl/status/statusor.h" #include "common/expr.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { -std::unique_ptr CreateDirectContainerAccessStep( - std::unique_ptr container_step, - std::unique_ptr key_step, bool enable_optional_types, - int64_t expr_id); - // Factory method for Select - based Execution step -absl::StatusOr> CreateContainerAccessStep( - const cel::CallExpr& call, int64_t expr_id, - bool enable_optional_types = false); +absl::StatusOr> CreateContainerAccessStep( + const cel::CallExpr& call, bool enable_optional_types = false); } // namespace google::api::expr::runtime diff --git a/eval/eval/container_access_step_test.cc b/eval/eval/container_access_step_test.cc index 25bf72223..346ada853 100644 --- a/eval/eval/container_access_step_test.cc +++ b/eval/eval/container_access_step_test.cc @@ -9,14 +9,16 @@ #include "cel/expr/syntax.pb.h" #include "google/protobuf/struct.pb.h" #include "absl/base/nullability.h" +#include "absl/log/absl_check.h" #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "base/builtins.h" #include "base/type_provider.h" #include "common/ast.h" #include "common/expr.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" #include "eval/eval/ident_step.h" #include "eval/public/activation.h" #include "eval/public/cel_attribute.h" @@ -33,12 +35,14 @@ #include "parser/parser.h" #include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" +#include "runtime/runtime_options.h" #include "google/protobuf/arena.h" namespace google::api::expr::runtime { namespace { +using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::Expr; using ::cel::SourceInfo; @@ -74,31 +78,27 @@ CelValue EvaluateAttributeHelper( container_expr.mutable_ident_expr().set_name("container"); key_expr.mutable_ident_expr().set_name("key"); - if (use_recursive_impl) { - path.push_back(std::make_unique( - CreateDirectContainerAccessStep(CreateDirectIdentStep("container", 1), - CreateDirectIdentStep("key", 2), - /*enable_optional_types=*/false, 3), - 3)); - } else { - path.push_back(std::move(CreateIdentStep("container", 1).value())); - path.push_back(std::move(CreateIdentStep("key", 2).value())); - path.push_back(std::move(CreateContainerAccessStep(call, 3).value())); - } + path.push_back(ExpressionStep::MakeIdentifierStep(0, 1)); + path.push_back(ExpressionStep::MakeIdentifierStep(1, 2)); + absl::StatusOr> access_step = + CreateContainerAccessStep(call); + ABSL_CHECK_OK(access_step); // Crash OK + path.push_back(ExpressionStep::MakeGenericStep(*std::move(access_step), 3)); cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; options.enable_heterogeneous_equality = false; CelExpressionFlatImpl cel_expr( - env, - FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env, FlatExpression(std::move(path), /*comprehension_slot_count=*/2, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, + IdentifierTable{{"container", 0}, {"key", 1}})); Activation activation; activation.InsertValue("container", container); activation.InsertValue("key", key); - activation.set_unknown_attribute_patterns(patterns); + ABSL_CHECK_OK(activation.SetUnknownAttributePatterns(patterns)); // Crash OK auto result = cel_expr.Evaluate(activation, arena); return *result; } @@ -227,8 +227,9 @@ TEST_P(ContainerAccessStepUniformityTest, TestMapKeyAccess) { TEST_P(ContainerAccessStepUniformityTest, TestBoolKeyType) { CelMapBuilder cel_map; - ASSERT_OK(cel_map.Add(CelValue::CreateBool(true), - CelValue::CreateStringView("value_true"))); + ASSERT_THAT(cel_map.Add(CelValue::CreateBool(true), + CelValue::CreateStringView("value_true")), + IsOk()); CelValue result = EvaluateAttribute(CelValue::CreateMap(&cel_map), CelValue::CreateBool(true), @@ -267,7 +268,7 @@ TEST_F(ContainerAccessStepTest, TestInvalidReceiverCreateContainerAccessStep) { Expr& extra_arg = call.mutable_args().emplace_back(); extra_arg.mutable_const_expr().set_bool_value(true); - EXPECT_THAT(CreateContainerAccessStep(call, 0).status(), + EXPECT_THAT(CreateContainerAccessStep(call).status(), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Invalid argument count"))); } @@ -285,7 +286,7 @@ TEST_F(ContainerAccessStepTest, TestInvalidGlobalCreateContainerAccessStep) { Expr& extra_arg = call.mutable_args().emplace_back(); extra_arg.mutable_const_expr().set_bool_value(true); - EXPECT_THAT(CreateContainerAccessStep(call, 0).status(), + EXPECT_THAT(CreateContainerAccessStep(call).status(), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Invalid argument count"))); } @@ -312,19 +313,6 @@ TEST_F(ContainerAccessStepTest, TestListIndexAccessUnknown) { ASSERT_TRUE(result.IsUnknownSet()); } -TEST_F(ContainerAccessStepTest, TestListUnknownKey) { - ContainerBackedListImpl cel_list({CelValue::CreateInt64(1), - CelValue::CreateInt64(2), - CelValue::CreateInt64(3)}); - - UnknownSet unknown_set; - CelValue result = - EvaluateAttribute(CelValue::CreateList(&cel_list), - CelValue::CreateUnknownSet(&unknown_set), true, true); - - ASSERT_TRUE(result.IsUnknownSet()); -} - TEST_F(ContainerAccessStepTest, TestMapInvalidKey) { const std::string kKey0 = "testkey0"; const std::string kKey1 = "testkey1"; @@ -344,31 +332,6 @@ TEST_F(ContainerAccessStepTest, TestMapInvalidKey) { HasSubstr("Invalid map key type: 'double'"))); } -TEST_F(ContainerAccessStepTest, TestMapUnknownKey) { - const std::string kKey0 = "testkey0"; - const std::string kKey1 = "testkey1"; - const std::string kKey2 = "testkey2"; - Struct cel_struct; - (*cel_struct.mutable_fields())[kKey0].set_string_value("value0"); - (*cel_struct.mutable_fields())[kKey1].set_string_value("value1"); - (*cel_struct.mutable_fields())[kKey2].set_string_value("value2"); - - UnknownSet unknown_set; - CelValue result = - EvaluateAttribute(CelProtoWrapper::CreateMessage(&cel_struct, &arena_), - CelValue::CreateUnknownSet(&unknown_set), true, true); - - ASSERT_TRUE(result.IsUnknownSet()); -} - -TEST_F(ContainerAccessStepTest, TestUnknownContainer) { - UnknownSet unknown_set; - CelValue result = EvaluateAttribute(CelValue::CreateUnknownSet(&unknown_set), - CelValue::CreateInt64(1), true, true); - - ASSERT_TRUE(result.IsUnknownSet()); -} - TEST_F(ContainerAccessStepTest, TestInvalidContainerType) { CelValue result = EvaluateAttribute(CelValue::CreateInt64(1), CelValue::CreateInt64(0), true, true); diff --git a/eval/eval/create_list_step.cc b/eval/eval/create_list_step.cc index bb977ce94..30ab3c714 100644 --- a/eval/eval/create_list_step.cc +++ b/eval/eval/create_list_step.cc @@ -4,99 +4,91 @@ #include #include #include -#include #include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" #include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/types/optional.h" -#include "common/casting.h" -#include "common/expr.h" +#include "absl/types/span.h" +#include "common/internal/attribute_trail.h" +#include "common/optional_ref.h" #include "common/value.h" #include "common/values/list_value_builder.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/attribute_utility.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" -#include "internal/status_macros.h" namespace google::api::expr::runtime { namespace { -using ::cel::Cast; using ::cel::ErrorValue; -using ::cel::InstanceOf; using ::cel::ListValueBuilderPtr; -using ::cel::UnknownValue; using ::cel::Value; using ::cel::common_internal::NewListValueBuilder; -class CreateListStep : public ExpressionStepBase { - public: - CreateListStep(int64_t expr_id, int list_size, - absl::flat_hash_set optional_indices) - : ExpressionStepBase(expr_id), - list_size_(list_size), - optional_indices_(std::move(optional_indices)) {} +constexpr size_t kMaxSmallListSize = 58; - absl::Status Evaluate(ExecutionFrame* frame) const override; +template +struct StepTraits; - private: - absl::Status DoEvaluate(ExecutionFrame* frame, Value* result) const; +template <> +struct StepTraits { + static bool is_optional(const SmallListStepInfo& step, size_t index) { + ABSL_DCHECK_LT(index, kMaxSmallListSize); + return ((step.optional_bit_set >> index) & 1) == 1; + } - int list_size_; - absl::flat_hash_set optional_indices_; + static size_t num_elements(const SmallListStepInfo& step) { + return step.list_size; + } }; -absl::Status CreateListStep::Evaluate(ExecutionFrame* frame) const { - if (list_size_ < 0) { - return absl::Status(absl::StatusCode::kInternal, - "CreateListStep: list size is <0"); +template <> +struct StepTraits { + static bool is_optional(const ListStepInfo& step, size_t index) { + return step.is_optional(index); } - if (!frame->value_stack().HasEnough(list_size_)) { - return absl::Status(absl::StatusCode::kInternal, - "CreateListStep: stack underflow"); + static size_t num_elements(const ListStepInfo& step) { + return step.num_elements(); } +}; - Value result; - CEL_RETURN_IF_ERROR(DoEvaluate(frame, &result)); +template +void EvaluateListStepImpl(const StepInfo& step, ExecutionFrame& frame) { + const size_t num_elements = StepTraits::num_elements(step); - frame->value_stack().PopAndPush(list_size_, std::move(result)); - return absl::OkStatus(); -} + if (!frame.value_stack().HasEnough(num_elements)) { + frame.Abort(absl::InternalError("CreateListStep: stack underflow")); + return; + } -absl::Status CreateListStep::DoEvaluate(ExecutionFrame* frame, - Value* result) const { - auto args = frame->value_stack().GetSpan(list_size_); + absl::Span args = frame.value_stack().GetSpan(num_elements); - for (const auto& arg : args) { - if (arg.IsError()) { - *result = arg; - return absl::OkStatus(); + for (size_t i = 0; i < num_elements; ++i) { + if (args[i].IsError()) { + frame.value_stack().SwapAndPop(num_elements, i); + return; } } - if (frame->enable_unknowns()) { - absl::optional unknown_set = - frame->attribute_utility().IdentifyAndMergeUnknowns( - args, frame->value_stack().GetAttributeSpan(list_size_), - /*use_partial=*/true); - if (unknown_set.has_value()) { - *result = std::move(*unknown_set); - return absl::OkStatus(); + if (frame.enable_unknowns()) { + if (auto unknown = cel::common_internal::PartiallyIdentifyAndMergeUnknowns( + args, frame.value_stack().GetAttributeSpan(num_elements), + frame.unknown_tree()); + unknown.has_value()) { + frame.value_stack().PopAndPush(num_elements, *unknown); + return; } } - ListValueBuilderPtr builder = NewListValueBuilder(frame->arena()); - builder->Reserve(args.size()); + ListValueBuilderPtr builder = NewListValueBuilder(frame.arena()); + builder->Reserve(num_elements); - for (size_t i = 0; i < args.size(); ++i) { - const auto& arg = args[i]; - if (optional_indices_.contains(static_cast(i))) { - if (auto optional_arg = arg.AsOptional(); optional_arg) { + for (size_t i = 0; i < num_elements; ++i) { + const Value& arg = args[i]; + if (StepTraits::is_optional(step, i)) { + if (cel::optional_ref optional_arg = + arg.AsOptional(); + optional_arg.has_value()) { if (!optional_arg->HasValue()) { continue; } @@ -104,180 +96,59 @@ absl::Status CreateListStep::DoEvaluate(ExecutionFrame* frame, optional_arg->Value(&optional_arg_value); if (optional_arg_value.IsError()) { // Error should never be in optional, but better safe than sorry. - *result = std::move(optional_arg_value); - return absl::OkStatus(); + frame.value_stack().PopAndPush(num_elements, + std::move(optional_arg_value)); + return; + } + if (absl::Status status = builder->Add(std::move(optional_arg_value)); + !status.ok()) { + frame.Abort(std::move(status)); + return; } - CEL_RETURN_IF_ERROR(builder->Add(std::move(optional_arg_value))); } else { - *result = cel::TypeConversionError(arg.GetTypeName(), "optional_type"); - return absl::OkStatus(); + frame.value_stack().PopAndPush( + num_elements, + cel::TypeConversionError(arg.GetTypeName(), "optional_type", + frame.arena())); + return; } } else { - CEL_RETURN_IF_ERROR(builder->Add(arg)); - } - } - - *result = std::move(*builder).Build(); - return absl::OkStatus(); -} - -absl::flat_hash_set MakeOptionalIndicesSet( - const cel::ListExpr& create_list_expr) { - absl::flat_hash_set optional_indices; - for (size_t i = 0; i < create_list_expr.elements().size(); ++i) { - if (create_list_expr.elements()[i].optional()) { - optional_indices.insert(static_cast(i)); - } - } - return optional_indices; -} - -class CreateListDirectStep : public DirectExpressionStep { - public: - CreateListDirectStep( - std::vector> elements, - absl::flat_hash_set optional_indices, int64_t expr_id) - : DirectExpressionStep(expr_id), - elements_(std::move(elements)), - optional_indices_(std::move(optional_indices)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const override { - ListValueBuilderPtr builder = NewListValueBuilder(frame.arena()); - builder->Reserve(elements_.size()); - - AttributeUtility::Accumulator unknowns = - frame.attribute_utility().CreateAccumulator(); - AttributeTrail tmp_attr; - - for (size_t i = 0; i < elements_.size(); ++i) { - const auto& element = elements_[i]; - CEL_RETURN_IF_ERROR(element->Evaluate(frame, result, tmp_attr)); - - if (result.IsError()) { - return absl::OkStatus(); - } - - if (frame.attribute_tracking_enabled()) { - if (frame.missing_attribute_errors_enabled()) { - if (frame.attribute_utility().CheckForMissingAttribute(tmp_attr)) { - CEL_ASSIGN_OR_RETURN( - result, frame.attribute_utility().CreateMissingAttributeError( - tmp_attr.attribute())); - return absl::OkStatus(); - } - } - if (frame.unknown_processing_enabled()) { - if (result.IsUnknown()) { - unknowns.Add(result.GetUnknown()); - } - if (frame.attribute_utility().CheckForUnknown(tmp_attr, - /*use_partial=*/true)) { - unknowns.Add(tmp_attr); - } - } + if (absl::Status status = builder->Add(arg); !status.ok()) { + frame.Abort(std::move(status)); + return; } - - if (!unknowns.IsEmpty()) { - // We found an unknown, there is no point in attempting to create a - // list. Instead iterate through the remaining elements and look for - // more unknowns. - continue; - } - - // Conditionally add if optional. - if (optional_indices_.contains(static_cast(i))) { - if (auto optional_arg = result.AsOptional(); optional_arg) { - if (!optional_arg->HasValue()) { - continue; - } - Value optional_arg_value; - optional_arg->Value(&optional_arg_value); - if (optional_arg_value.IsError()) { - // Error should never be in optional, but better safe than sorry. - result = std::move(optional_arg_value); - return absl::OkStatus(); - } - CEL_RETURN_IF_ERROR(builder->Add(std::move(optional_arg_value))); - continue; - } - result = - cel::TypeConversionError(result.GetTypeName(), "optional_type"); - return absl::OkStatus(); - } - - // Otherwise just add. - CEL_RETURN_IF_ERROR(builder->Add(std::move(result))); } - - if (!unknowns.IsEmpty()) { - result = std::move(unknowns).Build(); - return absl::OkStatus(); - } - result = std::move(*builder).Build(); - - return absl::OkStatus(); } - private: - std::vector> elements_; - absl::flat_hash_set optional_indices_; -}; - -class MutableListStep : public ExpressionStepBase { - public: - explicit MutableListStep(int64_t expr_id) : ExpressionStepBase(expr_id) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override; -}; - -absl::Status MutableListStep::Evaluate(ExecutionFrame* frame) const { - frame->value_stack().Push(cel::CustomListValue( - cel::common_internal::NewMutableListValue(frame->arena()), - frame->arena())); - return absl::OkStatus(); -} - -class DirectMutableListStep : public DirectExpressionStep { - public: - explicit DirectMutableListStep(int64_t expr_id) - : DirectExpressionStep(expr_id) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override; -}; - -absl::Status DirectMutableListStep::Evaluate( - ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const { - result = cel::CustomListValue( - cel::common_internal::NewMutableListValue(frame.arena()), frame.arena()); - return absl::OkStatus(); + frame.value_stack().PopAndPush(num_elements, std::move(*builder).Build()); } } // namespace -std::unique_ptr CreateDirectListStep( - std::vector> deps, - absl::flat_hash_set optional_indices, int64_t expr_id) { - return std::make_unique( - std::move(deps), std::move(optional_indices), expr_id); -} - -absl::StatusOr> CreateCreateListStep( - const cel::ListExpr& create_list_expr, int64_t expr_id) { - return std::make_unique( - expr_id, create_list_expr.elements().size(), - MakeOptionalIndicesSet(create_list_expr)); +void EvaluateListStep(const ListStepInfo& step, ExecutionFrame& frame) { + EvaluateListStepImpl(step, frame); } -std::unique_ptr CreateMutableListStep(int64_t expr_id) { - return std::make_unique(expr_id); +void EvaluateSmallListStep(const SmallListStepInfo& step, + ExecutionFrame& frame) { + EvaluateListStepImpl(step, frame); } -std::unique_ptr CreateDirectMutableListStep( +ExpressionStep CreateCreateListStep( + size_t size, absl::flat_hash_set optional_indices, int64_t expr_id) { - return std::make_unique(expr_id); + if (size <= kMaxSmallListSize) { + uint64_t optional_bit_set = 0; + for (size_t index : optional_indices) { + ABSL_DCHECK_LT(index, size); + optional_bit_set |= (uint64_t{1} << index); + } + return ExpressionStep::MakeCreateSmallListStep( + SmallListStepInfo{optional_bit_set, size}, expr_id); + } + return ExpressionStep::MakeCreateListStep( + std::make_unique(size, std::move(optional_indices)), + expr_id); } } // namespace google::api::expr::runtime diff --git a/eval/eval/create_list_step.h b/eval/eval/create_list_step.h index b60a5e9c8..dc8e57ed4 100644 --- a/eval/eval/create_list_step.h +++ b/eval/eval/create_list_step.h @@ -1,39 +1,49 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_CREATE_LIST_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_CREATE_LIST_STEP_H_ +#include #include -#include -#include +#include #include "absl/container/flat_hash_set.h" -#include "absl/status/statusor.h" -#include "common/expr.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" namespace google::api::expr::runtime { -// Factory method for CreateList that evaluates recursively. -std::unique_ptr CreateDirectListStep( - std::vector> deps, - absl::flat_hash_set optional_indices, int64_t expr_id); +class ExecutionFrame; +class ExpressionStep; + +// Small list step info is used for lists with 58 or fewer elements with 58 bits +// used as a bitset to indicate which elements are optional. +// List literals are generally small so this covers most cases. +struct SmallListStepInfo { + uint64_t optional_bit_set : 58; + size_t list_size : 6; +}; + +class ListStepInfo { + public: + explicit ListStepInfo(size_t num_elements) : num_elements_(num_elements) {} + ListStepInfo(size_t num_elements, absl::flat_hash_set opt_fields) + : num_elements_(num_elements), opt_fields_(std::move(opt_fields)) {} + + bool is_optional(size_t index) const { return opt_fields_.contains(index); } + + size_t num_elements() const { return num_elements_; } + + private: + size_t num_elements_; + absl::flat_hash_set opt_fields_; +}; + +void EvaluateListStep(const ListStepInfo& step, ExecutionFrame& frame); +void EvaluateSmallListStep(const SmallListStepInfo& step, + ExecutionFrame& frame); // Factory method for CreateList which constructs an immutable list. -absl::StatusOr> CreateCreateListStep( - const cel::ListExpr& create_list_expr, int64_t expr_id); - -// Factory method for CreateList which constructs a mutable list. -// -// This is intended for the list construction step is generated for a -// list-building comprehension (rather than a user authored expression). -std::unique_ptr CreateMutableListStep(int64_t expr_id); - -// Factory method for CreateList which constructs a mutable list. -// -// This is intended for the list construction step is generated for a -// list-building comprehension (rather than a user authored expression). -std::unique_ptr CreateDirectMutableListStep( - int64_t expr_id); +// The optional indices are assumed to be valid and in range [0, size). +ExpressionStep CreateCreateListStep( + size_t size, absl::flat_hash_set optional_indices = {}, + int64_t expr_id = -1); } // namespace google::api::expr::runtime diff --git a/eval/eval/create_list_step_test.cc b/eval/eval/create_list_step_test.cc index 990003823..370482017 100644 --- a/eval/eval/create_list_step_test.cc +++ b/eval/eval/create_list_step_test.cc @@ -1,5 +1,6 @@ #include "eval/eval/create_list_step.h" +#include #include #include #include @@ -14,14 +15,9 @@ #include "base/attribute.h" #include "base/attribute_set.h" #include "base/type_provider.h" -#include "common/casting.h" #include "common/expr.h" #include "common/value.h" -#include "common/value_testing.h" -#include "eval/eval/attribute_trail.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/const_value_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/ident_step.h" #include "eval/internal/interop.h" @@ -32,12 +28,8 @@ #include "eval/public/unknown_attribute_set.h" #include "internal/status_macros.h" #include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" #include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" -#include "runtime/internal/runtime_type_provider.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -46,27 +38,14 @@ namespace google::api::expr::runtime { namespace { using ::absl_testing::IsOk; -using ::absl_testing::IsOkAndHolds; -using ::absl_testing::StatusIs; using ::cel::Attribute; -using ::cel::AttributeQualifier; using ::cel::AttributeSet; -using ::cel::Cast; -using ::cel::ErrorValue; using ::cel::Expr; -using ::cel::InstanceOf; -using ::cel::IntValue; -using ::cel::ListValue; using ::cel::TypeProvider; -using ::cel::UnknownValue; -using ::cel::Value; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::runtime_internal::RuntimeEnv; -using ::cel::test::IntValueIs; using ::testing::Eq; -using ::testing::HasSubstr; using ::testing::Not; -using ::testing::UnorderedElementsAre; // Helper method. Creates simple pipeline containing Select step and runs it. absl::StatusOr RunExpression( @@ -80,16 +59,11 @@ absl::StatusOr RunExpression( for (auto value : values) { auto& expr0 = create_list.mutable_elements().emplace_back().mutable_expr(); expr0.mutable_const_expr().set_int64_value(value); - CEL_ASSIGN_OR_RETURN( - auto const_step, - CreateConstValueStep(cel::interop_internal::CreateIntValue(value), - /*expr_id=*/-1)); - path.push_back(std::move(const_step)); + path.push_back(ExpressionStep::MakeConstant( + cel::interop_internal::CreateIntValue(value))); } - CEL_ASSIGN_OR_RETURN(auto step, - CreateCreateListStep(create_list, dummy_expr.id())); - path.push_back(std::move(step)); + path.push_back(CreateCreateListStep(values.size(), {}, dummy_expr.id())); cel::RuntimeOptions options; if (enable_unknowns) { options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; @@ -99,7 +73,8 @@ absl::StatusOr RunExpression( FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, {})); Activation activation; return cel_expr.Evaluate(activation, arena); @@ -114,6 +89,7 @@ absl::StatusOr RunExpressionWithCelValues( Expr dummy_expr; Activation activation; + IdentifierTable identifiers; auto& create_list = dummy_expr.mutable_list_expr(); int ind = 0; for (auto value : values) { @@ -122,25 +98,24 @@ absl::StatusOr RunExpressionWithCelValues( expr0.set_id(ind); expr0.mutable_ident_expr().set_name(var_name); - CEL_ASSIGN_OR_RETURN(auto ident_step, - CreateIdentStep(var_name, /*expr_id=*/-1)); - path.push_back(std::move(ident_step)); + path.push_back(ExpressionStep::MakeIdentifierStep(identifiers.size())); + identifiers.push_back(Identifier{var_name, identifiers.size()}); activation.InsertValue(var_name, value); } - CEL_ASSIGN_OR_RETURN(auto step0, - CreateCreateListStep(create_list, dummy_expr.id())); - path.push_back(std::move(step0)); + path.push_back(CreateCreateListStep(values.size(), {}, dummy_expr.id())); cel::RuntimeOptions options; if (enable_unknowns) { options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; } + const size_t slot_count = identifiers.size(); CelExpressionFlatImpl cel_expr( - env, - FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env, FlatExpression(std::move(path), + /*comprehension_slot_count=*/slot_count, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, std::move(identifiers))); return cel_expr.Evaluate(activation, arena); } @@ -164,15 +139,13 @@ TEST(CreateListStepTest, TestCreateListStackUnderflow) { auto& expr0 = create_list.mutable_elements().emplace_back().mutable_expr(); expr0.mutable_const_expr().set_int64_value(1); - ASSERT_OK_AND_ASSIGN(auto step0, - CreateCreateListStep(create_list, dummy_expr.id())); - path.push_back(std::move(step0)); + path.push_back(CreateCreateListStep(1, {}, dummy_expr.id())); auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl cel_expr( env, FlatExpression(std::move(path), /*comprehension_slot_count=*/0, env->type_registry.GetComposedTypeProvider(), - cel::RuntimeOptions{})); + cel::RuntimeOptions{}, /*arena=*/nullptr, {})); Activation activation; google::protobuf::Arena arena; @@ -272,279 +245,6 @@ TEST(CreateListStepTest, CreateListHundredAnd2Unknowns) { EXPECT_THAT(result_set->unknown_attributes().size(), Eq(2)); } -TEST(CreateDirectListStep, Basic) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep(IntValue(1), -1)); - deps.push_back(CreateConstValueDirectStep(IntValue(2), -1)); - auto step = CreateDirectListStep(std::move(deps), {}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_OK(step->Evaluate(frame, result, attr)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).Size(), IsOkAndHolds(2)); -} - -TEST(CreateDirectListStep, ForwardFirstError) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep( - cel::ErrorValue(absl::InternalError("test1")), -1)); - deps.push_back(CreateConstValueDirectStep( - cel::ErrorValue(absl::InternalError("test2")), -1)); - auto step = CreateDirectListStep(std::move(deps), {}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_OK(step->Evaluate(frame, result, attr)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), - StatusIs(absl::StatusCode::kInternal, "test1")); -} - -std::vector UnknownAttrNames(const UnknownValue& v) { - std::vector names; - names.reserve(v.attribute_set().size()); - - for (const auto& attr : v.attribute_set()) { - EXPECT_OK(attr.AsString().status()); - names.push_back(attr.AsString().value_or("")); - } - return names; -} - -TEST(CreateDirectListStep, MergeUnknowns) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - AttributeSet attr_set1({Attribute("var1")}); - AttributeSet attr_set2({Attribute("var2")}); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep( - cel::UnknownValue(cel::Unknown(std::move(attr_set1))), -1)); - deps.push_back(CreateConstValueDirectStep( - cel::UnknownValue(cel::Unknown(std::move(attr_set2))), -1)); - auto step = CreateDirectListStep(std::move(deps), {}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_OK(step->Evaluate(frame, result, attr)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(UnknownAttrNames(Cast(result)), - UnorderedElementsAre("var1", "var2")); -} - -TEST(CreateDirectListStep, ErrorBeforeUnknown) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - AttributeSet attr_set1({Attribute("var1")}); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep( - cel::ErrorValue(absl::InternalError("test1")), -1)); - deps.push_back(CreateConstValueDirectStep( - cel::ErrorValue(absl::InternalError("test2")), -1)); - auto step = CreateDirectListStep(std::move(deps), {}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_OK(step->Evaluate(frame, result, attr)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), - StatusIs(absl::StatusCode::kInternal, "test1")); -} - -class SetAttrDirectStep : public DirectExpressionStep { - public: - explicit SetAttrDirectStep(Attribute attr) - : DirectExpressionStep(-1), attr_(std::move(attr)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attr) const override { - result = cel::NullValue(); - attr = AttributeTrail(attr_); - return absl::OkStatus(); - } - - private: - cel::Attribute attr_; -}; - -TEST(CreateDirectListStep, MissingAttribute) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - options.enable_missing_attribute_errors = true; - - activation.SetMissingPatterns({cel::AttributePattern( - "var1", {cel::AttributeQualifierPattern::OfString("field1")})}); - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep(cel::NullValue(), -1)); - deps.push_back(std::make_unique( - Attribute("var1", {AttributeQualifier::OfString("field1")}))); - auto step = CreateDirectListStep(std::move(deps), {}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_OK(step->Evaluate(frame, result, attr)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT( - Cast(result).NativeValue(), - StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("var1.field1"))); -} - -TEST(CreateDirectListStep, OptionalPresentSet) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep(IntValue(1), -1)); - deps.push_back(CreateConstValueDirectStep( - cel::OptionalValue::Of(IntValue(2), &arena), -1)); - auto step = CreateDirectListStep(std::move(deps), {1}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_OK(step->Evaluate(frame, result, attr)); - - ASSERT_TRUE(InstanceOf(result)); - auto list = Cast(result); - EXPECT_THAT(list.Size(), IsOkAndHolds(2)); - EXPECT_THAT(list.Get(0, cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena), - IsOkAndHolds(IntValueIs(1))); - EXPECT_THAT(list.Get(1, cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena), - IsOkAndHolds(IntValueIs(2))); -} - -TEST(CreateDirectListStep, OptionalAbsentNotSet) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep(IntValue(1), -1)); - deps.push_back(CreateConstValueDirectStep(cel::OptionalValue::None(), -1)); - auto step = CreateDirectListStep(std::move(deps), {1}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_OK(step->Evaluate(frame, result, attr)); - - ASSERT_TRUE(InstanceOf(result)); - auto list = Cast(result); - EXPECT_THAT(list.Size(), IsOkAndHolds(1)); - EXPECT_THAT(list.Get(0, cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena), - IsOkAndHolds(IntValueIs(1))); -} - -TEST(CreateDirectListStep, PartialUnknown) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - activation.SetUnknownPatterns({cel::AttributePattern( - "var1", {cel::AttributeQualifierPattern::OfString("field1")})}); - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep(cel::IntValue(1), -1)); - deps.push_back(std::make_unique(Attribute("var1", {}))); - auto step = CreateDirectListStep(std::move(deps), {}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_OK(step->Evaluate(frame, result, attr)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(UnknownAttrNames(Cast(result)), - UnorderedElementsAre("var1")); -} - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/eval/create_map_step.cc b/eval/eval/create_map_step.cc index 451181e75..275d5ac36 100644 --- a/eval/eval/create_map_step.cc +++ b/eval/eval/create_map_step.cc @@ -18,33 +18,25 @@ #include #include #include -#include #include "absl/container/flat_hash_set.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "common/casting.h" +#include "common/internal/attribute_trail.h" #include "common/value.h" #include "common/values/map_value_builder.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" #include "internal/status_macros.h" namespace google::api::expr::runtime { namespace { -using ::cel::Cast; -using ::cel::ErrorValue; -using ::cel::ErrorValueAssign; using ::cel::ErrorValueReturn; -using ::cel::InstanceOf; using ::cel::MapValueBuilderPtr; -using ::cel::UnknownValue; using ::cel::Value; using ::cel::common_internal::NewMapValueBuilder; using ::cel::common_internal::NewMutableMapValue; @@ -52,13 +44,12 @@ using ::cel::common_internal::NewMutableMapValue; // `CreateStruct` implementation for map. class CreateStructStepForMap final : public ExpressionStepBase { public: - CreateStructStepForMap(int64_t expr_id, size_t entry_count, + CreateStructStepForMap(size_t entry_count, absl::flat_hash_set optional_indices) - : ExpressionStepBase(expr_id), - entry_count_(entry_count), + : entry_count_(entry_count), optional_indices_(std::move(optional_indices)) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: absl::StatusOr DoEvaluate(ExecutionFrame* frame) const; @@ -78,11 +69,11 @@ absl::StatusOr CreateStructStepForMap::DoEvaluate( } if (frame->enable_unknowns()) { - absl::optional unknown_set = - frame->attribute_utility().IdentifyAndMergeUnknowns( - args, frame->value_stack().GetAttributeSpan(args.size()), true); - if (unknown_set.has_value()) { - return *unknown_set; + if (auto unknown = cel::common_internal::PartiallyIdentifyAndMergeUnknowns( + args, frame->value_stack().GetAttributeSpan(args.size()), + frame->unknown_tree()); + unknown.has_value()) { + return *unknown; } } @@ -91,7 +82,8 @@ absl::StatusOr CreateStructStepForMap::DoEvaluate( for (size_t i = 0; i < entry_count_; i += 1) { const auto& map_key = args[2 * i]; - CEL_RETURN_IF_ERROR(cel::CheckMapKey(map_key)).With(ErrorValueReturn()); + CEL_RETURN_IF_ERROR(cel::CheckMapKey(map_key)) + .With(ErrorValueReturn(frame->arena())); const auto& map_value = args[(2 * i) + 1]; if (optional_indices_.contains(static_cast(i))) { if (auto optional_map_value = map_value.AsOptional(); @@ -109,7 +101,7 @@ absl::StatusOr CreateStructStepForMap::DoEvaluate( builder->Put(map_key, std::move(optional_map_value_value))); } else { return cel::TypeConversionError(map_value.DebugString(), - "optional_type"); + "optional_type", frame->arena()); } } else { CEL_RETURN_IF_ERROR(builder->Put(map_key, map_value)); @@ -119,171 +111,44 @@ absl::StatusOr CreateStructStepForMap::DoEvaluate( return std::move(*builder).Build(); } -absl::Status CreateStructStepForMap::Evaluate(ExecutionFrame* frame) const { +void CreateStructStepForMap::Evaluate(ExecutionFrame* frame) const { if (frame->value_stack().size() < 2 * entry_count_) { - return absl::InternalError("CreateStructStepForMap: stack underflow"); + frame->Abort( + absl::InternalError("CreateStructStepForMap: stack underflow")); + return; } - CEL_ASSIGN_OR_RETURN(auto result, DoEvaluate(frame)); - - frame->value_stack().PopAndPush(2 * entry_count_, std::move(result)); - - return absl::OkStatus(); -} - -class DirectCreateMapStep : public DirectExpressionStep { - public: - DirectCreateMapStep(std::vector> deps, - absl::flat_hash_set optional_indices, - int64_t expr_id) - : DirectExpressionStep(expr_id), - deps_(std::move(deps)), - optional_indices_(std::move(optional_indices)), - entry_count_(deps_.size() / 2) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const override; - - private: - std::vector> deps_; - absl::flat_hash_set optional_indices_; - size_t entry_count_; -}; - -absl::Status DirectCreateMapStep::Evaluate( - ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const { - auto unknowns = frame.attribute_utility().CreateAccumulator(); - - MapValueBuilderPtr builder = NewMapValueBuilder(frame.arena()); - builder->Reserve(entry_count_); - - for (size_t i = 0; i < entry_count_; i += 1) { - Value key; - Value value; - AttributeTrail tmp_attr; - int map_key_index = 2 * i; - int map_value_index = map_key_index + 1; - CEL_RETURN_IF_ERROR(deps_[map_key_index]->Evaluate(frame, key, tmp_attr)); - - if (key.IsError()) { - result = std::move(key); - return absl::OkStatus(); - } - - if (frame.unknown_processing_enabled()) { - if (key.IsUnknown()) { - unknowns.Add(key.GetUnknown()); - } else if (frame.attribute_utility().CheckForUnknownPartial(tmp_attr)) { - unknowns.Add(tmp_attr); - } - } - - CEL_RETURN_IF_ERROR(cel::CheckMapKey(key)).With(ErrorValueAssign(result)); - - CEL_RETURN_IF_ERROR( - deps_[map_value_index]->Evaluate(frame, value, tmp_attr)); - - if (value.IsError()) { - result = std::move(value); - return absl::OkStatus(); - } - - if (frame.unknown_processing_enabled()) { - if (value.IsUnknown()) { - unknowns.Add(value.GetUnknown()); - } else if (frame.attribute_utility().CheckForUnknownPartial(tmp_attr)) { - unknowns.Add(tmp_attr); - } - } - - // Preserve the stack machine behavior of forwarding unknowns before - // errors. - if (!unknowns.IsEmpty()) { - continue; - } - - if (optional_indices_.contains(static_cast(i))) { - if (auto optional_map_value = value.AsOptional(); optional_map_value) { - if (!optional_map_value->HasValue()) { - continue; - } - Value optional_map_value_value; - optional_map_value->Value(&optional_map_value_value); - if (optional_map_value_value.IsError()) { - // Error should never be in optional, but better safe than sorry. - result = optional_map_value_value; - return absl::OkStatus(); - } - CEL_RETURN_IF_ERROR( - builder->Put(std::move(key), std::move(optional_map_value_value))); - continue; - } - result = cel::TypeConversionError(value.DebugString(), "optional_type"); - return absl::OkStatus(); - } - - CEL_RETURN_IF_ERROR(builder->Put(std::move(key), std::move(value))); - } - - if (!unknowns.IsEmpty()) { - result = std::move(unknowns).Build(); - return absl::OkStatus(); + absl::StatusOr result = DoEvaluate(frame); + if (!result.ok()) { + frame->Abort(std::move(result).status()); + return; } - result = std::move(*builder).Build(); - return absl::OkStatus(); + frame->value_stack().PopAndPush(2 * entry_count_, *std::move(result)); } -class MutableMapStep final : public ExpressionStep { +class MutableMapStep final : public ExpressionStepBase { public: - explicit MutableMapStep(int64_t expr_id) : ExpressionStep(expr_id) {} + MutableMapStep() = default; - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { frame->value_stack().Push(cel::CustomMapValue( NewMutableMapValue(frame->arena()), frame->arena())); - return absl::OkStatus(); - } -}; - -class DirectMutableMapStep final : public DirectExpressionStep { - public: - explicit DirectMutableMapStep(int64_t expr_id) - : DirectExpressionStep(expr_id) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - result = - cel::CustomMapValue(NewMutableMapValue(frame.arena()), frame.arena()); - return absl::OkStatus(); } }; } // namespace -std::unique_ptr CreateDirectCreateMapStep( - std::vector> deps, - absl::flat_hash_set optional_indices, int64_t expr_id) { - return std::make_unique( - std::move(deps), std::move(optional_indices), expr_id); -} - -absl::StatusOr> CreateCreateStructStepForMap( - size_t entry_count, absl::flat_hash_set optional_indices, - int64_t expr_id) { +absl::StatusOr> +CreateCreateStructStepForMap(size_t entry_count, + absl::flat_hash_set optional_indices) { // Make map-creating step. - return std::make_unique(expr_id, entry_count, + return std::make_unique(entry_count, std::move(optional_indices)); } -absl::StatusOr> CreateMutableMapStep( - int64_t expr_id) { - return std::make_unique(expr_id); -} - -std::unique_ptr CreateDirectMutableMapStep( - int64_t expr_id) { - return std::make_unique(expr_id); +std::unique_ptr CreateMutableMapStep() { + return std::make_unique(); } } // namespace google::api::expr::runtime diff --git a/eval/eval/create_map_step.h b/eval/eval/create_map_step.h index cf5e94644..2f9729999 100644 --- a/eval/eval/create_map_step.h +++ b/eval/eval/create_map_step.h @@ -18,41 +18,23 @@ #include #include #include -#include #include "absl/container/flat_hash_set.h" #include "absl/status/statusor.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { -// Creates an expression step that evaluates a create map expression. -// -// Deps must have an even number of elements, that alternate key, value pairs. -// (key1, value1, key2, value2...). -std::unique_ptr CreateDirectCreateMapStep( - std::vector> deps, - absl::flat_hash_set optional_indices, int64_t expr_id); - // Creates an `ExpressionStep` which performs `CreateStruct` for a map. -absl::StatusOr> CreateCreateStructStepForMap( - size_t entry_count, absl::flat_hash_set optional_indices, - int64_t expr_id); - -// Factory method for CreateMap which constructs a mutable map. -// -// This is intended for the map construction step is generated for a -// map-building comprehension (rather than a user authored expression). -absl::StatusOr> CreateMutableMapStep( - int64_t expr_id); +absl::StatusOr> +CreateCreateStructStepForMap(size_t entry_count, + absl::flat_hash_set optional_indices); // Factory method for CreateMap which constructs a mutable map. // // This is intended for the map construction step is generated for a // map-building comprehension (rather than a user authored expression). -std::unique_ptr CreateDirectMutableMapStep( - int64_t expr_id); +std::unique_ptr CreateMutableMapStep(); } // namespace google::api::expr::runtime diff --git a/eval/eval/create_map_step_test.cc b/eval/eval/create_map_step_test.cc index dbc9adb5a..ff2148627 100644 --- a/eval/eval/create_map_step_test.cc +++ b/eval/eval/create_map_step_test.cc @@ -29,7 +29,6 @@ #include "base/type_provider.h" #include "common/expr.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/ident_step.h" #include "eval/public/activation.h" @@ -56,7 +55,7 @@ using ::google::protobuf::Arena; absl::StatusOr CreateStackMachineProgram( const std::vector>& values, - Activation& activation) { + Activation& activation, IdentifierTable& identifiers) { ExecutionPath path; Expr expr1; @@ -71,14 +70,10 @@ absl::StatusOr CreateStackMachineProgram( std::string key_name = absl::StrCat("key", index); std::string value_name = absl::StrCat("value", index); - CEL_ASSIGN_OR_RETURN(auto step_key, - CreateIdentStep(key_name, /*expr_id=*/-1)); - - CEL_ASSIGN_OR_RETURN(auto step_value, - CreateIdentStep(value_name, /*expr _id=*/-1)); - - path.push_back(std::move(step_key)); - path.push_back(std::move(step_value)); + path.push_back(ExpressionStep::MakeIdentifierStep(identifiers.size())); + identifiers.push_back(Identifier{key_name, identifiers.size()}); + path.push_back(ExpressionStep::MakeIdentifierStep(identifiers.size())); + identifiers.push_back(Identifier{value_name, identifiers.size()}); activation.InsertValue(key_name, item.first); activation.InsertValue(value_name, item.second); @@ -87,35 +82,9 @@ absl::StatusOr CreateStackMachineProgram( index++; } - CEL_ASSIGN_OR_RETURN( - auto step1, CreateCreateStructStepForMap(values.size(), {}, expr1.id())); - path.push_back(std::move(step1)); - return path; -} - -absl::StatusOr CreateRecursiveProgram( - const std::vector>& values, - Activation& activation) { - ExecutionPath path; - - int index = 0; - std::vector> deps; - for (const auto& item : values) { - std::string key_name = absl::StrCat("key", index); - std::string value_name = absl::StrCat("value", index); - - deps.push_back(CreateDirectIdentStep(key_name, -1)); - - deps.push_back(CreateDirectIdentStep(value_name, -1)); - - activation.InsertValue(key_name, item.first); - activation.InsertValue(value_name, item.second); - - index++; - } - path.push_back(std::make_unique( - CreateDirectCreateMapStep(std::move(deps), {}, -1), -1)); - + CEL_ASSIGN_OR_RETURN(auto step1, + CreateCreateStructStepForMap(values.size(), {})); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step1), expr1.id())); return path; } @@ -127,22 +96,22 @@ absl::StatusOr RunCreateMapExpression( const std::vector>& values, google::protobuf::Arena* arena, bool enable_unknowns, bool enable_recursive_program) { Activation activation; + IdentifierTable identifiers; ExecutionPath path; - if (enable_recursive_program) { - CEL_ASSIGN_OR_RETURN(path, CreateRecursiveProgram(values, activation)); - } else { - CEL_ASSIGN_OR_RETURN(path, CreateStackMachineProgram(values, activation)); - } + CEL_ASSIGN_OR_RETURN( + path, CreateStackMachineProgram(values, activation, identifiers)); cel::RuntimeOptions options; if (enable_unknowns) { options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; } + const size_t slot_count = identifiers.size(); CelExpressionFlatImpl cel_expr( - env, - FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env, FlatExpression(std::move(path), + /*comprehension_slot_count=*/slot_count, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, std::move(identifiers))); return cel_expr.Evaluate(activation, arena); } @@ -174,25 +143,6 @@ TEST_P(CreateMapStepTest, TestCreateEmptyMap) { ASSERT_EQ(cel_map->size(), 0); } -// Test message creation if unknown argument is passed -TEST(CreateMapStepTest, TestMapCreateWithUnknown) { - absl_nonnull std::shared_ptr env = NewTestingRuntimeEnv(); - Arena arena; - UnknownSet unknown_set; - std::vector> entries; - - std::vector kKeys = {"test2", "test1"}; - - entries.push_back( - {CelValue::CreateString(&kKeys[0]), CelValue::CreateInt64(2)}); - entries.push_back({CelValue::CreateString(&kKeys[1]), - CelValue::CreateUnknownSet(&unknown_set)}); - - ASSERT_OK_AND_ASSIGN(CelValue result, RunCreateMapExpression( - env, entries, &arena, true, false)); - ASSERT_TRUE(result.IsUnknownSet()); -} - TEST(CreateMapStepTest, TestMapCreateWithError) { absl_nonnull std::shared_ptr env = NewTestingRuntimeEnv(); Arena arena; @@ -210,41 +160,6 @@ TEST(CreateMapStepTest, TestMapCreateWithError) { EXPECT_THAT(*result.ErrorOrDie(), StatusIs(absl::StatusCode::kCancelled)); } -TEST(CreateMapStepTest, TestMapCreateWithErrorRecursiveProgram) { - absl_nonnull std::shared_ptr env = NewTestingRuntimeEnv(); - Arena arena; - UnknownSet unknown_set; - absl::Status error = absl::CancelledError(); - std::vector> entries; - entries.push_back({CelValue::CreateStringView("foo"), - CelValue::CreateUnknownSet(&unknown_set)}); - entries.push_back( - {CelValue::CreateStringView("bar"), CelValue::CreateError(&error)}); - - ASSERT_OK_AND_ASSIGN(CelValue result, RunCreateMapExpression( - env, entries, &arena, true, true)); - ASSERT_TRUE(result.IsError()); - EXPECT_THAT(*result.ErrorOrDie(), StatusIs(absl::StatusCode::kCancelled)); -} - -TEST(CreateMapStepTest, TestMapCreateWithUnknownRecursiveProgram) { - absl_nonnull std::shared_ptr env = NewTestingRuntimeEnv(); - Arena arena; - UnknownSet unknown_set; - std::vector> entries; - - std::vector kKeys = {"test2", "test1"}; - - entries.push_back( - {CelValue::CreateString(&kKeys[0]), CelValue::CreateInt64(2)}); - entries.push_back({CelValue::CreateString(&kKeys[1]), - CelValue::CreateUnknownSet(&unknown_set)}); - - ASSERT_OK_AND_ASSIGN(CelValue result, RunCreateMapExpression( - env, entries, &arena, true, true)); - ASSERT_TRUE(result.IsUnknownSet()); -} - // Test that String Map is created successfully. TEST_P(CreateMapStepTest, TestCreateStringMap) { Arena arena; diff --git a/eval/eval/create_struct_step.cc b/eval/eval/create_struct_step.cc index 5d042baf5..c7ac0f22e 100644 --- a/eval/eval/create_struct_step.cc +++ b/eval/eval/create_struct_step.cc @@ -26,37 +26,32 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/casting.h" +#include "common/internal/attribute_trail.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" #include "internal/status_macros.h" namespace google::api::expr::runtime { namespace { -using ::cel::Cast; using ::cel::ErrorValue; -using ::cel::InstanceOf; using ::cel::StructValueBuilderInterface; -using ::cel::UnknownValue; using ::cel::Value; // `CreateStruct` implementation for message/struct. class CreateStructStepForStruct final : public ExpressionStepBase { public: - CreateStructStepForStruct(int64_t expr_id, std::string name, - std::vector entries, + CreateStructStepForStruct(std::string name, std::vector entries, absl::flat_hash_set optional_indices) - : ExpressionStepBase(expr_id), + : ExpressionStepBase(), name_(std::move(name)), entries_(std::move(entries)), optional_indices_(std::move(optional_indices)) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: absl::StatusOr DoEvaluate(ExecutionFrame* frame) const; @@ -79,12 +74,11 @@ absl::StatusOr CreateStructStepForStruct::DoEvaluate( } if (frame->enable_unknowns()) { - absl::optional unknown_set = - frame->attribute_utility().IdentifyAndMergeUnknowns( + if (auto unknown = cel::common_internal::PartiallyIdentifyAndMergeUnknowns( args, frame->value_stack().GetAttributeSpan(entries_size), - /*use_partial=*/true); - if (unknown_set.has_value()) { - return *unknown_set; + frame->unknown_tree()); + unknown.has_value()) { + return *unknown; } } @@ -92,8 +86,9 @@ absl::StatusOr CreateStructStepForStruct::DoEvaluate( frame->type_provider().NewValueBuilder( name_, frame->message_factory(), frame->arena())); if (builder == nullptr) { - return ErrorValue( - absl::NotFoundError(absl::StrCat("Unable to find builder: ", name_))); + return ErrorValue::From( + absl::NotFoundError(absl::StrCat("Unable to find builder: ", name_)), + frame->arena()); } for (int i = 0; i < entries_size; ++i) { @@ -117,7 +112,8 @@ absl::StatusOr CreateStructStepForStruct::DoEvaluate( return std::move(*error_value); } } else { - return cel::TypeConversionError(arg.DebugString(), "optional_type"); + return cel::TypeConversionError(arg.DebugString(), "optional_type", + frame->arena()); } } else { CEL_ASSIGN_OR_RETURN(absl::optional error_value, @@ -131,140 +127,27 @@ absl::StatusOr CreateStructStepForStruct::DoEvaluate( return std::move(*builder).Build(); } -absl::Status CreateStructStepForStruct::Evaluate(ExecutionFrame* frame) const { +void CreateStructStepForStruct::Evaluate(ExecutionFrame* frame) const { if (frame->value_stack().size() < entries_.size()) { - return absl::InternalError("CreateStructStepForStruct: stack underflow"); + frame->Abort( + absl::InternalError("CreateStructStepForStruct: stack underflow")); + return; } - CEL_ASSIGN_OR_RETURN(Value result, DoEvaluate(frame)); - frame->value_stack().PopAndPush(entries_.size(), std::move(result)); - - return absl::OkStatus(); -} - -class DirectCreateStructStep : public DirectExpressionStep { - public: - DirectCreateStructStep( - int64_t expr_id, std::string name, std::vector field_keys, - std::vector> deps, - absl::flat_hash_set optional_indices) - : DirectExpressionStep(expr_id), - name_(std::move(name)), - field_keys_(std::move(field_keys)), - deps_(std::move(deps)), - optional_indices_(std::move(optional_indices)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& trail) const override; - - private: - std::string name_; - std::vector field_keys_; - std::vector> deps_; - absl::flat_hash_set optional_indices_; -}; - -absl::Status DirectCreateStructStep::Evaluate(ExecutionFrameBase& frame, - Value& result, - AttributeTrail& trail) const { - Value field_value; - AttributeTrail field_attr; - auto unknowns = frame.attribute_utility().CreateAccumulator(); - - CEL_ASSIGN_OR_RETURN(auto builder, - frame.type_provider().NewValueBuilder( - name_, frame.message_factory(), frame.arena())); - if (builder == nullptr) { - result = cel::ErrorValue( - absl::NotFoundError(absl::StrCat("Unable to find builder: ", name_))); - return absl::OkStatus(); + absl::StatusOr result = DoEvaluate(frame); + if (!result.ok()) { + frame->Abort(std::move(result).status()); + return; } - - for (int i = 0; i < field_keys_.size(); i++) { - CEL_RETURN_IF_ERROR(deps_[i]->Evaluate(frame, field_value, field_attr)); - - // TODO(uncreated-issue/67): if the value is an error, we should be able to return - // early, however some client tests depend on the error message the struct - // impl returns in the stack machine version. - if (field_value.IsError()) { - result = std::move(field_value); - return absl::OkStatus(); - } - - if (frame.unknown_processing_enabled()) { - if (field_value.IsUnknown()) { - unknowns.Add(field_value.GetUnknown()); - } else if (frame.attribute_utility().CheckForUnknownPartial(field_attr)) { - unknowns.Add(field_attr); - } - } - - if (!unknowns.IsEmpty()) { - continue; - } - - if (optional_indices_.contains(static_cast(i))) { - if (auto optional_arg = field_value.AsOptional(); optional_arg) { - if (!optional_arg->HasValue()) { - continue; - } - Value optional_arg_value; - optional_arg->Value(&optional_arg_value); - if (optional_arg_value.IsError()) { - // Error should never be in optional, but better safe than sorry. - result = std::move(optional_arg_value); - return absl::OkStatus(); - } - CEL_ASSIGN_OR_RETURN( - absl::optional error_value, - builder->SetFieldByName(field_keys_[i], - std::move(optional_arg_value))); - if (error_value) { - result = std::move(*error_value); - return absl::OkStatus(); - } - continue; - } else { - result = cel::TypeConversionError(field_value.DebugString(), - "optional_type"); - return absl::OkStatus(); - } - } - - CEL_ASSIGN_OR_RETURN( - absl::optional error_value, - builder->SetFieldByName(field_keys_[i], std::move(field_value))); - if (error_value) { - result = std::move(*error_value); - return absl::OkStatus(); - } - } - - if (!unknowns.IsEmpty()) { - result = std::move(unknowns).Build(); - return absl::OkStatus(); - } - - CEL_ASSIGN_OR_RETURN(result, std::move(*builder).Build()); - return absl::OkStatus(); + frame->value_stack().PopAndPush(entries_.size(), *std::move(result)); } } // namespace -std::unique_ptr CreateDirectCreateStructStep( - std::string resolved_name, std::vector field_keys, - std::vector> deps, - absl::flat_hash_set optional_indices, int64_t expr_id) { - return std::make_unique( - expr_id, std::move(resolved_name), std::move(field_keys), std::move(deps), - std::move(optional_indices)); -} - -std::unique_ptr CreateCreateStructStep( +std::unique_ptr CreateCreateStructStep( std::string name, std::vector field_keys, - absl::flat_hash_set optional_indices, int64_t expr_id) { + absl::flat_hash_set optional_indices) { // MakeOptionalIndicesSet(create_struct_expr) return std::make_unique( - expr_id, std::move(name), std::move(field_keys), - std::move(optional_indices)); + std::move(name), std::move(field_keys), std::move(optional_indices)); } } // namespace google::api::expr::runtime diff --git a/eval/eval/create_struct_step.h b/eval/eval/create_struct_step.h index eb80634f8..1dd42273a 100644 --- a/eval/eval/create_struct_step.h +++ b/eval/eval/create_struct_step.h @@ -21,23 +21,15 @@ #include #include "absl/container/flat_hash_set.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { // Creates an `ExpressionStep` which performs `CreateStruct` for a // message/struct. -std::unique_ptr CreateDirectCreateStructStep( +std::unique_ptr CreateCreateStructStep( std::string name, std::vector field_keys, - std::vector> deps, - absl::flat_hash_set optional_indices, int64_t expr_id); - -// Creates an `ExpressionStep` which performs `CreateStruct` for a -// message/struct. -std::unique_ptr CreateCreateStructStep( - std::string name, std::vector field_keys, - absl::flat_hash_set optional_indices, int64_t expr_id); + absl::flat_hash_set optional_indices); } // namespace google::api::expr::runtime diff --git a/eval/eval/create_struct_step_test.cc b/eval/eval/create_struct_step_test.cc index cd9db9bd9..555f22376 100644 --- a/eval/eval/create_struct_step_test.cc +++ b/eval/eval/create_struct_step_test.cc @@ -29,11 +29,8 @@ #include "absl/strings/string_view.h" #include "absl/types/span.h" #include "base/type_provider.h" -#include "common/expr.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/ident_step.h" #include "eval/public/activation.h" #include "eval/public/cel_type_registry.h" #include "eval/public/cel_value.h" @@ -58,7 +55,6 @@ namespace { using ::absl_testing::IsOk; using ::absl_testing::StatusIs; -using ::cel::Expr; using ::cel::TypeProvider; using ::cel::internal::test::EqualsProto; using ::cel::runtime_internal::NewTestingRuntimeEnv; @@ -73,34 +69,12 @@ using ::testing::Pointwise; absl::StatusOr MakeStackMachinePath(absl::string_view field) { ExecutionPath path; - CEL_ASSIGN_OR_RETURN(auto step0, CreateIdentStep("message", /*expr_id=*/-1)); - auto step1 = CreateCreateStructStep("google.api.expr.runtime.TestMessage", {std::string(field)}, - /*optional_indices=*/{}, - - /*id=*/-1); + /*optional_indices=*/{}); - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); - - return path; -} - -absl::StatusOr MakeRecursivePath(absl::string_view field) { - ExecutionPath path; - - std::vector> deps; - deps.push_back(CreateDirectIdentStep("message", -1)); - - auto step1 = - CreateDirectCreateStructStep("google.api.expr.runtime.TestMessage", - {std::string(field)}, std::move(deps), - /*optional_indices=*/{}, - - /*id=*/-1); - - path.push_back(std::make_unique(std::move(step1), -1)); + path.push_back(ExpressionStep::MakeIdentifierStep(0)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step1))); return path; } @@ -126,16 +100,12 @@ absl::StatusOr RunExpression( } ExecutionPath path; - if (enable_recursive_planning) { - CEL_ASSIGN_OR_RETURN(path, MakeRecursivePath(field)); - } else { - CEL_ASSIGN_OR_RETURN(path, MakeStackMachinePath(field)); - } + CEL_ASSIGN_OR_RETURN(path, MakeStackMachinePath(field)); CelExpressionFlatImpl cel_expr( - env, - FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env, FlatExpression(std::move(path), /*comprehension_slot_count=*/1, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, IdentifierTable{{"message", 0}})); Activation activation; activation.InsertValue("message", value); @@ -200,28 +170,17 @@ TEST_P(CreateCreateStructStepTest, TestEmptyMessageCreation) { auto adapter = env_->legacy_type_registry.FindTypeAdapter( "google.api.expr.runtime.TestMessage"); - ASSERT_TRUE(adapter.has_value() && adapter->mutation_apis() != nullptr); + ASSERT_TRUE(adapter.has_value() && adapter->access_apis() != nullptr); ASSERT_OK_AND_ASSIGN(auto maybe_type, env_->type_registry.GetComposedTypeProvider().FindType( "google.api.expr.runtime.TestMessage")); ASSERT_TRUE(maybe_type.has_value()); - if (enable_recursive_planning()) { - auto step = - CreateDirectCreateStructStep("google.api.expr.runtime.TestMessage", - /*fields=*/{}, - /*deps=*/{}, - /*optional_indices=*/{}, - /*id=*/-1); - path.push_back( - std::make_unique(std::move(step), /*id=*/-1)); - } else { + auto step = CreateCreateStructStep("google.api.expr.runtime.TestMessage", /*fields=*/{}, - /*optional_indices=*/{}, - /*id=*/-1); - path.push_back(std::move(step)); - } + /*optional_indices=*/{}); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step))); cel::RuntimeOptions options; if (enable_unknowns(), enable_recursive_planning()) { @@ -230,7 +189,8 @@ TEST_P(CreateCreateStructStepTest, TestEmptyMessageCreation) { CelExpressionFlatImpl cel_expr( env_, FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env_->type_registry.GetComposedTypeProvider(), options)); + env_->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, {})); Activation activation; ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); @@ -270,34 +230,6 @@ TEST(CreateCreateStructStepTest, TestMessageCreateErrorRecursive) { StatusIs(absl::StatusCode::kCancelled)); } -// Test message creation if unknown argument is passed -TEST(CreateCreateStructStepTest, TestMessageCreateWithUnknown) { - absl_nonnull std::shared_ptr env = NewTestingRuntimeEnv(); - Arena arena; - TestMessage test_msg; - UnknownSet unknown_set; - - auto eval_status = - RunExpression(env, "bool_value", CelValue::CreateUnknownSet(&unknown_set), - &arena, true, /*enable_recursive_planning=*/false); - ASSERT_OK(eval_status); - ASSERT_TRUE(eval_status->IsUnknownSet()); -} - -// Test message creation if unknown argument is passed -TEST(CreateCreateStructStepTest, TestMessageCreateWithUnknownRecursive) { - absl_nonnull std::shared_ptr env = NewTestingRuntimeEnv(); - Arena arena; - TestMessage test_msg; - UnknownSet unknown_set; - - auto eval_status = - RunExpression(env, "bool_value", CelValue::CreateUnknownSet(&unknown_set), - &arena, true, /*enable_recursive_planning=*/true); - ASSERT_OK(eval_status); - ASSERT_TRUE(eval_status->IsUnknownSet()) << eval_status->DebugString(); -} - // Test that fields of type bool are set correctly TEST_P(CreateCreateStructStepTest, TestSetBoolField) { TestMessage test_msg; @@ -647,9 +579,7 @@ TEST_P(CreateCreateStructStepTest, TestSetStringMapField) { entries.push_back( {CelValue::CreateString(&kKeys[1]), CelValue::CreateInt64(1)}); - auto cel_map = - *CreateContainerBackedMap(absl::Span>( - entries.data(), entries.size())); + auto cel_map = *CreateContainerBackedMap(absl::MakeSpan(entries)); ASSERT_NO_FATAL_FAILURE(RunExpressionAndGetMessage( env_, "string_int32_map", CelValue::CreateMap(cel_map.get()), &arena_, @@ -673,9 +603,7 @@ TEST_P(CreateCreateStructStepTest, TestSetInt64MapField) { entries.push_back( {CelValue::CreateInt64(kKeys[1]), CelValue::CreateInt64(2)}); - auto cel_map = - *CreateContainerBackedMap(absl::Span>( - entries.data(), entries.size())); + auto cel_map = *CreateContainerBackedMap(absl::MakeSpan(entries)); ASSERT_NO_FATAL_FAILURE(RunExpressionAndGetMessage( env_, "int64_int32_map", CelValue::CreateMap(cel_map.get()), &arena_, @@ -699,9 +627,7 @@ TEST_P(CreateCreateStructStepTest, TestSetUInt64MapField) { entries.push_back( {CelValue::CreateUint64(kKeys[1]), CelValue::CreateInt64(2)}); - auto cel_map = - *CreateContainerBackedMap(absl::Span>( - entries.data(), entries.size())); + auto cel_map = *CreateContainerBackedMap(absl::MakeSpan(entries)); ASSERT_NO_FATAL_FAILURE(RunExpressionAndGetMessage( env_, "uint64_int32_map", CelValue::CreateMap(cel_map.get()), &arena_, diff --git a/eval/eval/direct_expression_step.cc b/eval/eval/direct_expression_step.cc deleted file mode 100644 index 2d7fc6fc0..000000000 --- a/eval/eval/direct_expression_step.cc +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include "eval/eval/direct_expression_step.h" - -#include - -#include "absl/status/status.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/evaluator_core.h" -#include "internal/status_macros.h" - -namespace google::api::expr::runtime { - -absl::Status WrappedDirectStep::Evaluate(ExecutionFrame* frame) const { - cel::Value result; - AttributeTrail attribute_trail; - CEL_RETURN_IF_ERROR(impl_->Evaluate(*frame, result, attribute_trail)); - frame->value_stack().Push(std::move(result), std::move(attribute_trail)); - return absl::OkStatus(); -} - -} // namespace google::api::expr::runtime diff --git a/eval/eval/direct_expression_step.h b/eval/eval/direct_expression_step.h deleted file mode 100644 index f11479065..000000000 --- a/eval/eval/direct_expression_step.h +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_DIRECT_EXPRESSION_STEP_H_ -#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_DIRECT_EXPRESSION_STEP_H_ - -#include -#include -#include -#include - -#include "absl/status/status.h" -#include "absl/types/optional.h" -#include "common/native_type.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/evaluator_core.h" - -namespace google::api::expr::runtime { - -// Represents a directly evaluated CEL expression. -// -// Subexpressions assign to values on the C++ program stack and call their -// dependencies directly. -// -// This reduces the setup overhead for evaluation and minimizes value churn -// to / from a heap based value stack managed by the CEL runtime, but can't be -// used for arbitrarily nested expressions. -class DirectExpressionStep { - public: - explicit DirectExpressionStep(int64_t expr_id) : expr_id_(expr_id) {} - DirectExpressionStep() : expr_id_(-1) {} - - virtual ~DirectExpressionStep() = default; - - int64_t expr_id() const { return expr_id_; } - bool comes_from_ast() const { return expr_id_ >= 0; } - - virtual absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& attribute) const = 0; - - // Return a type id for this node. - // - // Users must not make any assumptions about the type if the default value is - // returned. - virtual cel::NativeTypeId GetNativeTypeId() const { - return cel::NativeTypeId(); - } - - // Implementations optionally support inspecting the program tree. - virtual absl::optional> - GetDependencies() const { - return absl::nullopt; - } - - // Implementations optionally support extracting the program tree. - // - // Extract prevents the callee from functioning, and is only intended for use - // when replacing a given expression step. - virtual absl::optional>> - ExtractDependencies() { - return absl::nullopt; - }; - - protected: - int64_t expr_id_; -}; - -// Wrapper for direct steps to work with the stack machine impl. -class WrappedDirectStep : public ExpressionStep { - public: - WrappedDirectStep(std::unique_ptr impl, int64_t expr_id) - : ExpressionStep(expr_id, false), impl_(std::move(impl)) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override; - - cel::NativeTypeId GetNativeTypeId() const override { - return cel::NativeTypeId::For(); - } - - const DirectExpressionStep* wrapped() const { return impl_.get(); } - - private: - std::unique_ptr impl_; -}; - -} // namespace google::api::expr::runtime - -#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_DIRECT_EXPRESSION_STEP_H_ diff --git a/eval/eval/equality_steps.cc b/eval/eval/equality_steps.cc index d720302e4..8e4646cf8 100644 --- a/eval/eval/equality_steps.cc +++ b/eval/eval/equality_steps.cc @@ -13,19 +13,16 @@ // limitations under the License. #include "eval/eval/equality_steps.h" -#include -#include #include #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/types/span.h" #include "base/builtins.h" #include "common/value.h" #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" #include "internal/number.h" #include "internal/status_macros.h" #include "runtime/internal/errors.h" @@ -57,11 +54,12 @@ absl::StatusOr EvaluateEquality( } if (frame.unknown_processing_enabled()) { - auto accu = frame.attribute_utility().CreateAccumulator(); - accu.MaybeAdd(lhs, lhs_attr); - accu.MaybeAdd(rhs, rhs_attr); - if (!accu.IsEmpty()) { - return std::move(accu).Build(); + UnknownAccumulator accumulator(frame.unknown_tree()); + accumulator.MaybeAdd(lhs, lhs_attr); + accumulator.MaybeAdd(rhs, rhs_attr); + if (auto accumulated = std::move(accumulator).Accumulate(); + accumulated.has_value()) { + return *accumulated; } } @@ -69,66 +67,14 @@ absl::StatusOr EvaluateEquality( ValueEqualImpl(lhs, rhs, frame.descriptor_pool(), frame.message_factory(), frame.arena())); if (!is_equal.has_value()) { - return cel::ErrorValue(cel::runtime_internal::CreateNoMatchingOverloadError( - negation ? cel::builtin::kInequal : cel::builtin::kEqual)); + return cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError( + negation ? cel::builtin::kInequal : cel::builtin::kEqual), + frame.arena()); } return negation ? BoolValue(!*is_equal) : BoolValue(*is_equal); } -class DirectEqualityStep : public DirectExpressionStep { - public: - explicit DirectEqualityStep(std::unique_ptr lhs, - std::unique_ptr rhs, - bool negation, int64_t expr_id) - : DirectExpressionStep(expr_id), - lhs_(std::move(lhs)), - rhs_(std::move(rhs)), - negation_(negation) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const override { - AttributeTrail lhs_attr; - CEL_RETURN_IF_ERROR(lhs_->Evaluate(frame, result, lhs_attr)); - - Value rhs_result; - AttributeTrail rhs_attr; - CEL_RETURN_IF_ERROR(rhs_->Evaluate(frame, rhs_result, rhs_attr)); - CEL_ASSIGN_OR_RETURN( - result, EvaluateEquality(frame, result, lhs_attr, rhs_result, rhs_attr, - negation_)); - return absl::OkStatus(); - } - - private: - std::unique_ptr lhs_; - std::unique_ptr rhs_; - bool negation_; -}; - -class IterativeEqualityStep : public ExpressionStepBase { - public: - explicit IterativeEqualityStep(bool negation, int64_t expr_id) - : ExpressionStepBase(expr_id), negation_(negation) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - auto args = frame->value_stack().GetSpan(2); - auto attrs = frame->value_stack().GetAttributeSpan(2); - - CEL_ASSIGN_OR_RETURN(Value result, - EvaluateEquality(*frame, args[0], attrs[0], args[1], - attrs[1], negation_)); - - frame->value_stack().PopAndPush(2, std::move(result)); - return absl::OkStatus(); - } - - private: - bool negation_; -}; - absl::StatusOr EvaluateInMap(ExecutionFrameBase& frame, const Value& item, const MapValue& container) { @@ -140,9 +86,10 @@ absl::StatusOr EvaluateInMap(ExecutionFrameBase& frame, case ValueKind::kDouble: break; default: - return cel::ErrorValue( + return cel::ErrorValue::From( cel::runtime_internal::CreateNoMatchingOverloadError( - cel::builtin::kIn)); + cel::builtin::kIn), + frame.arena()); } Value result; CEL_RETURN_IF_ERROR(container.Has(item, frame.descriptor_pool(), @@ -196,11 +143,12 @@ absl::StatusOr EvaluateIn(ExecutionFrameBase& frame, const Value& item, } if (frame.unknown_processing_enabled()) { - auto accu = frame.attribute_utility().CreateAccumulator(); - accu.MaybeAdd(item, item_attr); - accu.MaybeAdd(container, container_attr); - if (!accu.IsEmpty()) { - return std::move(accu).Build(); + UnknownAccumulator accumulator(frame.unknown_tree()); + accumulator.MaybeAdd(item, item_attr); + accumulator.MaybeAdd(container, container_attr); + if (auto accumulated = std::move(accumulator).Accumulate(); + accumulated.has_value()) { + return *accumulated; } } if (container.IsList()) { @@ -210,84 +158,49 @@ absl::StatusOr EvaluateIn(ExecutionFrameBase& frame, const Value& item, if (container.IsMap()) { return EvaluateInMap(frame, item, container.GetMap()); } - return cel::ErrorValue( - cel::runtime_internal::CreateNoMatchingOverloadError(cel::builtin::kIn)); + return cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(cel::builtin::kIn), + frame.arena()); } -class DirectInStep : public DirectExpressionStep { - public: - explicit DirectInStep(std::unique_ptr item, - std::unique_ptr container, - int64_t expr_id) - : DirectExpressionStep(expr_id), - item_(std::move(item)), - container_(std::move(container)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const override { - AttributeTrail item_attr; - CEL_RETURN_IF_ERROR(item_->Evaluate(frame, result, item_attr)); +} // namespace - Value container_result; - AttributeTrail container_attr; - CEL_RETURN_IF_ERROR( - container_->Evaluate(frame, container_result, container_attr)); - CEL_ASSIGN_OR_RETURN(result, EvaluateIn(frame, result, item_attr, - container_result, container_attr)); - return absl::OkStatus(); +void EvaluateFastEqualStep(bool negation, ExecutionFrame& frame) { + if (!frame.value_stack().HasEnough(2)) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; } + absl::Span args = frame.value_stack().GetSpan(2); + absl::Span attrs = + frame.value_stack().GetAttributeSpan(2); - private: - std::unique_ptr item_; - std::unique_ptr container_; -}; - -class IterativeInStep : public ExpressionStepBase { - public: - explicit IterativeInStep(int64_t expr_id) : ExpressionStepBase(expr_id) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - - auto args = frame->value_stack().GetSpan(2); - auto attrs = frame->value_stack().GetAttributeSpan(2); - - CEL_ASSIGN_OR_RETURN( - Value result, EvaluateIn(*frame, args[0], attrs[0], args[1], attrs[1])); - frame->value_stack().PopAndPush(2, std::move(result)); - return absl::OkStatus(); + absl::StatusOr result = + EvaluateEquality(frame, args[0], attrs[0], args[1], attrs[1], negation); + if (!result.ok()) { + frame.Abort(std::move(result).status()); + return; } -}; - -} // namespace - -// Factory method for recursive _==_ and _!=_ Execution step -std::unique_ptr CreateDirectEqualityStep( - std::unique_ptr lhs, - std::unique_ptr rhs, bool negation, int64_t expr_id) { - return std::make_unique(std::move(lhs), std::move(rhs), - negation, expr_id); -} - -// Factory method for iterative _==_ and _!=_ Execution step -std::unique_ptr CreateEqualityStep(bool negation, - int64_t expr_id) { - return std::make_unique(negation, expr_id); + frame.value_stack().PopAndPush(2, *std::move(result)); } -// Factory method for recursive @in Execution step -std::unique_ptr CreateDirectInStep( - std::unique_ptr item, - std::unique_ptr container, int64_t expr_id) { - return std::make_unique(std::move(item), std::move(container), - expr_id); -} +void EvaluateFastInStep(ExecutionFrame& frame) { + if (!frame.value_stack().HasEnough(2)) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + absl::Span args = frame.value_stack().GetSpan(2); + absl::Span attrs = + frame.value_stack().GetAttributeSpan(2); -// Factory method for iterative @in Execution step -std::unique_ptr CreateInStep(int64_t expr_id) { - return std::make_unique(expr_id); + absl::StatusOr result = + EvaluateIn(frame, args[0], attrs[0], args[1], attrs[1]); + if (!result.ok()) { + frame.Abort(std::move(result).status()); + return; + } + frame.value_stack().PopAndPush(2, *std::move(result)); } } // namespace google::api::expr::runtime diff --git a/eval/eval/equality_steps.h b/eval/eval/equality_steps.h index eb3bec4ca..18b87cc79 100644 --- a/eval/eval/equality_steps.h +++ b/eval/eval/equality_steps.h @@ -15,30 +15,13 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_EQUALITY_STEPS_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_EQUALITY_STEPS_H_ -#include -#include - -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" - namespace google::api::expr::runtime { -// Factory method for recursive _==_/_!=_ Execution step -std::unique_ptr CreateDirectEqualityStep( - std::unique_ptr lhs, - std::unique_ptr rhs, bool negation, int64_t expr_id); - -// Factory method for iterative _==_/_!=_ Execution step -std::unique_ptr CreateEqualityStep(bool negation, - int64_t expr_id); +class ExecutionFrame; -// Factory method for recursive @in Execution step -std::unique_ptr CreateDirectInStep( - std::unique_ptr item, - std::unique_ptr container, int64_t expr_id); +void EvaluateFastEqualStep(bool negation, ExecutionFrame& frame); -// Factory method for iterative @in Execution step -std::unique_ptr CreateInStep(int64_t expr_id); +void EvaluateFastInStep(ExecutionFrame& frame); } // namespace google::api::expr::runtime diff --git a/eval/eval/equality_steps_test.cc b/eval/eval/equality_steps_test.cc index 168ce7603..33e1014cc 100644 --- a/eval/eval/equality_steps_test.cc +++ b/eval/eval/equality_steps_test.cc @@ -16,19 +16,19 @@ #include #include -#include #include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" -#include "absl/status/status_matchers.h" #include "base/attribute.h" +#include "base/attribute_set.h" +#include "common/internal/attribute_trail.h" +#include "common/unknown.h" #include "common/value.h" #include "common/value_kind.h" #include "common/value_testing.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" #include "internal/testing_message_factory.h" @@ -40,8 +40,6 @@ namespace google::api::expr::runtime { namespace { -using ::absl_testing::IsOk; -using ::cel::Attribute; using ::cel::DoubleValue; using ::cel::ErrorValue; using ::cel::IntValue; @@ -51,148 +49,27 @@ using ::cel::ValueKind; using ::cel::test::BoolValueIs; using ::cel::test::ValueKindIs; -class ValueStep : public ExpressionStep, public DirectExpressionStep { +class ValueStep : public ExpressionStepLogic { public: - ValueStep(Value value, Attribute attr) - : ExpressionStep(-1), - DirectExpressionStep(-1), - value_(std::move(value)), - attr_(std::move(attr)) {} explicit ValueStep(Value value) - : ExpressionStep(-1), - DirectExpressionStep(-1), - value_(std::move(value)), - attr_() {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - frame->value_stack().Push(value_, attr_); - return absl::OkStatus(); - } - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const override { - result = value_; - attribute_trail = attr_; - return absl::OkStatus(); + : ExpressionStepLogic(), value_(std::move(value)) {} + + void Evaluate(ExecutionFrame* frame) const override { + if (value_.IsUnknown()) { + frame->value_stack().Push(cel::common_internal::GraftUnknowns( + cel::common_internal::MakeUnknownValue( + cel::Unknown(cel::AttributeSet({cel::Attribute("foo")})), + frame->arena()), + frame->unknown_tree())); + } else { + frame->value_stack().Push(value_); + } } private: Value value_; - AttributeTrail attr_; }; -TEST(RecursiveTest, PartialAttrUnknown) { - cel::Activation activation; - google::protobuf::Arena arena; - cel::RuntimeOptions opts; - opts.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - // A little contrived for simplicity, but this is for cases where e.g. - // `msg == Msg{}` but msg.foo is unknown. - auto plan = CreateDirectEqualityStep( - std::make_unique(IntValue(1), cel::Attribute("foo")), - std::make_unique(IntValue(2)), false, -1); - - activation.SetUnknownPatterns({cel::AttributePattern( - "foo", {cel::AttributeQualifierPattern::OfString("bar")})}); - - ExecutionFrameBase frame(activation, opts, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - cel::Value result; - AttributeTrail attribute_trail; - ASSERT_THAT(plan->Evaluate(frame, result, attribute_trail), IsOk()); - - EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); -} - -TEST(RecursiveTest, PartialAttrUnknownDisabled) { - cel::Activation activation; - google::protobuf::Arena arena; - cel::RuntimeOptions opts; - opts.unknown_processing = cel::UnknownProcessingOptions::kDisabled; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - auto plan = CreateDirectEqualityStep( - std::make_unique(IntValue(1), cel::Attribute("foo")), - std::make_unique(IntValue(2)), false, -1); - - activation.SetUnknownPatterns({cel::AttributePattern( - "foo", {cel::AttributeQualifierPattern::OfString("bar")})}); - ExecutionFrameBase frame(activation, opts, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - cel::Value result; - AttributeTrail attribute_trail; - ASSERT_THAT(plan->Evaluate(frame, result, attribute_trail), IsOk()); - - EXPECT_THAT(result, BoolValueIs(false)); -} - -TEST(IterativeTest, PartialAttrUnknown) { - cel::Activation activation; - google::protobuf::Arena arena; - cel::RuntimeOptions opts; - opts.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - FlatExpressionEvaluatorState state( - /*value_stack_size=*/5, - /*comprehension_slot_count=*/0, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - std::vector> steps; - steps.push_back( - std::make_unique(IntValue(1), cel::Attribute("foo"))); - steps.push_back(std::make_unique(IntValue(2))); - steps.push_back(CreateEqualityStep(false, -1)); - - activation.SetUnknownPatterns({cel::AttributePattern( - "foo", {cel::AttributeQualifierPattern::OfString("bar")})}); - - ExecutionFrame frame(steps, activation, opts, state); - - ASSERT_OK_AND_ASSIGN(Value result, frame.Evaluate()); - - EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); -} - -TEST(IterativeTest, PartialAttrUnknownDisabled) { - cel::Activation activation; - google::protobuf::Arena arena; - cel::RuntimeOptions opts; - opts.unknown_processing = cel::UnknownProcessingOptions::kDisabled; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - FlatExpressionEvaluatorState state( - /*value_stack_size=*/5, - /*comprehension_slot_count=*/0, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - std::vector> steps; - steps.push_back( - std::make_unique(IntValue(1), cel::Attribute("foo"))); - steps.push_back(std::make_unique(IntValue(2))); - steps.push_back(CreateEqualityStep(false, -1)); - - activation.SetUnknownPatterns({cel::AttributePattern( - "foo", {cel::AttributeQualifierPattern::OfString("bar")})}); - ExecutionFrame frame(steps, activation, opts, state); - - ASSERT_OK_AND_ASSIGN(Value result, frame.Evaluate()); - - EXPECT_THAT(result, BoolValueIs(false)); -} - enum class InputType { kInt1, kInt2, kDouble1, kList, kMap, kError, kUnknown }; enum class OutputType { kBoolTrue, kBoolFalse, kError, kUnknown }; @@ -227,45 +104,7 @@ Value MakeValue(InputType type, google::protobuf::Arena* absl_nonnull arena) { } case InputType::kError: default: - return ErrorValue(absl::InternalError("error")); - } -} - -TEST_P(EqualsTest, Recursive) { - const EqualsTestCase& test_case = GetParam(); - cel::Activation activation; - google::protobuf::Arena arena; - cel::RuntimeOptions opts; - opts.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - auto plan = CreateDirectEqualityStep( - std::make_unique(MakeValue(test_case.lhs, &arena)), - std::make_unique(MakeValue(test_case.rhs, &arena)), - test_case.negation, -1); - - ExecutionFrameBase frame(activation, opts, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - cel::Value result; - AttributeTrail attribute_trail; - ASSERT_THAT(plan->Evaluate(frame, result, attribute_trail), IsOk()); - - switch (test_case.expected_result) { - case OutputType::kBoolTrue: - EXPECT_THAT(result, BoolValueIs(true)); - break; - case OutputType::kBoolFalse: - EXPECT_THAT(result, BoolValueIs(false)); - break; - case OutputType::kError: - EXPECT_THAT(result, ValueKindIs(ValueKind::kError)); - break; - case OutputType::kUnknown: - EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); - break; + return ErrorValue::From(absl::InternalError("error"), arena); } } @@ -280,16 +119,17 @@ TEST_P(EqualsTest, Iterative) { FlatExpressionEvaluatorState state( /*value_stack_size=*/5, - /*comprehension_slot_count=*/0, type_provider, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); - std::vector> steps; - steps.push_back( - std::make_unique(MakeValue(test_case.lhs, &arena))); - steps.push_back( - std::make_unique(MakeValue(test_case.rhs, &arena))); - steps.push_back(CreateEqualityStep(test_case.negation, -1)); + ExecutionPath steps; + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(MakeValue(test_case.lhs, &arena)))); + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(MakeValue(test_case.rhs, &arena)))); + steps.push_back(test_case.negation ? ExpressionStep::MakeFastNotEqualStep() + : ExpressionStep::MakeFastEqualStep()); ExecutionFrame frame(steps, activation, opts, state); @@ -413,43 +253,6 @@ struct InTestCase { class InTest : public ::testing::TestWithParam {}; -TEST_P(InTest, Recursive) { - const InTestCase& test_case = GetParam(); - cel::Activation activation; - google::protobuf::Arena arena; - cel::RuntimeOptions opts; - opts.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - auto plan = CreateDirectInStep( - std::make_unique(MakeValue(test_case.lhs, &arena)), - std::make_unique(MakeValue(test_case.rhs, &arena)), -1); - - ExecutionFrameBase frame(activation, opts, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - cel::Value result; - AttributeTrail attribute_trail; - ASSERT_THAT(plan->Evaluate(frame, result, attribute_trail), IsOk()); - - switch (test_case.expected_result) { - case OutputType::kBoolTrue: - EXPECT_THAT(result, BoolValueIs(true)); - break; - case OutputType::kBoolFalse: - EXPECT_THAT(result, BoolValueIs(false)); - break; - case OutputType::kError: - EXPECT_THAT(result, ValueKindIs(ValueKind::kError)); - break; - case OutputType::kUnknown: - EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); - break; - } -} - TEST_P(InTest, Iterative) { const InTestCase& test_case = GetParam(); cel::Activation activation; @@ -461,16 +264,16 @@ TEST_P(InTest, Iterative) { FlatExpressionEvaluatorState state( /*value_stack_size=*/5, - /*comprehension_slot_count=*/0, type_provider, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); - std::vector> steps; - steps.push_back( - std::make_unique(MakeValue(test_case.lhs, &arena))); - steps.push_back( - std::make_unique(MakeValue(test_case.rhs, &arena))); - steps.push_back(CreateInStep(-1)); + ExecutionPath steps; + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(MakeValue(test_case.lhs, &arena)))); + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(MakeValue(test_case.rhs, &arena)))); + steps.push_back(ExpressionStep::MakeFastInStep()); ExecutionFrame frame(steps, activation, opts, state); diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index 05dbed854..94b4021d6 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -14,10 +14,16 @@ #include "eval/eval/evaluator_core.h" +#include #include -#include +#include +#include +#include #include +#include "absl/base/attributes.h" +#include "absl/base/casts.h" +#include "absl/base/config.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/log/absl_check.h" @@ -25,26 +31,79 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "base/attribute.h" +#include "base/builtins.h" +#include "common/internal/attribute_trail.h" #include "common/value.h" +#include "common/value_kind.h" +#include "common/values/list_value_builder.h" +#include "eval/eval/attribute_trail.h" +#include "eval/eval/bitset_steps.h" +#include "eval/eval/comprehension_slots.h" +#include "eval/eval/comprehension_step.h" +#include "eval/eval/create_list_step.h" +#include "eval/eval/equality_steps.h" +#include "eval/eval/ident_step.h" +#include "eval/eval/iterator_stack.h" +#include "eval/eval/lazy_init_step.h" +#include "internal/status_macros.h" #include "runtime/activation_interface.h" +#include "runtime/internal/errors.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" +// ABSL_UNREACHABLE does hardening shenanigans that can mess up compiler +// optimizations. We use this in the interpreter loop which is extremely hot +// and we cannot accept the interference hardening can do to compiler +// optimizations. +#if defined(__cpp_lib_unreachable) && __cpp_lib_unreachable >= 202202L +#define CEL_INTERNAL_UNREACHABLE_IMPL() std::unreachable() +#elif defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_unreachable) +#define CEL_INTERNAL_UNREACHABLE_IMPL() __builtin_unreachable() +#elif ABSL_HAVE_BUILTIN(__builtin_assume) +#define CEL_INTERNAL_UNREACHABLE_IMPL() __builtin_assume(false) +#elif defined(_MSC_VER) +#define CEL_INTERNAL_UNREACHABLE_IMPL() __assume(false) +#else +#define CEL_INTERNAL_UNREACHABLE_IMPL() ((void)0) +#endif + +#define CEL_INTERNAL_UNREACHABLE() \ + do { \ + /* NOLINTNEXTLINE: misc-static-assert */ \ + assert(false && "unreachable"); \ + CEL_INTERNAL_UNREACHABLE_IMPL(); \ + } while (false) + namespace google::api::expr::runtime { +struct ExpressionStepData { + [[nodiscard]] + ABSL_ATTRIBUTE_ALWAYS_INLINE static const ExpressionStep::Data& Get( + const ExpressionStep& step ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return step.u_; + } +}; + void FlatExpressionEvaluatorState::Reset() { + if (!dirty_) { + return; + } value_stack_.Clear(); iterator_stack_.Clear(); comprehension_slots_.Reset(); + dirty_ = false; } -const ExpressionStep* ExecutionFrame::Next() { +const ExpressionStep* ExecutionFrame::NextSlow() { while (true) { const size_t end_pos = execution_path_.size(); if (ABSL_PREDICT_TRUE(pc_ < end_pos)) { - const auto* step = execution_path_[pc_++].get(); + const auto* step = &execution_path_[pc_++]; ABSL_ASSUME(step != nullptr); return step; } @@ -66,6 +125,13 @@ const ExpressionStep* ExecutionFrame::Next() { } } +void ExecutionFrame::AbortJumpOutOfRange(int offset) { + Abort(absl::Status( + absl::StatusCode::kInternal, + absl::StrCat("Jump address out of range: position: ", pc_, + ", offset: ", offset, ", range: ", execution_path_.size()))); +} + namespace { // This class abuses the fact that `absl::Status` is trivially destructible when @@ -101,44 +167,847 @@ class EvaluationStatus final { alignas(absl::Status) char status_[sizeof(absl::Status)]; }; +int64_t OneMinusArgCount(size_t count) { + ABSL_DCHECK(count <= + static_cast(std::numeric_limits::max())); + return 1 - static_cast(count); +} + +absl::Status LookupIdent(absl::string_view name, ExecutionFrame& frame, + cel::Value& result, AttributeTrail& attribute) { + if (frame.attribute_tracking_enabled() && + (AttributeLeadMatch)(frame.unknown_attributes(), frame.known_attributes(), + frame.missing_attributes(), name, result, attribute, + frame.unknown_tree())) { + return absl::OkStatus(); + } + + CEL_ASSIGN_OR_RETURN( + bool found, frame.activation().FindVariable(name, frame.descriptor_pool(), + frame.message_factory(), + frame.arena(), &result)); + + if (found) { + if (ABSL_PREDICT_FALSE(frame.attribute_tracking_enabled() && + result.IsUnknown())) { + // This really only happens in our own tests... + result = cel::common_internal::GraftUnknowns(result.GetUnknown(), + frame.unknown_tree()); + attribute = AttributeTrail(); + } + return absl::OkStatus(); + } + + if (frame.attribute_tracking_enabled()) { + attribute = AttributeTrail(); + } + result = cel::ErrorValue::From( + cel::runtime_internal::CreateError(absl::StrCat( + "No value with name \"", name, "\" found in Activation")), + frame.arena()); + + return absl::OkStatus(); +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateIdentifierStep( + ExecutionFrame& frame, const IdentStepInfo& step) { + ComprehensionSlot* slot = frame.comprehension_slots().Get(step.slot_index); + if (slot->Has()) { + frame.value_stack().Push(slot->value(), slot->attribute()); + return; + } + if (ABSL_PREDICT_FALSE(step.ident_index >= frame.identifiers().size())) { + frame.Abort(absl::InternalError( + absl::StrCat("identifier index out of range: ", step.ident_index))); + return; + } + const Identifier& ident = frame.identifiers()[step.ident_index]; + frame.value_stack().Push(cel::NullValue()); + if (absl::Status status = + LookupIdent(ident.name, frame, frame.value_stack().Peek(), + frame.value_stack().PeekAttribute()); + !status.ok()) { + frame.Abort(std::move(status)); + return; + } + slot->Set(frame.value_stack().Peek(), frame.value_stack().PeekAttribute()); +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateReadSlotStep( + ExecutionFrame& frame, size_t slot_index) { + const ComprehensionSlots::Slot* slot = + frame.comprehension_slots().Get(slot_index); + if (ABSL_PREDICT_FALSE(!slot->Has())) { + frame.Abort(absl::InternalError(absl::StrCat( + "Comprehension variable read out of scope: ", slot_index))); + return; + } + frame.value_stack().Push(slot->value(), slot->attribute()); +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateBoolJumpStep( + ExecutionFrame& frame, const BoolJumpStepInfo& step, bool target) { + ABSL_DCHECK(step.set) << "BoolJumpStep did not have a value set."; + if (ABSL_PREDICT_FALSE(!frame.value_stack().HasEnough(step.arg_count))) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + const cel::Value& value = frame.value_stack().Peek(); + if (value.IsBool() && value.GetBool().NativeValue() == target) { + frame.value_stack().SwapAndPop(step.arg_count, step.arg_count - 1); + frame.JumpToOrAbort(step.offset); + } + // No-op if the value is not a bool or the value is not the target. + // Cleanup will happen if we hit a later jump or we fall-through. +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateTernaryJumpStep( + ExecutionFrame& frame, const TernaryJumpStepInfo& step) { + ABSL_DCHECK(step.set) << "TernaryJumpStep did not have a value set."; + if (ABSL_PREDICT_FALSE(!frame.value_stack().HasEnough(1))) { + frame.Abort(absl::InternalError("TernaryJumpStep: value stack underflow")); + return; + } + const cel::Value& condition = frame.value_stack().Peek(); + switch (condition.kind()) { + case cel::ValueKind::kBool: + if (!condition.GetBool().NativeValue()) { + frame.JumpToOrAbort(step.jump_to_second_offset); + } + frame.value_stack().Pop(); + break; + default: + frame.value_stack().PopAndPush(cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError( + ""), + frame.arena())); + ABSL_FALLTHROUGH_INTENDED; + case cel::ValueKind::kError: + case cel::ValueKind::kUnknown: + // Propagate the error or unknown set. + frame.JumpToOrAbort(step.error_offset); + break; + } +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateMutableListAppendStep( + ExecutionFrame& frame) { + if (ABSL_PREDICT_FALSE(!frame.value_stack().HasEnough(2))) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + absl::Span args = frame.value_stack().GetSpan(2); + if (args[0].IsError()) { + frame.value_stack().Pop(1); + return; + } + if (args[1].IsError()) { + frame.value_stack().SwapAndPop(2, 1); + return; + } + if (frame.unknown_processing_enabled()) { + if (auto unknown = cel::common_internal::PartiallyIdentifyAndMergeUnknowns( + args, frame.value_stack().GetAttributeSpan(2), + frame.unknown_tree()); + unknown.has_value()) { + frame.value_stack().PopAndPush(2, *unknown); + return; + } + } + if (const cel::common_internal::MutableListValue* mutable_list_value = + cel::common_internal::AsMutableListValue(args[0]); + ABSL_PREDICT_TRUE(mutable_list_value != nullptr)) { + absl::Status status = mutable_list_value->Append(args[1]); + if (ABSL_PREDICT_FALSE(!status.ok())) { + frame.Abort(std::move(status)); + return; + } + frame.value_stack().Pop(1); + return; + } + frame.Abort( + absl::InvalidArgumentError("Unexpected call to runtime list append.")); +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateNotStep( + ExecutionFrame& frame) { + if (ABSL_PREDICT_FALSE(!frame.value_stack().HasEnough(1))) { + frame.Abort(absl::InternalError("Value stack underflow")); + return; + } + const cel::Value& operand = frame.value_stack().Peek(); + + switch (operand.kind()) { + case cel::ValueKind::kBool: + frame.value_stack().PopAndPush( + cel::BoolValue{!operand.GetBool().NativeValue()}); + break; + case cel::ValueKind::kUnknown: + case cel::ValueKind::kError: + // just forward. + break; + default: + frame.value_stack().PopAndPush(cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError( + cel::builtin::kNot), + frame.arena())); + break; + } +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateNotStrictlyFalseStep( + ExecutionFrame& frame) { + if (ABSL_PREDICT_FALSE(!frame.value_stack().HasEnough(1))) { + frame.Abort(absl::InternalError("Value stack underflow")); + return; + } + const cel::Value& operand = frame.value_stack().Peek(); + + switch (operand.kind()) { + case cel::ValueKind::kBool: + // just forward. + break; + case cel::ValueKind::kUnknown: + case cel::ValueKind::kError: + frame.value_stack().PopAndPush(cel::TrueValue()); + break; + default: + frame.value_stack().PopAndPush(cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError( + cel::builtin::kNot), + frame.arena())); + break; + } +} + +enum class BoolLogicKind { + kAnd, + kOr, +}; + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateBoolLogicStep( + ExecutionFrame& frame, size_t num_args, BoolLogicKind kind) { + if (ABSL_PREDICT_FALSE(!frame.value_stack().HasEnough(num_args))) { + frame.Abort(absl::InternalError("Value stack underflow")); + return; + } + + const bool shortcircuit = kind == BoolLogicKind::kOr; + const absl::string_view op_name = + kind == BoolLogicKind::kOr ? cel::builtin::kOr : cel::builtin::kAnd; + absl::Span args = frame.value_stack().GetSpan(num_args); + std::optional error_pos; + + for (size_t i = 0; i < args.size(); i++) { + const cel::Value& arg = args[i]; + switch (arg.kind()) { + case cel::ValueKind::kBool: + if (arg.GetBool() == shortcircuit) { + frame.value_stack().PopAndPush(num_args, + cel::BoolValue(shortcircuit)); + return; + } + break; + case cel::ValueKind::kUnknown: + break; + case cel::ValueKind::kError: + default: + if (!error_pos.has_value()) { + error_pos = i; + } + break; + } + } + + // As opposed to regular function, logical operation treat Unknowns with + // higher precedence than error. This is due to the fact that after Unknown + // is resolved to actual value, it may short-circuit and thus hide the + // error. + if (frame.enable_unknowns()) { + // Check if unknown? + UnknownAccumulator accumulator(frame.unknown_tree()); + for (const auto& arg : args) { + accumulator.MaybeAdd(arg); + } + if (auto accumulated = std::move(accumulator).Accumulate(); + accumulated.has_value()) { + frame.value_stack().PopAndPush(num_args, *accumulated); + return; + } + } + + if (!error_pos.has_value()) { + frame.value_stack().PopAndPush(num_args, cel::BoolValue(!shortcircuit)); + return; + } + + cel::Value result = args[error_pos.value()]; + if (!result.IsError()) { + result = cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(op_name), + frame.arena()); + } + frame.value_stack().PopAndPush(num_args, std::move(result)); +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateLazyInitStep( + ExecutionFrame& frame, const LazyInitStepInfo& step) { + ComprehensionSlot* slot = frame.comprehension_slots().Get(step.slot_index); + if (slot->Has()) { + frame.value_stack().Push(slot->value(), slot->attribute()); + } else { + frame.Call(step.slot_index, step.subexpression_index); + } +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateAssignSlotAndPop( + ExecutionFrame& frame, size_t slot_index) { + if (ABSL_PREDICT_FALSE(!frame.value_stack().HasEnough(1))) { + frame.Abort(absl::InternalError("Stack underflow assigning lazy value")); + return; + } + ComprehensionSlot* slot = frame.comprehension_slots().Get(slot_index); + slot->Set(frame.value_stack().Peek(), frame.value_stack().PeekAttribute()); + frame.value_stack().Pop(1); +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateClearSlotStep( + ExecutionFrame& frame, const ClearSlotStepInfo& step) { + for (size_t i = 0; i < step.slot_count; ++i) { + frame.comprehension_slots().ClearSlot(step.slot_index + i); + } +} + +cel::AttributeQualifier AttributeQualifierFromValue(const cel::Value& v) { + switch (v.kind()) { + case cel::ValueKind::kString: + return cel::AttributeQualifier::OfString(v.GetString().ToString()); + case cel::ValueKind::kInt64: + return cel::AttributeQualifier::OfInt(v.GetInt().NativeValue()); + case cel::ValueKind::kUint64: + return cel::AttributeQualifier::OfUint(v.GetUint().NativeValue()); + case cel::ValueKind::kBool: + return cel::AttributeQualifier::OfBool(v.GetBool().NativeValue()); + default: + // Non-matching qualifier. + return cel::AttributeQualifier(); + } +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateComprehensionInitStep( + ExecutionFrame* frame, const ComprehensionInitStepInfo& step) { + if (ABSL_PREDICT_FALSE(!frame->value_stack().HasEnough(1))) { + frame->Abort(absl::InternalError("Value stack underflow")); + return; + } + + const cel::Value& top = frame->value_stack().Peek(); + if (top.IsError() || top.IsUnknown()) { + frame->JumpToOrAbort(step.error_jump_offset); + return; + } + + if (frame->enable_unknowns() && top.IsMap()) { + const AttributeTrail& top_attr = frame->value_stack().PeekAttribute(); + if (auto unknown = top_attr.PartialUnknownMatch(frame->unknown_tree()); + unknown.has_value()) { + frame->value_stack().PopAndPush(*unknown); + frame->JumpToOrAbort(step.error_jump_offset); + return; + } + } + + absl::StatusOr iterator; + switch (top.kind()) { + case cel::ValueKind::kList: + iterator = top.GetList().NewIterator(); + break; + case cel::ValueKind::kMap: + iterator = top.GetMap().NewIterator(); + break; + default: + // Replace with an error and jump past + // ComprehensionFinishStep. + frame->value_stack().PopAndPush(cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(""), + frame->arena())); + frame->JumpToOrAbort(step.error_jump_offset); + return; + } + + if (!iterator.ok()) { + frame->Abort(std::move(iterator).status()); + return; + } + if (step.v2) { + frame->iterator_stack().Push(*std::move(iterator), step.iter_slot(), + step.iter2_slot(), step.accu_slot()); + } else { + frame->iterator_stack().Push(*std::move(iterator), step.iter_slot(), + step.accu_slot()); + } +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateComprehensionNextStep1( + ExecutionFrame* frame, const ComprehensionNextStepInfo& step) { + if (ABSL_PREDICT_FALSE(frame->iterator_stack().empty())) { + frame->Abort(absl::InternalError("Iterator stack underflow")); + return; + } + const cel::runtime_internal::IteratorStack::Entry& entry = + *frame->iterator_stack().Peek(); + if (ABSL_PREDICT_FALSE(!frame->value_stack().HasEnough(2))) { + frame->Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + + { + cel::Value& accu_var = frame->value_stack().Peek(); + AttributeTrail& accu_var_attr = frame->value_stack().PeekAttribute(); + frame->comprehension_slots().Set(entry.accu_slot, std::move(accu_var), + std::move(accu_var_attr)); + frame->value_stack().Pop(1); + } + + ComprehensionSlots::Slot* iter_slot = + frame->comprehension_slots().Get(entry.iter_slot); + ABSL_DCHECK(iter_slot != nullptr); + iter_slot->Set(); + + if (frame->enable_unknowns()) { + cel::Value key_or_value; + cel::Value* key; + cel::Value* value; + switch (frame->value_stack().Peek().kind()) { + case cel::ValueKind::kList: + key = &key_or_value; + value = iter_slot->mutable_value(); + break; + case cel::ValueKind::kMap: + key = iter_slot->mutable_value(); + value = nullptr; + break; + default: + ABSL_UNREACHABLE(); + } + absl::StatusOr ok = entry.iterator->Next2(frame->descriptor_pool(), + frame->message_factory(), + frame->arena(), key, value); + if (ABSL_PREDICT_FALSE(!ok.ok())) { + frame->Abort(std::move(ok).status()); + return; + } + if (!*ok) { + iter_slot->Clear(); + frame->JumpToOrAbort(step.jump_offset); + return; + } + absl::Status inc_status = frame->IncrementIterations(); + if (ABSL_PREDICT_FALSE(!inc_status.ok())) { + frame->Abort(std::move(inc_status)); + return; + } + cel::AttributeQualifier qualifier = AttributeQualifierFromValue(*key); + if (qualifier) { + static_cast( + frame->value_stack().PeekAttribute().Match( + qualifier, *iter_slot->mutable_value(), + *iter_slot->mutable_attribute(), frame->unknown_tree())); + } + } else { + absl::StatusOr ok = entry.iterator->Next1( + frame->descriptor_pool(), frame->message_factory(), frame->arena(), + iter_slot->mutable_value()); + if (ABSL_PREDICT_FALSE(!ok.ok())) { + frame->Abort(std::move(ok).status()); + return; + } + if (!*ok) { + iter_slot->Clear(); + frame->JumpToOrAbort(step.jump_offset); + return; + } + absl::Status inc_status = frame->IncrementIterations(); + if (ABSL_PREDICT_FALSE(!inc_status.ok())) { + frame->Abort(std::move(inc_status)); + return; + } + } +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateComprehensionNextStep2( + ExecutionFrame* frame, const ComprehensionNextStepInfo& step) { + if (ABSL_PREDICT_FALSE(frame->iterator_stack().empty())) { + frame->Abort(absl::InternalError("Iterator stack underflow")); + return; + } + const cel::runtime_internal::IteratorStack::Entry& entry = + *frame->iterator_stack().Peek(); + if (ABSL_PREDICT_FALSE(!frame->value_stack().HasEnough(2))) { + frame->Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + + { + cel::Value& accu_var = frame->value_stack().Peek(); + AttributeTrail& accu_var_attr = frame->value_stack().PeekAttribute(); + frame->comprehension_slots().Set(entry.accu_slot, std::move(accu_var), + std::move(accu_var_attr)); + frame->value_stack().Pop(1); + } + + ComprehensionSlots::Slot* iter_slot = + frame->comprehension_slots().Get(entry.iter_slot); + ABSL_DCHECK(iter_slot != nullptr); + iter_slot->Set(); + + ComprehensionSlots::Slot* iter2_slot = + frame->comprehension_slots().Get(entry.iter2_slot); + ABSL_DCHECK(iter2_slot != nullptr); + iter2_slot->Set(); + + absl::StatusOr ok = entry.iterator->Next2( + frame->descriptor_pool(), frame->message_factory(), frame->arena(), + iter_slot->mutable_value(), iter2_slot->mutable_value()); + if (ABSL_PREDICT_FALSE(!ok.ok())) { + frame->Abort(std::move(ok).status()); + return; + } + if (!*ok) { + iter_slot->Clear(); + iter2_slot->Clear(); + frame->JumpToOrAbort(step.jump_offset); + return; + } + absl::Status inc_status = frame->IncrementIterations(); + if (ABSL_PREDICT_FALSE(!inc_status.ok())) { + frame->Abort(std::move(inc_status)); + return; + } + if (frame->enable_unknowns()) { + cel::AttributeQualifier qualifier = + AttributeQualifierFromValue(iter_slot->value()); + if (qualifier) { + static_cast( + frame->value_stack().PeekAttribute().Match( + qualifier, *iter2_slot->mutable_value(), + *iter_slot->mutable_attribute(), frame->unknown_tree())); + *iter2_slot->mutable_attribute() = iter_slot->attribute(); + } + } +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateComprehensionCondStep1( + ExecutionFrame* frame, const ComprehensionCondStepInfo& step) { + if (ABSL_PREDICT_FALSE(frame->iterator_stack().empty())) { + frame->Abort(absl::InternalError("Iterator stack underflow")); + return; + } + const cel::runtime_internal::IteratorStack::Entry& entry = + *frame->iterator_stack().Peek(); + if (ABSL_PREDICT_FALSE(!frame->value_stack().HasEnough(2))) { + frame->Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + const cel::Value& top = frame->value_stack().Peek(); + switch (top.kind()) { + case cel::ValueKind::kBool: + break; + case cel::ValueKind::kError: + ABSL_FALLTHROUGH_INTENDED; + case cel::ValueKind::kUnknown: { + frame->value_stack().SwapAndPop(2, 1); + frame->comprehension_slots().ClearSlot(entry.iter_slot); + frame->comprehension_slots().ClearSlot(entry.accu_slot); + frame->iterator_stack().Pop(); + frame->JumpToOrAbort(step.error_jump_offset); + return; + } + default: { + frame->value_stack().PopAndPush( + 2, cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError( + ""), + frame->arena())); + frame->comprehension_slots().ClearSlot(entry.iter_slot); + frame->comprehension_slots().ClearSlot(entry.accu_slot); + frame->iterator_stack().Pop(); + frame->JumpToOrAbort(step.error_jump_offset); + return; + } + } + const bool loop_condition = absl::implicit_cast(top.GetBool()); + const bool short_circuiting = frame->options().short_circuiting; + frame->value_stack().Pop(1); // loop_condition + if (!loop_condition && short_circuiting) { + frame->JumpToOrAbort(step.jump_offset); + return; + } +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateComprehensionCondStep2( + ExecutionFrame* frame, const ComprehensionCondStepInfo& step) { + if (ABSL_PREDICT_FALSE(frame->iterator_stack().empty())) { + frame->Abort(absl::InternalError("Iterator stack underflow")); + return; + } + const cel::runtime_internal::IteratorStack::Entry& entry = + *frame->iterator_stack().Peek(); + if (ABSL_PREDICT_FALSE(!frame->value_stack().HasEnough(2))) { + frame->Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + const cel::Value& top = frame->value_stack().Peek(); + switch (top.kind()) { + case cel::ValueKind::kBool: + break; + case cel::ValueKind::kError: + ABSL_FALLTHROUGH_INTENDED; + case cel::ValueKind::kUnknown: { + frame->value_stack().SwapAndPop(2, 1); + frame->comprehension_slots().ClearSlot(entry.iter_slot); + frame->comprehension_slots().ClearSlot(entry.iter2_slot); + frame->comprehension_slots().ClearSlot(entry.accu_slot); + frame->iterator_stack().Pop(); + frame->JumpToOrAbort(step.error_jump_offset); + return; + } + default: { + frame->value_stack().PopAndPush( + 2, cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError( + ""), + frame->arena())); + frame->comprehension_slots().ClearSlot(entry.iter_slot); + frame->comprehension_slots().ClearSlot(entry.iter2_slot); + frame->comprehension_slots().ClearSlot(entry.accu_slot); + frame->iterator_stack().Pop(); + frame->JumpToOrAbort(step.error_jump_offset); + return; + } + } + const bool loop_condition = absl::implicit_cast(top.GetBool()); + const bool short_circuiting = frame->options().short_circuiting; + frame->value_stack().Pop(1); // loop_condition + if (!loop_condition && short_circuiting) { + frame->JumpToOrAbort(step.jump_offset); + return; + } +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateComprehensionFinishStep( + ExecutionFrame& frame, size_t accu_slot) { + if (ABSL_PREDICT_FALSE(!frame.value_stack().HasEnough(2))) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + frame.value_stack().SwapAndPop(2, 1); + frame.comprehension_slots().ClearSlot(accu_slot); + frame.iterator_stack().Pop(); +} + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateStep( + ExecutionFrame& frame, const ExpressionStep& step) { + switch (step.kind()) { + case ExpressionStepKind::kGenericLogic: + ExpressionStepData::Get(step).logic->Evaluate(&frame); + break; + case ExpressionStepKind::kIntConstant: + frame.value_stack().Push( + cel::IntValue(ExpressionStepData::Get(step).int_val)); + break; + case ExpressionStepKind::kBoolConstant: + frame.value_stack().Push( + cel::BoolValue(ExpressionStepData::Get(step).bool_val)); + break; + case ExpressionStepKind::kDoubleConstant: + frame.value_stack().Push( + cel::DoubleValue(ExpressionStepData::Get(step).double_val)); + break; + case ExpressionStepKind::kNullConstant: + frame.value_stack().Push(cel::NullValue()); + break; + case ExpressionStepKind::kUintConstant: + frame.value_stack().Push( + cel::UintValue(ExpressionStepData::Get(step).uint_val)); + break; + case ExpressionStepKind::kOtherConstant: + frame.value_stack().Push(*ExpressionStepData::Get(step).other_val); + break; + case ExpressionStepKind::kLazyInit: + EvaluateLazyInitStep(frame, ExpressionStepData::Get(step).lazy_init); + break; + case ExpressionStepKind::kAssignSlotAndPop: + EvaluateAssignSlotAndPop(frame, ExpressionStepData::Get(step).slot_index); + break; + case ExpressionStepKind::kClearSlots: + EvaluateClearSlotStep(frame, ExpressionStepData::Get(step).clear_slots); + break; + case ExpressionStepKind::kBooleanNot: + EvaluateNotStep(frame); + break; + case ExpressionStepKind::kNotStrictlyFalse: + EvaluateNotStrictlyFalseStep(frame); + break; + case ExpressionStepKind::kBooleanOr: + EvaluateBoolLogicStep(frame, ExpressionStepData::Get(step).arg_count, + BoolLogicKind::kOr); + break; + case ExpressionStepKind::kBooleanAnd: + EvaluateBoolLogicStep(frame, ExpressionStepData::Get(step).arg_count, + BoolLogicKind::kAnd); + break; + case ExpressionStepKind::kComprehensionFinish: + EvaluateComprehensionFinishStep(frame, + ExpressionStepData::Get(step).slot_index); + break; + case ExpressionStepKind::kComprehensionNext: + EvaluateComprehensionNextStep1(&frame, + ExpressionStepData::Get(step).next_step); + break; + case ExpressionStepKind::kComprehensionNext2: + EvaluateComprehensionNextStep2(&frame, + ExpressionStepData::Get(step).next_step); + break; + case ExpressionStepKind::kComprehensionCond: + EvaluateComprehensionCondStep1(&frame, + ExpressionStepData::Get(step).cond_step); + break; + case ExpressionStepKind::kComprehensionCond2: + EvaluateComprehensionCondStep2(&frame, + ExpressionStepData::Get(step).cond_step); + break; + case ExpressionStepKind::kReadSlot: + EvaluateReadSlotStep(frame, ExpressionStepData::Get(step).slot_index); + break; + case ExpressionStepKind::kBooleanOrJump: + EvaluateBoolJumpStep(frame, ExpressionStepData::Get(step).bool_jump_step, + true); + break; + case ExpressionStepKind::kBooleanAndJump: + EvaluateBoolJumpStep(frame, ExpressionStepData::Get(step).bool_jump_step, + false); + break; + case ExpressionStepKind::kTernaryJump: + EvaluateTernaryJumpStep(frame, + ExpressionStepData::Get(step).ternary_jump_step); + break; + case ExpressionStepKind::kFixedJump: + ABSL_DCHECK(ExpressionStepData::Get(step).fixed_jump_step.set) + << "FixedJumpStep did not have a value set."; + frame.JumpToOrAbort(ExpressionStepData::Get(step).fixed_jump_step.offset); + break; + case ExpressionStepKind::kIdentifier: + EvaluateIdentifierStep(frame, ExpressionStepData::Get(step).ident_step); + break; + case ExpressionStepKind::kEagerFunction: + ExpressionStepData::Get(step).eager_function_step->Evaluate(frame); + break; + case ExpressionStepKind::kLazyFunction: + ExpressionStepData::Get(step).lazy_function_step->Evaluate(frame); + break; + case ExpressionStepKind::kFastIn: + EvaluateFastInStep(frame); + break; + case ExpressionStepKind::kFastEqual: + EvaluateFastEqualStep(/*negation=*/false, frame); + break; + case ExpressionStepKind::kFastNotEqual: + EvaluateFastEqualStep(/*negation=*/true, frame); + break; + case ExpressionStepKind::kNewMutableList: + frame.value_stack().Push(cel::CustomListValue( + cel::common_internal::NewMutableListValue(frame.arena()), + frame.arena())); + break; + case ExpressionStepKind::kMutableListAppend: + EvaluateMutableListAppendStep(frame); + break; + case ExpressionStepKind::kCreateList: + EvaluateListStep(*ExpressionStepData::Get(step).create_list_step, frame); + break; + case ExpressionStepKind::kCreateSmallList: + EvaluateSmallListStep( + ExpressionStepData::Get(step).create_small_list_step, frame); + break; + case ExpressionStepKind::kBitIsSet: + EvaluateBitIsSetStep(ExpressionStepData::Get(step).bitmask, frame); + break; + case ExpressionStepKind::kBitAndIsNonZero: + EvaluateBitAndIsNonZeroStep(ExpressionStepData::Get(step).bitmask, frame); + break; + case ExpressionStepKind::kBitAndEqual: + EvaluateBitAndEqualStep(ExpressionStepData::Get(step).bitmask, frame); + break; + case ExpressionStepKind::kBitEqual: + EvaluateBitEqualStep(ExpressionStepData::Get(step).bitmask, frame); + break; + case ExpressionStepKind::kComprehensionInit: + EvaluateComprehensionInitStep(&frame, + ExpressionStepData::Get(step).init_step); + break; + case ExpressionStepKind::kMovedFrom: + frame.Abort(absl::InternalError( + "ExpressionStep::Evaluate called on moved-from step")); + break; + default: + CEL_INTERNAL_UNREACHABLE(); + } +} + } // namespace +void ExecutionFrame::Interpret() { + for (const ExpressionStep* expr = Next(); ABSL_PREDICT_TRUE(expr != nullptr); + expr = Next()) { + EvaluateStep(*this, *expr); + } +} + +void ExecutionFrame::InterpretWithCallback(EvaluationListener& listener) { + for (const ExpressionStep* expr = Next(); ABSL_PREDICT_TRUE(expr != nullptr); + expr = Next()) { + EvaluateStep(*this, *expr); + if (pc_ == 0 || !expr->comes_from_ast() || !abort_status().ok()) { + // Skip if we just started a Call, if the step doesn't map to an AST + // id, or if evaluation was aborted. + continue; + } + + if (ABSL_PREDICT_FALSE(value_stack().empty())) { + ABSL_LOG(ERROR) << "Stack is empty after a ExpressionStep.Evaluate. " + "Try to disable short-circuiting."; + continue; + } + if (EvaluationStatus status(listener(expr->id(), value_stack().Peek(), + descriptor_pool(), message_factory(), + arena())); + !status.ok()) { + Abort(std::move(status).Consume()); + return; + } + } +} + absl::StatusOr ExecutionFrame::Evaluate( EvaluationListener& listener) { const size_t initial_stack_size = value_stack().size(); if (!listener) { - for (const ExpressionStep* expr = Next(); - ABSL_PREDICT_TRUE(expr != nullptr); expr = Next()) { - if (EvaluationStatus status(expr->Evaluate(this)); !status.ok()) { - return std::move(status).Consume(); - } - } + Interpret(); } else { - for (const ExpressionStep* expr = Next(); - ABSL_PREDICT_TRUE(expr != nullptr); expr = Next()) { - if (EvaluationStatus status(expr->Evaluate(this)); !status.ok()) { - return std::move(status).Consume(); - } - - if (pc_ == 0 || !expr->comes_from_ast()) { - // Skip if we just started a Call or if the step doesn't map to an - // AST id. - continue; - } + InterpretWithCallback(listener); + } - if (ABSL_PREDICT_FALSE(value_stack().empty())) { - ABSL_LOG(ERROR) << "Stack is empty after a ExpressionStep.Evaluate. " - "Try to disable short-circuiting."; - continue; - } - if (EvaluationStatus status(listener(expr->id(), value_stack().Peek(), - descriptor_pool(), message_factory(), - arena())); - !status.ok()) { - return std::move(status).Consume(); - } - } + if (!abort_status().ok()) { + return std::move(abort_status()); } const size_t final_stack_size = value_stack().size(); @@ -158,9 +1027,17 @@ FlatExpressionEvaluatorState FlatExpression::MakeEvaluatorState( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) const { - return FlatExpressionEvaluatorState(path_.size(), comprehension_slots_size_, - type_provider_, descriptor_pool, - message_factory, arena); + return FlatExpressionEvaluatorState( + value_stack_size_, comprehension_slots_size_, iterator_stack_size_, + type_provider_, descriptor_pool, message_factory, arena); +} + +FlatExpressionEvaluatorState FlatExpression::MakeEvaluatorState( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nullable message_factory) const { + return FlatExpressionEvaluatorState( + value_stack_size_, comprehension_slots_size_, iterator_stack_size_, + type_provider_, descriptor_pool, message_factory); } absl::StatusOr FlatExpression::EvaluateWithCallback( @@ -168,11 +1045,204 @@ absl::StatusOr FlatExpression::EvaluateWithCallback( const cel::EmbedderContext* absl_nullable embedder_context, EvaluationListener listener, FlatExpressionEvaluatorState& state) const { state.Reset(); - ExecutionFrame frame(subexpressions_, activation, options_, state, - std::move(listener), embedder_context); + std::move(listener), embedder_context, identifiers_); return frame.Evaluate(frame.callback()); } +ExpressionStep ExpressionStep::MakeConstant(const cel::Value& value, + int64_t id) { + if (id < 0 || id > std::numeric_limits::max()) { + id = -1; + } + int32_t id32 = static_cast(id); + switch (value.kind()) { + case cel::ValueKind::kBool: { + ExpressionStep step(ExpressionStepKind::kBoolConstant, id32); + step.u_.bool_val = value.GetBool().NativeValue(); + return step; + } + case cel::ValueKind::kInt: { + ExpressionStep step(ExpressionStepKind::kIntConstant, id32); + step.u_.int_val = value.GetInt().NativeValue(); + return step; + } + case cel::ValueKind::kUint: { + ExpressionStep step(ExpressionStepKind::kUintConstant, id32); + step.u_.uint_val = value.GetUint().NativeValue(); + return step; + } + case cel::ValueKind::kDouble: { + ExpressionStep step(ExpressionStepKind::kDoubleConstant, id32); + step.u_.double_val = value.GetDouble().NativeValue(); + return step; + } + case cel::ValueKind::kNull: + return ExpressionStep(ExpressionStepKind::kNullConstant, id32); + default: { + ExpressionStep step(ExpressionStepKind::kOtherConstant, id32); + step.u_.other_val = new cel::Value(value); + return step; + } + } +} + +bool GetIfConstant(const ExpressionStep& step, cel::Value& out) { + switch (step.header_.kind) { + case ExpressionStepKind::kIntConstant: + out = cel::IntValue(step.u_.int_val); + return true; + case ExpressionStepKind::kBoolConstant: + out = cel::BoolValue(step.u_.bool_val); + return true; + case ExpressionStepKind::kDoubleConstant: + out = cel::DoubleValue(step.u_.double_val); + return true; + case ExpressionStepKind::kNullConstant: + out = cel::NullValue(); + return true; + case ExpressionStepKind::kUintConstant: + out = cel::UintValue(step.u_.uint_val); + return true; + case ExpressionStepKind::kOtherConstant: + out = *step.u_.other_val; + return true; + default: + return false; + } +} + +bool IsConstant(const ExpressionStep& step) { + switch (step.header_.kind) { + case ExpressionStepKind::kIntConstant: + case ExpressionStepKind::kBoolConstant: + case ExpressionStepKind::kDoubleConstant: + case ExpressionStepKind::kNullConstant: + case ExpressionStepKind::kUintConstant: + case ExpressionStepKind::kOtherConstant: + return true; + default: + return false; + } +} + +ComprehensionCondStepInfo* GetIfComprehensionCondStep(ExpressionStep& step) { + if (step.header_.kind == ExpressionStepKind::kComprehensionCond || + step.header_.kind == ExpressionStepKind::kComprehensionCond2) { + return &step.u_.cond_step; + } + return nullptr; +} + +ComprehensionNextStepInfo* GetIfComprehensionNextStep(ExpressionStep& step) { + if (step.header_.kind == ExpressionStepKind::kComprehensionNext || + step.header_.kind == ExpressionStepKind::kComprehensionNext2) { + return &step.u_.next_step; + } + return nullptr; +} + +ComprehensionInitStepInfo* GetIfComprehensionInitStep(ExpressionStep& step) { + if (step.header_.kind == ExpressionStepKind::kComprehensionInit) { + return &step.u_.init_step; + } + return nullptr; +} + +BoolJumpStepInfo* GetIfBoolJumpStep(ExpressionStep& step) { + if (step.header_.kind == ExpressionStepKind::kBooleanOrJump || + step.header_.kind == ExpressionStepKind::kBooleanAndJump) { + return &step.u_.bool_jump_step; + } + return nullptr; +} + +TernaryJumpStepInfo* GetIfTernaryJumpStep(ExpressionStep& step) { + if (step.header_.kind == ExpressionStepKind::kTernaryJump) { + return &step.u_.ternary_jump_step; + } + return nullptr; +} + +FixedJumpStepInfo* GetIfFixedJumpStep(ExpressionStep& step) { + if (step.header_.kind == ExpressionStepKind::kFixedJump) { + return &step.u_.fixed_jump_step; + } + return nullptr; +} + +IdentStepInfo* GetIfIdentStep(ExpressionStep& step) { + if (step.header_.kind == ExpressionStepKind::kIdentifier) { + return &step.u_.ident_step; + } + return nullptr; +} + +const IdentStepInfo* GetIfIdentStep(const ExpressionStep& step) { + if (step.header_.kind == ExpressionStepKind::kIdentifier) { + return &step.u_.ident_step; + } + return nullptr; +} + +std::optional ExpressionStep::StackDelta() const { + switch (header_.kind) { + case ExpressionStepKind::kMovedFrom: + return 0; + case ExpressionStepKind::kGenericLogic: + if (header_.stack_delta == std::numeric_limits::max()) { + // Impractical, but technically allowed. + return std::nullopt; + } + return header_.stack_delta; + case ExpressionStepKind::kIntConstant: + case ExpressionStepKind::kBoolConstant: + case ExpressionStepKind::kDoubleConstant: + case ExpressionStepKind::kNullConstant: + case ExpressionStepKind::kUintConstant: + case ExpressionStepKind::kOtherConstant: + case ExpressionStepKind::kLazyInit: + case ExpressionStepKind::kReadSlot: + case ExpressionStepKind::kIdentifier: + case ExpressionStepKind::kNewMutableList: + case ExpressionStepKind::kComprehensionInit: + return 1; + case ExpressionStepKind::kAssignSlotAndPop: + case ExpressionStepKind::kComprehensionFinish: + case ExpressionStepKind::kComprehensionNext: + case ExpressionStepKind::kComprehensionNext2: + case ExpressionStepKind::kComprehensionCond: + case ExpressionStepKind::kComprehensionCond2: + case ExpressionStepKind::kFastIn: + case ExpressionStepKind::kFastEqual: + case ExpressionStepKind::kFastNotEqual: + case ExpressionStepKind::kMutableListAppend: + return -1; + case ExpressionStepKind::kClearSlots: + case ExpressionStepKind::kBooleanNot: + case ExpressionStepKind::kNotStrictlyFalse: + case ExpressionStepKind::kBooleanOrJump: + case ExpressionStepKind::kBooleanAndJump: + case ExpressionStepKind::kTernaryJump: + case ExpressionStepKind::kFixedJump: + case ExpressionStepKind::kBitIsSet: + case ExpressionStepKind::kBitAndIsNonZero: + case ExpressionStepKind::kBitAndEqual: + case ExpressionStepKind::kBitEqual: + return 0; + case ExpressionStepKind::kBooleanOr: + case ExpressionStepKind::kBooleanAnd: + return OneMinusArgCount(u_.arg_count); + case ExpressionStepKind::kEagerFunction: + return OneMinusArgCount(u_.eager_function_step->num_arguments()); + case ExpressionStepKind::kLazyFunction: + return OneMinusArgCount(u_.lazy_function_step->num_arguments()); + case ExpressionStepKind::kCreateList: + return OneMinusArgCount(u_.create_list_step->num_elements()); + case ExpressionStepKind::kCreateSmallList: + return OneMinusArgCount(u_.create_small_list_step.list_size); + } + return std::nullopt; +} } // namespace google::api::expr::runtime diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index 575abfa05..e9a96f52c 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -17,26 +17,37 @@ #include #include +#include #include +#include #include #include +#include "absl/base/attributes.h" +#include "absl/base/no_destructor.h" #include "absl/base/nullability.h" +#include "absl/base/optimization.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/span.h" +#include "base/attribute_matcher.h" #include "base/type_provider.h" -#include "common/native_type.h" +#include "common/internal/unknowns.h" #include "common/value.h" -#include "eval/eval/attribute_utility.h" +#include "eval/eval/bitset_steps.h" #include "eval/eval/comprehension_slots.h" +#include "eval/eval/comprehension_step.h" +#include "eval/eval/create_list_step.h" +#include "eval/eval/equality_steps.h" #include "eval/eval/evaluator_stack.h" +#include "eval/eval/expression_step_logic.h" +#include "eval/eval/function_step.h" +#include "eval/eval/ident_step.h" #include "eval/eval/iterator_stack.h" +#include "eval/eval/lazy_init_step.h" #include "runtime/activation_interface.h" -#include "runtime/internal/activation_attribute_matcher_access.h" #include "runtime/runtime.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -54,25 +65,94 @@ class ExecutionFrame; using EvaluationListener = cel::TraceableProgram::EvaluationListener; -// Class Expression represents single execution step. +enum class ExpressionStepKind : uint16_t { + kMovedFrom = 0, + kGenericLogic = 1, + kIntConstant = 2, + kBoolConstant = 3, + kDoubleConstant = 4, + kNullConstant = 5, + kUintConstant = 6, + // Any constant that can't be inlined. + kOtherConstant = 7, + // Lazy subexpressions. + kLazyInit = 8, + kAssignSlotAndPop = 9, + kClearSlots = 10, + // Core boolean logic. + kBooleanNot = 11, + kNotStrictlyFalse = 12, + kBooleanOr = 13, + kBooleanAnd = 14, + // Comprehension steps. + kComprehensionFinish = 15, + kComprehensionNext = 16, + kComprehensionCond = 17, + kComprehensionNext2 = 18, + kComprehensionCond2 = 19, + kReadSlot = 20, + // Jump steps. + kBooleanOrJump = 21, + kBooleanAndJump = 22, + kTernaryJump = 23, + kFixedJump = 24, + // Identifier + kIdentifier = 25, + // Functions calls. + kEagerFunction = 26, + kLazyFunction = 27, + // fast built-ins. These are used if we know they haven't been extended. + // otherwise we use normal function call steps. + kFastIn = 28, + kFastEqual = 29, + kFastNotEqual = 30, + // Special built-in steps for mutable lists implementing map/filter. + kNewMutableList = 31, + kMutableListAppend = 32, + // Create list step. + kCreateList = 33, + kCreateSmallList = 34, + // Specialized bit operations implementing set checks. + // Each step contains a constant uint64_t bitmask. + kBitIsSet = 35, // In (member) (1 << arg) & mask != 0 + kBitAndIsNonZero = 36, // Any (intersect) & mask != 0 + kBitAndEqual = 37, // All (contains) & mask == mask + kBitEqual = 38, // Equivalent == mask + // Comprehension init step. + kComprehensionInit = 39, +}; + +struct BoolJumpStepInfo { + size_t arg_count : 32; + bool set : 1; + int offset : 31; +}; + +struct TernaryJumpStepInfo { + bool set : 1; + int error_offset : 31; + int jump_to_second_offset : 32; +}; + +struct FixedJumpStepInfo { + bool set : 1; + int32_t reserved : 31; + int offset : 32; +}; + +// Friend of ExpressionStep that allows the interpreter loop in the source file +// to access the internal expression step data. +struct ExpressionStepData; + class ExpressionStep { public: - explicit ExpressionStep(int64_t id, bool comes_from_ast = true) - : id_(id), comes_from_ast_(comes_from_ast) {} - + // Move-only. ExpressionStep(const ExpressionStep&) = delete; ExpressionStep& operator=(const ExpressionStep&) = delete; + ExpressionStep(ExpressionStep&&); + ExpressionStep& operator=(ExpressionStep&&); - virtual ~ExpressionStep() = default; - - // Performs actual evaluation. - // Values are passed between Expression objects via EvaluatorStack, which is - // supplied with context. - // Also, Expression gets values supplied by caller though Activation - // interface. - // ExpressionStep instances can in specific cases - // modify execution order(perform jumps). - virtual absl::Status Evaluate(ExecutionFrame* context) const = 0; + ~ExpressionStep(); // Returns corresponding expression object ID. // Requires that the input expression has IDs assigned to sub-expressions, @@ -80,26 +160,352 @@ class ExpressionStep { // expression associated (e.g. a jump step), or if there is no ID assigned to // the corresponding expression. Useful for error scenarios where information // from Expr object is needed to create CelError. - int64_t id() const { return id_; } + int64_t id() const { + return header_.id >= 0 ? static_cast(header_.id) : -1; + } + + ExpressionStepKind kind() const { return header_.kind; } // Returns if the execution step comes from AST. - bool comes_from_ast() const { return comes_from_ast_; } + bool comes_from_ast() const { return header_.id >= 0; } + + const ExpressionStepLogic* GetGenericStep() const; + bool IsGenericStep() const; + + // Returns the worst-case change in value stack size when evaluating this + // step, or std::nullopt if the delta overflows. + std::optional StackDelta() const; + + static ExpressionStep MakeGenericStep( + std::unique_ptr logic, int64_t id = -1, + int64_t stack_delta = 1) { + ExpressionStep step(ExpressionStepKind::kGenericLogic, id); + if (stack_delta < std::numeric_limits::min() || + stack_delta >= std::numeric_limits::max()) { + step.header_.stack_delta = std::numeric_limits::max(); + } else { + step.header_.stack_delta = static_cast(stack_delta); + } + step.u_.logic = logic.release(); + return step; + } + + static ExpressionStep MakeConstant(const cel::Value& value, int64_t id = -1); + + static ExpressionStep MakeLazyInitStep(size_t slot_index, + size_t subexpression_index, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kLazyInit, id); + ABSL_DCHECK_LE(slot_index, std::numeric_limits::max()); + ABSL_DCHECK_LE(subexpression_index, std::numeric_limits::max()); + step.u_.lazy_init = LazyInitStepInfo{slot_index, subexpression_index}; + return step; + } + + static ExpressionStep MakeAssignSlotAndPopStep(size_t slot_index, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kAssignSlotAndPop, id); + ABSL_DCHECK_LE(slot_index, std::numeric_limits::max()); + step.u_.slot_index = slot_index; + return step; + } + + static ExpressionStep MakeClearSlotStep(size_t slot_index, int64_t id = -1) { + return MakeClearSlotsStep(slot_index, 1, id); + } + + static ExpressionStep MakeClearSlotsStep(size_t slot_index, size_t slot_count, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kClearSlots, id); + ABSL_DCHECK_LE(slot_index, std::numeric_limits::max()); + ABSL_DCHECK_LE(slot_count, std::numeric_limits::max()); + step.u_.clear_slots = ClearSlotStepInfo{slot_index, slot_count}; + return step; + } + + static ExpressionStep MakeBooleanNotStep(int64_t id = -1) { + return ExpressionStep(ExpressionStepKind::kBooleanNot, id); + } + + static ExpressionStep MakeNotStrictlyFalseStep(int64_t id = -1) { + return ExpressionStep(ExpressionStepKind::kNotStrictlyFalse, id); + } + + static ExpressionStep MakeBooleanOrStep(size_t num_args, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kBooleanOr, id); + ABSL_DCHECK_LT(num_args, std::numeric_limits::max()); + step.u_.arg_count = num_args; + return step; + } + + static ExpressionStep MakeBooleanAndStep(size_t num_args, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kBooleanAnd, id); + ABSL_DCHECK_LT(num_args, std::numeric_limits::max()); + step.u_.arg_count = num_args; + return step; + } + + static ExpressionStep MakeComprehensionFinishStep(size_t accu_slot, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kComprehensionFinish, id); + step.u_.slot_index = accu_slot; + return step; + } + + static ExpressionStep MakeComprehensionNextStep(int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kComprehensionNext, id); + step.u_.next_step = ComprehensionNextStepInfo(); + return step; + } + + static ExpressionStep MakeComprehensionNext2Step(int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kComprehensionNext2, id); + step.u_.next_step = ComprehensionNextStepInfo(); + return step; + } + + static ExpressionStep MakeComprehensionCondStep(int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kComprehensionCond, id); + step.u_.cond_step = ComprehensionCondStepInfo(); + return step; + } + + static ExpressionStep MakeComprehensionCond2Step(int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kComprehensionCond2, id); + step.u_.cond_step = ComprehensionCondStepInfo(); + return step; + } + + static ExpressionStep MakeReadSlotStep(size_t slot_index, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kReadSlot, id); + ABSL_DCHECK_LT(slot_index, std::numeric_limits::max()); + step.u_.slot_index = slot_index; + return step; + } + + static ExpressionStep MakeBooleanOrJumpStep(size_t arg_count, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kBooleanOrJump, id); + ABSL_DCHECK_LT(arg_count, std::numeric_limits::max()); + step.u_.bool_jump_step = BoolJumpStepInfo{arg_count, false, 0}; + return step; + } + + static ExpressionStep MakeBooleanAndJumpStep(size_t arg_count, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kBooleanAndJump, id); + ABSL_DCHECK_LT(arg_count, std::numeric_limits::max()); + step.u_.bool_jump_step = BoolJumpStepInfo{arg_count, false, 0}; + return step; + } + + static ExpressionStep MakeTernaryJumpStep(int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kTernaryJump, id); + step.u_.ternary_jump_step = TernaryJumpStepInfo{false, 0, 0}; + return step; + } + + static ExpressionStep MakeFixedJumpStep(int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kFixedJump, id); + step.u_.fixed_jump_step = FixedJumpStepInfo{false, 0, 0}; + return step; + } + + static ExpressionStep MakeIdentifierStep(size_t ident_index, + size_t slot_index, int64_t id) { + ExpressionStep step(ExpressionStepKind::kIdentifier, id); + ABSL_DCHECK_LE(ident_index, std::numeric_limits::max()); + ABSL_DCHECK_LE(slot_index, std::numeric_limits::max()); + step.u_.ident_step = IdentStepInfo{static_cast(ident_index), + static_cast(slot_index)}; + return step; + } + + static ExpressionStep MakeIdentifierStep(size_t ident_index, + int64_t id = -1) { + return MakeIdentifierStep(ident_index, ident_index, id); + } + + static ExpressionStep MakeIdentStep(size_t ident_index, int64_t id = -1) { + return MakeIdentifierStep(ident_index, id); + } + + static ExpressionStep MakeFastInStep(int64_t id = -1) { + return ExpressionStep(ExpressionStepKind::kFastIn, id); + } + + static ExpressionStep MakeFastEqualStep(int64_t id = -1) { + return ExpressionStep(ExpressionStepKind::kFastEqual, id); + } + + static ExpressionStep MakeFastNotEqualStep(int64_t id = -1) { + return ExpressionStep(ExpressionStepKind::kFastNotEqual, id); + } + + static ExpressionStep MakeNewMutableListStep(int64_t id = -1) { + return ExpressionStep(ExpressionStepKind::kNewMutableList, id); + } + + static ExpressionStep MakeMutableListAppendStep(int64_t id = -1) { + return ExpressionStep(ExpressionStepKind::kMutableListAppend, id); + } + + static ExpressionStep MakeEagerFunctionStep( + std::unique_ptr step_impl, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kEagerFunction, id); + step.u_.eager_function_step = step_impl.release(); + return step; + } + + static ExpressionStep MakeLazyFunctionStep( + std::unique_ptr step_impl, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kLazyFunction, id); + step.u_.lazy_function_step = step_impl.release(); + return step; + } + + static ExpressionStep MakeCreateListStep( + std::unique_ptr step_impl, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kCreateList, id); + step.u_.create_list_step = step_impl.release(); + return step; + } + + static ExpressionStep MakeCreateSmallListStep(SmallListStepInfo info, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kCreateSmallList, id); + step.u_.create_small_list_step = info; + return step; + } + + static ExpressionStep MakeBitIsSetStep(uint64_t bitmask, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kBitIsSet, id); + step.u_.bitmask = bitmask; + return step; + } + + static ExpressionStep MakeBitAndIsNonZeroStep(uint64_t bitmask, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kBitAndIsNonZero, id); + step.u_.bitmask = bitmask; + return step; + } + + static ExpressionStep MakeBitAndEqualStep(uint64_t bitmask, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kBitAndEqual, id); + step.u_.bitmask = bitmask; + return step; + } + + static ExpressionStep MakeBitEqualStep(uint64_t bitmask, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kBitEqual, id); + step.u_.bitmask = bitmask; + return step; + } - // Return the type of the underlying expression step for special handling in - // the planning phase. This should only be overridden by special cases, and - // callers must not make any assumptions about the default case. - virtual cel::NativeTypeId GetNativeTypeId() const { - return cel::NativeTypeId(); + static ExpressionStep MakeComprehensionInitStep(uint32_t first_slot, + ComprehensionVersion version, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kComprehensionInit, id); + step.u_.init_step = ComprehensionInitStepInfo{ + .first_slot = first_slot, + .v2 = version == ComprehensionVersion::kV2, + }; + return step; } private: - const int64_t id_; - const bool comes_from_ast_; + friend struct ExpressionStepData; + + struct Header { + ExpressionStepKind kind; + // Note: This can be moved if steps are all migrated. + int16_t stack_delta; + int32_t id; + }; + + union Data { + std::nullptr_t empty; + ExpressionStepLogic* logic; + int64_t int_val; + uint64_t uint_val; + double double_val; + bool bool_val; + cel::Value* other_val; + LazyInitStepInfo lazy_init; + size_t slot_index; + ClearSlotStepInfo clear_slots; + size_t arg_count; + ComprehensionCondStepInfo cond_step; + ComprehensionNextStepInfo next_step; + BoolJumpStepInfo bool_jump_step; + TernaryJumpStepInfo ternary_jump_step; + FixedJumpStepInfo fixed_jump_step; + EagerFunctionStep* eager_function_step; + LazyFunctionStep* lazy_function_step; + IdentStepInfo ident_step; + ListStepInfo* create_list_step; + SmallListStepInfo create_small_list_step; + uint64_t bitmask; + ComprehensionInitStepInfo init_step; + + Data() : empty(nullptr) {} + ~Data() {} + }; + + ExpressionStep() : header_{ExpressionStepKind::kMovedFrom, 0, -1} {} + ExpressionStep(ExpressionStepKind kind, int32_t id) : header_{kind, 0, id} { + header_ = {kind, 0, id}; + } + ExpressionStep(ExpressionStepKind kind, int64_t id) : ExpressionStep() { + if (id < 0 || id > std::numeric_limits::max()) { + id = -1; + } + header_ = {kind, 0, static_cast(id)}; + } + ExpressionStep(ExpressionStepKind kind, int32_t id, + std::unique_ptr logic) + : ExpressionStep(kind, id) { + u_.logic = logic.release(); + } + + friend void swap(ExpressionStep& lhs, ExpressionStep& rhs) { + using std::swap; + swap(lhs.header_, rhs.header_); + swap(lhs.u_, rhs.u_); + } + + friend bool GetIfConstant(const ExpressionStep& step, cel::Value& out); + friend bool IsConstant(const ExpressionStep& step); + friend ComprehensionCondStepInfo* GetIfComprehensionCondStep( + ExpressionStep& step); + friend ComprehensionNextStepInfo* GetIfComprehensionNextStep( + ExpressionStep& step); + friend ComprehensionInitStepInfo* GetIfComprehensionInitStep( + ExpressionStep& step); + friend BoolJumpStepInfo* GetIfBoolJumpStep(ExpressionStep& step); + friend TernaryJumpStepInfo* GetIfTernaryJumpStep(ExpressionStep& step); + friend FixedJumpStepInfo* GetIfFixedJumpStep(ExpressionStep& step); + friend IdentStepInfo* GetIfIdentStep(ExpressionStep& step); + friend const IdentStepInfo* GetIfIdentStep(const ExpressionStep& step); + + Header header_; + // Note: ptr members are 'owned' by the step. + // ~ExpressionStep() is responsible for freeing. + Data u_; }; -using ExecutionPath = std::vector>; -using ExecutionPathView = - absl::Span>; +#ifndef _MSC_VER +// Keep the core instruction size small to get better memory locality for the +// main program. +// +// MSVC does not support some of the bit-field packing used here so it will be +// larger. +static_assert(sizeof(ExpressionStep) == 16); +#endif + +using ExecutionPath = std::vector; +using ExecutionPathView = absl::Span; // Class that wraps the state that needs to be allocated for expression // evaluation. This can be reused to save on allocations. @@ -107,22 +513,34 @@ class FlatExpressionEvaluatorState { public: FlatExpressionEvaluatorState( size_t value_stack_size, size_t comprehension_slot_count, - const cel::TypeProvider& type_provider, + size_t iterator_stack_size, const cel::TypeProvider& type_provider, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nullable message_factory = nullptr) + : value_stack_(value_stack_size), + iterator_stack_(iterator_stack_size), + comprehension_slots_(comprehension_slot_count), + type_provider_(type_provider), + descriptor_pool_(descriptor_pool), + message_factory_(message_factory), + unknown_tree_() {} + + FlatExpressionEvaluatorState( + size_t value_stack_size, size_t comprehension_slot_count, + size_t iterator_stack_size, const cel::TypeProvider& type_provider, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) : value_stack_(value_stack_size), - // We currently use comprehension_slot_count because it is less of an - // over estimate than value_stack_size. In future we should just - // calculate the correct capacity. - iterator_stack_(comprehension_slot_count), + iterator_stack_(iterator_stack_size), comprehension_slots_(comprehension_slot_count), type_provider_(type_provider), descriptor_pool_(descriptor_pool), message_factory_(message_factory), - arena_(arena) {} + arena_(arena), + unknown_tree_(arena) {} void Reset(); + void MarkDirty() { dirty_ = true; } EvaluatorStack& value_stack() { return value_stack_; } @@ -139,10 +557,23 @@ class FlatExpressionEvaluatorState { } google::protobuf::MessageFactory* absl_nonnull message_factory() { + ABSL_DCHECK(message_factory_ != nullptr); return message_factory_; } - google::protobuf::Arena* absl_nonnull arena() { return arena_; } + google::protobuf::Arena* absl_nonnull arena() { + ABSL_DCHECK(arena_ != nullptr); + return arena_; + } + + cel::common_internal::UnknownTree* unknown_tree() { return &unknown_tree_; } + + void Rebind(google::protobuf::Arena* absl_nonnull arena, + google::protobuf::MessageFactory* absl_nonnull message_factory) { + arena_ = arena; + message_factory_ = message_factory; + unknown_tree_.Reset(arena); + } private: EvaluatorStack value_stack_; @@ -150,8 +581,10 @@ class FlatExpressionEvaluatorState { ComprehensionSlots comprehension_slots_; const cel::TypeProvider& type_provider_; const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool_; - google::protobuf::MessageFactory* absl_nonnull message_factory_; - google::protobuf::Arena* absl_nonnull arena_; + google::protobuf::MessageFactory* absl_nullability_unknown message_factory_ = nullptr; + google::protobuf::Arena* absl_nullability_unknown arena_ = nullptr; + cel::common_internal::UnknownTree unknown_tree_; + bool dirty_ = false; }; // Context needed for evaluation. This is sufficient for supporting @@ -165,7 +598,8 @@ class ExecutionFrameBase { const cel::TypeProvider& type_provider, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena) + google::protobuf::Arena* absl_nonnull arena, + cel::common_internal::UnknownTree* unknown_tree) : activation_(&activation), callback_(), options_(&options), @@ -174,19 +608,32 @@ class ExecutionFrameBase { message_factory_(message_factory), arena_(arena), embedder_context_(nullptr), - attribute_utility_(activation.GetUnknownAttributes(), - activation.GetMissingAttributes()), + unknown_tree_(unknown_tree), + unknown_attributes_(options_->unknown_processing != + cel::UnknownProcessingOptions::kDisabled + ? activation.GetUnknownAttributeMatcher() + : cel::EmptyAttributeMatcher()), + known_attributes_(options_->unknown_processing != + cel::UnknownProcessingOptions::kDisabled + ? activation.GetKnownAttributeMatcher() + : cel::EmptyAttributeMatcher()), + missing_attributes_(options_->enable_missing_attribute_errors + ? activation.GetMissingAttributeMatcher() + : cel::EmptyAttributeMatcher()), slots_(&ComprehensionSlots::GetEmptyInstance()), max_iterations_(options.comprehension_max_iterations), - iterations_(0) { - if (unknown_processing_enabled()) { - if (auto matcher = cel::runtime_internal:: - ActivationAttributeMatcherAccess::GetAttributeMatcher(activation); - matcher != nullptr) { - attribute_utility_.set_matcher(matcher); - } - } - } + iterations_(0), + attribute_tracking_enabled_( + options_->unknown_processing != + cel::UnknownProcessingOptions::kDisabled || + options_->enable_missing_attribute_errors), + missing_attribute_errors_enabled_( + options_->enable_missing_attribute_errors), + unknown_processing_enabled_(options_->unknown_processing != + cel::UnknownProcessingOptions::kDisabled), + unknown_function_results_enabled_( + options_->unknown_processing == + cel::UnknownProcessingOptions::kAttributeAndFunction) {} ExecutionFrameBase(const cel::ActivationInterface& activation, EvaluationListener callback, @@ -196,7 +643,8 @@ class ExecutionFrameBase { google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena, const cel::EmbedderContext* absl_nullable embedder_context, - ComprehensionSlots& slots) + ComprehensionSlots& slots, + cel::common_internal::UnknownTree* unknown_tree) : activation_(&activation), callback_(std::move(callback)), options_(&options), @@ -205,19 +653,32 @@ class ExecutionFrameBase { message_factory_(message_factory), arena_(arena), embedder_context_(embedder_context), - attribute_utility_(activation.GetUnknownAttributes(), - activation.GetMissingAttributes()), + unknown_tree_(unknown_tree), + unknown_attributes_(options_->unknown_processing != + cel::UnknownProcessingOptions::kDisabled + ? activation.GetUnknownAttributeMatcher() + : cel::EmptyAttributeMatcher()), + known_attributes_(options_->unknown_processing != + cel::UnknownProcessingOptions::kDisabled + ? activation.GetKnownAttributeMatcher() + : cel::EmptyAttributeMatcher()), + missing_attributes_(options_->enable_missing_attribute_errors + ? activation.GetMissingAttributeMatcher() + : cel::EmptyAttributeMatcher()), slots_(&slots), max_iterations_(options.comprehension_max_iterations), - iterations_(0) { - if (unknown_processing_enabled()) { - if (auto matcher = cel::runtime_internal:: - ActivationAttributeMatcherAccess::GetAttributeMatcher(activation); - matcher != nullptr) { - attribute_utility_.set_matcher(matcher); - } - } - } + iterations_(0), + attribute_tracking_enabled_( + options_->unknown_processing != + cel::UnknownProcessingOptions::kDisabled || + options_->enable_missing_attribute_errors), + missing_attribute_errors_enabled_( + options_->enable_missing_attribute_errors), + unknown_processing_enabled_(options_->unknown_processing != + cel::UnknownProcessingOptions::kDisabled), + unknown_function_results_enabled_( + options_->unknown_processing == + cel::UnknownProcessingOptions::kAttributeAndFunction) {} const cel::ActivationInterface& activation() const { return *activation_; } @@ -241,28 +702,36 @@ class ExecutionFrameBase { return embedder_context_; } - const AttributeUtility& attribute_utility() const { - return attribute_utility_; + cel::common_internal::UnknownTree* absl_nonnull unknown_tree() const { + return unknown_tree_; } bool attribute_tracking_enabled() const { - return options_->unknown_processing != - cel::UnknownProcessingOptions::kDisabled || - options_->enable_missing_attribute_errors; + return attribute_tracking_enabled_; } bool missing_attribute_errors_enabled() const { - return options_->enable_missing_attribute_errors; + return missing_attribute_errors_enabled_; } bool unknown_processing_enabled() const { - return options_->unknown_processing != - cel::UnknownProcessingOptions::kDisabled; + return unknown_processing_enabled_; } bool unknown_function_results_enabled() const { - return options_->unknown_processing == - cel::UnknownProcessingOptions::kAttributeAndFunction; + return unknown_function_results_enabled_; + } + + const cel::AttributeMatcher& unknown_attributes() const { + return unknown_attributes_; + } + + const cel::AttributeMatcher& known_attributes() const { + return known_attributes_; + } + + const cel::AttributeMatcher& missing_attributes() const { + return missing_attributes_; } ComprehensionSlots& comprehension_slots() { return *slots_; } @@ -281,6 +750,8 @@ class ExecutionFrameBase { return absl::OkStatus(); } + absl::Status& abort_status() { return abort_status_; } + protected: const cel::ActivationInterface* absl_nonnull activation_; EvaluationListener callback_; @@ -290,10 +761,18 @@ class ExecutionFrameBase { google::protobuf::MessageFactory* absl_nonnull message_factory_; google::protobuf::Arena* absl_nonnull arena_; const cel::EmbedderContext* absl_nullable embedder_context_; - AttributeUtility attribute_utility_; + cel::common_internal::UnknownTree* absl_nonnull unknown_tree_; + const cel::AttributeMatcher& unknown_attributes_; + const cel::AttributeMatcher& known_attributes_; + const cel::AttributeMatcher& missing_attributes_; ComprehensionSlots* absl_nonnull slots_; const int max_iterations_; int iterations_; + absl::Status abort_status_; + const bool attribute_tracking_enabled_; + const bool missing_attribute_errors_enabled_; + const bool unknown_processing_enabled_; + const bool unknown_function_results_enabled_; }; // ExecutionFrame manages the context needed for expression evaluation. @@ -309,37 +788,53 @@ class ExecutionFrame : public ExecutionFrameBase { ExecutionPathView flat, const cel::ActivationInterface& activation, const cel::RuntimeOptions& options, FlatExpressionEvaluatorState& state, EvaluationListener callback = EvaluationListener(), - const cel::EmbedderContext* absl_nullable embedder_context = nullptr) + const cel::EmbedderContext* absl_nullable embedder_context = nullptr, + const IdentifierTable& identifiers = EmptyIdentifierTable()) : ExecutionFrameBase(activation, std::move(callback), options, state.type_provider(), state.descriptor_pool(), state.message_factory(), state.arena(), - embedder_context, state.comprehension_slots()), + embedder_context, state.comprehension_slots(), + state.unknown_tree()), pc_(0UL), execution_path_(flat), value_stack_(&state.value_stack()), iterator_stack_(&state.iterator_stack()), - subexpressions_() {} + subexpressions_(&execution_path_, 1), + identifiers_(&identifiers) { + state.MarkDirty(); + } ExecutionFrame( absl::Span subexpressions, const cel::ActivationInterface& activation, const cel::RuntimeOptions& options, FlatExpressionEvaluatorState& state, EvaluationListener callback = EvaluationListener(), - const cel::EmbedderContext* absl_nullable embedder_context = nullptr) + const cel::EmbedderContext* absl_nullable embedder_context = nullptr, + const IdentifierTable& identifiers = EmptyIdentifierTable()) : ExecutionFrameBase(activation, std::move(callback), options, state.type_provider(), state.descriptor_pool(), state.message_factory(), state.arena(), - embedder_context, state.comprehension_slots()), + embedder_context, state.comprehension_slots(), + state.unknown_tree()), pc_(0UL), execution_path_(subexpressions[0]), value_stack_(&state.value_stack()), iterator_stack_(&state.iterator_stack()), - subexpressions_(subexpressions) { + subexpressions_(subexpressions), + identifiers_(&identifiers) { ABSL_DCHECK(!subexpressions.empty()); + state.MarkDirty(); } // Returns next expression to evaluate. - const ExpressionStep* Next(); + ABSL_ATTRIBUTE_ALWAYS_INLINE const ExpressionStep* Next() { + if (ABSL_PREDICT_TRUE(pc_ < execution_path_.size())) { + const ExpressionStep* step = &execution_path_[pc_++]; + ABSL_ASSUME(step != nullptr); + return step; + } + return NextSlow(); + } // Evaluate the execution frame to completion. absl::StatusOr Evaluate(EvaluationListener& listener); @@ -350,19 +845,17 @@ class ExecutionFrame : public ExecutionFrameBase { // // Offset applies after normal pc increment. For example, JumpTo(0) is a // no-op, JumpTo(1) skips the expected next step. - absl::Status JumpTo(int offset) { + ABSL_ATTRIBUTE_ALWAYS_INLINE void JumpToOrAbort(int offset) { ABSL_DCHECK_LE(offset, static_cast(execution_path_.size())); ABSL_DCHECK_GE(offset, -static_cast(pc_)); int new_pc = static_cast(pc_) + offset; - if (new_pc < 0 || new_pc > static_cast(execution_path_.size())) { - return absl::Status(absl::StatusCode::kInternal, - absl::StrCat("Jump address out of range: position: ", - pc_, ", offset: ", offset, - ", range: ", execution_path_.size())); + if (ABSL_PREDICT_FALSE(new_pc < 0 || + new_pc > static_cast(execution_path_.size()))) { + AbortJumpOutOfRange(offset); + return; } pc_ = static_cast(new_pc); - return absl::OkStatus(); } // Move pc to a subexpression. @@ -375,7 +868,7 @@ class ExecutionFrame : public ExecutionFrameBase { void Call(size_t slot_index, size_t subexpression_index) { ABSL_DCHECK_LT(subexpression_index, subexpressions_.size()); ExecutionPathView subexpression = subexpressions_[subexpression_index]; - ABSL_DCHECK(subexpression != execution_path_); + ABSL_DCHECK(subexpression.data() != execution_path_.data()); size_t return_pc = pc_; // return pc == size() is supported (a tail call). ABSL_DCHECK_LE(return_pc, execution_path_.size()); @@ -391,6 +884,8 @@ class ExecutionFrame : public ExecutionFrameBase { return *iterator_stack_; } + const IdentifierTable& identifiers() const { return *identifiers_; } + bool enable_attribute_tracking() const { return attribute_tracking_enabled(); } @@ -418,7 +913,21 @@ class ExecutionFrame : public ExecutionFrameBase { return *activation_; } + void Abort(absl::Status status) { + ABSL_DCHECK(!subexpressions_.empty()); + ABSL_DCHECK(!status.ok()); + abort_status_.Update(std::move(status)); + call_stack_.clear(); + execution_path_ = subexpressions_[0]; + pc_ = execution_path_.size(); + } + private: + static const IdentifierTable& EmptyIdentifierTable() { + static const absl::NoDestructor kEmpty; + return *kEmpty; + } + struct SubFrame { size_t return_pc; size_t slot_index; @@ -426,11 +935,27 @@ class ExecutionFrame : public ExecutionFrameBase { size_t expected_stack_size; }; + // Runs steps until the program completes or is aborted. + void Interpret(); + + // Same as Interpret(), but also calls `listener` with the result of each + // step that maps to an AST node. + void InterpretWithCallback(EvaluationListener& listener); + + // Slow path of Next(): returns from a completed subexpression, or returns + // nullptr at the end of the program. + ABSL_ATTRIBUTE_NOINLINE const ExpressionStep* NextSlow(); + + // Error path of JumpToOrAbort(). + ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_COLD void AbortJumpOutOfRange( + int offset); + size_t pc_; // pc_ - Program Counter. Current position on execution path. ExecutionPathView execution_path_; EvaluatorStack* absl_nonnull const value_stack_; cel::runtime_internal::IteratorStack* absl_nonnull const iterator_stack_; absl::Span subexpressions_; + const IdentifierTable* absl_nonnull identifiers_; std::vector call_stack_; }; @@ -443,26 +968,36 @@ class FlatExpression { FlatExpression(ExecutionPath path, size_t comprehension_slots_size, const cel::TypeProvider& type_provider, const cel::RuntimeOptions& options, - absl_nullable std::shared_ptr arena = nullptr) + absl_nullable std::shared_ptr arena, + IdentifierTable identifiers, + std::optional value_stack_size = std::nullopt) : path_(std::move(path)), subexpressions_({path_}), comprehension_slots_size_(comprehension_slots_size), + iterator_stack_size_(comprehension_slots_size), + value_stack_size_(value_stack_size.value_or(path_.size())), type_provider_(type_provider), options_(options), - arena_(std::move(arena)) {} + arena_(std::move(arena)), + identifiers_(std::move(identifiers)) {} FlatExpression(ExecutionPath path, std::vector subexpressions, - size_t comprehension_slots_size, + size_t comprehension_slots_size, size_t iterator_stack_size, const cel::TypeProvider& type_provider, const cel::RuntimeOptions& options, - absl_nullable std::shared_ptr arena = nullptr) + absl_nullable std::shared_ptr arena, + IdentifierTable identifiers, + std::optional value_stack_size = std::nullopt) : path_(std::move(path)), subexpressions_(std::move(subexpressions)), comprehension_slots_size_(comprehension_slots_size), + iterator_stack_size_(iterator_stack_size), + value_stack_size_(value_stack_size.value_or(path_.size())), type_provider_(type_provider), options_(options), - arena_(std::move(arena)) {} + arena_(std::move(arena)), + identifiers_(std::move(identifiers)) {} // Move-only FlatExpression(FlatExpression&&) = default; @@ -475,6 +1010,10 @@ class FlatExpression { google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) const; + FlatExpressionEvaluatorState MakeEvaluatorState( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nullable message_factory = nullptr) const; + // Evaluate the expression. // // A status may be returned if an unexpected error occurs. Recoverable errors @@ -494,23 +1033,180 @@ class FlatExpression { return subexpressions_; } + const IdentifierTable& identifiers() const { return identifiers_; } + const cel::RuntimeOptions& options() const { return options_; } size_t comprehension_slots_size() const { return comprehension_slots_size_; } + size_t iterator_stack_size() const { return iterator_stack_size_; } + + size_t value_stack_size() const { return value_stack_size_; } + const cel::TypeProvider& type_provider() const { return type_provider_; } private: ExecutionPath path_; std::vector subexpressions_; size_t comprehension_slots_size_; + size_t iterator_stack_size_; + size_t value_stack_size_; const cel::TypeProvider& type_provider_; cel::RuntimeOptions options_; // Arena used during planning phase, may hold constant values so should be // kept alive. absl_nullable std::shared_ptr arena_; + IdentifierTable identifiers_; }; +// Helper functions for checking ExpressionStep kinds. Used for program +// optimization. + +// Checks if the step is a constant and if so, writes the value into `out`. +// Returns true if the step is a constant, false otherwise. +bool GetIfConstant(const ExpressionStep& step, cel::Value& out); + +// Checks if the step is a constant. +bool IsConstant(const ExpressionStep& step); + +ComprehensionCondStepInfo* GetIfComprehensionCondStep(ExpressionStep& step); +ComprehensionNextStepInfo* GetIfComprehensionNextStep(ExpressionStep& step); +ComprehensionInitStepInfo* GetIfComprehensionInitStep(ExpressionStep& step); + +BoolJumpStepInfo* GetIfBoolJumpStep(ExpressionStep& step); +BoolJumpStepInfo* GetIfBoolJumpStep(ExpressionStep* step); + +TernaryJumpStepInfo* GetIfTernaryJumpStep(ExpressionStep& step); +TernaryJumpStepInfo* GetIfTernaryJumpStep(ExpressionStep* step); + +FixedJumpStepInfo* GetIfFixedJumpStep(ExpressionStep& step); +FixedJumpStepInfo* GetIfFixedJumpStep(ExpressionStep* step); + +IdentStepInfo* GetIfIdentStep(ExpressionStep& step); +const IdentStepInfo* GetIfIdentStep(const ExpressionStep& step); +IdentStepInfo* GetIfIdentStep(ExpressionStep* step); +const IdentStepInfo* GetIfIdentStep(const ExpressionStep* step); + +// Implementation details. + +inline BoolJumpStepInfo* GetIfBoolJumpStep(ExpressionStep* step) { + if (step == nullptr) { + return nullptr; + } + return GetIfBoolJumpStep(*step); +} + +inline TernaryJumpStepInfo* GetIfTernaryJumpStep(ExpressionStep* step) { + if (step == nullptr) { + return nullptr; + } + return GetIfTernaryJumpStep(*step); +} + +inline FixedJumpStepInfo* GetIfFixedJumpStep(ExpressionStep* step) { + if (step == nullptr) { + return nullptr; + } + return GetIfFixedJumpStep(*step); +} + +inline IdentStepInfo* GetIfIdentStep(ExpressionStep* step) { + if (step == nullptr) { + return nullptr; + } + return GetIfIdentStep(*step); +} + +inline const IdentStepInfo* GetIfIdentStep(const ExpressionStep* step) { + if (step == nullptr) { + return nullptr; + } + return GetIfIdentStep(*step); +} + +inline ExpressionStep::~ExpressionStep() { + switch (header_.kind) { + case ExpressionStepKind::kGenericLogic: + delete u_.logic; + break; + case ExpressionStepKind::kOtherConstant: + delete u_.other_val; + break; + case ExpressionStepKind::kEagerFunction: + delete u_.eager_function_step; + break; + case ExpressionStepKind::kLazyFunction: + delete u_.lazy_function_step; + break; + case ExpressionStepKind::kCreateList: + delete u_.create_list_step; + break; + case ExpressionStepKind::kMovedFrom: + case ExpressionStepKind::kIntConstant: + case ExpressionStepKind::kBoolConstant: + case ExpressionStepKind::kDoubleConstant: + case ExpressionStepKind::kNullConstant: + case ExpressionStepKind::kUintConstant: + case ExpressionStepKind::kLazyInit: + case ExpressionStepKind::kAssignSlotAndPop: + case ExpressionStepKind::kClearSlots: + case ExpressionStepKind::kBooleanNot: + case ExpressionStepKind::kNotStrictlyFalse: + case ExpressionStepKind::kBooleanOr: + case ExpressionStepKind::kBooleanAnd: + case ExpressionStepKind::kComprehensionFinish: + case ExpressionStepKind::kComprehensionCond: + case ExpressionStepKind::kComprehensionNext: + case ExpressionStepKind::kComprehensionCond2: + case ExpressionStepKind::kComprehensionNext2: + case ExpressionStepKind::kReadSlot: + case ExpressionStepKind::kBooleanOrJump: + case ExpressionStepKind::kBooleanAndJump: + case ExpressionStepKind::kTernaryJump: + case ExpressionStepKind::kFixedJump: + case ExpressionStepKind::kIdentifier: + case ExpressionStepKind::kFastIn: + case ExpressionStepKind::kFastEqual: + case ExpressionStepKind::kFastNotEqual: + case ExpressionStepKind::kNewMutableList: + case ExpressionStepKind::kMutableListAppend: + case ExpressionStepKind::kCreateSmallList: + case ExpressionStepKind::kBitIsSet: + case ExpressionStepKind::kBitAndIsNonZero: + case ExpressionStepKind::kBitAndEqual: + case ExpressionStepKind::kBitEqual: + case ExpressionStepKind::kComprehensionInit: + break; + default: + ABSL_UNREACHABLE(); + } + header_.kind = ExpressionStepKind::kMovedFrom; + u_.empty = nullptr; +} + +inline const ExpressionStepLogic* ExpressionStep::GetGenericStep() const { + ABSL_DCHECK_EQ(header_.kind, ExpressionStepKind::kGenericLogic); + return u_.logic; +} + +inline bool ExpressionStep::IsGenericStep() const { + return header_.kind == ExpressionStepKind::kGenericLogic; +} + +inline ExpressionStep::ExpressionStep(ExpressionStep&& other) + : ExpressionStep() { + using std::swap; + swap(other, *this); +} + +inline ExpressionStep& ExpressionStep::operator=(ExpressionStep&& other) { + using std::swap; + ExpressionStep tmp; + swap(*this, tmp); + swap(other, *this); + return *this; +} + } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_EVALUATOR_CORE_H_ diff --git a/eval/eval/evaluator_core_test.cc b/eval/eval/evaluator_core_test.cc index 8d61c4659..4ee559441 100644 --- a/eval/eval/evaluator_core_test.cc +++ b/eval/eval/evaluator_core_test.cc @@ -1,15 +1,23 @@ #include "eval/eval/evaluator_core.h" +#include #include +#include #include +#include #include +#include #include "cel/expr/syntax.pb.h" +#include "absl/container/flat_hash_set.h" #include "absl/status/status.h" #include "base/type_provider.h" #include "common/value.h" #include "eval/compiler/cel_expression_builder_flat_impl.h" #include "eval/eval/cel_expression_flat_impl.h" +#include "eval/eval/create_list_step.h" +#include "eval/eval/expression_step_logic.h" +#include "eval/eval/function_step.h" #include "eval/internal/interop.h" #include "eval/public/activation.h" #include "eval/public/builtin_func_registrar.h" @@ -18,6 +26,8 @@ #include "internal/testing_descriptor_pool.h" #include "internal/testing_message_factory.h" #include "runtime/activation.h" +#include "runtime/function_overload_reference.h" +#include "runtime/function_registry.h" #include "runtime/internal/runtime_env_testing.h" #include "runtime/internal/runtime_type_provider.h" #include "runtime/runtime_options.h" @@ -25,6 +35,8 @@ namespace google::api::expr::runtime { +using ::absl_testing::IsOk; +using ::absl_testing::StatusIs; using ::cel::IntValue; using ::cel::TypeProvider; using ::cel::interop_internal::CreateIntValue; @@ -32,33 +44,33 @@ using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::expr::Expr; using ::google::api::expr::runtime::RegisterBuiltinFunctions; using ::testing::_; +using ::testing::ElementsAre; using ::testing::Eq; +using ::testing::HasSubstr; // Fake expression implementation // Pushes int64(0) on top of value stack. -class FakeConstExpressionStep : public ExpressionStep { +class FakeConstExpressionStep : public ExpressionStepLogic { public: - FakeConstExpressionStep() : ExpressionStep(0, true) {} + FakeConstExpressionStep() = default; - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { frame->value_stack().Push(CreateIntValue(0)); - return absl::OkStatus(); } }; // Fake expression implementation // Increments argument on top of the stack. -class FakeIncrementExpressionStep : public ExpressionStep { +class FakeIncrementExpressionStep : public ExpressionStepLogic { public: - FakeIncrementExpressionStep() : ExpressionStep(0, true) {} + FakeIncrementExpressionStep() = default; - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { auto value = frame->value_stack().Peek(); frame->value_stack().Pop(1); EXPECT_TRUE(value->Is()); int64_t val = value.GetInt().NativeValue(); frame->value_stack().Push(CreateIntValue(val + 1)); - return absl::OkStatus(); } }; @@ -67,65 +79,238 @@ TEST(EvaluatorCoreTest, ExecutionFrameNext) { google::protobuf::Arena arena; cel::runtime_internal::RuntimeTypeProvider type_provider( cel::internal::GetTestingDescriptorPool()); - auto const_step = std::make_unique(); - auto incr_step1 = std::make_unique(); - auto incr_step2 = std::make_unique(); - path.push_back(std::move(const_step)); - path.push_back(std::move(incr_step1)); - path.push_back(std::move(incr_step2)); - - auto dummy_expr = std::make_unique(); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kDisabled; cel::Activation activation; FlatExpressionEvaluatorState state( path.size(), - /*comprehension_slots_size=*/0, type_provider, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); ExecutionFrame frame(path, activation, options, state); - EXPECT_THAT(frame.Next(), Eq(path[0].get())); - EXPECT_THAT(frame.Next(), Eq(path[1].get())); - EXPECT_THAT(frame.Next(), Eq(path[2].get())); + EXPECT_THAT(frame.Next(), Eq(&path[0])); + EXPECT_THAT(frame.Next(), Eq(&path[1])); + EXPECT_THAT(frame.Next(), Eq(&path[2])); EXPECT_THAT(frame.Next(), Eq(nullptr)); } -TEST(EvaluatorCoreTest, SimpleEvaluatorTest) { +TEST(EvaluatorCoreTest, JumpOutOfRangeAborts) { ExecutionPath path; - auto const_step = std::make_unique(); - auto incr_step1 = std::make_unique(); - auto incr_step2 = std::make_unique(); + google::protobuf::Arena arena; + cel::runtime_internal::RuntimeTypeProvider type_provider( + cel::internal::GetTestingDescriptorPool()); - path.push_back(std::move(const_step)); - path.push_back(std::move(incr_step1)); - path.push_back(std::move(incr_step2)); + path.push_back(ExpressionStep::MakeFixedJumpStep()); + FixedJumpStepInfo* jump_info = GetIfFixedJumpStep(path.back()); + ASSERT_NE(jump_info, nullptr); + jump_info->set = true; + jump_info->offset = 1; + + cel::RuntimeOptions options; + cel::Activation activation; + FlatExpressionEvaluatorState state( + path.size(), + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, + cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena); + ExecutionFrame frame(path, activation, options, state); + + EXPECT_THAT(frame.Evaluate(), + StatusIs(absl::StatusCode::kInternal, + HasSubstr("Jump address out of range"))); +} + +TEST(EvaluatorCoreTest, TestStepDataAccessors) { + ExpressionStep fixed_jump = ExpressionStep::MakeFixedJumpStep(); + EXPECT_NE(GetIfFixedJumpStep(fixed_jump), nullptr); + EXPECT_NE(GetIfFixedJumpStep(&fixed_jump), nullptr); + EXPECT_EQ(GetIfFixedJumpStep(static_cast(nullptr)), nullptr); + + ExpressionStep bool_or_jump = ExpressionStep::MakeBooleanOrJumpStep(2); + EXPECT_NE(GetIfBoolJumpStep(bool_or_jump), nullptr); + EXPECT_NE(GetIfBoolJumpStep(&bool_or_jump), nullptr); + EXPECT_EQ(GetIfBoolJumpStep(static_cast(nullptr)), nullptr); + + ExpressionStep bool_and_jump = ExpressionStep::MakeBooleanAndJumpStep(2); + EXPECT_NE(GetIfBoolJumpStep(bool_and_jump), nullptr); + EXPECT_NE(GetIfBoolJumpStep(&bool_and_jump), nullptr); + + ExpressionStep ternary_jump = ExpressionStep::MakeTernaryJumpStep(); + EXPECT_NE(GetIfTernaryJumpStep(ternary_jump), nullptr); + EXPECT_NE(GetIfTernaryJumpStep(&ternary_jump), nullptr); + EXPECT_EQ(GetIfTernaryJumpStep(static_cast(nullptr)), + nullptr); + + ExpressionStep ident_step = ExpressionStep::MakeIdentifierStep(1, 2, 3); + ASSERT_NE(GetIfIdentStep(ident_step), nullptr); + EXPECT_EQ(GetIfIdentStep(ident_step)->ident_index, 1); + EXPECT_EQ(GetIfIdentStep(ident_step)->slot_index, 2); + EXPECT_NE(GetIfIdentStep(&ident_step), nullptr); + EXPECT_EQ(GetIfIdentStep(static_cast(nullptr)), nullptr); + + ExpressionStep comp_cond = ExpressionStep::MakeComprehensionCondStep(); + EXPECT_NE(GetIfComprehensionCondStep(comp_cond), nullptr); + + ExpressionStep comp_cond2 = ExpressionStep::MakeComprehensionCond2Step(); + EXPECT_NE(GetIfComprehensionCondStep(comp_cond2), nullptr); + + ExpressionStep comp_next = ExpressionStep::MakeComprehensionNextStep(); + EXPECT_NE(GetIfComprehensionNextStep(comp_next), nullptr); + + ExpressionStep comp_next2 = ExpressionStep::MakeComprehensionNext2Step(); + EXPECT_NE(GetIfComprehensionNextStep(comp_next2), nullptr); + + ExpressionStep const_step = ExpressionStep::MakeConstant(cel::IntValue(42)); + EXPECT_TRUE(IsConstant(const_step)); + cel::Value value; + EXPECT_TRUE(GetIfConstant(const_step, value)); + EXPECT_TRUE(value.IsInt()); + EXPECT_EQ(value.GetInt().NativeValue(), 42); + + ExpressionStep generic_step = ExpressionStep::MakeGenericStep( + std::make_unique()); + EXPECT_TRUE(generic_step.IsGenericStep()); + EXPECT_NE(generic_step.GetGenericStep(), nullptr); + + // Negative checks for mismatched step kinds. + EXPECT_EQ(GetIfFixedJumpStep(bool_or_jump), nullptr); + EXPECT_EQ(GetIfBoolJumpStep(fixed_jump), nullptr); + EXPECT_EQ(GetIfTernaryJumpStep(fixed_jump), nullptr); + EXPECT_EQ(GetIfComprehensionCondStep(fixed_jump), nullptr); + EXPECT_EQ(GetIfComprehensionNextStep(fixed_jump), nullptr); + EXPECT_FALSE(IsConstant(fixed_jump)); + EXPECT_FALSE(GetIfConstant(fixed_jump, value)); + EXPECT_FALSE(fixed_jump.IsGenericStep()); +} + +TEST(EvaluatorCoreTest, SimpleEvaluatorTest) { + ExecutionPath path; + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( env, FlatExpression(std::move(path), 0, env->type_registry.GetComposedTypeProvider(), - cel::RuntimeOptions{})); + cel::RuntimeOptions{}, /*arena=*/nullptr, {})); Activation activation; google::protobuf::Arena arena; auto status = impl.Evaluate(activation, &arena); - EXPECT_OK(status); + ASSERT_THAT(status, IsOk()); auto value = status.value(); EXPECT_TRUE(value.IsInt64()); EXPECT_THAT(value.Int64OrDie(), Eq(2)); } +TEST(EvaluatorCoreTest, MakeConstant) { + cel::runtime_internal::RuntimeTypeProvider type_provider( + cel::internal::GetTestingDescriptorPool()); + google::protobuf::Arena arena; + cel::Activation activation; + cel::RuntimeOptions options; + + auto evaluate_constant = + [&](const cel::Value& value) -> absl::StatusOr { + ExecutionPath path; + path.push_back(ExpressionStep::MakeConstant(value)); + FlatExpression expr(std::move(path), 0, type_provider, options, + /*arena=*/nullptr, {}); + auto state = expr.MakeEvaluatorState( + cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena); + return expr.EvaluateWithCallback(activation, nullptr, nullptr, state); + }; + + ASSERT_OK_AND_ASSIGN(auto bool_val, evaluate_constant(cel::BoolValue(true))); + EXPECT_TRUE(bool_val.IsBool()); + EXPECT_TRUE(bool_val.GetBool().NativeValue()); + + ASSERT_OK_AND_ASSIGN(auto int_val, evaluate_constant(cel::IntValue(42))); + EXPECT_TRUE(int_val.IsInt()); + EXPECT_EQ(int_val.GetInt().NativeValue(), 42); + + ASSERT_OK_AND_ASSIGN(auto uint_val, evaluate_constant(cel::UintValue(100))); + EXPECT_TRUE(uint_val.IsUint()); + EXPECT_EQ(uint_val.GetUint().NativeValue(), 100); + + ASSERT_OK_AND_ASSIGN(auto double_val, + evaluate_constant(cel::DoubleValue(3.14))); + EXPECT_TRUE(double_val.IsDouble()); + EXPECT_DOUBLE_EQ(double_val.GetDouble().NativeValue(), 3.14); + + ASSERT_OK_AND_ASSIGN(auto null_val, evaluate_constant(cel::NullValue())); + EXPECT_TRUE(null_val.IsNull()); + + ASSERT_OK_AND_ASSIGN(auto str_val, + evaluate_constant(cel::StringValue::Literal("hello"))); + EXPECT_TRUE(str_val.IsString()); + EXPECT_EQ(str_val.GetString().ToString(), "hello"); + + auto step_with_id = ExpressionStep::MakeConstant(cel::IntValue(1), 123); + EXPECT_EQ(step_with_id.id(), 123); + EXPECT_TRUE(step_with_id.comes_from_ast()); +} + class MockTraceCallback { public: MOCK_METHOD(void, Call, (int64_t expr_id, const CelValue& value, google::protobuf::Arena*)); }; +TEST(EvaluatorCoreTest, TraceFilterById) { + ExecutionPath path; + // Step with ID 0 should trigger trace callback. + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(), /*id=*/0)); + // Steps with large IDs (> int32_t max) should not trigger trace callback. + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(), + /*id=*/static_cast(std::numeric_limits::max()) + 1)); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(), + /*id=*/std::numeric_limits::max())); + // Steps with negative IDs should not trigger trace callback. + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(), /*id=*/-1)); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(), /*id=*/-100)); + + auto env = NewTestingRuntimeEnv(); + CelExpressionFlatImpl impl( + env, FlatExpression(std::move(path), 0, + env->type_registry.GetComposedTypeProvider(), + cel::RuntimeOptions{}, /*arena=*/nullptr, {})); + + Activation activation; + google::protobuf::Arena arena; + + std::vector traced_ids; + auto eval_status = impl.Trace( + activation, &arena, + [&](int64_t expr_id, const CelValue& value, google::protobuf::Arena* arena) { + traced_ids.push_back(expr_id); + return absl::OkStatus(); + }); + ASSERT_THAT(eval_status, IsOk()); + EXPECT_THAT(traced_ids, ElementsAre(0)); +} + TEST(EvaluatorCoreTest, TraceTest) { Expr expr; cel::expr::SourceInfo source_info; @@ -186,7 +371,7 @@ TEST(EvaluatorCoreTest, TraceTest) { cel::RuntimeOptions options; options.short_circuiting = false; CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); - ASSERT_OK(RegisterBuiltinFunctions(builder.GetRegistry())); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); ASSERT_OK_AND_ASSIGN(auto cel_expr, builder.CreateExpression(&expr, &source_info)); @@ -218,7 +403,95 @@ TEST(EvaluatorCoreTest, TraceTest) { callback.Call(expr_id, value, arena); return absl::OkStatus(); }); - ASSERT_OK(eval_status); + ASSERT_THAT(eval_status, IsOk()); +} + +TEST(EvaluatorCoreTest, StepStackDelta) { + EXPECT_EQ(ExpressionStep::MakeConstant(cel::IntValue(1)).StackDelta(), 1); + EXPECT_EQ(ExpressionStep::MakeConstant(cel::BoolValue(true)).StackDelta(), 1); + EXPECT_EQ(ExpressionStep::MakeConstant(cel::DoubleValue(1.0)).StackDelta(), + 1); + EXPECT_EQ(ExpressionStep::MakeConstant(cel::NullValue()).StackDelta(), 1); + EXPECT_EQ(ExpressionStep::MakeConstant(cel::UintValue(1)).StackDelta(), 1); + EXPECT_EQ( + ExpressionStep::MakeConstant(cel::StringValue::Literal("a")).StackDelta(), + 1); + + ExpressionStep moved_step = ExpressionStep::MakeConstant(cel::IntValue(1)); + ExpressionStep dest_step = std::move(moved_step); + EXPECT_EQ(moved_step.StackDelta(), 0); // NOLINT(bugprone-use-after-move) + EXPECT_EQ(dest_step.StackDelta(), 1); + + EXPECT_EQ(ExpressionStep::MakeLazyInitStep(0, 1).StackDelta(), 1); + EXPECT_EQ(ExpressionStep::MakeAssignSlotAndPopStep(0).StackDelta(), -1); + EXPECT_EQ(ExpressionStep::MakeClearSlotsStep(0, 2).StackDelta(), 0); + EXPECT_EQ(ExpressionStep::MakeReadSlotStep(0).StackDelta(), 1); + + EXPECT_EQ(ExpressionStep::MakeBooleanNotStep().StackDelta(), 0); + EXPECT_EQ(ExpressionStep::MakeNotStrictlyFalseStep().StackDelta(), 0); + EXPECT_EQ(ExpressionStep::MakeBooleanOrStep(3).StackDelta(), -2); + EXPECT_EQ(ExpressionStep::MakeBooleanAndStep(2).StackDelta(), -1); + + EXPECT_EQ(ExpressionStep::MakeComprehensionFinishStep(0).StackDelta(), -1); + EXPECT_EQ(ExpressionStep::MakeComprehensionNextStep().StackDelta(), -1); + EXPECT_EQ(ExpressionStep::MakeComprehensionNext2Step().StackDelta(), -1); + EXPECT_EQ(ExpressionStep::MakeComprehensionCondStep().StackDelta(), -1); + EXPECT_EQ(ExpressionStep::MakeComprehensionCond2Step().StackDelta(), -1); + + EXPECT_EQ(ExpressionStep::MakeBooleanOrJumpStep(2).StackDelta(), 0); + EXPECT_EQ(ExpressionStep::MakeBooleanAndJumpStep(2).StackDelta(), 0); + EXPECT_EQ(ExpressionStep::MakeTernaryJumpStep().StackDelta(), 0); + EXPECT_EQ(ExpressionStep::MakeFixedJumpStep().StackDelta(), 0); + + EXPECT_EQ(ExpressionStep::MakeIdentifierStep(0).StackDelta(), 1); + EXPECT_EQ(ExpressionStep::MakeFastInStep().StackDelta(), -1); + EXPECT_EQ(ExpressionStep::MakeFastEqualStep().StackDelta(), -1); + EXPECT_EQ(ExpressionStep::MakeFastNotEqualStep().StackDelta(), -1); + EXPECT_EQ(ExpressionStep::MakeNewMutableListStep().StackDelta(), 1); + EXPECT_EQ(ExpressionStep::MakeMutableListAppendStep().StackDelta(), -1); + + EXPECT_EQ(ExpressionStep::MakeCreateSmallListStep(SmallListStepInfo{0, 4}) + .StackDelta(), + -3); + EXPECT_EQ( + ExpressionStep::MakeCreateListStep( + std::make_unique(5, absl::flat_hash_set{})) + .StackDelta(), + -4); + + EXPECT_EQ(ExpressionStep::MakeEagerFunctionStep( + std::make_unique( + std::vector{}, "fn", + /*num_args=*/3, /*receiver_style=*/false, /*expr_id=*/1)) + .StackDelta(), + -2); + EXPECT_EQ(ExpressionStep::MakeLazyFunctionStep( + std::make_unique( + std::vector{}, "fn", + /*num_args=*/2, /*receiver_style=*/false, /*expr_id=*/1)) + .StackDelta(), + -1); + + EXPECT_EQ(ExpressionStep::MakeGenericStep( + std::make_unique()) + .StackDelta(), + 1); + EXPECT_EQ(ExpressionStep::MakeGenericStep( + std::make_unique(), /*id=*/-1, + /*stack_delta=*/-2) + .StackDelta(), + -2); + EXPECT_EQ(ExpressionStep::MakeGenericStep( + std::make_unique(), /*id=*/-1, + /*stack_delta=*/std::numeric_limits::max()) + .StackDelta(), + std::nullopt); + EXPECT_EQ(ExpressionStep::MakeGenericStep( + std::make_unique(), /*id=*/-1, + /*stack_delta=*/ + static_cast(std::numeric_limits::min()) - 1) + .StackDelta(), + std::nullopt); } } // namespace google::api::expr::runtime diff --git a/eval/eval/evaluator_stack.cc b/eval/eval/evaluator_stack.cc index 47c625dac..24da595e1 100644 --- a/eval/eval/evaluator_stack.cc +++ b/eval/eval/evaluator_stack.cc @@ -3,14 +3,12 @@ #include #include #include -#include -#include #include "absl/base/dynamic_annotations.h" #include "absl/base/nullability.h" #include "absl/log/absl_log.h" +#include "common/internal/attribute_trail.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" #include "internal/new.h" namespace google::api::expr::runtime { @@ -27,7 +25,8 @@ void EvaluatorStack::Grow() { void EvaluatorStack::Reserve(size_t size) { static_assert(alignof(cel::Value) <= __STDCPP_DEFAULT_NEW_ALIGNMENT__); - static_assert(alignof(AttributeTrail) <= __STDCPP_DEFAULT_NEW_ALIGNMENT__); + static_assert(alignof(cel::common_internal::AttributeTrail) <= + __STDCPP_DEFAULT_NEW_ALIGNMENT__); if (max_size_ >= size) { return; @@ -39,10 +38,12 @@ void EvaluatorStack::Reserve(size_t size) { reinterpret_cast(data); cel::Value* absl_nullability_unknown values = values_begin; - AttributeTrail* absl_nullability_unknown attributes_begin = - reinterpret_cast(reinterpret_cast(data) + - AttributesBytesOffset(size)); - AttributeTrail* absl_nullability_unknown attributes = attributes_begin; + cel::common_internal::AttributeTrail* absl_nullability_unknown + attributes_begin = + reinterpret_cast( + reinterpret_cast(data) + AttributesBytesOffset(size)); + cel::common_internal::AttributeTrail* absl_nullability_unknown attributes = + attributes_begin; if (max_size_ > 0) { const size_t n = this->size(); @@ -55,13 +56,9 @@ void EvaluatorStack::Reserve(size_t size) { attributes_begin + size, attributes + m); for (size_t i = 0; i < m; ++i) { - ::new (static_cast(values++)) - cel::Value(std::move(values_begin_[i])); - ::new (static_cast(attributes++)) - AttributeTrail(std::move(attributes_begin_[i])); + *values++ = values_begin_[i]; + *attributes++ = attributes_begin_[i]; } - std::destroy_n(values_begin_, n); - std::destroy_n(attributes_begin_, n); ABSL_ANNOTATE_CONTIGUOUS_CONTAINER(values_begin_, values_begin_ + max_size_, values_, values_begin_ + max_size_); diff --git a/eval/eval/evaluator_stack.h b/eval/eval/evaluator_stack.h index b6abd1f76..9ab0c84b3 100644 --- a/eval/eval/evaluator_stack.h +++ b/eval/eval/evaluator_stack.h @@ -3,7 +3,6 @@ #include #include -#include #include #include @@ -13,10 +12,9 @@ #include "absl/base/optimization.h" #include "absl/log/absl_check.h" #include "absl/meta/type_traits.h" -#include "absl/types/optional.h" #include "absl/types/span.h" +#include "common/internal/attribute_trail.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" #include "internal/align.h" #include "internal/new.h" @@ -34,9 +32,6 @@ class EvaluatorStack { ~EvaluatorStack() { if (max_size() > 0) { - const size_t n = size(); - std::destroy_n(values_begin_, n); - std::destroy_n(attributes_begin_, n); cel::internal::SizedDelete(data_, SizeBytes(max_size_)); } } @@ -97,10 +92,6 @@ class EvaluatorStack { // Dumps the entire stack state as is. void Clear() { if (max_size() > 0) { - const size_t n = size(); - std::destroy_n(values_begin_, n); - std::destroy_n(attributes_begin_, n); - ABSL_ANNOTATE_CONTIGUOUS_CONTAINER( values_begin_, values_begin_ + max_size_, values_, values_begin_); ABSL_ANNOTATE_CONTIGUOUS_CONTAINER(attributes_begin_, @@ -124,10 +115,12 @@ class EvaluatorStack { // Gets the last size attribute trails of the stack. // Checking that stack has enough elements is caller's responsibility. // Please note that calls to Push may invalidate returned Span object. - absl::Span GetAttributeSpan(size_t size) const { + absl::Span GetAttributeSpan( + size_t size) const { ABSL_DCHECK(HasEnough(size)); - return absl::Span(attributes_ - size, size); + return absl::Span( + attributes_ - size, size); } // Peeks the last element of the stack. @@ -148,7 +141,7 @@ class EvaluatorStack { // Peeks the last element of the attribute stack. // Checking that stack is not empty is caller's responsibility. - const AttributeTrail& PeekAttribute() const { + const cel::common_internal::AttributeTrail& PeekAttribute() const { ABSL_DCHECK(HasEnough(1)); return *(attributes_ - 1); @@ -156,7 +149,7 @@ class EvaluatorStack { // Peeks the last element of the attribute stack. // Checking that stack is not empty is caller's responsibility. - AttributeTrail& PeekAttribute() { + cel::common_internal::AttributeTrail& PeekAttribute() { ABSL_DCHECK(HasEnough(1)); return *(attributes_ - 1); @@ -166,9 +159,7 @@ class EvaluatorStack { ABSL_DCHECK(!empty()); --values_; - values_->~Value(); --attributes_; - attributes_->~AttributeTrail(); ABSL_ANNOTATE_CONTIGUOUS_CONTAINER(values_begin_, values_begin_ + max_size_, values_ + 1, values_); @@ -187,12 +178,14 @@ class EvaluatorStack { } } - template , - std::is_convertible>>> - void Push(V&& value, A&& attribute) { + template >>> + void Push(V&& value, cel::common_internal::AttributeTrail attribute) { ABSL_DCHECK(!full()); + ABSL_DCHECK(!attribute.IsFullMatch()) + << "Full matches should not be pushed onto the value stack, they " + "should be handled directly and converted to an unknown value or " + "missing error value"; if (ABSL_PREDICT_FALSE(full())) { Grow(); @@ -204,9 +197,8 @@ class EvaluatorStack { attributes_begin_ + max_size_, attributes_, attributes_ + 1); - ::new (static_cast(values_++)) cel::Value(std::forward(value)); - ::new (static_cast(attributes_++)) - AttributeTrail(std::forward(attribute)); + *values_++ = std::forward(value); + *attributes_++ = attribute; } template (value), absl::nullopt); + Push(std::forward(value), cel::common_internal::AttributeTrail()); } + void Push() { Push(cel::Value(), cel::common_internal::AttributeTrail()); } + // Equivalent to `PopAndPush(1, ...)`. - template , - std::is_convertible>>> - void PopAndPush(V&& value, A&& attribute) { + template >>> + void PopAndPush(V&& value, cel::common_internal::AttributeTrail attribute) { ABSL_DCHECK(!empty()); + ABSL_DCHECK(!attribute.IsFullMatch()) + << "Full matches should not be pushed onto the value stack, they " + "should be handled directly and converted to an unknown value or " + "missing error value"; *(values_ - 1) = std::forward(value); - *(attributes_ - 1) = std::forward(attribute); + *(attributes_ - 1) = attribute; } // Equivalent to `PopAndPush(1, ...)`. @@ -235,38 +231,35 @@ class EvaluatorStack { void PopAndPush(V&& value) { ABSL_DCHECK(!empty()); - PopAndPush(std::forward(value), absl::nullopt); + PopAndPush(std::forward(value), cel::common_internal::AttributeTrail()); } // Equivalent to `Pop(n)` followed by `Push(...)`. Both `V` and `A` MUST NOT // be located on the stack. If this is the case, use SwapAndPop instead. - template , - std::is_convertible>>> - void PopAndPush(size_t n, V&& value, A&& attribute) { + template >>> + void PopAndPush(size_t n, V&& value, + cel::common_internal::AttributeTrail attribute) { if (n > 0) { + ABSL_DCHECK(!attribute.IsFullMatch()) + << "Full matches should not be pushed onto the value stack, they " + "should be handled directly and converted to an unknown value or " + "missing error value"; if constexpr (std::is_same_v>) { ABSL_DCHECK(&value < values_begin_ || &value >= values_begin_ + max_size_) << "Attmpting to push a value about to be popped, use PopAndSwap " "instead."; } - if constexpr (std::is_same_v>) { - ABSL_DCHECK(&attribute < attributes_begin_ || - &attribute >= attributes_begin_ + max_size_) - << "Attmpting to push an attribute about to be popped, use " - "PopAndSwap instead."; - } Pop(n - 1); ABSL_DCHECK(!empty()); *(values_ - 1) = std::forward(value); - *(attributes_ - 1) = std::forward(attribute); + *(attributes_ - 1) = attribute; } else { - Push(std::forward(value), std::forward(attribute)); + Push(std::forward(value), attribute); } } @@ -275,12 +268,13 @@ class EvaluatorStack { template >> void PopAndPush(size_t n, V&& value) { - PopAndPush(n, std::forward(value), absl::nullopt); + PopAndPush(n, std::forward(value), + cel::common_internal::AttributeTrail()); } - // Swaps the `n - i` element (from the top of the stack) with the `n` element, - // and pops `n - 1` elements. This results in the `n - i` element being at the - // top of the stack. + // Given the top `n` the elements of the stack, swap the `i`th element with + // the 0th, then pop n - 1 elements (leaving the `i`th element as the new + // top). void SwapAndPop(size_t n, size_t i) { ABSL_DCHECK_GT(n, 0); ABSL_DCHECK_LT(i, n); @@ -305,7 +299,8 @@ class EvaluatorStack { } static size_t SizeBytes(size_t size) { - return AttributesBytesOffset(size) + (sizeof(AttributeTrail) * size); + return AttributesBytesOffset(size) + + (sizeof(cel::common_internal::AttributeTrail) * size); } void Grow(); @@ -315,8 +310,10 @@ class EvaluatorStack { cel::Value* absl_nullability_unknown values_ = nullptr; cel::Value* absl_nullability_unknown values_begin_ = nullptr; - AttributeTrail* absl_nullability_unknown attributes_ = nullptr; - AttributeTrail* absl_nullability_unknown attributes_begin_ = nullptr; + cel::common_internal::AttributeTrail* absl_nullability_unknown attributes_ = + nullptr; + cel::common_internal::AttributeTrail* absl_nullability_unknown + attributes_begin_ = nullptr; cel::Value* absl_nullability_unknown values_end_ = nullptr; void* absl_nullability_unknown data_ = nullptr; size_t max_size_ = 0; diff --git a/eval/eval/evaluator_stack_test.cc b/eval/eval/evaluator_stack_test.cc index 9ce862d8a..2fcbc4ae7 100644 --- a/eval/eval/evaluator_stack_test.cc +++ b/eval/eval/evaluator_stack_test.cc @@ -2,6 +2,7 @@ #include "base/attribute.h" #include "common/value.h" +#include "eval/eval/attribute_trail.h" #include "internal/testing.h" namespace google::api::expr::runtime { @@ -14,21 +15,17 @@ TEST(EvaluatorStackTest, StackPushPop) { EvaluatorStack stack(10); stack.Push(cel::IntValue(1)); stack.Push(cel::IntValue(2), AttributeTrail()); - stack.Push(cel::IntValue(3), AttributeTrail("name")); + stack.Push(cel::IntValue(3), AttributeTrail()); ASSERT_EQ(stack.Peek().GetInt().NativeValue(), 3); - ASSERT_FALSE(stack.PeekAttribute().empty()); - ASSERT_EQ(stack.PeekAttribute().attribute(), attribute); stack.Pop(1); ASSERT_EQ(stack.Peek().GetInt().NativeValue(), 2); - ASSERT_TRUE(stack.PeekAttribute().empty()); stack.Pop(1); ASSERT_EQ(stack.Peek().GetInt().NativeValue(), 1); - ASSERT_TRUE(stack.PeekAttribute().empty()); } // Test that inner stacks within value stack retain the equality of their sizes. diff --git a/eval/eval/expression_step_base.h b/eval/eval/expression_step_base.h index 5b2f72f8e..7f4425098 100644 --- a/eval/eval/expression_step_base.h +++ b/eval/eval/expression_step_base.h @@ -1,11 +1,18 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_EXPRESSION_STEP_BASE_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_EXPRESSION_STEP_BASE_H_ -#include "eval/eval/evaluator_core.h" +#include + +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { -using ExpressionStepBase = ExpressionStep; +class ExpressionStepBase : public ExpressionStepLogic { + public: + ExpressionStepBase() = default; + explicit ExpressionStepBase(int64_t /*expr_id*/, + bool /*comes_from_ast*/ = true) {} +}; } // namespace google::api::expr::runtime diff --git a/eval/eval/expression_step_logic.h b/eval/eval/expression_step_logic.h new file mode 100644 index 000000000..2b07af527 --- /dev/null +++ b/eval/eval/expression_step_logic.h @@ -0,0 +1,51 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_EXPRESSION_STEP_LOGIC_H_ +#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_EXPRESSION_STEP_LOGIC_H_ + +#include "common/native_type.h" + +namespace google::api::expr::runtime { + +class ExecutionFrame; + +// ExpressionStepLogic is the base class for generic expression steps that are +// not implemented directly in the evaluator core. +class ExpressionStepLogic { + public: + virtual ~ExpressionStepLogic() = default; + + // Performs actual evaluation. + // Values are passed between Expression objects via EvaluatorStack, which is + // supplied with context. + // Also, Expression gets values supplied by caller though Activation + // interface. + // ExpressionStep instances can in specific cases + // modify execution order(perform jumps). + // + // Unrecoverable errors are reported via ExecutionFrame::Abort. + virtual void Evaluate(ExecutionFrame* frame) const = 0; + + // Return the type of the underlying expression step for special handling in + // the planning phase. This should only be overridden by special cases, and + // callers must not make any assumptions about the default case. + virtual cel::NativeTypeId GetNativeTypeId() const { + return cel::NativeTypeId(); + } +}; + +} // namespace google::api::expr::runtime + +#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_EXPRESSION_STEP_LOGIC_H_ diff --git a/eval/eval/function_step.cc b/eval/eval/function_step.cc index 2a10e9674..6dbce79cd 100644 --- a/eval/eval/function_step.cc +++ b/eval/eval/function_step.cc @@ -4,27 +4,27 @@ #include #include #include +#include #include #include #include -#include "absl/container/inlined_vector.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "absl/types/span.h" -#include "common/casting.h" #include "common/expr.h" #include "common/function_descriptor.h" +#include "common/internal/attribute_trail.h" +#include "common/internal/unknowns.h" #include "common/kind.h" #include "common/value.h" #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" #include "eval/internal/errors.h" #include "internal/status_macros.h" #include "runtime/activation_interface.h" @@ -48,10 +48,11 @@ using ::cel::runtime_internal::CreateNoMatchingOverloadError; // errors or unknown sets must be allowed as a non-strict function. bool ShouldAcceptOverload(const cel::FunctionDescriptor& descriptor, absl::Span arguments) { - for (size_t i = 0; i < arguments.size(); i++) { - if (arguments[i]->Is() || - arguments[i]->Is()) { - return !descriptor.is_strict(); + if (descriptor.is_strict()) { + for (const auto& argument : arguments) { + if (argument.IsError() || argument.IsUnknown()) { + return false; + } } } return true; @@ -113,14 +114,12 @@ std::vector CheckForPartialUnknowns( std::vector result; result.reserve(args.size()); for (size_t i = 0; i < args.size(); i++) { - const AttributeTrail& trail = attrs.subspan(i, 1)[0]; - - if (frame->attribute_utility().CheckForUnknown(trail, - /*use_partial=*/true)) { - result.push_back( - frame->attribute_utility().CreateUnknownSet(trail.attribute())); + const AttributeTrail& trail = attrs[i]; + if (auto unknown = trail.PartialUnknownMatch(frame->unknown_tree()); + unknown.has_value()) { + result.push_back(*unknown); } else { - result.push_back(args.at(i)); + result.push_back(args[i]); } } @@ -146,38 +145,6 @@ bool IsUnknownFunctionResultError(const Value& result) { // resolve to a single function implementation and a descriptor or none. using ResolveResult = absl::optional; -// Implementation of ExpressionStep that finds suitable CelFunction overload and -// invokes it. Abstract base class standardizes behavior between lazy and eager -// function bindings. Derived classes provide ResolveFunction behavior. -class AbstractFunctionStep : public ExpressionStepBase { - public: - // Constructs FunctionStep that uses overloads specified. - AbstractFunctionStep(const std::string& name, size_t num_arguments, - bool receiver_style, int64_t expr_id) - : ExpressionStepBase(expr_id), - name_(name), - num_arguments_(num_arguments), - receiver_style_(receiver_style) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override; - - // Handles overload resolution and updating result appropriately. - // Shouldn't update frame state. - // - // A non-ok result is an unrecoverable error, either from an illegal - // evaluation state or forwarded from an extension function. Errors where - // evaluation can reasonably condition are returned in the result as a - // cel::ErrorValue. - absl::StatusOr DoEvaluate(ExecutionFrame* frame) const; - - virtual absl::StatusOr ResolveFunction( - absl::Span args, const ExecutionFrame* frame) const = 0; - - protected: - std::string name_; - size_t num_arguments_; - bool receiver_style_; -}; inline absl::StatusOr Invoke( const cel::FunctionOverloadReference& overload, int64_t expr_id, @@ -188,13 +155,21 @@ inline absl::StatusOr Invoke( context.set_embedder_context(frame.embedder_context()); } - CEL_ASSIGN_OR_RETURN(Value result, - overload.implementation.Invoke(args, context)); + absl::StatusOr result = overload.implementation.Invoke(args, context); + if (!result.ok()) { + return result; + } if (frame.unknown_function_results_enabled() && - IsUnknownFunctionResultError(result)) { - return frame.attribute_utility().CreateUnknownSet(overload.descriptor, - expr_id, args); + IsUnknownFunctionResultError(*result)) { + return cel::common_internal::MakeUnknownValue( + cel::common_internal::UnknownValueRep{ + .root = frame.unknown_tree()->Root(), + .attributes = nullptr, + .functions = cel::common_internal::CreateUnknownFunctionSet( + frame.unknown_tree()->Root()->SetFunction( + frame.unknown_tree(), overload.descriptor.name()), + frame.unknown_tree()->GetArena())}); } return result; } @@ -208,98 +183,55 @@ Value NoOverloadResult(absl::string_view name, // should be propagated along execution path. for (size_t i = 0; i < args.size(); i++) { const auto& arg = args[i]; - if (cel::InstanceOf(arg)) { + if (arg.IsError()) { return arg; } } if (frame.unknown_processing_enabled()) { // Already converted partial unknowns to unknown sets so just merge. - absl::optional unknown_set = - frame.attribute_utility().MergeUnknowns(args); - if (unknown_set.has_value()) { - return *unknown_set; + UnknownAccumulator accumulator(frame.unknown_tree()); + for (const auto& arg : args) { + accumulator.MaybeAdd(arg); + } + if (auto accumulated = std::move(accumulator).Accumulate(); + accumulated.has_value()) { + return *accumulated; } } // If no errors or unknowns in input args, create new CelError for missing // overload. - std::string signature; if (receiver_style) { if (args.empty()) { // Should not be possible, but return a sensible error in case of logic // error. - return ErrorValue( - CreateNoMatchingOverloadError(absl::StrCat("().", name, "()"))); + return ErrorValue::From( + CreateNoMatchingOverloadError(absl::StrCat("().", name, "()")), + frame.arena()); } - return ErrorValue(CreateNoMatchingOverloadError(absl::StrCat( - "(", - ToLegacyKindName(cel::KindToString(ValueKindToKind(args[0].kind()))), - ").", name, CallArgTypeString(args.subspan(1))))); + return ErrorValue::From( + CreateNoMatchingOverloadError(absl::StrCat( + "(", + ToLegacyKindName( + cel::KindToString(ValueKindToKind(args[0].kind()))), + ").", name, CallArgTypeString(args.subspan(1)))), + frame.arena()); } - return cel::ErrorValue(CreateNoMatchingOverloadError( - absl::StrCat(name, CallArgTypeString(args)))); + return cel::ErrorValue::From(CreateNoMatchingOverloadError( + absl::StrCat(name, CallArgTypeString(args))), + frame.arena()); } -absl::StatusOr AbstractFunctionStep::DoEvaluate( - ExecutionFrame* frame) const { - // Create Span object that contains input arguments to the function. - auto input_args = frame->value_stack().GetSpan(num_arguments_); - - std::vector unknowns_args; - // Preprocess args. If an argument is partially unknown, convert it to an - // unknown attribute set. - if (frame->enable_unknowns()) { - auto input_attrs = frame->value_stack().GetAttributeSpan(num_arguments_); - unknowns_args = CheckForPartialUnknowns(frame, input_args, input_attrs); - input_args = absl::MakeConstSpan(unknowns_args); - } - - // Derived class resolves to a single function overload or none. - CEL_ASSIGN_OR_RETURN(ResolveResult matched_function, - ResolveFunction(input_args, frame)); - - // Overload found and is allowed to consume the arguments. - if (matched_function.has_value() && - ShouldAcceptOverload(matched_function->descriptor, input_args)) { - return Invoke(*matched_function, id(), input_args, *frame); - } - - return NoOverloadResult(name_, input_args, receiver_style_, *frame); -} - -absl::Status AbstractFunctionStep::Evaluate(ExecutionFrame* frame) const { - if (!frame->value_stack().HasEnough(num_arguments_)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - - // DoEvaluate may return a status for non-recoverable errors (e.g. - // unexpected typing, illegal expression state). Application errors that can - // reasonably be handled as a cel error will appear in the result value. - CEL_ASSIGN_OR_RETURN(auto result, DoEvaluate(frame)); - - frame->value_stack().PopAndPush(num_arguments_, std::move(result)); - - return absl::OkStatus(); -} - -absl::StatusOr ResolveStatic( +ResolveResult ResolveStatic( absl::Span input_args, absl::Span overloads) { - ResolveResult result = absl::nullopt; - for (const auto& overload : overloads) { if (ArgumentKindsMatch(overload.descriptor, input_args)) { - // More than one overload matches our arguments. - if (result.has_value()) { - return absl::Status(absl::StatusCode::kInternal, - "Cannot resolve overloads"); - } - - result.emplace(overload); + return overload; } } - return result; + return std::nullopt; } absl::StatusOr ResolveLazy( @@ -307,7 +239,7 @@ absl::StatusOr ResolveLazy( bool receiver_style, absl::Span providers, const ExecutionFrameBase& frame) { - ResolveResult result = absl::nullopt; + ResolveResult result = std::nullopt; std::vector arg_types(input_args.size()); @@ -315,7 +247,7 @@ absl::StatusOr ResolveLazy( input_args.begin(), input_args.end(), arg_types.begin(), [](const cel::Value& value) { return ValueKindToKind(value->kind()); }); - cel::FunctionDescriptor matcher{name, receiver_style, arg_types}; + cel::FunctionDescriptor matcher{name, receiver_style, std::move(arg_types)}; const cel::ActivationInterface& activation = frame.activation(); for (auto provider : providers) { @@ -342,48 +274,7 @@ absl::StatusOr ResolveLazy( return result; } -class EagerFunctionStep : public AbstractFunctionStep { - public: - EagerFunctionStep(std::vector overloads, - const std::string& name, size_t num_args, - bool receiver_style, int64_t expr_id) - : AbstractFunctionStep(name, num_args, receiver_style, expr_id), - overloads_(std::move(overloads)) {} - - absl::StatusOr ResolveFunction( - absl::Span input_args, - const ExecutionFrame* frame) const override { - return ResolveStatic(input_args, overloads_); - } - private: - std::vector overloads_; -}; - -class LazyFunctionStep : public AbstractFunctionStep { - public: - // Constructs LazyFunctionStep that attempts to lookup function implementation - // at runtime. - LazyFunctionStep(const std::string& name, size_t num_args, - bool receiver_style, - std::vector providers, - int64_t expr_id) - : AbstractFunctionStep(name, num_args, receiver_style, expr_id), - providers_(std::move(providers)) {} - - absl::StatusOr ResolveFunction( - absl::Span input_args, - const ExecutionFrame* frame) const override; - - private: - std::vector providers_; -}; - -absl::StatusOr LazyFunctionStep::ResolveFunction( - absl::Span input_args, - const ExecutionFrame* frame) const { - return ResolveLazy(input_args, name_, receiver_style_, providers_, *frame); -} class StaticResolver { public: @@ -419,119 +310,112 @@ class LazyResolver { bool receiver_style_; }; -template -class DirectFunctionStepImpl : public DirectExpressionStep { - public: - DirectFunctionStepImpl( - int64_t expr_id, const std::string& name, - std::vector> arg_steps, - bool receiver_style, Resolver&& resolver) - : DirectExpressionStep(expr_id), - name_(name), - arg_steps_(std::move(arg_steps)), - receiver_style_(receiver_style), - resolver_(std::forward(resolver)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& trail) const override { - absl::InlinedVector args; - absl::InlinedVector arg_trails; - - args.resize(arg_steps_.size()); - arg_trails.resize(arg_steps_.size()); - - for (size_t i = 0; i < arg_steps_.size(); i++) { - CEL_RETURN_IF_ERROR( - arg_steps_[i]->Evaluate(frame, args[i], arg_trails[i])); - } +} // namespace - if (frame.unknown_processing_enabled()) { - for (size_t i = 0; i < arg_trails.size(); i++) { - if (frame.attribute_utility().CheckForUnknown(arg_trails[i], - /*use_partial=*/true)) { - args[i] = frame.attribute_utility().CreateUnknownSet( - arg_trails[i].attribute()); - } - } - } +template +void EvaluateFunctionStep(const Step* step, ExecutionFrame& frame) { + if (!frame.value_stack().HasEnough(step->num_arguments_)) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } - CEL_ASSIGN_OR_RETURN(ResolveResult resolved_function, - resolver_.Resolve(frame, args)); + // Create Span object that contains input arguments to the function. + absl::Span input_args = + frame.value_stack().GetSpan(step->num_arguments_); - if (resolved_function.has_value() && - ShouldAcceptOverload(resolved_function->descriptor, args)) { - CEL_ASSIGN_OR_RETURN(result, - Invoke(*resolved_function, expr_id_, args, frame)); + std::vector unknowns_args; + // Preprocess args. If an argument is partially unknown, convert it to an + // unknown attribute set. + if (frame.enable_unknowns()) { + absl::Span input_attrs = + frame.value_stack().GetAttributeSpan(step->num_arguments_); + unknowns_args = CheckForPartialUnknowns(&frame, input_args, input_attrs); + input_args = absl::MakeConstSpan(unknowns_args); + } - return absl::OkStatus(); + // Derived class resolves to a single function overload or none. + // The extra complexity here is to avoid an unnecessary absl::StatusOr + // wrapping ResolveResult for the common case (static resolution). + ResolveResult resolved_function; + if constexpr (std::is_same_v, + decltype(step->ResolveFunction(input_args, + frame))>) { + absl::StatusOr matched_function = + step->ResolveFunction(input_args, frame); + if (!matched_function.ok()) { + frame.Abort(std::move(matched_function).status()); + return; + } + if (matched_function->has_value()) { + resolved_function.emplace(**matched_function); + } + } else { + ResolveResult matched_function = step->ResolveFunction(input_args, frame); + if (matched_function.has_value()) { + resolved_function.emplace(*matched_function); } - - result = NoOverloadResult(name_, args, receiver_style_, frame); - - return absl::OkStatus(); } - absl::optional> GetDependencies() - const override { - std::vector dependencies; - dependencies.reserve(arg_steps_.size()); - for (const auto& arg_step : arg_steps_) { - dependencies.push_back(arg_step.get()); + // Overload found and is allowed to consume the arguments. + if (resolved_function.has_value() && + ShouldAcceptOverload(resolved_function->descriptor, input_args)) { + absl::StatusOr result = + Invoke(*resolved_function, step->expr_id_, input_args, frame); + if (!result.ok()) { + frame.Abort(std::move(result).status()); + return; + } + if (frame.enable_unknowns()) { + cel::common_internal::GraftUnknownsAt(*result, frame.unknown_tree()); } - return dependencies; + frame.value_stack().PopAndPush(step->num_arguments_, *std::move(result)); + return; } - absl::optional>> - ExtractDependencies() override { - return std::move(arg_steps_); - } + frame.value_stack().PopAndPush( + step->num_arguments_, + NoOverloadResult(step->name_, input_args, step->receiver_style_, frame)); +} - private: - friend Resolver; - std::string name_; - std::vector> arg_steps_; - bool receiver_style_; - Resolver resolver_; -}; +void EagerFunctionStep::Evaluate(ExecutionFrame& frame) const { + EvaluateFunctionStep(this, frame); +} -} // namespace +void LazyFunctionStep::Evaluate(ExecutionFrame& frame) const { + EvaluateFunctionStep(this, frame); +} -std::unique_ptr CreateDirectFunctionStep( - int64_t expr_id, const cel::CallExpr& call, - std::vector> deps, - std::vector overloads) { - return std::make_unique>( - expr_id, call.function(), std::move(deps), call.has_target(), - StaticResolver(std::move(overloads))); +ResolveResult EagerFunctionStep::ResolveFunction( + absl::Span input_args, + const ExecutionFrameBase& frame) const { + return ResolveStatic(input_args, overloads_); } -std::unique_ptr CreateDirectLazyFunctionStep( - int64_t expr_id, const cel::CallExpr& call, - std::vector> deps, - std::vector providers) { - return std::make_unique>( - expr_id, call.function(), std::move(deps), call.has_target(), - LazyResolver(std::move(providers), call.function(), call.has_target())); +absl::StatusOr LazyFunctionStep::ResolveFunction( + absl::Span input_args, + const ExecutionFrameBase& frame) const { + return ResolveLazy(input_args, name_, receiver_style_, providers_, frame); } -absl::StatusOr> CreateFunctionStep( +std::unique_ptr CreateLazyFunctionStep( const cel::CallExpr& call_expr, int64_t expr_id, std::vector lazy_overloads) { bool receiver_style = call_expr.has_target(); size_t num_args = call_expr.args().size() + (receiver_style ? 1 : 0); - const std::string& name = call_expr.function(); - return std::make_unique(name, num_args, receiver_style, - std::move(lazy_overloads), expr_id); + return std::make_unique(std::move(lazy_overloads), + call_expr.function(), num_args, + receiver_style, expr_id); } -absl::StatusOr> CreateFunctionStep( +std::unique_ptr CreateFunctionStep( const cel::CallExpr& call_expr, int64_t expr_id, std::vector overloads) { bool receiver_style = call_expr.has_target(); size_t num_args = call_expr.args().size() + (receiver_style ? 1 : 0); - const std::string& name = call_expr.function(); - return std::make_unique(std::move(overloads), name, - num_args, receiver_style, expr_id); + return std::make_unique(std::move(overloads), + call_expr.function(), num_args, + receiver_style, expr_id); } } // namespace google::api::expr::runtime diff --git a/eval/eval/function_step.h b/eval/eval/function_step.h index 9f664dc09..4587da2c5 100644 --- a/eval/eval/function_step.h +++ b/eval/eval/function_step.h @@ -1,48 +1,110 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_FUNCTION_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_FUNCTION_STEP_H_ +#include #include #include +#include +#include +#include #include +#include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/types/span.h" #include "common/expr.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "common/value.h" +#include "eval/eval/expression_step_logic.h" #include "runtime/function_overload_reference.h" #include "runtime/function_registry.h" namespace google::api::expr::runtime { -// Factory method for Call-based execution step where the function has been -// statically resolved from a set of eagerly functions configured in the -// CelFunctionRegistry. -std::unique_ptr CreateDirectFunctionStep( - int64_t expr_id, const cel::CallExpr& call, - std::vector> deps, - std::vector overloads); - -// Factory method for Call-based execution step where the function has been -// statically resolved from a set of lazy functions configured in the -// CelFunctionRegistry. -std::unique_ptr CreateDirectLazyFunctionStep( - int64_t expr_id, const cel::CallExpr& call, - std::vector> deps, - std::vector providers); +class LazyFunctionStep; +class EagerFunctionStep; +class ExecutionFrameBase; +class ExecutionFrame; // Factory method for Call-based execution step where the function will be // resolved at runtime (lazily) from an input Activation. -absl::StatusOr> CreateFunctionStep( +std::unique_ptr CreateLazyFunctionStep( const cel::CallExpr& call, int64_t expr_id, std::vector lazy_overloads); // Factory method for Call-based execution step where the function has been // statically resolved from a set of eagerly functions configured in the // CelFunctionRegistry. -absl::StatusOr> CreateFunctionStep( +std::unique_ptr CreateFunctionStep( const cel::CallExpr& call, int64_t expr_id, std::vector overloads); +// Common base class for EagerFunctionStep and LazyFunctionStep. +class FunctionStepBase { + public: + size_t num_arguments() const { return num_arguments_; } + + private: + friend class EagerFunctionStep; + friend class LazyFunctionStep; + template + friend void EvaluateFunctionStep(const Step* step, ExecutionFrame& frame); + + // Constructs FunctionStep that uses overloads specified. + FunctionStepBase(const std::string& name, size_t num_arguments, + bool receiver_style, int64_t expr_id) + : name_(name), + num_arguments_(num_arguments), + receiver_style_(receiver_style), + expr_id_(expr_id) {} + + std::string name_; + size_t num_arguments_; + bool receiver_style_; + int64_t expr_id_; +}; + +class EagerFunctionStep : public FunctionStepBase { + public: + EagerFunctionStep(std::vector overloads, + const std::string& name, size_t num_args, + bool receiver_style, int64_t expr_id) + : FunctionStepBase(name, num_args, receiver_style, expr_id), + overloads_(std::move(overloads)) {} + + void Evaluate(ExecutionFrame& frame) const; + + private: + template + friend void EvaluateFunctionStep(const Step* step, ExecutionFrame& frame); + + std::optional ResolveFunction( + absl::Span input_args, + const ExecutionFrameBase& frame) const; + + std::vector overloads_; +}; + +class LazyFunctionStep : public FunctionStepBase { + public: + LazyFunctionStep(std::vector providers, + const std::string& name, size_t num_args, + bool receiver_style, int64_t expr_id) + : FunctionStepBase(name, num_args, receiver_style, expr_id), + providers_(std::move(providers)) {} + + void Evaluate(ExecutionFrame& frame) const; + + private: + template + friend void EvaluateFunctionStep(const Step* step, ExecutionFrame& frame); + + absl::StatusOr> ResolveFunction( + absl::Span input_args, + const ExecutionFrameBase& frame) const; + + std::vector providers_; +}; + } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_FUNCTION_STEP_H_ diff --git a/eval/eval/function_step_test.cc b/eval/eval/function_step_test.cc index 3d3bae34d..c42bf0604 100644 --- a/eval/eval/function_step_test.cc +++ b/eval/eval/function_step_test.cc @@ -12,17 +12,15 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/types/span.h" -#include "base/builtins.h" +#include "base/attribute.h" +#include "base/attribute_set.h" #include "base/type_provider.h" #include "common/constant.h" #include "common/expr.h" #include "common/kind.h" #include "common/value.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/const_value_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/ident_step.h" #include "eval/internal/interop.h" #include "eval/public/activation.h" #include "eval/public/cel_attribute.h" @@ -35,11 +33,8 @@ #include "eval/public/testing/matchers.h" #include "eval/testutil/test_message.pb.h" #include "internal/testing.h" -#include "runtime/function_overload_reference.h" -#include "runtime/function_registry.h" #include "runtime/internal/runtime_env_testing.h" #include "runtime/runtime_options.h" -#include "runtime/standard_functions.h" #include "google/protobuf/arena.h" namespace google::api::expr::runtime { @@ -55,7 +50,6 @@ using ::cel::TypeProvider; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::testing::Eq; using ::testing::Not; -using ::testing::Truly; int GetExprId() { static int id = 0; @@ -170,7 +164,8 @@ class SinkFunction : public CelFunction { // Create and initialize a registry with some default functions. void AddDefaults(CelFunctionRegistry& registry) { - static UnknownSet* unknown_set = new UnknownSet(); + static UnknownSet* unknown_set = + new UnknownSet(cel::AttributeSet({cel::Attribute("foo")})); EXPECT_TRUE(registry .Register(std::make_unique( CelValue::CreateInt64(3), "Const3")) @@ -212,30 +207,20 @@ std::vector ArgumentMatcher(const CallExpr& call) { : call.args().size()); } -std::unique_ptr CreateExpressionImpl( - const cel::RuntimeOptions& options, - std::unique_ptr expr) { - ExecutionPath path; - path.push_back(std::make_unique(std::move(expr), -1)); - - auto env = NewTestingRuntimeEnv(); - return std::make_unique( - env, - FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); -} - -absl::StatusOr> MakeTestFunctionStep( +absl::StatusOr MakeTestFunctionStep( const CallExpr& call, const CelFunctionRegistry& registry) { auto argument_matcher = ArgumentMatcher(call); auto lazy_overloads = registry.ModernFindLazyOverloads( call.function(), call.has_target(), argument_matcher); + int id = GetExprId(); if (!lazy_overloads.empty()) { - return CreateFunctionStep(call, GetExprId(), lazy_overloads); + return ExpressionStep::MakeLazyFunctionStep( + CreateLazyFunctionStep(call, id, std::move(lazy_overloads)), id); } auto overloads = registry.FindStaticOverloads( call.function(), call.has_target(), argument_matcher); - return CreateFunctionStep(call, GetExprId(), overloads); + return ExpressionStep::MakeEagerFunctionStep( + CreateFunctionStep(call, id, std::move(overloads)), id); } // Test common functions with varying levels of unknown support. @@ -251,7 +236,8 @@ class FunctionStepTest return std::make_unique( env, FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, {})); } }; @@ -396,17 +382,13 @@ TEST_P(FunctionStepTest, TestNoMatchingOverloadsUnexpectedArgCount) { ASSERT_OK_AND_ASSIGN(auto step1, MakeTestFunctionStep(call1, registry)); ASSERT_OK_AND_ASSIGN(auto step2, MakeTestFunctionStep(call1, registry)); - ASSERT_OK_AND_ASSIGN( - auto step3, - CreateFunctionStep(add_call, -1, - registry.FindStaticOverloads( - add_call.function(), false, - {cel::Kind::kInt64, cel::Kind::kInt64}))); - path.push_back(std::move(step0)); path.push_back(std::move(step1)); path.push_back(std::move(step2)); - path.push_back(std::move(step3)); + path.push_back(ExpressionStep::MakeEagerFunctionStep(CreateFunctionStep( + add_call, -1, + registry.FindStaticOverloads(add_call.function(), false, + {cel::Kind::kInt64, cel::Kind::kInt64})))); std::unique_ptr impl = GetExpression(std::move(path)); @@ -535,13 +517,11 @@ TEST_P(FunctionStepTest, LazyFunctionOverloadingTest) { lt_call.mutable_args().emplace_back(); lt_call.set_function("_<_"); - ASSERT_OK_AND_ASSIGN( - auto step0, - CreateConstValueStep(cel::interop_internal::CreateIntValue(20), -1)); + auto step0 = + ExpressionStep::MakeConstant(cel::interop_internal::CreateIntValue(20)); ASSERT_OK_AND_ASSIGN(auto step1, MakeTestFunctionStep(call1, registry)); - ASSERT_OK_AND_ASSIGN( - auto step2, - CreateConstValueStep(cel::interop_internal::CreateDoubleValue(21.9), -1)); + auto step2 = ExpressionStep::MakeConstant( + cel::interop_internal::CreateDoubleValue(21.9)); ASSERT_OK_AND_ASSIGN(auto step3, MakeTestFunctionStep(call2, registry)); ASSERT_OK_AND_ASSIGN(auto step4, MakeTestFunctionStep(lt_call, registry)); @@ -625,15 +605,18 @@ INSTANTIATE_TEST_SUITE_P( class FunctionStepTestUnknowns : public testing::TestWithParam { public: - std::unique_ptr GetExpression(ExecutionPath&& path) { + std::unique_ptr GetExpression( + ExecutionPath&& path, IdentifierTable identifiers = {}) { cel::RuntimeOptions options; options.unknown_processing = GetParam(); auto env = NewTestingRuntimeEnv(); + const size_t slot_count = identifiers.size(); return std::make_unique( env, - FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + FlatExpression(std::move(path), /*comprehension_slot_count=*/slot_count, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, std::move(identifiers))); } }; @@ -675,14 +658,14 @@ TEST_P(FunctionStepTestUnknowns, PartialUnknownHandlingTest) { IdentExpr ident1; ident1.set_name("param"); CallExpr call1 = SinkFunction::MakeCall(); - - ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep("param", GetExprId())); + ExpressionStep step0 = ExpressionStep::MakeIdentifierStep(0, GetExprId()); ASSERT_OK_AND_ASSIGN(auto step1, MakeTestFunctionStep(call1, registry)); path.push_back(std::move(step0)); path.push_back(std::move(step1)); - std::unique_ptr impl = GetExpression(std::move(path)); + std::unique_ptr impl = + GetExpression(std::move(path), IdentifierTable{{"param", 0}}); Activation activation; TestMessage msg; @@ -695,7 +678,7 @@ TEST_P(FunctionStepTestUnknowns, PartialUnknownHandlingTest) { // Set attribute pattern that marks attribute "param[true]" as unknown. // It should result in "param" being handled as partially unknown, which is // is handled as fully unknown when used as function input argument. - activation.set_unknown_attribute_patterns({pattern}); + ASSERT_THAT(activation.SetUnknownAttributePatterns({pattern}), IsOk()); ASSERT_OK_AND_ASSIGN(CelValue value, impl->Evaluate(activation, &arena)); ASSERT_TRUE(value.IsUnknownSet()); @@ -770,10 +753,10 @@ TEST(FunctionStepTestUnknownFunctionResults, CaptureArgs) { cel::UnknownProcessingOptions::kAttributeAndFunction; auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( - env, - FlatExpression(std::move(path), - /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env, FlatExpression(std::move(path), + /*comprehension_slot_count=*/0, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, {})); Activation activation; google::protobuf::Arena arena; @@ -820,10 +803,10 @@ TEST(FunctionStepTestUnknownFunctionResults, MergeDownCaptureArgs) { cel::UnknownProcessingOptions::kAttributeAndFunction; auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( - env, - FlatExpression(std::move(path), - /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env, FlatExpression(std::move(path), + /*comprehension_slot_count=*/0, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, {})); Activation activation; google::protobuf::Arena arena; @@ -870,10 +853,10 @@ TEST(FunctionStepTestUnknownFunctionResults, MergeCaptureArgs) { cel::UnknownProcessingOptions::kAttributeAndFunction; auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( - env, - FlatExpression(std::move(path), - /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env, FlatExpression(std::move(path), + /*comprehension_slot_count=*/0, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, {})); Activation activation; google::protobuf::Arena arena; @@ -888,7 +871,7 @@ TEST(FunctionStepTestUnknownFunctionResults, UnknownVsErrorPrecedenceTest) { CelError error0 = absl::CancelledError(); CelValue error_value = CelValue::CreateError(&error0); - UnknownSet unknown_set; + UnknownSet unknown_set(cel::AttributeSet({cel::Attribute("foo")})); CelValue unknown_value = CelValue::CreateUnknownSet(&unknown_set); ASSERT_OK(registry.Register( @@ -915,10 +898,10 @@ TEST(FunctionStepTestUnknownFunctionResults, UnknownVsErrorPrecedenceTest) { cel::UnknownProcessingOptions::kAttributeAndFunction; auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( - env, - FlatExpression(std::move(path), - /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env, FlatExpression(std::move(path), + /*comprehension_slot_count=*/0, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, {})); Activation activation; google::protobuf::Arena arena; @@ -985,19 +968,19 @@ class NullFunction : public CelFunction { TEST(FunctionStepStrictnessTest, IfFunctionStrictAndGivenUnknownSkipsInvocation) { - UnknownSet unknown_set; + UnknownSet unknown_set(cel::AttributeSet({cel::Attribute("foo")})); CelFunctionRegistry registry; - ASSERT_OK(registry.Register(std::make_unique( - CelValue::CreateUnknownSet(&unknown_set), "ConstUnknown"))); - ASSERT_OK(registry.Register(std::make_unique( - CelValue::Type::kUnknownSet, /*is_strict=*/true))); + ASSERT_THAT(registry.Register(std::make_unique( + CelValue::CreateUnknownSet(&unknown_set), "ConstUnknown")), + IsOk()); + ASSERT_THAT(registry.Register(std::make_unique( + CelValue::Type::kUnknownSet, /*is_strict=*/true)), + IsOk()); ExecutionPath path; CallExpr call0 = ConstFunction::MakeCall("ConstUnknown"); CallExpr call1 = SinkFunction::MakeCall(); - ASSERT_OK_AND_ASSIGN(std::unique_ptr step0, - MakeTestFunctionStep(call0, registry)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr step1, - MakeTestFunctionStep(call1, registry)); + ASSERT_OK_AND_ASSIGN(auto step0, MakeTestFunctionStep(call0, registry)); + ASSERT_OK_AND_ASSIGN(auto step1, MakeTestFunctionStep(call1, registry)); path.push_back(std::move(step0)); path.push_back(std::move(step1)); cel::RuntimeOptions options; @@ -1005,10 +988,10 @@ TEST(FunctionStepStrictnessTest, cel::UnknownProcessingOptions::kAttributeAndFunction; auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( - env, - FlatExpression(std::move(path), - /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env, FlatExpression(std::move(path), + /*comprehension_slot_count=*/0, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, {})); Activation activation; google::protobuf::Arena arena; ASSERT_OK_AND_ASSIGN(CelValue value, impl.Evaluate(activation, &arena)); @@ -1016,19 +999,19 @@ TEST(FunctionStepStrictnessTest, } TEST(FunctionStepStrictnessTest, IfFunctionNonStrictAndGivenUnknownInvokesIt) { - UnknownSet unknown_set; + UnknownSet unknown_set(cel::AttributeSet({cel::Attribute("foo")})); CelFunctionRegistry registry; - ASSERT_OK(registry.Register(std::make_unique( - CelValue::CreateUnknownSet(&unknown_set), "ConstUnknown"))); - ASSERT_OK(registry.Register(std::make_unique( - CelValue::Type::kUnknownSet, /*is_strict=*/false))); + ASSERT_THAT(registry.Register(std::make_unique( + CelValue::CreateUnknownSet(&unknown_set), "ConstUnknown")), + IsOk()); + ASSERT_THAT(registry.Register(std::make_unique( + CelValue::Type::kUnknownSet, /*is_strict=*/false)), + IsOk()); ExecutionPath path; CallExpr call0 = ConstFunction::MakeCall("ConstUnknown"); CallExpr call1 = SinkFunction::MakeCall(); - ASSERT_OK_AND_ASSIGN(std::unique_ptr step0, - MakeTestFunctionStep(call0, registry)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr step1, - MakeTestFunctionStep(call1, registry)); + ASSERT_OK_AND_ASSIGN(auto step0, MakeTestFunctionStep(call0, registry)); + ASSERT_OK_AND_ASSIGN(auto step1, MakeTestFunctionStep(call1, registry)); path.push_back(std::move(step0)); path.push_back(std::move(step1)); Expr placeholder_expr; @@ -1037,185 +1020,15 @@ TEST(FunctionStepStrictnessTest, IfFunctionNonStrictAndGivenUnknownInvokesIt) { cel::UnknownProcessingOptions::kAttributeAndFunction; auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( - env, - FlatExpression(std::move(path), - /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env, FlatExpression(std::move(path), + /*comprehension_slot_count=*/0, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, {})); Activation activation; google::protobuf::Arena arena; ASSERT_OK_AND_ASSIGN(CelValue value, impl.Evaluate(activation, &arena)); ASSERT_THAT(value, test::IsCelInt64(Eq(0))); } -class DirectFunctionStepTest : public testing::Test { - public: - DirectFunctionStepTest() = default; - - void SetUp() override { - ASSERT_OK(cel::RegisterStandardFunctions(registry_, options_)); - } - - std::vector GetOverloads( - absl::string_view name, int64_t arguments_size) { - std::vector matcher; - matcher.resize(arguments_size, cel::Kind::kAny); - return registry_.FindStaticOverloads(name, false, matcher); - } - - // Helper for shorthand constructing direct expr deps. - // - // Works around copies in init-list construction. - std::vector> MakeDeps( - std::unique_ptr dep, - std::unique_ptr dep2) { - std::vector> result; - result.reserve(2); - result.push_back(std::move(dep)); - result.push_back(std::move(dep2)); - return result; - }; - - protected: - cel::FunctionRegistry registry_; - cel::RuntimeOptions options_; - google::protobuf::Arena arena_; -}; - -TEST_F(DirectFunctionStepTest, SimpleCall) { - cel::IntValue(1); - - CallExpr call; - call.set_function(cel::builtin::kAdd); - call.mutable_args().emplace_back(); - call.mutable_args().emplace_back(); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep(cel::IntValue(1))); - deps.push_back(CreateConstValueDirectStep(cel::IntValue(1))); - - auto expr = CreateDirectFunctionStep(-1, call, std::move(deps), - GetOverloads(cel::builtin::kAdd, 2)); - - auto plan = CreateExpressionImpl(options_, std::move(expr)); - - Activation activation; - ASSERT_OK_AND_ASSIGN(auto value, plan->Evaluate(activation, &arena_)); - - EXPECT_THAT(value, test::IsCelInt64(2)); -} - -TEST_F(DirectFunctionStepTest, RecursiveCall) { - cel::IntValue(1); - - CallExpr call; - call.set_function(cel::builtin::kAdd); - call.mutable_args().emplace_back(); - call.mutable_args().emplace_back(); - - auto overloads = GetOverloads(cel::builtin::kAdd, 2); - - auto MakeLeaf = [&]() { - return CreateDirectFunctionStep( - -1, call, - MakeDeps(CreateConstValueDirectStep(cel::IntValue(1)), - CreateConstValueDirectStep(cel::IntValue(1))), - overloads); - }; - - auto expr = CreateDirectFunctionStep( - -1, call, - MakeDeps(CreateDirectFunctionStep( - -1, call, MakeDeps(MakeLeaf(), MakeLeaf()), overloads), - CreateDirectFunctionStep( - -1, call, MakeDeps(MakeLeaf(), MakeLeaf()), overloads)), - overloads); - - auto plan = CreateExpressionImpl(options_, std::move(expr)); - - Activation activation; - ASSERT_OK_AND_ASSIGN(auto value, plan->Evaluate(activation, &arena_)); - - EXPECT_THAT(value, test::IsCelInt64(8)); -} - -TEST_F(DirectFunctionStepTest, ErrorHandlingCall) { - cel::IntValue(1); - - CallExpr add_call; - add_call.set_function(cel::builtin::kAdd); - add_call.mutable_args().emplace_back(); - add_call.mutable_args().emplace_back(); - - CallExpr div_call; - div_call.set_function(cel::builtin::kDivide); - div_call.mutable_args().emplace_back(); - div_call.mutable_args().emplace_back(); - - auto add_overloads = GetOverloads(cel::builtin::kAdd, 2); - auto div_overloads = GetOverloads(cel::builtin::kDivide, 2); - - auto error_expr = CreateDirectFunctionStep( - -1, div_call, - MakeDeps(CreateConstValueDirectStep(cel::IntValue(1)), - CreateConstValueDirectStep(cel::IntValue(0))), - div_overloads); - - auto expr = CreateDirectFunctionStep( - -1, add_call, - MakeDeps(std::move(error_expr), - CreateConstValueDirectStep(cel::IntValue(1))), - add_overloads); - - auto plan = CreateExpressionImpl(options_, std::move(expr)); - - Activation activation; - ASSERT_OK_AND_ASSIGN(auto value, plan->Evaluate(activation, &arena_)); - - EXPECT_THAT(value, - test::IsCelError(StatusIs(absl::StatusCode::kInvalidArgument, - testing::HasSubstr("divide by zero")))); -} - -TEST_F(DirectFunctionStepTest, NoOverload) { - cel::IntValue(1); - - CallExpr call; - call.set_function(cel::builtin::kAdd); - call.mutable_args().emplace_back(); - call.mutable_args().emplace_back(); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep(cel::IntValue(1))); - deps.push_back(CreateConstValueDirectStep(cel::StringValue("2"))); - - auto expr = CreateDirectFunctionStep(-1, call, std::move(deps), - GetOverloads(cel::builtin::kAdd, 2)); - - auto plan = CreateExpressionImpl(options_, std::move(expr)); - - Activation activation; - ASSERT_OK_AND_ASSIGN(auto value, plan->Evaluate(activation, &arena_)); - - EXPECT_THAT(value, Truly(CheckNoMatchingOverloadError)); -} - -TEST_F(DirectFunctionStepTest, NoOverload0Args) { - cel::IntValue(1); - - CallExpr call; - call.set_function(cel::builtin::kAdd); - - std::vector> deps; - auto expr = CreateDirectFunctionStep(-1, call, std::move(deps), - GetOverloads(cel::builtin::kAdd, 2)); - - auto plan = CreateExpressionImpl(options_, std::move(expr)); - - Activation activation; - ASSERT_OK_AND_ASSIGN(auto value, plan->Evaluate(activation, &arena_)); - - EXPECT_THAT(value, Truly(CheckNoMatchingOverloadError)); -} - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/eval/ident_step.cc b/eval/eval/ident_step.cc deleted file mode 100644 index 7ec1a3031..000000000 --- a/eval/eval/ident_step.cc +++ /dev/null @@ -1,174 +0,0 @@ -#include "eval/eval/ident_step.h" - -#include -#include -#include -#include -#include - -#include "absl/base/nullability.h" -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/string_view.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/comprehension_slots.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" -#include "eval/internal/errors.h" -#include "internal/status_macros.h" - -namespace google::api::expr::runtime { - -namespace { - -using ::cel::Value; -using ::cel::runtime_internal::CreateError; - -class IdentStep : public ExpressionStepBase { - public: - IdentStep(absl::string_view name, int64_t expr_id) - : ExpressionStepBase(expr_id), name_(name) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override; - - private: - std::string name_; -}; - -absl::Status LookupIdent(absl::string_view name, ExecutionFrameBase& frame, - Value& result, AttributeTrail& attribute) { - if (frame.attribute_tracking_enabled()) { - attribute = AttributeTrail(std::string(name)); - if (frame.missing_attribute_errors_enabled() && - frame.attribute_utility().CheckForMissingAttribute(attribute)) { - CEL_ASSIGN_OR_RETURN( - result, frame.attribute_utility().CreateMissingAttributeError( - attribute.attribute())); - return absl::OkStatus(); - } - if (frame.unknown_processing_enabled() && - frame.attribute_utility().CheckForUnknownExact(attribute)) { - result = - frame.attribute_utility().CreateUnknownSet(attribute.attribute()); - return absl::OkStatus(); - } - } - - CEL_ASSIGN_OR_RETURN( - auto found, frame.activation().FindVariable(name, frame.descriptor_pool(), - frame.message_factory(), - frame.arena(), &result)); - - if (found) { - return absl::OkStatus(); - } - - result = cel::ErrorValue(CreateError( - absl::StrCat("No value with name \"", name, "\" found in Activation"))); - - return absl::OkStatus(); -} - -absl::Status IdentStep::Evaluate(ExecutionFrame* frame) const { - Value value; - AttributeTrail attribute; - - CEL_RETURN_IF_ERROR(LookupIdent(name_, *frame, value, attribute)); - - frame->value_stack().Push(std::move(value), std::move(attribute)); - - return absl::OkStatus(); -} - -absl::StatusOr LookupSlot( - absl::string_view name, size_t slot_index, ExecutionFrameBase& frame) { - ComprehensionSlots::Slot* slot = frame.comprehension_slots().Get(slot_index); - if (!slot->Has()) { - return absl::InternalError( - absl::StrCat("Comprehension variable accessed out of scope: ", name)); - } - return slot; -} - -class SlotStep : public ExpressionStepBase { - public: - SlotStep(absl::string_view name, size_t slot_index, int64_t expr_id) - : ExpressionStepBase(expr_id), name_(name), slot_index_(slot_index) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - CEL_ASSIGN_OR_RETURN(const ComprehensionSlots::Slot* slot, - LookupSlot(name_, slot_index_, *frame)); - frame->value_stack().Push(slot->value(), slot->attribute()); - return absl::OkStatus(); - } - - private: - std::string name_; - - size_t slot_index_; -}; - -class DirectIdentStep : public DirectExpressionStep { - public: - DirectIdentStep(absl::string_view name, int64_t expr_id) - : DirectExpressionStep(expr_id), name_(name) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - return LookupIdent(name_, frame, result, attribute); - } - - private: - std::string name_; -}; - -class DirectSlotStep : public DirectExpressionStep { - public: - DirectSlotStep(absl::string_view name, size_t slot_index, int64_t expr_id) - : DirectExpressionStep(expr_id), - name_(std::move(name)), - slot_index_(slot_index) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - CEL_ASSIGN_OR_RETURN(const ComprehensionSlots::Slot* slot, - LookupSlot(name_, slot_index_, frame)); - - if (frame.attribute_tracking_enabled()) { - attribute = slot->attribute(); - } - result = slot->value(); - return absl::OkStatus(); - } - - private: - std::string name_; - size_t slot_index_; -}; - -} // namespace - -std::unique_ptr CreateDirectIdentStep( - absl::string_view identifier, int64_t expr_id) { - return std::make_unique(identifier, expr_id); -} - -std::unique_ptr CreateDirectSlotIdentStep( - absl::string_view identifier, size_t slot_index, int64_t expr_id) { - return std::make_unique(identifier, slot_index, expr_id); -} - -absl::StatusOr> CreateIdentStep( - const absl::string_view name, int64_t expr_id) { - return std::make_unique(name, expr_id); -} - -absl::StatusOr> CreateIdentStepForSlot( - const absl::string_view name, size_t slot_index, int64_t expr_id) { - return std::make_unique(name, slot_index, expr_id); -} - -} // namespace google::api::expr::runtime diff --git a/eval/eval/ident_step.h b/eval/eval/ident_step.h index d1bdde388..7ec2049d5 100644 --- a/eval/eval/ident_step.h +++ b/eval/eval/ident_step.h @@ -3,28 +3,24 @@ #include #include -#include - -#include "absl/status/statusor.h" -#include "absl/strings/string_view.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include +#include namespace google::api::expr::runtime { -std::unique_ptr CreateDirectIdentStep( - absl::string_view identifier, int64_t expr_id); +class ExecutionFrame; -std::unique_ptr CreateDirectSlotIdentStep( - absl::string_view identifier, size_t slot_index, int64_t expr_id); +struct Identifier { + std::string name; + size_t slot_index = 0; +}; -// Factory method for Ident - based Execution step -absl::StatusOr> CreateIdentStep( - absl::string_view name, int64_t expr_id); +using IdentifierTable = std::vector; -// Factory method for identifier that has been assigned to a slot. -absl::StatusOr> CreateIdentStepForSlot( - absl::string_view name, size_t slot_index, int64_t expr_id); +struct IdentStepInfo { + uint32_t ident_index; + uint32_t slot_index; +}; } // namespace google::api::expr::runtime diff --git a/eval/eval/ident_step_test.cc b/eval/eval/ident_step_test.cc index ce10d7d98..1afa88057 100644 --- a/eval/eval/ident_step_test.cc +++ b/eval/eval/ident_step_test.cc @@ -1,27 +1,26 @@ #include "eval/eval/ident_step.h" #include +#include #include #include #include #include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" #include "base/type_provider.h" -#include "common/casting.h" #include "common/memory.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" #include "eval/eval/cel_expression_flat_impl.h" #include "eval/eval/evaluator_core.h" #include "eval/public/activation.h" +#include "eval/public/base_activation.h" #include "eval/public/cel_attribute.h" +#include "eval/public/cel_function.h" #include "eval/public/cel_value.h" #include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" +#include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" -#include "runtime/internal/runtime_type_provider.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -29,41 +28,34 @@ namespace google::api::expr::runtime { namespace { +using ::absl_testing::IsOk; using ::absl_testing::StatusIs; -using ::cel::Cast; -using ::cel::ErrorValue; -using ::cel::InstanceOf; -using ::cel::IntValue; using ::cel::MemoryManagerRef; using ::cel::RuntimeOptions; using ::cel::TypeProvider; -using ::cel::UnknownValue; -using ::cel::Value; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::google::protobuf::Arena; using ::testing::Eq; using ::testing::HasSubstr; -using ::testing::SizeIs; TEST(IdentStepTest, TestIdentStep) { - ASSERT_OK_AND_ASSIGN(auto step, CreateIdentStep("name0", /*id=*/-1)); - ExecutionPath path; - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeIdentifierStep(0)); auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( - env, FlatExpression(std::move(path), /*comprehension_slot_count=*/0, + env, FlatExpression(std::move(path), /*comprehension_slot_count=*/1, env->type_registry.GetComposedTypeProvider(), - cel::RuntimeOptions{})); + cel::RuntimeOptions{}, /*arena=*/nullptr, + IdentifierTable{{"name0", 0}})); Activation activation; Arena arena; std::string value("test"); activation.InsertValue("name0", CelValue::CreateString(&value)); - auto status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + absl::StatusOr status0 = impl.Evaluate(activation, &arena); + ASSERT_THAT(status0, IsOk()); CelValue result = status0.value(); @@ -71,50 +63,119 @@ TEST(IdentStepTest, TestIdentStep) { EXPECT_THAT(result.StringOrDie().value(), Eq("test")); } -TEST(IdentStepTest, TestIdentStepNameNotFound) { - ASSERT_OK_AND_ASSIGN(auto step, CreateIdentStep("name0", /*id=*/-1)); +TEST(IdentStepTest, TestIdentStepWithSlotCachesLookup) { + ExecutionPath path; + // Evaluate "name0" twice and check equality (or return the second after + // popping the first viaComprehensionFinish / ReadSlot). Here we read the + // identifier step twice and then read slot 1 directly. + path.push_back(ExpressionStep::MakeIdentifierStep( + /*ident_index=*/0, /*slot_index=*/1, /*id=*/-1)); + path.push_back(ExpressionStep::MakeAssignSlotAndPopStep(0)); + path.push_back(ExpressionStep::MakeIdentifierStep( + /*ident_index=*/0, /*slot_index=*/1, /*id=*/-1)); + + auto env = NewTestingRuntimeEnv(); + CelExpressionFlatImpl impl( + env, FlatExpression(std::move(path), /*comprehension_slot_count=*/2, + env->type_registry.GetComposedTypeProvider(), + cel::RuntimeOptions{}, /*arena=*/nullptr, + IdentifierTable{{"name0", /*slot_index=*/1}})); + + class CountingActivation : public BaseActivation { + public: + CountingActivation(int* counter, const std::string* val) + : counter_(counter), val_(val) {} + std::vector FindFunctionOverloads( + absl::string_view) const override { + return {}; + } + std::optional FindValue(absl::string_view name, + Arena*) const override { + if (name == "name0") { + (*counter_)++; + return CelValue::CreateString(val_); + } + return std::nullopt; + } + + private: + int* counter_; + const std::string* val_; + }; + Arena arena; + int lookup_calls = 0; + std::string value("cached_value"); + CountingActivation activation(&lookup_calls, &value); + + absl::StatusOr status0 = impl.Evaluate(activation, &arena); + ASSERT_THAT(status0, IsOk()); + ASSERT_TRUE(status0->IsString()); + EXPECT_THAT(status0->StringOrDie().value(), Eq("cached_value")); + EXPECT_EQ(lookup_calls, 1); +} + +TEST(IdentStepTest, TestIdentStepNameNotFound) { ExecutionPath path; - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeIdentifierStep(0)); auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( - env, FlatExpression(std::move(path), /*comprehension_slot_count=*/0, + env, FlatExpression(std::move(path), /*comprehension_slot_count=*/1, env->type_registry.GetComposedTypeProvider(), - cel::RuntimeOptions{})); + cel::RuntimeOptions{}, /*arena=*/nullptr, + IdentifierTable{{"name0", 0}})); Activation activation; Arena arena; std::string value("test"); - auto status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + absl::StatusOr status0 = impl.Evaluate(activation, &arena); + ASSERT_THAT(status0, IsOk()); CelValue result = status0.value(); ASSERT_TRUE(result.IsError()); } -TEST(IdentStepTest, DisableMissingAttributeErrorsOK) { - ASSERT_OK_AND_ASSIGN(auto step, CreateIdentStep("name0", /*id=*/-1)); +TEST(IdentStepTest, TestIdentStepIndexOutOfRange) { + ExecutionPath path; + path.push_back(ExpressionStep::MakeIdentifierStep(0)); + std::shared_ptr env = + NewTestingRuntimeEnv(); + CelExpressionFlatImpl impl( + env, FlatExpression(std::move(path), /*comprehension_slots_size=*/1, + env->type_registry.GetComposedTypeProvider(), + cel::RuntimeOptions{}, /*arena=*/nullptr, + IdentifierTable{})); + + Activation activation; + Arena arena; + + EXPECT_THAT(impl.Evaluate(activation, &arena), + StatusIs(absl::StatusCode::kInternal, + HasSubstr("identifier index out of range"))); +} + +TEST(IdentStepTest, DisableMissingAttributeErrorsOK) { ExecutionPath path; - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeIdentifierStep(0)); cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kDisabled; auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( - env, - FlatExpression(std::move(path), - /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env, FlatExpression(std::move(path), + /*comprehension_slot_count=*/1, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, IdentifierTable{{"name0", 0}})); Activation activation; Arena arena; std::string value("test"); activation.InsertValue("name0", CelValue::CreateString(&value)); - auto status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + absl::StatusOr status0 = impl.Evaluate(activation, &arena); + ASSERT_THAT(status0, IsOk()); CelValue result = status0.value(); @@ -122,19 +183,17 @@ TEST(IdentStepTest, DisableMissingAttributeErrorsOK) { EXPECT_THAT(result.StringOrDie().value(), Eq("test")); const CelAttributePattern pattern("name0", {}); - activation.set_missing_attribute_patterns({pattern}); + ASSERT_THAT(activation.SetMissingAttributePatterns({pattern}), IsOk()); status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + ASSERT_THAT(status0, IsOk()); EXPECT_THAT(status0->StringOrDie().value(), Eq("test")); } TEST(IdentStepTest, TestIdentStepMissingAttributeErrors) { - ASSERT_OK_AND_ASSIGN(auto step, CreateIdentStep("name0", /*expr_id=*/1)); - ExecutionPath path; - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeIdentifierStep(0)); cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kDisabled; @@ -142,18 +201,18 @@ TEST(IdentStepTest, TestIdentStepMissingAttributeErrors) { auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( - env, - FlatExpression(std::move(path), - /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env, FlatExpression(std::move(path), + /*comprehension_slot_count=*/1, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, IdentifierTable{{"name0", 0}})); Activation activation; Arena arena; std::string value("test"); activation.InsertValue("name0", CelValue::CreateString(&value)); - auto status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + absl::StatusOr status0 = impl.Evaluate(activation, &arena); + ASSERT_THAT(status0, IsOk()); CelValue result = status0.value(); @@ -161,30 +220,28 @@ TEST(IdentStepTest, TestIdentStepMissingAttributeErrors) { EXPECT_THAT(result.StringOrDie().value(), Eq("test")); CelAttributePattern pattern("name0", {}); - activation.set_missing_attribute_patterns({pattern}); + ASSERT_THAT(activation.SetMissingAttributePatterns({pattern}), IsOk()); status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + ASSERT_THAT(status0, IsOk()); EXPECT_EQ(status0->ErrorOrDie()->code(), absl::StatusCode::kInvalidArgument); EXPECT_EQ(status0->ErrorOrDie()->message(), "MissingAttributeError: name0"); } TEST(IdentStepTest, TestIdentStepUnknownAttribute) { - ASSERT_OK_AND_ASSIGN(auto step, CreateIdentStep("name0", /*expr_id=*/1)); - ExecutionPath path; - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeIdentifierStep(0)); // Expression with unknowns enabled. cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( - env, - FlatExpression(std::move(path), - /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); + env, FlatExpression(std::move(path), + /*comprehension_slot_count=*/1, + env->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, IdentifierTable{{"name0", 0}})); Activation activation; Arena arena; @@ -194,9 +251,9 @@ TEST(IdentStepTest, TestIdentStepUnknownAttribute) { std::vector unknown_patterns; unknown_patterns.push_back(CelAttributePattern("name_bad", {})); - activation.set_unknown_attribute_patterns(unknown_patterns); - auto status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), IsOk()); + absl::StatusOr status0 = impl.Evaluate(activation, &arena); + ASSERT_THAT(status0, IsOk()); CelValue result = status0.value(); @@ -205,112 +262,15 @@ TEST(IdentStepTest, TestIdentStepUnknownAttribute) { unknown_patterns.push_back(CelAttributePattern("name0", {})); - activation.set_unknown_attribute_patterns(unknown_patterns); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), IsOk()); status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + ASSERT_THAT(status0, IsOk()); result = status0.value(); ASSERT_TRUE(result.IsUnknownSet()); } -TEST(DirectIdentStepTest, Basic) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - cel::Activation activation; - RuntimeOptions options; - - activation.InsertOrAssignValue("var1", IntValue(42)); - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - Value result; - AttributeTrail trail; - - auto step = CreateDirectIdentStep("var1", -1); - - ASSERT_OK(step->Evaluate(frame, result, trail)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), Eq(42)); -} - -TEST(DirectIdentStepTest, UnknownAttribute) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - cel::Activation activation; - RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - - activation.InsertOrAssignValue("var1", IntValue(42)); - activation.SetUnknownPatterns({CreateCelAttributePattern("var1", {})}); - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - Value result; - AttributeTrail trail; - - auto step = CreateDirectIdentStep("var1", -1); - - ASSERT_OK(step->Evaluate(frame, result, trail)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).attribute_set(), SizeIs(1)); -} - -TEST(DirectIdentStepTest, MissingAttribute) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - cel::Activation activation; - RuntimeOptions options; - options.enable_missing_attribute_errors = true; - - activation.InsertOrAssignValue("var1", IntValue(42)); - activation.SetMissingPatterns({CreateCelAttributePattern("var1", {})}); - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - Value result; - AttributeTrail trail; - - auto step = CreateDirectIdentStep("var1", -1); - - ASSERT_OK(step->Evaluate(frame, result, trail)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), - StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("var1"))); -} - -TEST(DirectIdentStepTest, NotFound) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - cel::Activation activation; - RuntimeOptions options; - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - Value result; - AttributeTrail trail; - - auto step = CreateDirectIdentStep("var1", -1); - - ASSERT_OK(step->Evaluate(frame, result, trail)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), - StatusIs(absl::StatusCode::kUnknown, - HasSubstr("\"var1\" found in Activation"))); -} - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/eval/iterator_stack.h b/eval/eval/iterator_stack.h index 9b5daa889..ccb377dd3 100644 --- a/eval/eval/iterator_stack.h +++ b/eval/eval/iterator_stack.h @@ -20,6 +20,7 @@ #include #include "absl/base/nullability.h" +#include "absl/container/inlined_vector.h" #include "absl/log/absl_check.h" #include "common/value.h" @@ -27,8 +28,15 @@ namespace cel::runtime_internal { class IteratorStack final { public: + struct Entry { + absl_nonnull ValueIteratorPtr iterator; + size_t iter_slot; + size_t iter2_slot; + size_t accu_slot; + }; + explicit IteratorStack(size_t max_size) : max_size_(max_size) { - iterators_.reserve(max_size_); + entries_.reserve(max_size_); } IteratorStack(const IteratorStack&) = delete; @@ -37,38 +45,55 @@ class IteratorStack final { IteratorStack& operator=(const IteratorStack&) = delete; IteratorStack& operator=(IteratorStack&&) = delete; - size_t size() const { return iterators_.size(); } + size_t size() const { return entries_.size(); } - bool empty() const { return iterators_.empty(); } + bool empty() const { return entries_.empty(); } - bool full() const { return iterators_.size() == max_size_; } + bool full() const { return entries_.size() == max_size_; } size_t max_size() const { return max_size_; } - void Clear() { iterators_.clear(); } + void Clear() { entries_.clear(); } + + void Push(absl_nonnull ValueIteratorPtr iterator, size_t iter_slot, + size_t iter2_slot, size_t accu_slot) { + ABSL_DCHECK(!full()); + ABSL_DCHECK(iterator != nullptr); + + entries_.push_back( + Entry{std::move(iterator), iter_slot, iter2_slot, accu_slot}); + } - void Push(absl_nonnull ValueIteratorPtr iterator) { + void Push(absl_nonnull ValueIteratorPtr iterator, size_t iter_slot, + size_t accu_slot) { ABSL_DCHECK(!full()); ABSL_DCHECK(iterator != nullptr); - iterators_.push_back(std::move(iterator)); + entries_.push_back(Entry{std::move(iterator), iter_slot, 0, accu_slot}); + } + + ValueIterator* absl_nonnull PeekIterator() { + ABSL_DCHECK(!empty()); + + return entries_.back().iterator.get(); } - ValueIterator* absl_nonnull Peek() { + // Returns a pointer to the top entry in the stack. + // Invalidated by Pop() and Push(). + Entry* absl_nonnull Peek() { ABSL_DCHECK(!empty()); - ABSL_DCHECK(iterators_.back() != nullptr); - return iterators_.back().get(); + return &entries_.back(); } void Pop() { ABSL_DCHECK(!empty()); - iterators_.pop_back(); + entries_.pop_back(); } private: - std::vector iterators_; + absl::InlinedVector entries_; size_t max_size_; }; diff --git a/eval/eval/jump_step.cc b/eval/eval/jump_step.cc deleted file mode 100644 index a65789841..000000000 --- a/eval/eval/jump_step.cc +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2017 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "eval/eval/jump_step.h" - -#include -#include -#include - -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/types/optional.h" -#include "common/value.h" -#include "eval/internal/errors.h" - -namespace google::api::expr::runtime { - -namespace { - -using ::cel::BoolValue; -using ::cel::ErrorValue; -using ::cel::UnknownValue; -using ::cel::Value; -using ::cel::runtime_internal::CreateNoMatchingOverloadError; - -class JumpStep : public JumpStepBase { - public: - // Constructs FunctionStep that uses overloads specified. - JumpStep(absl::optional jump_offset, int64_t expr_id) - : JumpStepBase(jump_offset, expr_id) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - return Jump(frame); - } -}; - -class CondJumpStep : public JumpStepBase { - public: - // Constructs FunctionStep that uses overloads specified. - CondJumpStep(bool jump_condition, bool leave_on_stack, - absl::optional jump_offset, int64_t expr_id) - : JumpStepBase(jump_offset, expr_id), - jump_condition_(jump_condition), - leave_on_stack_(leave_on_stack) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - // Peek the top value - if (!frame->value_stack().HasEnough(1)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - - const auto& value = frame->value_stack().Peek(); - const auto should_jump = value.Is() && - jump_condition_ == value.GetBool().NativeValue(); - - if (!leave_on_stack_) { - frame->value_stack().Pop(1); - } - - if (should_jump) { - return Jump(frame); - } - - return absl::OkStatus(); - } - - private: - const bool jump_condition_; - const bool leave_on_stack_; -}; - -class BoolCheckJumpStep : public JumpStepBase { - public: - // Checks if the top value is a boolean: - // - no-op if it is a boolean - // - jump to the label if it is an error value - // - jump to the label if it is unknown value - // - jump to the label if it is neither an error nor a boolean, pops it and - // pushes "no matching overload" error - BoolCheckJumpStep(absl::optional jump_offset, int64_t expr_id) - : JumpStepBase(jump_offset, expr_id) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - // Peek the top value - if (!frame->value_stack().HasEnough(1)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - - const Value& value = frame->value_stack().Peek(); - - if (value->Is()) { - return absl::OkStatus(); - } - - if (value->Is() || value->Is()) { - return Jump(frame); - } - - // Neither bool, error, nor unknown set. - Value error_value = - cel::ErrorValue(CreateNoMatchingOverloadError("")); - - frame->value_stack().PopAndPush(std::move(error_value)); - return Jump(frame); - - return absl::OkStatus(); - } -}; - -} // namespace - -// Factory method for Conditional Jump step. -// Conditional Jump requires a boolean value to sit on the stack. -// It is compared to jump_condition, and if matched, jump is performed. -std::unique_ptr CreateCondJumpStep( - bool jump_condition, bool leave_on_stack, absl::optional jump_offset, - int64_t expr_id) { - return std::make_unique(jump_condition, leave_on_stack, - jump_offset, expr_id); -} - -// Factory method for Jump step. -std::unique_ptr CreateJumpStep(absl::optional jump_offset, - int64_t expr_id) { - return std::make_unique(jump_offset, expr_id); -} - -// Factory method for Conditional Jump step. -// Conditional Jump requires a value to sit on the stack. -// If this value is an error or unknown, a jump is performed. -std::unique_ptr CreateBoolCheckJumpStep( - absl::optional jump_offset, int64_t expr_id) { - return std::make_unique(jump_offset, expr_id); -} - -} // namespace google::api::expr::runtime diff --git a/eval/eval/jump_step.h b/eval/eval/jump_step.h deleted file mode 100644 index 55147da5f..000000000 --- a/eval/eval/jump_step.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2017 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_JUMP_STEP_H_ -#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_JUMP_STEP_H_ - -#include -#include - -#include "cel/expr/syntax.pb.h" -#include "absl/status/status.h" -#include "absl/types/optional.h" -#include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" - -namespace google::api::expr::runtime { - -class JumpStepBase : public ExpressionStepBase { - public: - JumpStepBase(absl::optional jump_offset, int64_t expr_id) - : ExpressionStepBase(expr_id, false), jump_offset_(jump_offset) {} - - void set_jump_offset(int offset) { jump_offset_ = offset; } - - absl::Status Jump(ExecutionFrame* frame) const { - if (!jump_offset_.has_value()) { - return absl::Status(absl::StatusCode::kInternal, "Jump offset not set"); - } - return frame->JumpTo(jump_offset_.value()); - } - - private: - absl::optional jump_offset_; -}; - -// Factory method for Jump step. -std::unique_ptr CreateJumpStep(absl::optional jump_offset, - int64_t expr_id); - -// Factory method for Conditional Jump step. -// Conditional Jump requires a boolean value to sit on the stack. -// It is compared to jump_condition, and if matched, jump is performed. -// leave on stack indicates whether value should be kept on top of the stack or -// removed. -std::unique_ptr CreateCondJumpStep( - bool jump_condition, bool leave_on_stack, absl::optional jump_offset, - int64_t expr_id); - -// Factory method for ErrorJump step. -// This step performs a Jump when an Error is on the top of the stack. -// Value is left on stack if it is a bool or an error. -std::unique_ptr CreateBoolCheckJumpStep( - absl::optional jump_offset, int64_t expr_id); - -} // namespace google::api::expr::runtime - -#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_JUMP_STEP_H_ diff --git a/eval/eval/lazy_init_step.cc b/eval/eval/lazy_init_step.cc deleted file mode 100644 index eb9be7796..000000000 --- a/eval/eval/lazy_init_step.cc +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "eval/eval/lazy_init_step.h" - -#include -#include -#include -#include - -#include "cel/expr/value.pb.h" -#include "absl/base/nullability.h" -#include "absl/status/status.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/comprehension_slots.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" -#include "internal/status_macros.h" - -namespace google::api::expr::runtime { - -namespace { - -using ::cel::Value; - -class LazyInitStep final : public ExpressionStepBase { - public: - LazyInitStep(size_t slot_index, size_t subexpression_index, int64_t expr_id) - : ExpressionStepBase(expr_id), - slot_index_(slot_index), - subexpression_index_(subexpression_index) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - ComprehensionSlot* slot = frame->comprehension_slots().Get(slot_index_); - if (slot->Has()) { - frame->value_stack().Push(slot->value(), slot->attribute()); - } else { - frame->Call(slot_index_, subexpression_index_); - } - return absl::OkStatus(); - } - - private: - const size_t slot_index_; - const size_t subexpression_index_; -}; - -class DirectLazyInitStep final : public DirectExpressionStep { - public: - DirectLazyInitStep(size_t slot_index, - const DirectExpressionStep* subexpression, int64_t expr_id) - : DirectExpressionStep(expr_id), - slot_index_(slot_index), - subexpression_(subexpression) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - ComprehensionSlot* slot = frame.comprehension_slots().Get(slot_index_); - if (slot->Has()) { - result = slot->value(); - attribute = slot->attribute(); - } else { - CEL_RETURN_IF_ERROR(subexpression_->Evaluate(frame, result, attribute)); - slot->Set(result, attribute); - } - return absl::OkStatus(); - } - - private: - const size_t slot_index_; - const DirectExpressionStep* absl_nonnull const subexpression_; -}; - -class BindStep : public DirectExpressionStep { - public: - BindStep(size_t slot_index, - std::unique_ptr subexpression, int64_t expr_id) - : DirectExpressionStep(expr_id), - slot_index_(slot_index), - subexpression_(std::move(subexpression)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - CEL_RETURN_IF_ERROR(subexpression_->Evaluate(frame, result, attribute)); - - frame.comprehension_slots().ClearSlot(slot_index_); - - return absl::OkStatus(); - } - - private: - size_t slot_index_; - std::unique_ptr subexpression_; -}; - -class AssignSlotAndPopStepStep final : public ExpressionStepBase { - public: - explicit AssignSlotAndPopStepStep(size_t slot_index) - : ExpressionStepBase(/*expr_id=*/-1, /*comes_from_ast=*/false), - slot_index_(slot_index) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - if (!frame->value_stack().HasEnough(1)) { - return absl::InternalError("Stack underflow assigning lazy value"); - } - - frame->comprehension_slots().Set(slot_index_, frame->value_stack().Peek(), - frame->value_stack().PeekAttribute()); - frame->value_stack().Pop(1); - - return absl::OkStatus(); - } - - private: - const size_t slot_index_; -}; - -class ClearSlotStep : public ExpressionStepBase { - public: - explicit ClearSlotStep(size_t slot_index, int64_t expr_id) - : ExpressionStepBase(expr_id), slot_index_(slot_index) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - frame->comprehension_slots().ClearSlot(slot_index_); - return absl::OkStatus(); - } - - private: - size_t slot_index_; -}; - -class ClearSlotsStep final : public ExpressionStepBase { - public: - explicit ClearSlotsStep(size_t slot_index, size_t slot_count, int64_t expr_id) - : ExpressionStepBase(expr_id), - slot_index_(slot_index), - slot_count_(slot_count) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - for (size_t i = 0; i < slot_count_; ++i) { - frame->comprehension_slots().ClearSlot(slot_index_ + i); - } - return absl::OkStatus(); - } - - private: - const size_t slot_index_; - const size_t slot_count_; -}; - -class BlockStep : public DirectExpressionStep { - public: - BlockStep(size_t slot_index, size_t slot_count, - std::unique_ptr subexpression, - int64_t expr_id) - : DirectExpressionStep(expr_id), - slot_index_(slot_index), - slot_count_(slot_count), - subexpression_(std::move(subexpression)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - CEL_RETURN_IF_ERROR(subexpression_->Evaluate(frame, result, attribute)); - - for (size_t i = 0; i < slot_count_; ++i) { - frame.comprehension_slots().ClearSlot(slot_index_ + i); - } - - return absl::OkStatus(); - } - - private: - size_t slot_index_; - size_t slot_count_; - std::unique_ptr subexpression_; -}; - -} // namespace - -std::unique_ptr CreateDirectBindStep( - size_t slot_index, std::unique_ptr expression, - int64_t expr_id) { - return std::make_unique(slot_index, std::move(expression), expr_id); -} - -std::unique_ptr CreateDirectBlockStep( - size_t slot_index, size_t slot_count, - std::unique_ptr expression, int64_t expr_id) { - return std::make_unique(slot_index, slot_count, - std::move(expression), expr_id); -} - -std::unique_ptr CreateDirectLazyInitStep( - size_t slot_index, const DirectExpressionStep* absl_nonnull subexpression, - int64_t expr_id) { - return std::make_unique(slot_index, subexpression, - expr_id); -} - -std::unique_ptr CreateLazyInitStep(size_t slot_index, - size_t subexpression_index, - int64_t expr_id) { - return std::make_unique(slot_index, subexpression_index, - expr_id); -} - -std::unique_ptr CreateAssignSlotAndPopStep(size_t slot_index) { - return std::make_unique(slot_index); -} - -std::unique_ptr CreateClearSlotStep(size_t slot_index, - int64_t expr_id) { - return std::make_unique(slot_index, expr_id); -} - -std::unique_ptr CreateClearSlotsStep(size_t slot_index, - size_t slot_count, - int64_t expr_id) { - ABSL_DCHECK_GT(slot_count, 0); - return std::make_unique(slot_index, slot_count, expr_id); -} - -} // namespace google::api::expr::runtime diff --git a/eval/eval/lazy_init_step.h b/eval/eval/lazy_init_step.h index 714308dfd..0cb3e1fef 100644 --- a/eval/eval/lazy_init_step.h +++ b/eval/eval/lazy_init_step.h @@ -38,49 +38,18 @@ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_LAZY_INIT_STEP_H_ #include -#include -#include - -#include "absl/base/nullability.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" namespace google::api::expr::runtime { -// Creates a step representing a Bind expression. -std::unique_ptr CreateDirectBindStep( - size_t slot_index, std::unique_ptr expression, - int64_t expr_id); - -// Creates a step representing a cel.@block expression. -std::unique_ptr CreateDirectBlockStep( - size_t slot_index, size_t slot_count, - std::unique_ptr expression, int64_t expr_id); - -// Creates a direct step representing accessing a lazily evaluated alias from -// a bind or block. -std::unique_ptr CreateDirectLazyInitStep( - size_t slot_index, const DirectExpressionStep* absl_nonnull subexpression, - int64_t expr_id); - -// Creates a step representing accessing a lazily evaluated alias from -// a bind or block. -std::unique_ptr CreateLazyInitStep(size_t slot_index, - size_t subexpression_index, - int64_t expr_id); - -// Helper step to assign a slot value from the top of stack on initialization. -std::unique_ptr CreateAssignSlotAndPopStep(size_t slot_index); - -// Helper step to clear a slot. -// Slots may be reused in different contexts so need to be cleared after a -// context is done. -std::unique_ptr CreateClearSlotStep(size_t slot_index, - int64_t expr_id); +struct LazyInitStepInfo { + size_t slot_index : 32; + size_t subexpression_index : 32; +}; -std::unique_ptr CreateClearSlotsStep(size_t slot_index, - size_t slot_count, - int64_t expr_id); +struct ClearSlotStepInfo { + size_t slot_index : 32; + size_t slot_count : 32 = 1; +}; } // namespace google::api::expr::runtime diff --git a/eval/eval/lazy_init_step_test.cc b/eval/eval/lazy_init_step_test.cc index b9bef90a1..d7211e597 100644 --- a/eval/eval/lazy_init_step_test.cc +++ b/eval/eval/lazy_init_step_test.cc @@ -19,7 +19,7 @@ #include "base/type_provider.h" #include "common/value.h" -#include "eval/eval/const_value_step.h" +#include "eval/eval/comprehension_slots.h" #include "eval/eval/evaluator_core.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" @@ -46,7 +46,8 @@ class LazyInitStepTest : public testing::Test { public: LazyInitStepTest() : type_provider_(cel::internal::GetTestingDescriptorPool()), - evaluator_state_(kValueStack, kComprehensionSlotCount, type_provider_, + evaluator_state_(kValueStack, kComprehensionSlotCount, + /*iterator_stack_size=*/0, type_provider_, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena_) {} @@ -58,92 +59,85 @@ class LazyInitStepTest : public testing::Test { Activation activation_; }; -TEST_F(LazyInitStepTest, CreateCheckInitStepDoesInit) { +TEST_F(LazyInitStepTest, MakeLazyInitDoesInit) { ExecutionPath path; ExecutionPath subpath; - path.push_back(CreateLazyInitStep(/*slot_index=*/0, - /*subexpression_index=*/1, -1)); + path.push_back(ExpressionStep::MakeLazyInitStep( + /*slot_index=*/0, /*subexpression_index=*/1)); - ASSERT_OK_AND_ASSIGN(subpath.emplace_back(), - CreateConstValueStep(cel::IntValue(42), -1, false)); + subpath.push_back(ExpressionStep::MakeConstant(cel::IntValue(42))); std::vector expression_table{path, subpath}; ExecutionFrame frame(expression_table, activation_, runtime_options_, evaluator_state_); - ASSERT_OK_AND_ASSIGN(auto value, frame.Evaluate()); + ASSERT_OK_AND_ASSIGN(cel::Value value, frame.Evaluate()); EXPECT_TRUE(value->Is() && value.GetInt().NativeValue() == 42); } -TEST_F(LazyInitStepTest, CreateCheckInitStepSkipInit) { +TEST_F(LazyInitStepTest, MakeLazyInitSkipInit) { ExecutionPath path; ExecutionPath subpath; - // This is the expected usage, but in this test we are just depending on the - // fact that these don't change the stack and fit the program layout - // requirements. - path.push_back(CreateLazyInitStep(/*slot_index=*/0, -1, -1)); + path.push_back(ExpressionStep::MakeLazyInitStep( + /*slot_index=*/0, /*subexpression_index=*/2)); - ASSERT_OK_AND_ASSIGN(subpath.emplace_back(), - CreateConstValueStep(cel::IntValue(42), -1, false)); + subpath.push_back(ExpressionStep::MakeConstant(cel::IntValue(42))); std::vector expression_table{path, subpath}; ExecutionFrame frame(expression_table, activation_, runtime_options_, evaluator_state_); frame.comprehension_slots().Set(0, cel::IntValue(42)); - ASSERT_OK_AND_ASSIGN(auto value, frame.Evaluate()); + ASSERT_OK_AND_ASSIGN(cel::Value value, frame.Evaluate()); EXPECT_TRUE(value->Is() && value.GetInt().NativeValue() == 42); } -TEST_F(LazyInitStepTest, CreateAssignSlotAndPopStepBasic) { +TEST_F(LazyInitStepTest, MakeAssignSlotAndPopBasic) { ExecutionPath path; - path.push_back(CreateAssignSlotAndPopStep(0)); + path.push_back(ExpressionStep::MakeAssignSlotAndPopStep(0)); ExecutionFrame frame(path, activation_, runtime_options_, evaluator_state_); frame.comprehension_slots().ClearSlot(0); frame.value_stack().Push(cel::IntValue(42)); - // This will error because no return value, step will still evaluate. frame.Evaluate().IgnoreError(); - auto* slot = frame.comprehension_slots().Get(0); + ComprehensionSlots::Slot* slot = frame.comprehension_slots().Get(0); ASSERT_TRUE(slot->Has()); EXPECT_TRUE(slot->value()->Is() && slot->value().GetInt().NativeValue() == 42); EXPECT_TRUE(frame.value_stack().empty()); } -TEST_F(LazyInitStepTest, CreateClearSlotStepBasic) { +TEST_F(LazyInitStepTest, MakeClearSlotBasic) { ExecutionPath path; - path.push_back(CreateClearSlotStep(0, -1)); + path.push_back(ExpressionStep::MakeClearSlotStep(0)); ExecutionFrame frame(path, activation_, runtime_options_, evaluator_state_); frame.comprehension_slots().Set(0, cel::IntValue(42)); - // This will error because no return value, step will still evaluate. frame.Evaluate().IgnoreError(); - auto* slot = frame.comprehension_slots().Get(0); + ComprehensionSlot* slot = frame.comprehension_slots().Get(0); ASSERT_FALSE(slot->Has()); } -TEST_F(LazyInitStepTest, CreateClearSlotsStepBasic) { +TEST_F(LazyInitStepTest, MakeClearSlotsBasic) { ExecutionPath path; - path.push_back(CreateClearSlotsStep(0, 2, -1)); + path.push_back(ExpressionStep::MakeClearSlotsStep(0, 2)); ExecutionFrame frame(path, activation_, runtime_options_, evaluator_state_); frame.comprehension_slots().Set(0, cel::IntValue(42)); frame.comprehension_slots().Set(1, cel::IntValue(42)); - // This will error because no return value, step will still evaluate. frame.Evaluate().IgnoreError(); EXPECT_FALSE(frame.comprehension_slots().Get(0)->Has()); diff --git a/eval/eval/logic_step.cc b/eval/eval/logic_step.cc deleted file mode 100644 index f844d8c05..000000000 --- a/eval/eval/logic_step.cc +++ /dev/null @@ -1,487 +0,0 @@ -#include "eval/eval/logic_step.h" - -#include -#include -#include -#include - -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/types/optional.h" -#include "absl/types/span.h" -#include "base/builtins.h" -#include "common/casting.h" -#include "common/value.h" -#include "common/value_kind.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" -#include "eval/internal/errors.h" -#include "internal/status_macros.h" -#include "runtime/internal/errors.h" - -namespace google::api::expr::runtime { - -namespace { - -using ::cel::BoolValue; -using ::cel::Cast; -using ::cel::ErrorValue; -using ::cel::InstanceOf; -using ::cel::UnknownValue; -using ::cel::Value; -using ::cel::ValueKind; -using ::cel::runtime_internal::CreateNoMatchingOverloadError; - -enum class OpType { kAnd, kOr }; - -// Shared logic for the fall through case (we didn't see the shortcircuit -// value). -absl::Status ReturnLogicResult(ExecutionFrameBase& frame, OpType op_type, - Value& lhs_result, Value& rhs_result, - AttributeTrail& attribute_trail, - AttributeTrail& rhs_attr) { - ValueKind lhs_kind = lhs_result.kind(); - ValueKind rhs_kind = rhs_result.kind(); - - if (frame.unknown_processing_enabled()) { - if (lhs_kind == ValueKind::kUnknown && rhs_kind == ValueKind::kUnknown) { - lhs_result = frame.attribute_utility().MergeUnknownValues( - Cast(lhs_result), Cast(rhs_result)); - // Clear attribute trail so this doesn't get re-identified as a new - // unknown and reset the accumulated attributes. - attribute_trail = AttributeTrail(); - return absl::OkStatus(); - } else if (lhs_kind == ValueKind::kUnknown) { - return absl::OkStatus(); - } else if (rhs_kind == ValueKind::kUnknown) { - lhs_result = std::move(rhs_result); - attribute_trail = std::move(rhs_attr); - return absl::OkStatus(); - } - } - - if (lhs_kind == ValueKind::kError) { - return absl::OkStatus(); - } else if (rhs_kind == ValueKind::kError) { - lhs_result = std::move(rhs_result); - attribute_trail = std::move(rhs_attr); - return absl::OkStatus(); - } - - if (lhs_kind == ValueKind::kBool && rhs_kind == ValueKind::kBool) { - return absl::OkStatus(); - } - - // Otherwise, add a no overload error. - attribute_trail = AttributeTrail(); - lhs_result = cel::ErrorValue(CreateNoMatchingOverloadError( - op_type == OpType::kOr ? cel::builtin::kOr : cel::builtin::kAnd)); - return absl::OkStatus(); -} - -class ExhaustiveDirectLogicStep : public DirectExpressionStep { - public: - explicit ExhaustiveDirectLogicStep(std::unique_ptr lhs, - std::unique_ptr rhs, - OpType op_type, int64_t expr_id) - : DirectExpressionStep(expr_id), - lhs_(std::move(lhs)), - rhs_(std::move(rhs)), - op_type_(op_type) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& attribute_trail) const override; - - private: - std::unique_ptr lhs_; - std::unique_ptr rhs_; - OpType op_type_; -}; - -absl::Status ExhaustiveDirectLogicStep::Evaluate( - ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& attribute_trail) const { - CEL_RETURN_IF_ERROR(lhs_->Evaluate(frame, result, attribute_trail)); - ValueKind lhs_kind = result.kind(); - - Value rhs_result; - AttributeTrail rhs_attr; - CEL_RETURN_IF_ERROR(rhs_->Evaluate(frame, rhs_result, attribute_trail)); - - ValueKind rhs_kind = rhs_result.kind(); - if (lhs_kind == ValueKind::kBool) { - bool lhs_bool = Cast(result).NativeValue(); - if ((op_type_ == OpType::kOr && lhs_bool) || - (op_type_ == OpType::kAnd && !lhs_bool)) { - return absl::OkStatus(); - } - } - - if (rhs_kind == ValueKind::kBool) { - bool rhs_bool = Cast(rhs_result).NativeValue(); - if ((op_type_ == OpType::kOr && rhs_bool) || - (op_type_ == OpType::kAnd && !rhs_bool)) { - result = std::move(rhs_result); - attribute_trail = std::move(rhs_attr); - return absl::OkStatus(); - } - } - - return ReturnLogicResult(frame, op_type_, result, rhs_result, attribute_trail, - rhs_attr); -} - -class DirectLogicStep : public DirectExpressionStep { - public: - explicit DirectLogicStep(std::unique_ptr lhs, - std::unique_ptr rhs, - OpType op_type, int64_t expr_id) - : DirectExpressionStep(expr_id), - lhs_(std::move(lhs)), - rhs_(std::move(rhs)), - op_type_(op_type) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& attribute_trail) const override; - - private: - std::unique_ptr lhs_; - std::unique_ptr rhs_; - OpType op_type_; -}; - -absl::Status DirectLogicStep::Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const { - CEL_RETURN_IF_ERROR(lhs_->Evaluate(frame, result, attribute_trail)); - ValueKind lhs_kind = result.kind(); - if (lhs_kind == ValueKind::kBool) { - bool lhs_bool = Cast(result).NativeValue(); - if ((op_type_ == OpType::kOr && lhs_bool) || - (op_type_ == OpType::kAnd && !lhs_bool)) { - return absl::OkStatus(); - } - } - - Value rhs_result; - AttributeTrail rhs_attr; - - CEL_RETURN_IF_ERROR(rhs_->Evaluate(frame, rhs_result, attribute_trail)); - - ValueKind rhs_kind = rhs_result.kind(); - - if (rhs_kind == ValueKind::kBool) { - bool rhs_bool = Cast(rhs_result).NativeValue(); - if ((op_type_ == OpType::kOr && rhs_bool) || - (op_type_ == OpType::kAnd && !rhs_bool)) { - result = std::move(rhs_result); - attribute_trail = std::move(rhs_attr); - return absl::OkStatus(); - } - } - - return ReturnLogicResult(frame, op_type_, result, rhs_result, attribute_trail, - rhs_attr); -} - -class LogicalOpStep : public ExpressionStepBase { - public: - // Constructs FunctionStep that uses overloads specified. - LogicalOpStep(OpType op_type, int64_t expr_id) - : ExpressionStepBase(expr_id), op_type_(op_type) { - shortcircuit_ = (op_type_ == OpType::kOr); - } - - absl::Status Evaluate(ExecutionFrame* frame) const override; - - private: - void Calculate(ExecutionFrame* frame, absl::Span args, - Value& result) const { - bool bool_args[2]; - bool has_bool_args[2]; - - for (size_t i = 0; i < args.size(); i++) { - has_bool_args[i] = args[i]->Is(); - if (has_bool_args[i]) { - bool_args[i] = args[i].GetBool().NativeValue(); - if (bool_args[i] == shortcircuit_) { - result = BoolValue{bool_args[i]}; - return; - } - } - } - - if (has_bool_args[0] && has_bool_args[1]) { - switch (op_type_) { - case OpType::kAnd: - result = BoolValue{bool_args[0] && bool_args[1]}; - return; - case OpType::kOr: - result = BoolValue{bool_args[0] || bool_args[1]}; - return; - } - } - - // As opposed to regular function, logical operation treat Unknowns with - // higher precedence than error. This is due to the fact that after Unknown - // is resolved to actual value, it may short-circuit and thus hide the - // error. - if (frame->enable_unknowns()) { - // Check if unknown? - absl::optional unknown_set = - frame->attribute_utility().MergeUnknowns(args); - if (unknown_set.has_value()) { - result = std::move(*unknown_set); - return; - } - } - - if (args[0]->Is()) { - result = args[0]; - return; - } else if (args[1]->Is()) { - result = args[1]; - return; - } - - // Fallback. - result = cel::ErrorValue(CreateNoMatchingOverloadError( - (op_type_ == OpType::kOr) ? cel::builtin::kOr : cel::builtin::kAnd)); - } - - const OpType op_type_; - bool shortcircuit_; -}; - -absl::Status LogicalOpStep::Evaluate(ExecutionFrame* frame) const { - // Must have 2 or more values on the stack. - if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - - // Create Span object that contains input arguments to the function. - auto args = frame->value_stack().GetSpan(2); - Value result; - Calculate(frame, args, result); - frame->value_stack().PopAndPush(args.size(), std::move(result)); - - return absl::OkStatus(); -} - -std::unique_ptr CreateDirectLogicStep( - std::unique_ptr lhs, - std::unique_ptr rhs, int64_t expr_id, OpType op_type, - bool shortcircuiting) { - if (shortcircuiting) { - return std::make_unique(std::move(lhs), std::move(rhs), - op_type, expr_id); - } else { - return std::make_unique( - std::move(lhs), std::move(rhs), op_type, expr_id); - } -} - -class DirectNotStep : public DirectExpressionStep { - public: - explicit DirectNotStep(std::unique_ptr operand, - int64_t expr_id) - : DirectExpressionStep(expr_id), operand_(std::move(operand)) {} - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const override; - - private: - std::unique_ptr operand_; -}; - -absl::Status DirectNotStep::Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const { - CEL_RETURN_IF_ERROR(operand_->Evaluate(frame, result, attribute_trail)); - - if (frame.unknown_processing_enabled()) { - if (frame.attribute_utility().CheckForUnknownPartial(attribute_trail)) { - result = frame.attribute_utility().CreateUnknownSet( - attribute_trail.attribute()); - return absl::OkStatus(); - } - } - - switch (result.kind()) { - case ValueKind::kBool: - result = BoolValue{!result.GetBool().NativeValue()}; - break; - case ValueKind::kUnknown: - case ValueKind::kError: - // just forward. - break; - default: - result = - cel::ErrorValue(CreateNoMatchingOverloadError(cel::builtin::kNot)); - break; - } - - return absl::OkStatus(); -} - -class IterativeNotStep : public ExpressionStepBase { - public: - explicit IterativeNotStep(int64_t expr_id) : ExpressionStepBase(expr_id) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override; -}; - -absl::Status IterativeNotStep::Evaluate(ExecutionFrame* frame) const { - if (!frame->value_stack().HasEnough(1)) { - return absl::InternalError("Value stack underflow"); - } - const Value& operand = frame->value_stack().Peek(); - - if (frame->unknown_processing_enabled()) { - const AttributeTrail& attribute_trail = - frame->value_stack().PeekAttribute(); - if (frame->attribute_utility().CheckForUnknownPartial(attribute_trail)) { - frame->value_stack().PopAndPush( - frame->attribute_utility().CreateUnknownSet( - attribute_trail.attribute())); - return absl::OkStatus(); - } - } - - switch (operand.kind()) { - case ValueKind::kBool: - frame->value_stack().PopAndPush( - BoolValue{!operand.GetBool().NativeValue()}); - break; - case ValueKind::kUnknown: - case ValueKind::kError: - // just forward. - break; - default: - frame->value_stack().PopAndPush( - cel::ErrorValue(CreateNoMatchingOverloadError(cel::builtin::kNot))); - break; - } - - return absl::OkStatus(); -} - -class DirectNotStrictlyFalseStep : public DirectExpressionStep { - public: - explicit DirectNotStrictlyFalseStep( - std::unique_ptr operand, int64_t expr_id) - : DirectExpressionStep(expr_id), operand_(std::move(operand)) {} - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const override; - - private: - std::unique_ptr operand_; -}; - -absl::Status DirectNotStrictlyFalseStep::Evaluate( - ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const { - CEL_RETURN_IF_ERROR(operand_->Evaluate(frame, result, attribute_trail)); - - switch (result.kind()) { - case ValueKind::kBool: - // just forward. - break; - case ValueKind::kUnknown: - case ValueKind::kError: - result = BoolValue(true); - break; - default: - result = - cel::ErrorValue(CreateNoMatchingOverloadError(cel::builtin::kNot)); - break; - } - - return absl::OkStatus(); -} - -class IterativeNotStrictlyFalseStep : public ExpressionStepBase { - public: - explicit IterativeNotStrictlyFalseStep(int64_t expr_id) - : ExpressionStepBase(expr_id) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override; -}; - -absl::Status IterativeNotStrictlyFalseStep::Evaluate( - ExecutionFrame* frame) const { - if (!frame->value_stack().HasEnough(1)) { - return absl::InternalError("Value stack underflow"); - } - const Value& operand = frame->value_stack().Peek(); - - switch (operand.kind()) { - case ValueKind::kBool: - // just forward. - break; - case ValueKind::kUnknown: - case ValueKind::kError: - frame->value_stack().PopAndPush(BoolValue(true)); - break; - default: - frame->value_stack().PopAndPush( - cel::ErrorValue(CreateNoMatchingOverloadError(cel::builtin::kNot))); - break; - } - - return absl::OkStatus(); -} - -} // namespace - -// Factory method for "And" Execution step -std::unique_ptr CreateDirectAndStep( - std::unique_ptr lhs, - std::unique_ptr rhs, int64_t expr_id, - bool shortcircuiting) { - return CreateDirectLogicStep(std::move(lhs), std::move(rhs), expr_id, - OpType::kAnd, shortcircuiting); -} - -// Factory method for "Or" Execution step -std::unique_ptr CreateDirectOrStep( - std::unique_ptr lhs, - std::unique_ptr rhs, int64_t expr_id, - bool shortcircuiting) { - return CreateDirectLogicStep(std::move(lhs), std::move(rhs), expr_id, - OpType::kOr, shortcircuiting); -} - -// Factory method for "And" Execution step -absl::StatusOr> CreateAndStep(int64_t expr_id) { - return std::make_unique(OpType::kAnd, expr_id); -} - -// Factory method for "Or" Execution step -absl::StatusOr> CreateOrStep(int64_t expr_id) { - return std::make_unique(OpType::kOr, expr_id); -} - -// Factory method for recursive logical not "!" Execution step -std::unique_ptr CreateDirectNotStep( - std::unique_ptr operand, int64_t expr_id) { - return std::make_unique(std::move(operand), expr_id); -} - -// Factory method for iterative logical not "!" Execution step -std::unique_ptr CreateNotStep(int64_t expr_id) { - return std::make_unique(expr_id); -} - -// Factory method for recursive logical "@not_strictly_false" Execution step. -std::unique_ptr CreateDirectNotStrictlyFalseStep( - std::unique_ptr operand, int64_t expr_id) { - return std::make_unique(std::move(operand), - expr_id); -} - -// Factory method for iterative logical "@not_strictly_false" Execution step. -std::unique_ptr CreateNotStrictlyFalseStep(int64_t expr_id) { - return std::make_unique(expr_id); -} - -} // namespace google::api::expr::runtime diff --git a/eval/eval/logic_step.h b/eval/eval/logic_step.h deleted file mode 100644 index d75ed3715..000000000 --- a/eval/eval/logic_step.h +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_LOGIC_STEP_H_ -#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_LOGIC_STEP_H_ - -#include -#include - -#include "absl/status/statusor.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" - -namespace google::api::expr::runtime { - -// Factory method for "And" Execution step -std::unique_ptr CreateDirectAndStep( - std::unique_ptr lhs, - std::unique_ptr rhs, int64_t expr_id, - bool shortcircuiting); - -// Factory method for "Or" Execution step -std::unique_ptr CreateDirectOrStep( - std::unique_ptr lhs, - std::unique_ptr rhs, int64_t expr_id, - bool shortcircuiting); - -// Factory method for "And" Execution step -absl::StatusOr> CreateAndStep(int64_t expr_id); - -// Factory method for "Or" Execution step -absl::StatusOr> CreateOrStep(int64_t expr_id); - -// Factory method for recursive logical not "!" Execution step -std::unique_ptr CreateDirectNotStep( - std::unique_ptr operand, int64_t expr_id); - -// Factory method for iterative logical not "!" Execution step -std::unique_ptr CreateNotStep(int64_t expr_id); - -// Factory method for recursive logical "@not_strictly_false" Execution step. -std::unique_ptr CreateDirectNotStrictlyFalseStep( - std::unique_ptr operand, int64_t expr_id); - -// Factory method for iterative logical "@not_strictly_false" Execution step. -std::unique_ptr CreateNotStrictlyFalseStep(int64_t expr_id); - -} // namespace google::api::expr::runtime - -#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_LOGIC_STEP_H_ diff --git a/eval/eval/logic_step_test.cc b/eval/eval/logic_step_test.cc index 17ca8ba0d..ef07b7236 100644 --- a/eval/eval/logic_step_test.cc +++ b/eval/eval/logic_step_test.cc @@ -1,42 +1,21 @@ -#include "eval/eval/logic_step.h" - #include #include -#include #include -#include #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" -#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "base/attribute.h" -#include "base/attribute_set.h" #include "base/type_provider.h" -#include "common/casting.h" #include "common/expr.h" -#include "common/unknown.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/const_value_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/ident_step.h" #include "eval/public/activation.h" -#include "eval/public/cel_attribute.h" #include "eval/public/cel_value.h" -#include "eval/public/unknown_attribute_set.h" -#include "eval/public/unknown_set.h" #include "internal/status_macros.h" #include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" #include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" -#include "runtime/internal/runtime_type_provider.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -45,20 +24,11 @@ namespace google::api::expr::runtime { namespace { using ::absl_testing::IsOk; -using ::cel::Attribute; -using ::cel::AttributeSet; -using ::cel::BoolValue; -using ::cel::Cast; using ::cel::Expr; -using ::cel::InstanceOf; -using ::cel::IntValue; using ::cel::TypeProvider; -using ::cel::UnknownValue; -using ::cel::Value; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::runtime_internal::RuntimeEnv; using ::google::protobuf::Arena; -using ::testing::Eq; class LogicStepTest : public testing::TestWithParam { public: @@ -67,14 +37,11 @@ class LogicStepTest : public testing::TestWithParam { absl::Status EvaluateLogic(CelValue arg0, CelValue arg1, bool is_or, CelValue* result, bool enable_unknown) { ExecutionPath path; - CEL_ASSIGN_OR_RETURN(auto step, CreateIdentStep("name0", /*expr_id=*/-1)); - path.push_back(std::move(step)); - - CEL_ASSIGN_OR_RETURN(step, CreateIdentStep("name1", /*expr_id=*/-1)); - path.push_back(std::move(step)); - - CEL_ASSIGN_OR_RETURN(step, (is_or) ? CreateOrStep(2) : CreateAndStep(2)); - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeIdentifierStep(0)); + path.push_back(ExpressionStep::MakeIdentifierStep(1)); + path.push_back( + (is_or) ? ExpressionStep::MakeBooleanOrStep(/*num_args=*/2, /*id=*/2) + : ExpressionStep::MakeBooleanAndStep(/*num_args=*/2, /*id=*/2)); auto dummy_expr = std::make_unique(); cel::RuntimeOptions options; @@ -84,8 +51,10 @@ class LogicStepTest : public testing::TestWithParam { } CelExpressionFlatImpl impl( env_, - FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env_->type_registry.GetComposedTypeProvider(), options)); + FlatExpression(std::move(path), /*comprehension_slot_count=*/2, + env_->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, + IdentifierTable{{"name0", 0}, {"name1", 1}})); Activation activation; activation.InsertValue("name0", arg0); @@ -215,435 +184,8 @@ TEST_P(LogicStepTest, TestOrLogicErrorHandling) { ASSERT_TRUE(result.BoolOrDie()); } -TEST_F(LogicStepTest, TestAndLogicUnknownHandling) { - CelValue result; - UnknownSet unknown_set; - CelError cel_error = absl::CancelledError(); - CelValue unknown_value = CelValue::CreateUnknownSet(&unknown_set); - CelValue error_value = CelValue::CreateError(&cel_error); - absl::Status status = EvaluateLogic(unknown_value, CelValue::CreateBool(true), - false, &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - status = EvaluateLogic(CelValue::CreateBool(true), unknown_value, false, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - status = EvaluateLogic(CelValue::CreateBool(false), unknown_value, false, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsBool()); - ASSERT_FALSE(result.BoolOrDie()); - - status = EvaluateLogic(unknown_value, CelValue::CreateBool(false), false, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsBool()); - ASSERT_FALSE(result.BoolOrDie()); - - status = EvaluateLogic(error_value, unknown_value, false, &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - status = EvaluateLogic(unknown_value, error_value, false, &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - CelAttribute attr0("name0", {}), attr1("name1", {}); - UnknownAttributeSet unknown_attr_set0({attr0}); - UnknownAttributeSet unknown_attr_set1({attr1}); - UnknownSet unknown_set0(unknown_attr_set0); - UnknownSet unknown_set1(unknown_attr_set1); - - EXPECT_THAT(unknown_attr_set0.size(), Eq(1)); - EXPECT_THAT(unknown_attr_set1.size(), Eq(1)); - - status = EvaluateLogic(CelValue::CreateUnknownSet(&unknown_set0), - CelValue::CreateUnknownSet(&unknown_set1), false, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - ASSERT_THAT(result.UnknownSetOrDie()->unknown_attributes().size(), Eq(2)); -} - -TEST_F(LogicStepTest, TestOrLogicUnknownHandling) { - CelValue result; - UnknownSet unknown_set; - CelError cel_error = absl::CancelledError(); - CelValue unknown_value = CelValue::CreateUnknownSet(&unknown_set); - CelValue error_value = CelValue::CreateError(&cel_error); - absl::Status status = EvaluateLogic( - unknown_value, CelValue::CreateBool(false), true, &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - status = EvaluateLogic(CelValue::CreateBool(false), unknown_value, true, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - status = EvaluateLogic(CelValue::CreateBool(true), unknown_value, true, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsBool()); - ASSERT_TRUE(result.BoolOrDie()); - - status = EvaluateLogic(unknown_value, CelValue::CreateBool(true), true, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsBool()); - ASSERT_TRUE(result.BoolOrDie()); - - status = EvaluateLogic(unknown_value, error_value, true, &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - status = EvaluateLogic(error_value, unknown_value, true, &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - CelAttribute attr0("name0", {}), attr1("name1", {}); - UnknownAttributeSet unknown_attr_set0({attr0}); - UnknownAttributeSet unknown_attr_set1({attr1}); - - UnknownSet unknown_set0(unknown_attr_set0); - UnknownSet unknown_set1(unknown_attr_set1); - - EXPECT_THAT(unknown_attr_set0.size(), Eq(1)); - EXPECT_THAT(unknown_attr_set1.size(), Eq(1)); - - status = EvaluateLogic(CelValue::CreateUnknownSet(&unknown_set0), - CelValue::CreateUnknownSet(&unknown_set1), true, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - ASSERT_THAT(result.UnknownSetOrDie()->unknown_attributes().size(), Eq(2)); -} - INSTANTIATE_TEST_SUITE_P(LogicStepTest, LogicStepTest, testing::Bool()); -enum class BinaryOp { kAnd, kOr }; -enum class UnaryOp { kNot, kNotStrictlyFalse }; - -enum class OpArg { - kTrue, - kFalse, - kUnknown, - kError, - // Arbitrary incorrect type - kInt -}; - -enum class OpResult { - kTrue, - kFalse, - kUnknown, - kError, -}; - -struct BinaryTestCase { - std::string name; - BinaryOp op; - OpArg arg0; - OpArg arg1; - OpResult result; -}; - -UnknownValue MakeUnknownValue(std::string attr) { - std::vector attrs; - attrs.push_back(Attribute(std::move(attr))); - return cel::UnknownValue(cel::Unknown(AttributeSet(attrs))); -} - -std::unique_ptr MakeArgStep(OpArg arg, - absl::string_view name) { - switch (arg) { - case OpArg::kTrue: - return CreateConstValueDirectStep(BoolValue(true)); - case OpArg::kFalse: - return CreateConstValueDirectStep(BoolValue(false)); - case OpArg::kUnknown: - return CreateConstValueDirectStep(MakeUnknownValue(std::string(name))); - case OpArg::kError: - return CreateConstValueDirectStep( - cel::ErrorValue(absl::InternalError(name))); - case OpArg::kInt: - return CreateConstValueDirectStep(IntValue(42)); - } -}; - -class DirectBinaryLogicStepTest - : public testing::TestWithParam> { - public: - DirectBinaryLogicStepTest() = default; - - bool ShortcircuitingEnabled() { return std::get<0>(GetParam()); } - const BinaryTestCase& GetTestCase() { return std::get<1>(GetParam()); } - - protected: - Arena arena_; -}; - -TEST_P(DirectBinaryLogicStepTest, TestCases) { - const BinaryTestCase& test_case = GetTestCase(); - - std::unique_ptr lhs = - MakeArgStep(test_case.arg0, "lhs"); - std::unique_ptr rhs = - MakeArgStep(test_case.arg1, "rhs"); - - std::unique_ptr op = - (test_case.op == BinaryOp::kAnd) - ? CreateDirectAndStep(std::move(lhs), std::move(rhs), -1, - ShortcircuitingEnabled()) - : CreateDirectOrStep(std::move(lhs), std::move(rhs), -1, - ShortcircuitingEnabled()); - - cel::Activation activation; - cel::RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value value; - AttributeTrail attr; - ASSERT_THAT(op->Evaluate(frame, value, attr), IsOk()); - - switch (test_case.result) { - case OpResult::kTrue: - ASSERT_TRUE(value.IsBool()); - EXPECT_TRUE(value.GetBool().NativeValue()); - break; - case OpResult::kFalse: - ASSERT_TRUE(value.IsBool()); - EXPECT_FALSE(value.GetBool().NativeValue()); - break; - case OpResult::kUnknown: - EXPECT_TRUE(value.IsUnknown()); - break; - case OpResult::kError: - EXPECT_TRUE(value.IsError()); - break; - } -} - -INSTANTIATE_TEST_SUITE_P( - DirectBinaryLogicStepTest, DirectBinaryLogicStepTest, - testing::Combine(testing::Bool(), - testing::ValuesIn>({ - { - "AndFalseFalse", - BinaryOp::kAnd, - OpArg::kFalse, - OpArg::kFalse, - OpResult::kFalse, - }, - { - "AndFalseTrue", - BinaryOp::kAnd, - OpArg::kFalse, - OpArg::kTrue, - OpResult::kFalse, - }, - { - "AndTrueFalse", - BinaryOp::kAnd, - OpArg::kTrue, - OpArg::kFalse, - OpResult::kFalse, - }, - { - "AndTrueTrue", - BinaryOp::kAnd, - OpArg::kTrue, - OpArg::kTrue, - OpResult::kTrue, - }, - - { - "AndTrueError", - BinaryOp::kAnd, - OpArg::kTrue, - OpArg::kError, - OpResult::kError, - }, - { - "AndErrorTrue", - BinaryOp::kAnd, - OpArg::kError, - OpArg::kTrue, - OpResult::kError, - }, - { - "AndFalseError", - BinaryOp::kAnd, - OpArg::kFalse, - OpArg::kError, - OpResult::kFalse, - }, - { - "AndErrorFalse", - BinaryOp::kAnd, - OpArg::kError, - OpArg::kFalse, - OpResult::kFalse, - }, - { - "AndErrorError", - BinaryOp::kAnd, - OpArg::kError, - OpArg::kError, - OpResult::kError, - }, - - { - "AndTrueUnknown", - BinaryOp::kAnd, - OpArg::kTrue, - OpArg::kUnknown, - OpResult::kUnknown, - }, - { - "AndUnknownTrue", - BinaryOp::kAnd, - OpArg::kUnknown, - OpArg::kTrue, - OpResult::kUnknown, - }, - { - "AndFalseUnknown", - BinaryOp::kAnd, - OpArg::kFalse, - OpArg::kUnknown, - OpResult::kFalse, - }, - { - "AndUnknownFalse", - BinaryOp::kAnd, - OpArg::kUnknown, - OpArg::kFalse, - OpResult::kFalse, - }, - { - "AndUnknownUnknown", - BinaryOp::kAnd, - OpArg::kUnknown, - OpArg::kUnknown, - OpResult::kUnknown, - }, - { - "AndUnknownError", - BinaryOp::kAnd, - OpArg::kUnknown, - OpArg::kError, - OpResult::kUnknown, - }, - { - "AndErrorUnknown", - BinaryOp::kAnd, - OpArg::kError, - OpArg::kUnknown, - OpResult::kUnknown, - }, - // Or cases are simplified since the logic generalizes - // and is covered by and cases. - })), - [](const testing::TestParamInfo& info) - -> std::string { - bool shortcircuiting_enabled = std::get<0>(info.param); - absl::string_view name = std::get<1>(info.param).name; - return absl::StrCat( - name, (shortcircuiting_enabled ? "ShortcircuitingEnabled" : "")); - }); - -struct UnaryTestCase { - std::string name; - UnaryOp op; - OpArg arg; - OpResult result; -}; - -class DirectUnaryLogicStepTest : public testing::TestWithParam { - public: - DirectUnaryLogicStepTest() = default; - - const UnaryTestCase& GetTestCase() { return GetParam(); } - - protected: - Arena arena_; -}; - -TEST_P(DirectUnaryLogicStepTest, TestCases) { - const UnaryTestCase& test_case = GetTestCase(); - - std::unique_ptr arg = MakeArgStep(test_case.arg, "arg"); - - std::unique_ptr op = - (test_case.op == UnaryOp::kNot) - ? CreateDirectNotStep(std::move(arg), -1) - : CreateDirectNotStrictlyFalseStep(std::move(arg), -1); - - cel::Activation activation; - cel::RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value value; - AttributeTrail attr; - ASSERT_THAT(op->Evaluate(frame, value, attr), IsOk()); - - switch (test_case.result) { - case OpResult::kTrue: - ASSERT_TRUE(value.IsBool()); - EXPECT_TRUE(value.GetBool().NativeValue()); - break; - case OpResult::kFalse: - ASSERT_TRUE(value.IsBool()); - EXPECT_FALSE(value.GetBool().NativeValue()); - break; - case OpResult::kUnknown: - EXPECT_TRUE(value.IsUnknown()); - break; - case OpResult::kError: - EXPECT_TRUE(value.IsError()); - break; - } -} - -INSTANTIATE_TEST_SUITE_P( - DirectUnaryLogicStepTest, DirectUnaryLogicStepTest, - testing::ValuesIn>( - {UnaryTestCase{"NotTrue", UnaryOp::kNot, OpArg::kTrue, - OpResult::kFalse}, - UnaryTestCase{"NotError", UnaryOp::kNot, OpArg::kError, - OpResult::kError}, - UnaryTestCase{"NotUnknown", UnaryOp::kNot, OpArg::kUnknown, - OpResult::kUnknown}, - UnaryTestCase{"NotInt", UnaryOp::kNot, OpArg::kInt, OpResult::kError}, - UnaryTestCase{"NotFalse", UnaryOp::kNot, OpArg::kFalse, - OpResult::kTrue}, - UnaryTestCase{"NotStrictlyFalseTrue", UnaryOp::kNotStrictlyFalse, - OpArg::kTrue, OpResult::kTrue}, - UnaryTestCase{"NotStrictlyFalseError", UnaryOp::kNotStrictlyFalse, - OpArg::kError, OpResult::kTrue}, - UnaryTestCase{"NotStrictlyFalseUnknown", UnaryOp::kNotStrictlyFalse, - OpArg::kUnknown, OpResult::kTrue}, - UnaryTestCase{"NotStrictlyFalseInt", UnaryOp::kNotStrictlyFalse, - OpArg::kInt, OpResult::kError}, - UnaryTestCase{"NotStrictlyFalseFalse", UnaryOp::kNotStrictlyFalse, - OpArg::kFalse, OpResult::kFalse}}), - [](const testing::TestParamInfo& info) - -> std::string { return info.param.name; }); - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/eval/optional_or_step.cc b/eval/eval/optional_or_step.cc index 1c52d91b6..c0b341b4f 100644 --- a/eval/eval/optional_or_step.cc +++ b/eval/eval/optional_or_step.cc @@ -14,106 +14,48 @@ #include "eval/eval/optional_or_step.h" -#include #include +#include #include #include "absl/base/optimization.h" #include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/types/optional.h" #include "absl/types/span.h" -#include "common/casting.h" #include "common/value.h" #include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" -#include "eval/eval/jump_step.h" -#include "internal/status_macros.h" +#include "eval/eval/expression_step_logic.h" #include "runtime/internal/errors.h" +#include "google/protobuf/arena.h" namespace google::api::expr::runtime { namespace { -using ::cel::As; using ::cel::ErrorValue; -using ::cel::InstanceOf; -using ::cel::OptionalValue; -using ::cel::UnknownValue; using ::cel::Value; using ::cel::runtime_internal::CreateNoMatchingOverloadError; enum class OptionalOrKind { kOrOptional, kOrValue }; -ErrorValue MakeNoOverloadError(OptionalOrKind kind) { +ErrorValue MakeNoOverloadError(OptionalOrKind kind, google::protobuf::Arena* arena) { switch (kind) { case OptionalOrKind::kOrOptional: - return ErrorValue(CreateNoMatchingOverloadError("or")); + return ErrorValue::From(CreateNoMatchingOverloadError("or"), arena); case OptionalOrKind::kOrValue: - return ErrorValue(CreateNoMatchingOverloadError("orValue")); + return ErrorValue::From(CreateNoMatchingOverloadError("orValue"), arena); } ABSL_UNREACHABLE(); } -// Implements short-circuiting for optional.or. -// Expected layout if short-circuiting enabled: -// -// +--------+-----------------------+-------------------------------+ -// | idx | Step | Stack After | -// +--------+-----------------------+-------------------------------+ -// | 1 | | OptionalValue | -// +--------+-----------------------+-------------------------------+ -// | 2 | Jump to 5 if present | OptionalValue | -// +--------+-----------------------+-------------------------------+ -// | 3 | | OptionalValue, OptionalValue | -// +--------+-----------------------+-------------------------------+ -// | 4 | optional.or | OptionalValue | -// +--------+-----------------------+-------------------------------+ -// | 5 | | ... | -// +--------------------------------+-------------------------------+ -// -// If implementing the orValue variant, the jump step handles unwrapping ( -// getting the result of optional.value()) -class OptionalHasValueJumpStep final : public JumpStepBase { - public: - OptionalHasValueJumpStep(int64_t expr_id, OptionalOrKind kind) - : JumpStepBase({}, expr_id), kind_(kind) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - if (!frame->value_stack().HasEnough(1)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - const auto& value = frame->value_stack().Peek(); - auto optional_value = As(value); - // We jump if the receiver is `optional_type` which has a value or the - // receiver is an error/unknown. Unlike `_||_` we are not commutative. If - // we run into an error/unknown, we skip the `else` branch. - const bool should_jump = - (optional_value.has_value() && optional_value->HasValue()) || - (!optional_value.has_value() && (cel::InstanceOf(value) || - cel::InstanceOf(value))); - if (should_jump) { - if (kind_ == OptionalOrKind::kOrValue && optional_value.has_value()) { - frame->value_stack().PopAndPush(optional_value->Value()); - } - return Jump(frame); - } - return absl::OkStatus(); - } - - private: - const OptionalOrKind kind_; -}; - class OptionalOrStep : public ExpressionStepBase { public: - explicit OptionalOrStep(int64_t expr_id, OptionalOrKind kind) - : ExpressionStepBase(expr_id), kind_(kind) {} + explicit OptionalOrStep(OptionalOrKind kind) + : ExpressionStepBase(), kind_(kind) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: const OptionalOrKind kind_; @@ -126,16 +68,16 @@ class OptionalOrStep : public ExpressionStepBase { absl::Status EvalOptionalOr(OptionalOrKind kind, const Value& lhs, const Value& rhs, const AttributeTrail& lhs_attr, const AttributeTrail& rhs_attr, Value& result, - AttributeTrail& result_attr) { - if (InstanceOf(lhs) || InstanceOf(lhs)) { + AttributeTrail& result_attr, google::protobuf::Arena* arena) { + if (lhs.IsError() || lhs.IsUnknown()) { result = lhs; result_attr = lhs_attr; return absl::OkStatus(); } - auto lhs_optional_value = As(lhs); + auto lhs_optional_value = lhs.AsOptional(); if (!lhs_optional_value.has_value()) { - result = MakeNoOverloadError(kind); + result = MakeNoOverloadError(kind, arena); result_attr = AttributeTrail(); return absl::OkStatus(); } @@ -150,9 +92,9 @@ absl::Status EvalOptionalOr(OptionalOrKind kind, const Value& lhs, return absl::OkStatus(); } - if (kind == OptionalOrKind::kOrOptional && !InstanceOf(rhs) && - !InstanceOf(rhs) && !InstanceOf(rhs)) { - result = MakeNoOverloadError(kind); + if (kind == OptionalOrKind::kOrOptional && !rhs.IsError() && + !rhs.IsUnknown() && !rhs.IsOptional()) { + result = MakeNoOverloadError(kind, arena); result_attr = AttributeTrail(); return absl::OkStatus(); } @@ -162,9 +104,10 @@ absl::Status EvalOptionalOr(OptionalOrKind kind, const Value& lhs, return absl::OkStatus(); } -absl::Status OptionalOrStep::Evaluate(ExecutionFrame* frame) const { +void OptionalOrStep::Evaluate(ExecutionFrame* frame) const { if (!frame->value_stack().HasEnough(2)) { - return absl::InternalError("Value stack underflow"); + frame->Abort(absl::InternalError("Value stack underflow")); + return; } absl::Span args = frame->value_stack().GetSpan(2); @@ -173,133 +116,52 @@ absl::Status OptionalOrStep::Evaluate(ExecutionFrame* frame) const { Value result; AttributeTrail result_attr; - CEL_RETURN_IF_ERROR(EvalOptionalOr(kind_, args[0], args[1], args_attr[0], - args_attr[1], result, result_attr)); + if (absl::Status status = + EvalOptionalOr(kind_, args[0], args[1], args_attr[0], args_attr[1], + result, result_attr, frame->arena()); + !status.ok()) { + frame->Abort(std::move(status)); + return; + } frame->value_stack().PopAndPush(2, std::move(result), std::move(result_attr)); - return absl::OkStatus(); -} - -class ExhaustiveDirectOptionalOrStep : public DirectExpressionStep { - public: - ExhaustiveDirectOptionalOrStep( - int64_t expr_id, std::unique_ptr optional, - std::unique_ptr alternative, OptionalOrKind kind) - - : DirectExpressionStep(expr_id), - kind_(kind), - optional_(std::move(optional)), - alternative_(std::move(alternative)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override; - - private: - OptionalOrKind kind_; - std::unique_ptr optional_; - std::unique_ptr alternative_; -}; - -absl::Status ExhaustiveDirectOptionalOrStep::Evaluate( - ExecutionFrameBase& frame, Value& result, AttributeTrail& attribute) const { - CEL_RETURN_IF_ERROR(optional_->Evaluate(frame, result, attribute)); - Value rhs; - AttributeTrail rhs_attr; - CEL_RETURN_IF_ERROR(alternative_->Evaluate(frame, rhs, rhs_attr)); - CEL_RETURN_IF_ERROR(EvalOptionalOr(kind_, result, rhs, attribute, rhs_attr, - result, attribute)); - return absl::OkStatus(); } -class DirectOptionalOrStep : public DirectExpressionStep { - public: - DirectOptionalOrStep(int64_t expr_id, - std::unique_ptr optional, - std::unique_ptr alternative, - OptionalOrKind kind) - - : DirectExpressionStep(expr_id), - kind_(kind), - optional_(std::move(optional)), - alternative_(std::move(alternative)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override; - - private: - OptionalOrKind kind_; - std::unique_ptr optional_; - std::unique_ptr alternative_; -}; - -absl::Status DirectOptionalOrStep::Evaluate(ExecutionFrameBase& frame, - Value& result, - AttributeTrail& attribute) const { - CEL_RETURN_IF_ERROR(optional_->Evaluate(frame, result, attribute)); - - if (InstanceOf(result) || InstanceOf(result)) { - // Forward the lhs error instead of attempting to evaluate the alternative - // (unlike CEL's commutative logic operators). - return absl::OkStatus(); - } +} // namespace - auto optional_value = As(static_cast(result)); - if (!optional_value.has_value()) { - result = MakeNoOverloadError(kind_); - return absl::OkStatus(); +void OptionalHasValueJumpStep::Evaluate(ExecutionFrame* frame) const { + if (!frame->value_stack().HasEnough(1)) { + frame->Abort(absl::InternalError("Value stack underflow")); + return; } - - if (optional_value->HasValue()) { - if (kind_ == OptionalOrKind::kOrValue) { - result = optional_value->Value(); + const Value& value = frame->value_stack().Peek(); + auto optional_value = value.AsOptional(); + // We jump if the receiver is `optional_type` which has a value or the + // receiver is an error/unknown. Unlike `_||_` we are not commutative. If + // we run into an error/unknown, we skip the `else` branch. + const bool should_jump = + (optional_value.has_value() && optional_value->HasValue()) || + (!optional_value.has_value() && (value.IsError() || value.IsUnknown())); + if (should_jump) { + if (is_or_value_ && optional_value.has_value()) { + frame->value_stack().PopAndPush(optional_value->Value()); } - return absl::OkStatus(); - } - - CEL_RETURN_IF_ERROR(alternative_->Evaluate(frame, result, attribute)); - - // If optional.or check that rhs is an optional. - // - // Otherwise, we don't know what type to expect so can't check anything. - if (kind_ == OptionalOrKind::kOrOptional) { - if (!InstanceOf(result) && !InstanceOf(result) && - !InstanceOf(result)) { - result = MakeNoOverloadError(kind_); + if (!jump_offset_.has_value()) { + frame->Abort(absl::InternalError("Jump offset not set")); + return; } + frame->JumpToOrAbort(*jump_offset_); } - - return absl::OkStatus(); } -} // namespace - -std::unique_ptr CreateOptionalHasValueJumpStep(bool or_value, - int64_t expr_id) { - return std::make_unique( - expr_id, - or_value ? OptionalOrKind::kOrValue : OptionalOrKind::kOrOptional); +std::unique_ptr CreateOptionalHasValueJumpStep( + bool or_value) { + return std::make_unique(or_value); } -std::unique_ptr CreateOptionalOrStep(bool is_or_value, - int64_t expr_id) { +std::unique_ptr CreateOptionalOrStep(bool is_or_value) { return std::make_unique( - expr_id, is_or_value ? OptionalOrKind::kOrValue : OptionalOrKind::kOrOptional); } -std::unique_ptr CreateDirectOptionalOrStep( - int64_t expr_id, std::unique_ptr optional, - std::unique_ptr alternative, bool is_or_value, - bool short_circuiting) { - auto kind = - is_or_value ? OptionalOrKind::kOrValue : OptionalOrKind::kOrOptional; - if (short_circuiting) { - return std::make_unique(expr_id, std::move(optional), - std::move(alternative), kind); - } else { - return std::make_unique( - expr_id, std::move(optional), std::move(alternative), kind); - } -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/optional_or_step.h b/eval/eval/optional_or_step.h index 59977c857..fc55a0ab7 100644 --- a/eval/eval/optional_or_step.h +++ b/eval/eval/optional_or_step.h @@ -15,15 +15,29 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_OPTIONAL_OR_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_OPTIONAL_OR_STEP_H_ -#include #include +#include -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/jump_step.h" +#include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { +class OptionalHasValueJumpStep final : public ExpressionStepBase { + public: + explicit OptionalHasValueJumpStep(bool is_or_value) + : is_or_value_(is_or_value) {} + + void set_jump_offset(int offset) { jump_offset_ = offset; } + + void Evaluate(ExecutionFrame* frame) const override; + + private: + const bool is_or_value_; + std::optional jump_offset_; +}; + // Factory method for OptionalHasValueJump step, used to implement // short-circuiting optional.or and optional.orValue. // @@ -31,20 +45,12 @@ namespace google::api::expr::runtime { // true, performs a jump. If `or_value` is true and we are jumping, // `optional.value` is called and the result replaces the optional at the top of // the stack. -std::unique_ptr CreateOptionalHasValueJumpStep(bool or_value, - int64_t expr_id); +std::unique_ptr CreateOptionalHasValueJumpStep( + bool or_value); // Factory method for OptionalOr step, used to implement optional.or and // optional.orValue. -std::unique_ptr CreateOptionalOrStep(bool is_or_value, - int64_t expr_id); - -// Creates a step implementing the short-circuiting optional.or or -// optional.orValue step. -std::unique_ptr CreateDirectOptionalOrStep( - int64_t expr_id, std::unique_ptr optional, - std::unique_ptr alternative, bool is_or_value, - bool short_circuiting); +std::unique_ptr CreateOptionalOrStep(bool is_or_value); } // namespace google::api::expr::runtime diff --git a/eval/eval/optional_or_step_test.cc b/eval/eval/optional_or_step_test.cc deleted file mode 100644 index 14f1c3bd9..000000000 --- a/eval/eval/optional_or_step_test.cc +++ /dev/null @@ -1,382 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "eval/eval/optional_or_step.h" - -#include - -#include "absl/memory/memory.h" -#include "absl/status/status.h" -#include "common/casting.h" -#include "common/value.h" -#include "common/value_kind.h" -#include "common/value_testing.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/const_value_step.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" -#include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" -#include "runtime/internal/errors.h" -#include "runtime/internal/runtime_type_provider.h" -#include "runtime/runtime_options.h" -#include "google/protobuf/arena.h" - -namespace google::api::expr::runtime { -namespace { - -using ::absl_testing::StatusIs; -using ::cel::Activation; -using ::cel::As; -using ::cel::ErrorValue; -using ::cel::InstanceOf; -using ::cel::IntValue; -using ::cel::OptionalValue; -using ::cel::RuntimeOptions; -using ::cel::UnknownValue; -using ::cel::Value; -using ::cel::ValueKind; -using ::cel::test::ErrorValueIs; -using ::cel::test::IntValueIs; -using ::cel::test::OptionalValueIs; -using ::cel::test::ValueKindIs; -using ::testing::HasSubstr; -using ::testing::NiceMock; - -class MockDirectStep : public DirectExpressionStep { - public: - MOCK_METHOD(absl::Status, Evaluate, - (ExecutionFrameBase & frame, Value& result, - AttributeTrail& scratch), - (const, override)); -}; - -std::unique_ptr MockNeverCalledDirectStep() { - auto* mock = new NiceMock(); - EXPECT_CALL(*mock, Evaluate).Times(0); - return absl::WrapUnique(mock); -} - -std::unique_ptr MockExpectCallDirectStep() { - auto* mock = new NiceMock(); - EXPECT_CALL(*mock, Evaluate) - .Times(1) - .WillRepeatedly( - [](ExecutionFrameBase& frame, Value& result, AttributeTrail& attr) { - result = ErrorValue(absl::InternalError("expected to be unused")); - return absl::OkStatus(); - }); - return absl::WrapUnique(mock); -} - -class OptionalOrTest : public testing::Test { - public: - OptionalOrTest() - : type_provider_(cel::internal::GetTestingDescriptorPool()) {} - - protected: - google::protobuf::Arena arena_; - cel::runtime_internal::RuntimeTypeProvider type_provider_; - Activation empty_activation_; -}; - -TEST_F(OptionalOrTest, OptionalOrLeftPresentShortcutRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, - CreateConstValueDirectStep(OptionalValue::Of(IntValue(42), &arena_)), - MockNeverCalledDirectStep(), - /*is_or_value=*/false, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, OptionalValueIs(IntValueIs(42))); -} - -TEST_F(OptionalOrTest, OptionalOrLeftErrorShortcutsRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, - CreateConstValueDirectStep(ErrorValue(absl::InternalError("error"))), - MockNeverCalledDirectStep(), - /*is_or_value=*/false, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - EXPECT_THAT(result, ValueKindIs(ValueKind::kError)); -} - -TEST_F(OptionalOrTest, OptionalOrLeftErrorExhaustiveRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, - CreateConstValueDirectStep(ErrorValue(absl::InternalError("error"))), - MockExpectCallDirectStep(), - /*is_or_value=*/false, - /*short_circuiting=*/false); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - EXPECT_THAT(result, ValueKindIs(ValueKind::kError)); -} - -TEST_F(OptionalOrTest, OptionalOrLeftUnknownShortcutsRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(UnknownValue()), - MockNeverCalledDirectStep(), - /*is_or_value=*/false, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); -} - -TEST_F(OptionalOrTest, OptionalOrLeftUnknownExhaustiveRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(UnknownValue()), - MockExpectCallDirectStep(), - /*is_or_value=*/false, - /*short_circuiting=*/false); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); -} - -TEST_F(OptionalOrTest, OptionalOrLeftAbsentReturnRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(OptionalValue::None()), - CreateConstValueDirectStep(OptionalValue::Of(IntValue(42), &arena_)), - /*is_or_value=*/false, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, OptionalValueIs(IntValueIs(42))); -} - -TEST_F(OptionalOrTest, OptionalOrLeftWrongType) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(IntValue(42)), - MockNeverCalledDirectStep(), - /*is_or_value=*/false, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, - ErrorValueIs(StatusIs( - absl::StatusCode::kUnknown, - HasSubstr(cel::runtime_internal::kErrNoMatchingOverload)))); -} - -TEST_F(OptionalOrTest, OptionalOrRightWrongType) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(OptionalValue::None()), - CreateConstValueDirectStep(IntValue(42)), - /*is_or_value=*/false, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, - ErrorValueIs(StatusIs( - absl::StatusCode::kUnknown, - HasSubstr(cel::runtime_internal::kErrNoMatchingOverload)))); -} - -TEST_F(OptionalOrTest, OptionalOrValueLeftPresentShortcutRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, - CreateConstValueDirectStep(OptionalValue::Of(IntValue(42), &arena_)), - MockNeverCalledDirectStep(), - /*is_or_value=*/true, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, IntValueIs(42)); -} - -TEST_F(OptionalOrTest, OptionalOrValueLeftPresentExhaustiveRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, - CreateConstValueDirectStep(OptionalValue::Of(IntValue(42), &arena_)), - MockExpectCallDirectStep(), - /*is_or_value=*/true, - /*short_circuiting=*/false); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, IntValueIs(42)); -} - -TEST_F(OptionalOrTest, OptionalOrValueLeftErrorShortcutsRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, - CreateConstValueDirectStep(ErrorValue(absl::InternalError("error"))), - MockNeverCalledDirectStep(), - /*is_or_value=*/true, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - EXPECT_THAT(result, ValueKindIs(ValueKind::kError)); -} - -TEST_F(OptionalOrTest, OptionalOrValueLeftUnknownShortcutsRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(UnknownValue()), - MockNeverCalledDirectStep(), true, true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); -} - -TEST_F(OptionalOrTest, OptionalOrValueLeftAbsentReturnRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(OptionalValue::None()), - CreateConstValueDirectStep(IntValue(42)), - /*is_or_value=*/true, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, IntValueIs(42)); -} - -TEST_F(OptionalOrTest, OptionalOrValueLeftWrongType) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(IntValue(42)), - MockNeverCalledDirectStep(), true, true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, - ErrorValueIs(StatusIs( - absl::StatusCode::kUnknown, - HasSubstr(cel::runtime_internal::kErrNoMatchingOverload)))); -} - -} // namespace -} // namespace google::api::expr::runtime diff --git a/eval/eval/regex_match_step.cc b/eval/eval/regex_match_step.cc index 2a06de1b8..4df758e76 100644 --- a/eval/eval/regex_match_step.cc +++ b/eval/eval/regex_match_step.cc @@ -14,7 +14,6 @@ #include "eval/eval/regex_match_step.h" -#include #include #include #include @@ -24,11 +23,9 @@ #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" -#include "internal/status_macros.h" +#include "eval/eval/expression_step_logic.h" #include "re2/re2.h" namespace google::api::expr::runtime { @@ -37,7 +34,6 @@ namespace { using ::cel::BoolValue; using ::cel::StringValue; -using ::cel::Value; inline constexpr int kNumRegexMatchArguments = 1; inline constexpr size_t kRegexMatchStepSubject = 0; @@ -59,77 +55,40 @@ struct MatchesVisitor final { class RegexMatchStep final : public ExpressionStepBase { public: - RegexMatchStep(int64_t expr_id, std::shared_ptr re2) - : ExpressionStepBase(expr_id, /*comes_from_ast=*/true), - re2_(std::move(re2)) {} + explicit RegexMatchStep(std::shared_ptr re2) + : ExpressionStepBase(), re2_(std::move(re2)) {} - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { if (!frame->value_stack().HasEnough(kNumRegexMatchArguments)) { - return absl::Status(absl::StatusCode::kInternal, - "Insufficient arguments supplied for regular " - "expression match"); + frame->Abort(absl::InternalError( + "Insufficient arguments supplied for regular expression match")); + return; } auto input_args = frame->value_stack().GetSpan(kNumRegexMatchArguments); const auto& subject = input_args[kRegexMatchStepSubject]; + if (subject.IsUnknown() || subject.IsError()) { + // Propagate unknowns and errors. + return; + } if (!subject->Is()) { - return absl::Status(absl::StatusCode::kInternal, - "First argument for regular " - "expression match must be a string"); + frame->Abort(absl::InternalError( + "First argument for regular expression match must be a string")); + return; } bool match = subject.GetString().NativeValue(MatchesVisitor{*re2_}); frame->value_stack().Pop(kNumRegexMatchArguments); frame->value_stack().Push(cel::BoolValue(match)); - return absl::OkStatus(); } private: const std::shared_ptr re2_; }; -class RegexMatchDirectStep final : public DirectExpressionStep { - public: - RegexMatchDirectStep(int64_t expr_id, - std::unique_ptr subject, - std::shared_ptr re2) - : DirectExpressionStep(expr_id), - subject_(std::move(subject)), - re2_(std::move(re2)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - AttributeTrail subject_attr; - CEL_RETURN_IF_ERROR(subject_->Evaluate(frame, result, subject_attr)); - if (result.IsError() || result.IsUnknown()) { - return absl::OkStatus(); - } - - if (!result.IsString()) { - return absl::Status(absl::StatusCode::kInternal, - "First argument for regular " - "expression match must be a string"); - } - bool match = result.GetString().NativeValue(MatchesVisitor{*re2_}); - result = BoolValue(match); - return absl::OkStatus(); - } - - private: - std::unique_ptr subject_; - const std::shared_ptr re2_; -}; - } // namespace -std::unique_ptr CreateDirectRegexMatchStep( - int64_t expr_id, std::unique_ptr subject, +absl::StatusOr> CreateRegexMatchStep( std::shared_ptr re2) { - return std::make_unique(expr_id, std::move(subject), - std::move(re2)); -} - -absl::StatusOr> CreateRegexMatchStep( - std::shared_ptr re2, int64_t expr_id) { - return std::make_unique(expr_id, std::move(re2)); + return std::make_unique(std::move(re2)); } } // namespace google::api::expr::runtime diff --git a/eval/eval/regex_match_step.h b/eval/eval/regex_match_step.h index 1d8a09118..d44559787 100644 --- a/eval/eval/regex_match_step.h +++ b/eval/eval/regex_match_step.h @@ -15,23 +15,16 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_REGEX_MATCH_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_REGEX_MATCH_STEP_H_ -#include #include #include "absl/status/statusor.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" #include "re2/re2.h" namespace google::api::expr::runtime { -std::unique_ptr CreateDirectRegexMatchStep( - int64_t expr_id, std::unique_ptr subject, +absl::StatusOr> CreateRegexMatchStep( std::shared_ptr re2); - -absl::StatusOr> CreateRegexMatchStep( - std::shared_ptr re2, int64_t expr_id); - } #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_REGEX_MATCH_STEP_H_ diff --git a/eval/eval/regex_match_step_test.cc b/eval/eval/regex_match_step_test.cc index 53b955b25..fa141365e 100644 --- a/eval/eval/regex_match_step_test.cc +++ b/eval/eval/regex_match_step_test.cc @@ -17,7 +17,6 @@ #include "cel/expr/checked.pb.h" #include "cel/expr/syntax.pb.h" #include "absl/status/status.h" -#include "absl/status/statusor.h" #include "eval/public/activation.h" #include "eval/public/builtin_func_registrar.h" #include "eval/public/cel_expr_builder_factory.h" diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index 420f3ac31..8a70e3b82 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -1,42 +1,41 @@ #include "eval/eval/select_step.h" -#include #include #include #include #include "absl/base/nullability.h" #include "absl/log/absl_check.h" -#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/expr.h" +#include "base/attribute.h" +#include "common/legacy_value.h" +#include "common/type.h" #include "common/value.h" #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" #include "internal/status_macros.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" +#undef GetMessage + namespace google::api::expr::runtime { namespace { using ::cel::BoolValue; using ::cel::ErrorValue; -using ::cel::MapValue; -using ::cel::NullValue; using ::cel::OptionalValue; using ::cel::ProtoWrapperTypeOptions; using ::cel::StringValue; -using ::cel::StructValue; using ::cel::Value; using ::cel::ValueKind; @@ -50,462 +49,440 @@ absl::Status InvalidSelectTargetError() { "Applying SELECT to non-message type"); } -absl::optional CheckForMarkedAttributes(const AttributeTrail& trail, - ExecutionFrameBase& frame) { - if (frame.unknown_processing_enabled() && - frame.attribute_utility().CheckForUnknownExact(trail)) { - return frame.attribute_utility().CreateUnknownSet(trail.attribute()); - } - - if (frame.missing_attribute_errors_enabled() && - frame.attribute_utility().CheckForMissingAttribute(trail)) { - auto result = frame.attribute_utility().CreateMissingAttributeError( - trail.attribute()); - - if (result.ok()) { - return std::move(result).value(); +absl::Status PerformHas(const Value& target, absl::string_view field, + const StringValue& field_value, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena, Value& result) { + switch (target.kind()) { + case ValueKind::kMap: { + CEL_RETURN_IF_ERROR(target.GetMap().Has(field_value, descriptor_pool, + message_factory, arena, &result)); + return absl::OkStatus(); } - // Invariant broken (an invalid CEL Attribute shouldn't match anything). - // Log and return a CelError. - ABSL_LOG(ERROR) << "Invalid attribute pattern matched select path: " - << result.status().ToString(); // NOLINT: OSS compatibility - return cel::ErrorValue(std::move(result).status()); + case ValueKind::kStruct: { + auto has_field = target.GetStruct().HasFieldByName(field); + if (!has_field.ok()) { + result = ErrorValue::From(std::move(has_field).status(), arena); + } else { + result = BoolValue{*has_field}; + } + return absl::OkStatus(); + } + default: + return InvalidSelectTargetError(); } - - return absl::nullopt; } -void TestOnlySelect(const StructValue& msg, const std::string& field, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, - Value* absl_nonnull result) { - absl::StatusOr has_field = msg.HasFieldByName(field); - - if (!has_field.ok()) { - *result = ErrorValue(std::move(has_field).status()); - return; +absl::Status PerformGet(const Value& target, absl::string_view field, + const StringValue& field_value, + ProtoWrapperTypeOptions unboxing_option, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena, Value& result) { + switch (target.kind()) { + case ValueKind::kMap: { + auto status = target.GetMap().Get(field_value, descriptor_pool, + message_factory, arena, &result); + if (!status.ok()) { + result = ErrorValue::From(std::move(status), arena); + } + return absl::OkStatus(); + } + case ValueKind::kStruct: { + auto status = target.GetStruct().GetFieldByName( + field, unboxing_option, descriptor_pool, message_factory, arena, + &result); + if (!status.ok()) { + result = ErrorValue::From(std::move(status), arena); + } + return absl::OkStatus(); + } + default: + return InvalidSelectTargetError(); } - *result = BoolValue{*has_field}; } -void TestOnlySelect(const MapValue& map, const StringValue& field_name, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, - Value* absl_nonnull result) { - // Field presence only supports string keys containing valid identifier - // characters. - absl::Status presence = - map.Has(field_name, descriptor_pool, message_factory, arena, result); - - if (!presence.ok()) { - *result = ErrorValue(std::move(presence)); - return; +absl::Status PerformOptionalGet(const Value& target, absl::string_view field, + const StringValue& field_value, + ProtoWrapperTypeOptions unboxing_option, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena, Value& result) { + switch (target.kind()) { + case ValueKind::kMap: { + CEL_ASSIGN_OR_RETURN( + bool found, target.GetMap().Find(field_value, descriptor_pool, + message_factory, arena, &result)); + if (!found) { + result = OptionalValue::None(); + return absl::OkStatus(); + } + ABSL_DCHECK(!result.IsUnknown()); + result = OptionalValue::Of(std::move(result), arena); + return absl::OkStatus(); + } + case ValueKind::kStruct: { + CEL_ASSIGN_OR_RETURN(bool found, + target.GetStruct().HasFieldByName(field)); + if (!found) { + result = OptionalValue::None(); + return absl::OkStatus(); + } + CEL_RETURN_IF_ERROR(target.GetStruct().GetFieldByName( + field, unboxing_option, descriptor_pool, message_factory, arena, + &result)); + + ABSL_DCHECK(!result.IsUnknown()); + result = OptionalValue::Of(std::move(result), arena); + return absl::OkStatus(); + } + default: + return InvalidSelectTargetError(); } - ABSL_DCHECK(!result->IsUnknown()); } // SelectStep performs message field access specified by Expr::Select // message. class SelectStep : public ExpressionStepBase { public: - SelectStep(StringValue value, bool test_field_presence, int64_t expr_id, + SelectStep(absl::string_view field, bool test_field_presence, bool enable_wrapper_type_null_unboxing, bool enable_optional_types) - : ExpressionStepBase(expr_id), - field_value_(std::move(value)), - field_(field_value_.ToString()), - test_field_presence_(test_field_presence), + : field_(field), unboxing_option_(enable_wrapper_type_null_unboxing ? ProtoWrapperTypeOptions::kUnsetNull : ProtoWrapperTypeOptions::kUnsetProtoDefault), + test_field_presence_(test_field_presence), enable_optional_types_(enable_optional_types) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; - - private: - absl::Status PerformTestOnlySelect(ExecutionFrame* frame, - const Value& arg) const; - absl::StatusOr PerformSelect(ExecutionFrame* frame, const Value& arg, - Value& result) const; + void Evaluate(ExecutionFrame* frame) const override; - cel::StringValue field_value_; + protected: std::string field_; - bool test_field_presence_; ProtoWrapperTypeOptions unboxing_option_; + bool test_field_presence_; bool enable_optional_types_; }; -absl::Status SelectStep::Evaluate(ExecutionFrame* frame) const { +void SelectStep::Evaluate(ExecutionFrame* frame) const { if (!frame->value_stack().HasEnough(1)) { - return absl::Status(absl::StatusCode::kInternal, - "No arguments supplied for Select-type expression"); + frame->Abort(absl::InternalError( + "No arguments supplied for Select-type expression")); + return; } - const Value& arg = frame->value_stack().Peek(); - const AttributeTrail& trail = frame->value_stack().PeekAttribute(); + Value& arg_and_result = frame->value_stack().Peek(); - if (arg.IsUnknown() || arg.IsError()) { + if (arg_and_result.IsUnknown() || arg_and_result.IsError()) { // Bubble up unknowns and errors. - return absl::OkStatus(); - } - - AttributeTrail result_trail; - - // Handle unknown resolution. - if (frame->enable_unknowns() || frame->enable_missing_attribute_errors()) { - result_trail = trail.Step(&field_); + return; } absl::optional optional_arg; - if (enable_optional_types_ && arg.IsOptional()) { - optional_arg = arg.GetOptional(); + if (enable_optional_types_ && arg_and_result.IsOptional()) { + optional_arg = arg_and_result.GetOptional(); } - if (!(optional_arg || arg->Is() || arg->Is())) { - frame->value_stack().PopAndPush(cel::ErrorValue(InvalidSelectTargetError()), - std::move(result_trail)); - return absl::OkStatus(); + if (!(optional_arg || arg_and_result.IsMap() || arg_and_result.IsStruct())) { + // Do not bother stepping the attribute trail if the target is incorrect, we + // would not have encountered it anyway. + arg_and_result = + cel::ErrorValue::From(InvalidSelectTargetError(), frame->arena()); + if (frame->attribute_tracking_enabled()) { + frame->value_stack().PeekAttribute().Clear(); + } + return; } - absl::optional marked_attribute_check = - CheckForMarkedAttributes(result_trail, *frame); - if (marked_attribute_check.has_value()) { - frame->value_stack().PopAndPush(std::move(marked_attribute_check).value(), - std::move(result_trail)); - return absl::OkStatus(); + // Handle unknown resolution. + if (frame->attribute_tracking_enabled() && + frame->value_stack().PeekAttribute().Match( + cel::AttributeQualifierView::OfString(field_), arg_and_result, + frame->unknown_tree())) { + return; } - // Handle test only Select. + Value result; if (test_field_presence_) { + const Value* target = &arg_and_result; if (optional_arg) { if (!optional_arg->HasValue()) { - frame->value_stack().PopAndPush(cel::BoolValue{false}); - return absl::OkStatus(); + arg_and_result = cel::FalseValue(); + return; } - Value value; - optional_arg->Value(&value); - return PerformTestOnlySelect(frame, value); + optional_arg->Value(&result); + target = &result; + } + if (absl::Status status = + PerformHas(*target, field_, cel::StringValue::WrapUnsafe(field_), + frame->descriptor_pool(), frame->message_factory(), + frame->arena(), result); + !status.ok()) { + frame->Abort(std::move(status)); + return; } - return PerformTestOnlySelect(frame, arg); + arg_and_result = result; + return; } - // Normal select path. - // Select steps can be applied to either maps or messages if (optional_arg) { if (!optional_arg->HasValue()) { - // Leave optional_arg at the top of the stack. Its empty. - return absl::OkStatus(); + arg_and_result = OptionalValue::None(); + return; } Value value; - Value result; - bool ok; optional_arg->Value(&value); - CEL_ASSIGN_OR_RETURN(ok, PerformSelect(frame, value, result)); - if (!ok) { - frame->value_stack().PopAndPush(cel::OptionalValue::None(), - std::move(result_trail)); - return absl::OkStatus(); + absl::Status status = + PerformOptionalGet(value, field_, cel::StringValue::WrapUnsafe(field_), + unboxing_option_, frame->descriptor_pool(), + frame->message_factory(), frame->arena(), result); + if (!status.ok()) { + result = ErrorValue::From(std::move(status), frame->arena()); } - frame->value_stack().PopAndPush( - cel::OptionalValue::Of(std::move(result), frame->arena()), - std::move(result_trail)); - return absl::OkStatus(); + arg_and_result = result; + return; } - // Normal select path. - // Select steps can be applied to either maps or messages - switch (arg.kind()) { - case ValueKind::kStruct: { - Value result; - auto status = arg.GetStruct().GetFieldByName( - field_, unboxing_option_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &result); - if (!status.ok()) { - result = ErrorValue(std::move(status)); - } - frame->value_stack().PopAndPush(std::move(result), - std::move(result_trail)); - return absl::OkStatus(); - } - case ValueKind::kMap: { - Value result; - auto status = - arg.GetMap().Get(field_value_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &result); - if (!status.ok()) { - result = ErrorValue(std::move(status)); - } - frame->value_stack().PopAndPush(std::move(result), - std::move(result_trail)); - return absl::OkStatus(); - } - default: - // Control flow should have returned earlier. - return InvalidSelectTargetError(); + if (absl::Status status = PerformGet( + arg_and_result, field_, cel::StringValue::WrapUnsafe(field_), + unboxing_option_, frame->descriptor_pool(), frame->message_factory(), + frame->arena(), result); + !status.ok()) { + frame->Abort(std::move(status)); + return; } + arg_and_result = result; } -absl::Status SelectStep::PerformTestOnlySelect(ExecutionFrame* frame, - const Value& arg) const { - switch (arg.kind()) { - case ValueKind::kMap: { - Value result; - TestOnlySelect(arg.GetMap(), field_value_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &result); - frame->value_stack().PopAndPush(std::move(result)); - return absl::OkStatus(); - } - case ValueKind::kMessage: { - Value result; - TestOnlySelect(arg.GetStruct(), field_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &result); - frame->value_stack().PopAndPush(std::move(result)); - return absl::OkStatus(); - } - default: - // Control flow should have returned earlier. - return InvalidSelectTargetError(); - } +bool CheckAttributeTrail(const std::string& field, ExecutionFrame* frame) { + return frame->attribute_tracking_enabled() && + frame->value_stack().PeekAttribute().Match( + cel::AttributeQualifierView::OfString(field), + frame->value_stack().Peek(), frame->unknown_tree()); } -absl::StatusOr SelectStep::PerformSelect(ExecutionFrame* frame, - const Value& arg, - Value& result) const { - switch (arg->kind()) { - case ValueKind::kStruct: { - const auto& struct_value = arg.GetStruct(); - CEL_ASSIGN_OR_RETURN(auto ok, struct_value.HasFieldByName(field_)); - if (!ok) { - result = NullValue{}; - return false; - } - CEL_RETURN_IF_ERROR(struct_value.GetFieldByName( - field_, unboxing_option_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &result)); - ABSL_DCHECK(!result.IsUnknown()); - return true; - } - case ValueKind::kMap: { - CEL_ASSIGN_OR_RETURN( - auto found, - arg.GetMap().Find(field_value_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &result)); - ABSL_DCHECK(!found || !result.IsUnknown()); - return found; - } - default: - // Control flow should have returned earlier. - return InvalidSelectTargetError(); +bool SupportsCachedFieldDescriptor( + const cel::ParsedMessageValue& parsed_message, + const google::protobuf::Descriptor* descriptor, + const google::protobuf::FieldDescriptor* field_descriptor) { + ABSL_DCHECK_EQ(field_descriptor->containing_type(), descriptor); + const google::protobuf::Descriptor* rt_descriptor = parsed_message.GetDescriptor(); + + if (rt_descriptor != descriptor) { + return false; } + + // Caller should have already checked this. Crash here if not instead of + // making proto crash. + ABSL_DCHECK_EQ(rt_descriptor->file()->pool(), + field_descriptor->file()->pool()); + + return true; } -class DirectSelectStep : public DirectExpressionStep { +class ProtoSelectStep : public SelectStep { public: - DirectSelectStep(int64_t expr_id, - std::unique_ptr operand, - StringValue field, bool test_only, - bool enable_wrapper_type_null_unboxing, - bool enable_optional_types) - : DirectExpressionStep(expr_id), - operand_(std::move(operand)), - field_value_(std::move(field)), - field_(field_value_.ToString()), - test_only_(test_only), - unboxing_option_(enable_wrapper_type_null_unboxing - ? ProtoWrapperTypeOptions::kUnsetNull - : ProtoWrapperTypeOptions::kUnsetProtoDefault), - enable_optional_types_(enable_optional_types) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - CEL_RETURN_IF_ERROR(operand_->Evaluate(frame, result, attribute)); + ProtoSelectStep(absl::string_view value, + bool enable_wrapper_type_null_unboxing, + bool enable_optional_types, + const google::protobuf::Descriptor* descriptor, + const google::protobuf::FieldDescriptor* field_descriptor) + : SelectStep(value, /*test_field_presence=*/false, + enable_wrapper_type_null_unboxing, enable_optional_types), + descriptor_(descriptor), + field_descriptor_(field_descriptor) { + ABSL_DCHECK(descriptor_ != nullptr); + ABSL_DCHECK(field_descriptor_ != nullptr); + } - if (result.IsError() || result.IsUnknown()) { - // Just forward. - return absl::OkStatus(); + void Evaluate(ExecutionFrame* frame) const override { + if (!frame->value_stack().HasEnough(1)) { + frame->Abort(absl::InternalError( + "No arguments supplied for Select-type expression")); + return; } - if (frame.attribute_tracking_enabled()) { - attribute = attribute.Step(&field_); - absl::optional value = CheckForMarkedAttributes(attribute, frame); - if (value.has_value()) { - result = std::move(value).value(); - return absl::OkStatus(); + const Value& arg = frame->value_stack().Peek(); + if (auto unwrapped = arg.AsParsedMessage(); + unwrapped.has_value() && + SupportsCachedFieldDescriptor(*unwrapped, descriptor_, + field_descriptor_)) { + EvaluateMessageFieldGet(frame, *unwrapped); + return; + } else if (const google::protobuf::Message* legacy_message = + cel::interop_internal::GetLegacyMessage(arg); + legacy_message != nullptr) { + auto parsed_message = cel::UnsafeParsedMessageValue(legacy_message); + // A little unfortunate, but need to special case for legacy values so we + // can minimize back and forth interop conversions. + if (SupportsCachedFieldDescriptor(parsed_message, descriptor_, + field_descriptor_)) { + EvaluateMessageFieldGet(frame, legacy_message); + return; } } + // If we get an unexpected value type, fall back to the generic + // implementation. + SelectStep::Evaluate(frame); + } - absl::optional optional_arg; + private: + void EvaluateMessageFieldGet( + ExecutionFrame* frame, + const cel::ParsedMessageValue& parsed_message) const; + void EvaluateMessageFieldGet( + ExecutionFrame* frame, + const google::protobuf::Message* absl_nonnull legacy_message) const; + + const google::protobuf::Descriptor* descriptor_; + const google::protobuf::FieldDescriptor* field_descriptor_; +}; - if (enable_optional_types_ && result.IsOptional()) { - optional_arg = result.GetOptional(); - } +void ProtoSelectStep::EvaluateMessageFieldGet( + ExecutionFrame* frame, + const cel::ParsedMessageValue& parsed_message) const { + if (CheckAttributeTrail(field_, frame)) { + return; + } + if (absl::Status status = parsed_message.GetField( + field_descriptor_, unboxing_option_, frame->descriptor_pool(), + frame->message_factory(), frame->arena(), + &frame->value_stack().Peek()); + !status.ok()) { + frame->Abort(std::move(status)); + } +} - switch (result.kind()) { - case ValueKind::kStruct: - case ValueKind::kMap: - break; - default: - if (optional_arg) { - break; - } - result = cel::ErrorValue(InvalidSelectTargetError()); - return absl::OkStatus(); - } +void ProtoSelectStep::EvaluateMessageFieldGet( + ExecutionFrame* frame, + const google::protobuf::Message* absl_nonnull legacy_message) const { + ABSL_DCHECK(legacy_message != nullptr); + if (CheckAttributeTrail(field_, frame)) { + return; + } + if (absl::Status status = cel::interop_internal::WrapLegacyMessageField( + legacy_message, field_descriptor_, unboxing_option_, + frame->descriptor_pool(), frame->message_factory(), frame->arena(), + &frame->value_stack().Peek()); + !status.ok()) { + frame->Abort(std::move(status)); + } +} - if (test_only_) { - if (optional_arg) { - if (!optional_arg->HasValue()) { - result = cel::BoolValue{false}; - return absl::OkStatus(); - } - Value value; - optional_arg->Value(&value); - PerformTestOnlySelect(frame, value, result); - return absl::OkStatus(); - } - PerformTestOnlySelect(frame, result, result); - return absl::OkStatus(); - } +class ProtoHasStep : public SelectStep { + public: + ProtoHasStep(absl::string_view field, bool enable_wrapper_type_null_unboxing, + bool enable_optional_types, const google::protobuf::Descriptor* descriptor, + const google::protobuf::FieldDescriptor* field_descriptor) + : SelectStep(field, /*test_field_presence=*/true, + enable_wrapper_type_null_unboxing, enable_optional_types), + descriptor_(descriptor), + field_descriptor_(field_descriptor) { + ABSL_DCHECK(descriptor_ != nullptr); + ABSL_DCHECK(field_descriptor_ != nullptr); + } - if (optional_arg) { - if (!optional_arg->HasValue()) { - // result is still buffer for the container. just return. - return absl::OkStatus(); - } - Value value; - optional_arg->Value(&value); - return PerformOptionalSelect(frame, value, result); + void Evaluate(ExecutionFrame* frame) const override { + if (!frame->value_stack().HasEnough(1)) { + frame->Abort(absl::InternalError( + "No arguments supplied for Select-type expression")); + return; } - auto status = PerformSelect(frame, result, result); - if (!status.ok()) { - result = ErrorValue(std::move(status)); + const Value& arg = frame->value_stack().Peek(); + if (auto unwrapped = arg.AsParsedMessage(); + unwrapped.has_value() && + SupportsCachedFieldDescriptor(*unwrapped, descriptor_, + field_descriptor_)) { + EvaluateHas(frame, *unwrapped); + return; + } else if (const google::protobuf::Message* legacy_message = + cel::interop_internal::GetLegacyMessage(arg); + legacy_message != nullptr) { + cel::ParsedMessageValue parsed_message = + cel::UnsafeParsedMessageValue(legacy_message); + if (SupportsCachedFieldDescriptor(parsed_message, descriptor_, + field_descriptor_)) { + EvaluateHas(frame, parsed_message); + return; + } } - return absl::OkStatus(); + // If we get an unexpected value type, fall back to the generic + // implementation. + SelectStep::Evaluate(frame); } private: - std::unique_ptr operand_; - - void PerformTestOnlySelect(ExecutionFrameBase& frame, const Value& value, - Value& result) const; - absl::Status PerformOptionalSelect(ExecutionFrameBase& frame, - const Value& value, Value& result) const; - absl::Status PerformSelect(ExecutionFrameBase& frame, const Value& value, - Value& result) const; - - // Field name in formats supported by each of the map and struct field access - // APIs. - // - // ToString or ValueManager::CreateString may force a copy so we do this at - // plan time. - StringValue field_value_; - std::string field_; + void EvaluateHas(ExecutionFrame* frame, + const cel::ParsedMessageValue& parsed_message) const; - // whether this is a has() expression. - bool test_only_; - ProtoWrapperTypeOptions unboxing_option_; - bool enable_optional_types_; + const google::protobuf::Descriptor* descriptor_; + const google::protobuf::FieldDescriptor* field_descriptor_; }; -void DirectSelectStep::PerformTestOnlySelect(ExecutionFrameBase& frame, - const cel::Value& value, - Value& result) const { - switch (value.kind()) { - case ValueKind::kMap: - TestOnlySelect(value.GetMap(), field_value_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), &result); - return; - case ValueKind::kMessage: - TestOnlySelect(value.GetStruct(), field_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), &result); - return; - default: - // Control flow should have returned earlier. - result = cel::ErrorValue(InvalidSelectTargetError()); - return; - } -} - -absl::Status DirectSelectStep::PerformOptionalSelect(ExecutionFrameBase& frame, - const Value& value, - Value& result) const { - switch (value.kind()) { - case ValueKind::kStruct: { - auto struct_value = value.GetStruct(); - CEL_ASSIGN_OR_RETURN(auto ok, struct_value.HasFieldByName(field_)); - if (!ok) { - result = OptionalValue::None(); - return absl::OkStatus(); - } - CEL_RETURN_IF_ERROR(struct_value.GetFieldByName( - field_, unboxing_option_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), &result)); - ABSL_DCHECK(!result.IsUnknown()); - result = OptionalValue::Of(std::move(result), frame.arena()); - return absl::OkStatus(); - } - case ValueKind::kMap: { - CEL_ASSIGN_OR_RETURN( - auto found, - value.GetMap().Find(field_value_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), &result)); - if (!found) { - result = OptionalValue::None(); - return absl::OkStatus(); - } - ABSL_DCHECK(!result.IsUnknown()); - result = OptionalValue::Of(std::move(result), frame.arena()); - return absl::OkStatus(); - } - default: - // Control flow should have returned earlier. - return InvalidSelectTargetError(); - } -} - -absl::Status DirectSelectStep::PerformSelect(ExecutionFrameBase& frame, - const cel::Value& value, - Value& result) const { - switch (value.kind()) { - case ValueKind::kStruct: - CEL_RETURN_IF_ERROR(value.GetStruct().GetFieldByName( - field_, unboxing_option_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), &result)); - ABSL_DCHECK(!result.IsUnknown()); - return absl::OkStatus(); - case ValueKind::kMap: - CEL_RETURN_IF_ERROR( - value.GetMap().Get(field_value_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), &result)); - ABSL_DCHECK(!result.IsUnknown()); - return absl::OkStatus(); - default: - // Control flow should have returned earlier. - return InvalidSelectTargetError(); +void ProtoHasStep::EvaluateHas( + ExecutionFrame* frame, + const cel::ParsedMessageValue& parsed_message) const { + if (CheckAttributeTrail(field_, frame)) { + return; } + frame->value_stack().Peek() = + BoolValue{parsed_message.HasField(field_descriptor_)}; } } // namespace -std::unique_ptr CreateDirectSelectStep( - std::unique_ptr operand, StringValue field, - bool test_only, int64_t expr_id, bool enable_wrapper_type_null_unboxing, - bool enable_optional_types) { - return std::make_unique( - expr_id, std::move(operand), std::move(field), test_only, - enable_wrapper_type_null_unboxing, enable_optional_types); +// Factory method for Select - based Execution step +absl::StatusOr> CreateSelectStep( + absl::string_view field, bool test_only, + bool enable_wrapper_type_null_unboxing, bool enable_optional_types) { + return std::make_unique(field, test_only, + enable_wrapper_type_null_unboxing, + enable_optional_types); } // Factory method for Select - based Execution step -absl::StatusOr> CreateSelectStep( - const cel::SelectExpr& select_expr, int64_t expr_id, +absl::StatusOr> CreateTypedSelectStep( + absl::string_view field, cel::StructType resolved_operand_type, + cel::StructTypeField resolved_field, bool test_only, bool enable_wrapper_type_null_unboxing, bool enable_optional_types) { - return std::make_unique( - cel::StringValue(select_expr.field()), select_expr.test_only(), expr_id, - enable_wrapper_type_null_unboxing, enable_optional_types); + if (!resolved_operand_type.IsMessage()) { + // The specialization only supports messages. Fallback to the generic + // implementation for other types. + // TODO(uncreated-issue/89): support optional select and chaining. + return CreateSelectStep(field, test_only, enable_wrapper_type_null_unboxing, + enable_optional_types); + } + const google::protobuf::Descriptor* descriptor = + resolved_operand_type.GetMessage().descriptor(); + + ABSL_DCHECK(resolved_field.IsMessage()); + const google::protobuf::FieldDescriptor* field_descriptor = + resolved_field.GetMessage().descriptor(); + + if (field_descriptor->file()->pool() != descriptor->file()->pool()) { + // The field descriptor is not in the same pool as the operand type. + // (this should only happen if an overlay extends the type). + // + // We don't have a way to determine if a runtime message is compatible + // with the resolved extension and proto's reflection implementation may + // crash. + // + // Fallback to the generic implementation. + return CreateSelectStep(field, test_only, enable_wrapper_type_null_unboxing, + enable_optional_types); + } + + if (test_only) { + return std::make_unique( + field, enable_wrapper_type_null_unboxing, enable_optional_types, + descriptor, field_descriptor); + } + + return std::make_unique( + field, enable_wrapper_type_null_unboxing, enable_optional_types, + descriptor, field_descriptor); } } // namespace google::api::expr::runtime diff --git a/eval/eval/select_step.h b/eval/eval/select_step.h index 6eaaf9487..9e5d2967b 100644 --- a/eval/eval/select_step.h +++ b/eval/eval/select_step.h @@ -1,28 +1,25 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_SELECT_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_SELECT_STEP_H_ -#include #include #include "absl/status/statusor.h" -#include "common/expr.h" -#include "common/value.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "absl/strings/string_view.h" +#include "common/type.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { -// Factory method for recursively evaluated select step. -std::unique_ptr CreateDirectSelectStep( - std::unique_ptr operand, cel::StringValue field, - bool test_only, int64_t expr_id, bool enable_wrapper_type_null_unboxing, - bool enable_optional_types = false); - -// Factory method for Select - based Execution step -absl::StatusOr> CreateSelectStep( - const cel::SelectExpr& select_expr, int64_t expr_id, +// Factory method for Select stack machine based Execution step +absl::StatusOr> CreateSelectStep( + absl::string_view field, bool test_only, bool enable_wrapper_type_null_unboxing, bool enable_optional_types = false); +absl::StatusOr> CreateTypedSelectStep( + absl::string_view field, cel::StructType resolved_operand_type, + cel::StructTypeField resolved_field, bool test_only, + bool enable_wrapper_type_null_unboxing, bool enable_optional_types); + } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_SELECT_STEP_H_ diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index ce532eabd..e2adb9d2c 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -8,46 +8,37 @@ #include "cel/expr/syntax.pb.h" #include "google/protobuf/wrappers.pb.h" #include "absl/base/nullability.h" -#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" #include "base/attribute.h" #include "base/attribute_set.h" #include "base/type_provider.h" -#include "common/casting.h" #include "common/expr.h" #include "common/legacy_value.h" -#include "common/value.h" -#include "common/value_testing.h" -#include "eval/eval/attribute_trail.h" +#include "common/type.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/const_value_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/ident_step.h" #include "eval/public/activation.h" #include "eval/public/cel_attribute.h" #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_map_impl.h" #include "eval/public/structs/cel_proto_wrapper.h" -#include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/trivial_legacy_type_info.h" -#include "eval/public/testing/matchers.h" +#include "eval/public/unknown_attribute_set.h" +#include "eval/public/unknown_set.h" #include "eval/testutil/test_extensions.pb.h" #include "eval/testutil/test_message.pb.h" -#include "extensions/protobuf/value.h" #include "internal/proto_matchers.h" #include "internal/status_macros.h" #include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" #include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" -#include "runtime/internal/runtime_type_provider.h" #include "runtime/runtime_options.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" +#include "google/protobuf/descriptor.h" namespace google::api::expr::runtime { @@ -56,69 +47,28 @@ namespace { using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::Attribute; -using ::cel::AttributeQualifier; using ::cel::AttributeSet; -using ::cel::BoolValue; -using ::cel::Cast; -using ::cel::ErrorValue; using ::cel::Expr; -using ::cel::InstanceOf; -using ::cel::IntValue; -using ::cel::OptionalValue; using ::cel::RuntimeOptions; using ::cel::TypeProvider; -using ::cel::UnknownValue; -using ::cel::Value; using ::cel::expr::conformance::proto3::TestAllTypes; -using ::cel::extensions::ProtoMessageToValue; using ::cel::internal::test::EqualsProto; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::runtime_internal::RuntimeEnv; -using ::cel::test::IntValueIs; -using ::testing::_; using ::testing::Eq; using ::testing::HasSubstr; -using ::testing::Return; -using ::testing::UnorderedElementsAre; struct RunExpressionOptions { bool enable_unknowns = false; bool enable_wrapper_type_null_unboxing = false; }; -// Simple implementation LegacyTypeAccessApis / LegacyTypeInfoApis that allows -// mocking for getters/setters. -class MockAccessor : public LegacyTypeAccessApis, public LegacyTypeInfoApis { - public: - MOCK_METHOD(absl::StatusOr, HasField, - (absl::string_view field_name, - const CelValue::MessageWrapper& value), - (const, override)); - MOCK_METHOD(absl::StatusOr, GetField, - (absl::string_view field_name, - const CelValue::MessageWrapper& instance, - ProtoWrapperTypeOptions unboxing_option, - cel::MemoryManagerRef memory_manager), - (const, override)); - MOCK_METHOD(absl::string_view, GetTypename, - (const CelValue::MessageWrapper& instance), (const, override)); - MOCK_METHOD(std::string, DebugString, - (const CelValue::MessageWrapper& instance), (const, override)); - MOCK_METHOD(std::vector, ListFields, - (const CelValue::MessageWrapper& value), (const, override)); - const LegacyTypeAccessApis* GetAccessApis( - const CelValue::MessageWrapper& instance) const override { - return this; - } -}; - class SelectStepTest : public testing::Test { public: SelectStepTest() : env_(NewTestingRuntimeEnv()) {} // Helper method. Creates simple pipeline containing Select step and runs it. absl::StatusOr RunExpression(const CelValue target, absl::string_view field, bool test, - absl::string_view unknown_path, RunExpressionOptions options) { ExecutionPath path; @@ -130,24 +80,27 @@ class SelectStepTest : public testing::Test { auto& ident = expr0.mutable_ident_expr(); ident.set_name("target"); - CEL_ASSIGN_OR_RETURN(auto step0, CreateIdentStep(ident.name(), expr0.id())); CEL_ASSIGN_OR_RETURN( auto step1, - CreateSelectStep(select, expr.id(), + CreateSelectStep(select.field(), select.test_only(), options.enable_wrapper_type_null_unboxing)); - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); + path.push_back(ExpressionStep::MakeIdentifierStep(0, expr0.id())); + path.push_back( + ExpressionStep::MakeGenericStep(std::move(step1), expr.id())); cel::RuntimeOptions runtime_options; if (options.enable_unknowns) { runtime_options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; } + // Force the creation of a message factory at the env level. + static_cast(env_->MutableMessageFactory()); CelExpressionFlatImpl cel_expr( - env_, FlatExpression(std::move(path), /*comprehension_slot_count=*/0, + env_, FlatExpression(std::move(path), /*comprehension_slot_count=*/1, env_->type_registry.GetComposedTypeProvider(), - runtime_options)); + runtime_options, /*arena=*/nullptr, + IdentifierTable{{ident.name(), 0}})); Activation activation; activation.InsertValue("target", target); @@ -158,35 +111,20 @@ class SelectStepTest : public testing::Test { absl::string_view field, bool test, RunExpressionOptions options) { return RunExpression(CelProtoWrapper::CreateMessage(message, &arena_), - field, test, "", options); + field, test, options); } absl::StatusOr RunExpression(const TestMessage* message, absl::string_view field, bool test, - absl::string_view unknown_path, RunExpressionOptions options) { return RunExpression(CelProtoWrapper::CreateMessage(message, &arena_), - field, test, unknown_path, options); - } - - absl::StatusOr RunExpression(const TestMessage* message, - absl::string_view field, bool test, - RunExpressionOptions options) { - return RunExpression(message, field, test, "", options); - } - - absl::StatusOr RunExpression(const CelMap* map_value, - absl::string_view field, bool test, - absl::string_view unknown_path, - RunExpressionOptions options) { - return RunExpression(CelValue::CreateMap(map_value), field, test, - unknown_path, options); + field, test, options); } absl::StatusOr RunExpression(const CelMap* map_value, absl::string_view field, bool test, RunExpressionOptions options) { - return RunExpression(map_value, field, test, "", options); + return RunExpression(CelValue::CreateMap(map_value), field, test, options); } protected: @@ -215,8 +153,7 @@ TEST_P(SelectStepConformanceTest, SelectTargetNotStructOrMap) { ASSERT_OK_AND_ASSIGN( CelValue result, RunExpression(CelValue::CreateStringView("some_value"), "some_field", - /*test=*/false, - /*unknown_path=*/"", options)); + /*test=*/false, options)); ASSERT_TRUE(result.IsError()); EXPECT_THAT(*result.ErrorOrDie(), @@ -326,54 +263,34 @@ TEST_F(SelectStepTest, MapPresenseIsErrorTest) { Expr& expr0 = select_map.mutable_operand(); auto& ident = expr0.mutable_ident_expr(); ident.set_name("target"); - - ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(select_map, expr1.id(), + CreateSelectStep(select_map.field(), select_map.test_only(), /*enable_wrapper_type_null_unboxing=*/false)); ASSERT_OK_AND_ASSIGN( auto step2, - CreateSelectStep(select, select_expr.id(), + CreateSelectStep(select.field(), select.test_only(), /*enable_wrapper_type_null_unboxing=*/false)); ExecutionPath path; - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); - path.push_back(std::move(step2)); + path.push_back(ExpressionStep::MakeIdentifierStep(0, expr0.id())); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step1), expr1.id())); + path.push_back( + ExpressionStep::MakeGenericStep(std::move(step2), select_expr.id())); CelExpressionFlatImpl cel_expr( - env_, FlatExpression(std::move(path), /*comprehension_slot_count=*/0, + env_, FlatExpression(std::move(path), /*comprehension_slot_count=*/1, env_->type_registry.GetComposedTypeProvider(), - cel::RuntimeOptions{})); + cel::RuntimeOptions{}, /*arena=*/nullptr, + IdentifierTable{{ident.name(), 0}})); Activation activation; activation.InsertValue("target", CelProtoWrapper::CreateMessage(&message, &arena_)); ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); - EXPECT_TRUE(result.IsError()); + ASSERT_TRUE(result.IsError()); EXPECT_EQ(result.ErrorOrDie()->code(), absl::StatusCode::kInvalidArgument); } -TEST_F(SelectStepTest, MapPresenseIsTrueWithUnknownTest) { - UnknownSet unknown_set; - std::string key1 = "key1"; - std::vector> key_values{ - {CelValue::CreateString(&key1), - CelValue::CreateUnknownSet(&unknown_set)}}; - - auto map_value = CreateContainerBackedMap( - absl::Span>(key_values)) - .value(); - - RunExpressionOptions options; - options.enable_unknowns = true; - - ASSERT_OK_AND_ASSIGN(CelValue result, - RunExpression(map_value.get(), "key1", true, options)); - ASSERT_TRUE(result.IsBool()); - EXPECT_EQ(result.BoolOrDie(), true); -} - TEST_P(SelectStepConformanceTest, FieldIsNotPresentInProtoTest) { TestMessage message; @@ -571,17 +488,36 @@ TEST_P(SelectStepConformanceTest, GlobalExtensionsMessageTest) { } TEST_P(SelectStepConformanceTest, GlobalExtensionsMessageUnsetTest) { - TestExtensions exts; + // Implementation details: + // The test environment is a dynamic descriptor pool with the same definition + // as the linked proto. + // Use a dynamic message with the expected factory and pool so we can expect + // the identity of the default message. + const google::protobuf::Descriptor* descriptor = + env_->descriptor_pool->FindMessageTypeByName( + TestExtensions::descriptor()->full_name()); + ASSERT_NE(descriptor, nullptr); + const google::protobuf::FieldDescriptor* field = + env_->descriptor_pool->FindExtensionByPrintableName( + descriptor, "google.api.expr.runtime.nested_ext"); + ASSERT_NE(field, nullptr); + ASSERT_TRUE(field->containing_type() == descriptor); + const auto* prototype = + env_->MutableMessageFactory()->GetPrototype(descriptor); + ASSERT_NE(prototype, nullptr); + const auto* msg_default = &prototype->GetReflection()->GetMessage( + *prototype, field, env_->MutableMessageFactory()); + RunExpressionOptions options; options.enable_unknowns = GetParam(); ASSERT_OK_AND_ASSIGN( CelValue result, - RunExpression(&exts, "google.api.expr.runtime.nested_ext", false, - options)); + RunExpression(CelProtoWrapper::CreateMessage(prototype, &arena_), + "google.api.expr.runtime.nested_ext", false, options)); ASSERT_TRUE(result.IsMessage()); - EXPECT_THAT(result.MessageOrDie(), Eq(&TestExtensions::default_instance())); + EXPECT_THAT(result.MessageOrDie(), Eq(msg_default)); } TEST_P(SelectStepConformanceTest, GlobalExtensionsWrapperTest) { @@ -677,85 +613,20 @@ TEST_P(SelectStepConformanceTest, NullMessageAccessor) { CelValue value = CelValue::CreateMessageWrapper( CelValue::MessageWrapper(&message, TrivialTypeInfo::GetInstance())); - ASSERT_OK_AND_ASSIGN(CelValue result, - RunExpression(value, "message_value", - /*test=*/false, - /*unknown_path=*/"", options)); + ASSERT_OK_AND_ASSIGN(CelValue result, RunExpression(value, "message_value", + /*test=*/false, options)); ASSERT_TRUE(result.IsError()); EXPECT_THAT(*result.ErrorOrDie(), StatusIs(absl::StatusCode::kNotFound)); // same for has ASSERT_OK_AND_ASSIGN(result, RunExpression(value, "message_value", - /*test=*/true, - /*unknown_path=*/"", options)); + /*test=*/true, options)); ASSERT_TRUE(result.IsError()); EXPECT_THAT(*result.ErrorOrDie(), StatusIs(absl::StatusCode::kNotFound)); } -TEST_P(SelectStepConformanceTest, CustomAccessor) { - TestMessage message; - TestMessage* message2 = message.mutable_message_value(); - message2->set_int32_value(1); - message2->set_string_value("test"); - RunExpressionOptions options; - options.enable_unknowns = GetParam(); - testing::NiceMock accessor; - CelValue value = CelValue::CreateMessageWrapper( - CelValue::MessageWrapper(&message, &accessor)); - - ON_CALL(accessor, GetField(_, _, _, _)) - .WillByDefault(Return(CelValue::CreateInt64(2))); - ON_CALL(accessor, HasField(_, _)).WillByDefault(Return(false)); - - ASSERT_OK_AND_ASSIGN(CelValue result, - RunExpression(value, "message_value", - /*test=*/false, - /*unknown_path=*/"", options)); - - EXPECT_THAT(result, test::IsCelInt64(2)); - - // testonly select (has) - ASSERT_OK_AND_ASSIGN(result, RunExpression(value, "message_value", - /*test=*/true, - /*unknown_path=*/"", options)); - - EXPECT_THAT(result, test::IsCelBool(false)); -} - -TEST_P(SelectStepConformanceTest, CustomAccessorErrorHandling) { - TestMessage message; - TestMessage* message2 = message.mutable_message_value(); - message2->set_int32_value(1); - message2->set_string_value("test"); - RunExpressionOptions options; - options.enable_unknowns = GetParam(); - testing::NiceMock accessor; - CelValue value = CelValue::CreateMessageWrapper( - CelValue::MessageWrapper(&message, &accessor)); - - ON_CALL(accessor, GetField(_, _, _, _)) - .WillByDefault(Return(absl::InternalError("bad data"))); - ON_CALL(accessor, HasField(_, _)) - .WillByDefault(Return(absl::NotFoundError("not found"))); - - // For get field, implementation may return an error-type cel value or a - // status (e.g. broken assumption using a core type). - ASSERT_OK_AND_ASSIGN(CelValue result, - RunExpression(value, "message_value", - /*test=*/false, - /*unknown_path=*/"", options)); - EXPECT_THAT(result, test::IsCelError(StatusIs(absl::StatusCode::kInternal))); - - // testonly select (has) errors are coerced to CelError. - ASSERT_OK_AND_ASSIGN(result, RunExpression(value, "message_value", - /*test=*/true, - /*unknown_path=*/"", options)); - - EXPECT_THAT(result, test::IsCelError(StatusIs(absl::StatusCode::kNotFound))); -} - TEST_P(SelectStepConformanceTest, SimpleEnumTest) { TestMessage message; message.set_enum_value(TestMessage::TEST_ENUM_1); @@ -833,14 +704,14 @@ TEST_P(SelectStepConformanceTest, CelErrorAsArgument) { auto& ident = expr0.mutable_ident_expr(); ident.set_name("message"); - ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(select, dummy_expr.id(), + CreateSelectStep(select.field(), select.test_only(), /*enable_wrapper_type_null_unboxing=*/false)); - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); + path.push_back(ExpressionStep::MakeIdentifierStep(0, expr0.id())); + path.push_back( + ExpressionStep::MakeGenericStep(std::move(step1), dummy_expr.id())); CelError error = absl::CancelledError(); @@ -850,8 +721,9 @@ TEST_P(SelectStepConformanceTest, CelErrorAsArgument) { } CelExpressionFlatImpl cel_expr( env_, - FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env_->type_registry.GetComposedTypeProvider(), options)); + FlatExpression(std::move(path), /*comprehension_slot_count=*/1, + env_->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, IdentifierTable{{ident.name(), 0}})); Activation activation; activation.InsertValue("message", CelValue::CreateError(&error)); @@ -874,19 +746,20 @@ TEST_F(SelectStepTest, DisableMissingAttributeOK) { auto& ident = expr0.mutable_ident_expr(); ident.set_name("message"); - ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(select, dummy_expr.id(), + CreateSelectStep(select.field(), select.test_only(), /*enable_wrapper_type_null_unboxing=*/false)); - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); + path.push_back(ExpressionStep::MakeIdentifierStep(0, expr0.id())); + path.push_back( + ExpressionStep::MakeGenericStep(std::move(step1), dummy_expr.id())); CelExpressionFlatImpl cel_expr( - env_, FlatExpression(std::move(path), /*comprehension_slot_count=*/0, + env_, FlatExpression(std::move(path), /*comprehension_slot_count=*/1, env_->type_registry.GetComposedTypeProvider(), - cel::RuntimeOptions{})); + cel::RuntimeOptions{}, /*arena=*/nullptr, + IdentifierTable{{ident.name(), 0}})); Activation activation; activation.InsertValue("message", CelProtoWrapper::CreateMessage(&message, &arena_)); @@ -896,7 +769,7 @@ TEST_F(SelectStepTest, DisableMissingAttributeOK) { EXPECT_EQ(result.BoolOrDie(), true); CelAttributePattern pattern("message", {}); - activation.set_missing_attribute_patterns({pattern}); + ASSERT_THAT(activation.SetMissingAttributePatterns({pattern}), IsOk()); ASSERT_OK_AND_ASSIGN(result, cel_expr.Evaluate(activation, &arena_)); EXPECT_EQ(result.BoolOrDie(), true); @@ -916,21 +789,22 @@ TEST_F(SelectStepTest, UnrecoverableUnknownValueProducesError) { auto& ident = expr0.mutable_ident_expr(); ident.set_name("message"); - ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(select, dummy_expr.id(), + CreateSelectStep(select.field(), select.test_only(), /*enable_wrapper_type_null_unboxing=*/false)); - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); + path.push_back(ExpressionStep::MakeIdentifierStep(0, expr0.id())); + path.push_back( + ExpressionStep::MakeGenericStep(std::move(step1), dummy_expr.id())); cel::RuntimeOptions options; options.enable_missing_attribute_errors = true; CelExpressionFlatImpl cel_expr( env_, - FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env_->type_registry.GetComposedTypeProvider(), options)); + FlatExpression(std::move(path), /*comprehension_slot_count=*/1, + env_->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, IdentifierTable{{ident.name(), 0}})); Activation activation; activation.InsertValue("message", CelProtoWrapper::CreateMessage(&message, &arena_)); @@ -942,7 +816,7 @@ TEST_F(SelectStepTest, UnrecoverableUnknownValueProducesError) { CelAttributePattern pattern("message", {CreateCelAttributeQualifierPattern( CelValue::CreateStringView("bool_value"))}); - activation.set_missing_attribute_patterns({pattern}); + ASSERT_THAT(activation.SetMissingAttributePatterns({pattern}), IsOk()); ASSERT_OK_AND_ASSIGN(result, cel_expr.Evaluate(activation, &arena_)); EXPECT_THAT(*result.ErrorOrDie(), @@ -964,30 +838,31 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { auto& ident = expr0.mutable_ident_expr(); ident.set_name("message"); - auto step0_status = CreateIdentStep(ident.name(), expr0.id()); auto step1_status = - CreateSelectStep(select, dummy_expr.id(), + CreateSelectStep(select.field(), select.test_only(), /*enable_wrapper_type_null_unboxing=*/false); - ASSERT_THAT(step0_status, IsOk()); ASSERT_THAT(step1_status, IsOk()); - path.push_back(*std::move(step0_status)); - path.push_back(*std::move(step1_status)); + path.push_back(ExpressionStep::MakeIdentifierStep(0, expr0.id())); + path.push_back(ExpressionStep::MakeGenericStep(std::move(*step1_status), + dummy_expr.id())); cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; CelExpressionFlatImpl cel_expr( env_, - FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env_->type_registry.GetComposedTypeProvider(), options)); + FlatExpression(std::move(path), /*comprehension_slot_count=*/1, + env_->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, IdentifierTable{{ident.name(), 0}})); { std::vector unknown_patterns; Activation activation; activation.InsertValue("message", CelProtoWrapper::CreateMessage(&message, &arena_)); - activation.set_unknown_attribute_patterns(unknown_patterns); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), + IsOk()); ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); @@ -1004,7 +879,8 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { Activation activation; activation.InsertValue("message", CelProtoWrapper::CreateMessage(&message, &arena_)); - activation.set_unknown_attribute_patterns(unknown_patterns); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), + IsOk()); ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); @@ -1019,7 +895,8 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { Activation activation; activation.InsertValue("message", CelProtoWrapper::CreateMessage(&message, &arena_)); - activation.set_unknown_attribute_patterns(unknown_patterns); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), + IsOk()); ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); @@ -1033,7 +910,8 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { Activation activation; activation.InsertValue("message", CelProtoWrapper::CreateMessage(&message, &arena_)); - activation.set_unknown_attribute_patterns(unknown_patterns); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), + IsOk()); ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); @@ -1048,7 +926,8 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { Activation activation; activation.InsertValue("message", CelProtoWrapper::CreateMessage(&message, &arena_)); - activation.set_unknown_attribute_patterns(unknown_patterns); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), + IsOk()); ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); @@ -1057,526 +936,125 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { } } -INSTANTIATE_TEST_SUITE_P(UnknownsEnabled, SelectStepConformanceTest, - testing::Bool()); +TEST_P(SelectStepConformanceTest, TypedSelectStepTest) { + cel::StructType resolved_operand_type( + (cel::MessageType(TestAllTypes::descriptor()))); + const google::protobuf::FieldDescriptor* field_desc = + TestAllTypes::descriptor()->FindFieldByName("single_int64"); + ASSERT_NE(field_desc, nullptr); + cel::StructTypeField resolved_field((cel::MessageTypeField(field_desc))); -class DirectSelectStepTest : public testing::Test { - public: - DirectSelectStepTest() - : type_provider_(cel::internal::GetTestingDescriptorPool()) {} - - cel::Value TestWrapMessage(const google::protobuf::Message* message) { - CelValue value = CelProtoWrapper::CreateMessage(message, &arena_); - auto result = cel::interop_internal::FromLegacyValue(&arena_, value); - ABSL_DCHECK_OK(result.status()); - return std::move(result).value(); - } + ASSERT_OK_AND_ASSIGN( + auto step1, CreateTypedSelectStep( + "single_int64", resolved_operand_type, resolved_field, + /*test_only=*/false, + /*enable_wrapper_type_null_unboxing=*/false, + /*enable_optional_types=*/false)); - std::vector AttributeStrings(const UnknownValue& v) { - std::vector result; - for (const Attribute& attr : v.attribute_set()) { - auto attr_str = attr.AsString(); - ABSL_DCHECK_OK(attr_str.status()); - result.push_back(std::move(attr_str).value()); - } - return result; + ExecutionPath path; + path.push_back(ExpressionStep::MakeIdentifierStep(0)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step1))); + cel::RuntimeOptions options; + if (GetParam()) { + options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; } - - protected: - google::protobuf::Arena arena_; - cel::runtime_internal::RuntimeTypeProvider type_provider_; -}; - -TEST_F(DirectSelectStepTest, SelectFromMap) { - cel::Activation activation; - RuntimeOptions options; - - auto step = CreateDirectSelectStep( - CreateDirectIdentStep("map_val", -1), cel::StringValue("one"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT(map_builder->Put(cel::StringValue("one"), IntValue(1)), IsOk()); - ASSERT_THAT(map_builder->Put(cel::StringValue("two"), IntValue(2)), IsOk()); - activation.InsertOrAssignValue("map_val", std::move(*map_builder).Build()); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(InstanceOf(result)); - - EXPECT_EQ(Cast(result).NativeValue(), 1); -} - -TEST_F(DirectSelectStepTest, HasMap) { - cel::Activation activation; - RuntimeOptions options; - - auto step = CreateDirectSelectStep( - CreateDirectIdentStep("map_val", -1), cel::StringValue("two"), - /*test_only=*/true, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT(map_builder->Put(cel::StringValue("one"), IntValue(1)), IsOk()); - ASSERT_THAT(map_builder->Put(cel::StringValue("two"), IntValue(2)), IsOk()); - activation.InsertOrAssignValue("map_val", std::move(*map_builder).Build()); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(InstanceOf(result)); - - EXPECT_TRUE(Cast(result).NativeValue()); -} - -TEST_F(DirectSelectStepTest, SelectFromOptionalMap) { - cel::Activation activation; - RuntimeOptions options; - - auto step = CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), - cel::StringValue("one"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); - - auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT(map_builder->Put(cel::StringValue("one"), IntValue(1)), IsOk()); - ASSERT_THAT(map_builder->Put(cel::StringValue("two"), IntValue(2)), IsOk()); - activation.InsertOrAssignValue( - "map_val", OptionalValue::Of(std::move(*map_builder).Build(), &arena_)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(static_cast(result)).Value(), - IntValueIs(1)); -} - -TEST_F(DirectSelectStepTest, SelectFromOptionalMapAbsent) { - cel::Activation activation; - RuntimeOptions options; - - auto step = CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), - cel::StringValue("three"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); - - auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT(map_builder->Put(cel::StringValue("one"), IntValue(1)), IsOk()); - ASSERT_THAT(map_builder->Put(cel::StringValue("two"), IntValue(2)), IsOk()); - activation.InsertOrAssignValue( - "map_val", OptionalValue::Of(std::move(*map_builder).Build(), &arena_)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_FALSE( - Cast(static_cast(result)).HasValue()); -} - -TEST_F(DirectSelectStepTest, SelectFromOptionalStruct) { - cel::Activation activation; - RuntimeOptions options; - - auto step = CreateDirectSelectStep(CreateDirectIdentStep("struct_val", -1), - cel::StringValue("single_int64"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); + CelExpressionFlatImpl cel_expr( + env_, + FlatExpression(std::move(path), /*comprehension_slot_count=*/1, + env_->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, IdentifierTable{{"message", 0}})); TestAllTypes message; - message.set_single_int64(1); - - ASSERT_OK_AND_ASSIGN( - Value struct_val, - ProtoMessageToValue(std::move(message), - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_)); - - activation.InsertOrAssignValue("struct_val", - OptionalValue::Of(struct_val, &arena_)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); + message.set_single_int64(42); + Activation activation; + activation.InsertValue("message", + CelProtoWrapper::CreateMessage(&message, &arena_)); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(static_cast(result)).Value(), - IntValueIs(1)); + ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); + ASSERT_TRUE(result.IsInt64()); + EXPECT_EQ(result.Int64OrDie(), 42); } -TEST_F(DirectSelectStepTest, SelectFromOptionalStructFieldNotSet) { - cel::Activation activation; - RuntimeOptions options; - - auto step = CreateDirectSelectStep(CreateDirectIdentStep("struct_val", -1), - cel::StringValue("single_string"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); - - TestAllTypes message; - message.set_single_int64(1); +TEST_P(SelectStepConformanceTest, TypedSelectStepPropagatesUnknown) { + cel::StructType resolved_operand_type( + (cel::MessageType(TestAllTypes::descriptor()))); + const google::protobuf::FieldDescriptor* field_desc = + TestAllTypes::descriptor()->FindFieldByName("single_int64"); + ASSERT_NE(field_desc, nullptr); + cel::StructTypeField resolved_field((cel::MessageTypeField(field_desc))); ASSERT_OK_AND_ASSIGN( - Value struct_val, - ProtoMessageToValue(std::move(message), - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_)); - - activation.InsertOrAssignValue("struct_val", - OptionalValue::Of(struct_val, &arena_)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_FALSE( - Cast(static_cast(result)).HasValue()); -} - -TEST_F(DirectSelectStepTest, SelectFromEmptyOptional) { - cel::Activation activation; - RuntimeOptions options; - - auto step = CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), - cel::StringValue("one"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); - - activation.InsertOrAssignValue("map_val", OptionalValue::None()); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_FALSE( - cel::Cast(static_cast(result)).HasValue()); -} - -TEST_F(DirectSelectStepTest, HasOptional) { - cel::Activation activation; - RuntimeOptions options; - - auto step = CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), - cel::StringValue("two"), - /*test_only=*/true, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); - - auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT(map_builder->Put(cel::StringValue("one"), IntValue(1)), IsOk()); - ASSERT_THAT(map_builder->Put(cel::StringValue("two"), IntValue(2)), IsOk()); - activation.InsertOrAssignValue( - "map_val", OptionalValue::Of(std::move(*map_builder).Build(), &arena_)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(InstanceOf(result)); - - EXPECT_TRUE(Cast(result).NativeValue()); -} - -TEST_F(DirectSelectStepTest, HasEmptyOptional) { - cel::Activation activation; - RuntimeOptions options; - - auto step = CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), - cel::StringValue("two"), - /*test_only=*/true, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); - - activation.InsertOrAssignValue("map_val", OptionalValue::None()); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(InstanceOf(result)); - - EXPECT_FALSE(Cast(result).NativeValue()); -} - -TEST_F(DirectSelectStepTest, SelectFromStruct) { - cel::Activation activation; - RuntimeOptions options; - - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("test_all_types", -1), - cel::StringValue("single_int64"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - TestAllTypes message; - message.set_single_int64(1); - activation.InsertOrAssignValue("test_all_types", TestWrapMessage(&message)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(InstanceOf(result)); - - EXPECT_EQ(Cast(result).NativeValue(), 1); -} - -TEST_F(DirectSelectStepTest, HasStruct) { - cel::Activation activation; - RuntimeOptions options; - - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("test_all_types", -1), - cel::StringValue("single_string"), - /*test_only=*/true, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - TestAllTypes message; - message.set_single_int64(1); - activation.InsertOrAssignValue("test_all_types", TestWrapMessage(&message)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - - // has(test_all_types.single_string) - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_FALSE(Cast(result).NativeValue()); -} - -TEST_F(DirectSelectStepTest, SelectFromUnsupportedType) { - cel::Activation activation; - RuntimeOptions options; - - auto step = CreateDirectSelectStep( - CreateDirectIdentStep("bool_val", -1), cel::StringValue("one"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - activation.InsertOrAssignValue("bool_val", BoolValue(false)); + auto step1, CreateTypedSelectStep( + "single_int64", resolved_operand_type, resolved_field, + /*test_only=*/false, + /*enable_wrapper_type_null_unboxing=*/false, + /*enable_optional_types=*/false)); - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(InstanceOf(result)); - - EXPECT_THAT(Cast(result).NativeValue(), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("Applying SELECT to non-message type"))); -} - -TEST_F(DirectSelectStepTest, AttributeUpdatedIfRequested) { - cel::Activation activation; - RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("test_all_types", -1), - cel::StringValue("single_int64"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - TestAllTypes message; - message.set_single_int64(1); - activation.InsertOrAssignValue("test_all_types", TestWrapMessage(&message)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); + ExecutionPath path; + path.push_back(ExpressionStep::MakeIdentifierStep(0)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step1))); + cel::RuntimeOptions options; + if (GetParam()) { + options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; + } + CelExpressionFlatImpl cel_expr( + env_, + FlatExpression(std::move(path), /*comprehension_slot_count=*/1, + env_->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, IdentifierTable{{"message", 0}})); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_EQ(Cast(result).NativeValue(), 1); + UnknownSet unknown_set(UnknownAttributeSet({CelAttribute("message", {})})); + Activation activation; + activation.InsertValue("message", CelValue::CreateUnknownSet(&unknown_set)); - ASSERT_OK_AND_ASSIGN(std::string attr_str, attr.attribute().AsString()); - EXPECT_EQ(attr_str, "test_all_types.single_int64"); + ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); + ASSERT_TRUE(result.IsUnknownSet()); } -TEST_F(DirectSelectStepTest, MissingAttributesToErrors) { - cel::Activation activation; - RuntimeOptions options; - options.enable_missing_attribute_errors = true; - - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("test_all_types", -1), - cel::StringValue("single_int64"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); +TEST_F(SelectStepTest, TypedSelectStepUnknownPatternResolvesToUnknown) { + cel::StructType resolved_operand_type( + (cel::MessageType(TestAllTypes::descriptor()))); + const google::protobuf::FieldDescriptor* field_desc = + TestAllTypes::descriptor()->FindFieldByName("single_int64"); + ASSERT_NE(field_desc, nullptr); + cel::StructTypeField resolved_field((cel::MessageTypeField(field_desc))); - TestAllTypes message; - message.set_single_int64(1); - activation.InsertOrAssignValue("test_all_types", TestWrapMessage(&message)); - activation.SetMissingPatterns({cel::AttributePattern( - "test_all_types", - {cel::AttributeQualifierPattern::OfString("single_int64")})}); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("test_all_types.single_int64"))); -} + ASSERT_OK_AND_ASSIGN( + auto step1, CreateTypedSelectStep( + "single_int64", resolved_operand_type, resolved_field, + /*test_only=*/false, + /*enable_wrapper_type_null_unboxing=*/false, + /*enable_optional_types=*/false)); -TEST_F(DirectSelectStepTest, IdentifiesUnknowns) { - cel::Activation activation; - RuntimeOptions options; + ExecutionPath path; + path.push_back(ExpressionStep::MakeIdentifierStep(0)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step1))); + cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("test_all_types", -1), - cel::StringValue("single_int64"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); + CelExpressionFlatImpl cel_expr( + env_, + FlatExpression(std::move(path), /*comprehension_slot_count=*/1, + env_->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, IdentifierTable{{"message", 0}})); TestAllTypes message; - message.set_single_int64(1); - activation.InsertOrAssignValue("test_all_types", TestWrapMessage(&message)); - activation.SetUnknownPatterns({cel::AttributePattern( - "test_all_types", - {cel::AttributeQualifierPattern::OfString("single_int64")})}); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); + message.set_single_int64(42); - ASSERT_TRUE(InstanceOf(result)); + std::vector unknown_patterns; + unknown_patterns.push_back(CelAttributePattern( + "message", {CelAttributeQualifierPattern::OfString("single_int64")})); - EXPECT_THAT(AttributeStrings(Cast(result)), - UnorderedElementsAre("test_all_types.single_int64")); -} - -TEST_F(DirectSelectStepTest, ForwardErrorValue) { - cel::Activation activation; - RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - - auto step = CreateDirectSelectStep( - CreateConstValueDirectStep(cel::ErrorValue(absl::InternalError("test1")), - -1), - cel::StringValue("single_int64"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); + Activation activation; + activation.InsertValue("message", + CelProtoWrapper::CreateMessage(&message, &arena_)); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), - StatusIs(absl::StatusCode::kInternal, HasSubstr("test1"))); + ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); + ASSERT_TRUE(result.IsUnknownSet()); } -TEST_F(DirectSelectStepTest, ForwardUnknownOperand) { - cel::Activation activation; - RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - - AttributeSet attr_set({Attribute("attr", {AttributeQualifier::OfInt(0)})}); - auto step = CreateDirectSelectStep( - CreateConstValueDirectStep( - cel::UnknownValue(cel::Unknown(std::move(attr_set))), -1), - cel::StringValue("single_int64"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - TestAllTypes message; - message.set_single_int64(1); - activation.InsertOrAssignValue("test_all_types", TestWrapMessage(&message)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(AttributeStrings(Cast(result)), - UnorderedElementsAre("attr[0]")); -} +INSTANTIATE_TEST_SUITE_P(UnknownsEnabled, SelectStepConformanceTest, + testing::Bool()); } // namespace diff --git a/eval/eval/shadowable_value_step.cc b/eval/eval/shadowable_value_step.cc index 1ebab2f1e..d257e7514 100644 --- a/eval/eval/shadowable_value_step.cc +++ b/eval/eval/shadowable_value_step.cc @@ -1,20 +1,16 @@ #include "eval/eval/shadowable_value_step.h" -#include #include #include #include -#include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" -#include "internal/status_macros.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { @@ -24,75 +20,40 @@ using ::cel::Value; class ShadowableValueStep : public ExpressionStepBase { public: - ShadowableValueStep(std::string identifier, cel::Value value, int64_t expr_id) - : ExpressionStepBase(expr_id), + ShadowableValueStep(std::string identifier, cel::Value value) + : ExpressionStepBase(), identifier_(std::move(identifier)), value_(std::move(value)) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: std::string identifier_; Value value_; }; -absl::Status ShadowableValueStep::Evaluate(ExecutionFrame* frame) const { +void ShadowableValueStep::Evaluate(ExecutionFrame* frame) const { cel::Value result; - CEL_ASSIGN_OR_RETURN(auto found, - frame->modern_activation().FindVariable( - identifier_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &result)); - if (found) { + absl::StatusOr found = frame->modern_activation().FindVariable( + identifier_, frame->descriptor_pool(), frame->message_factory(), + frame->arena(), &result); + if (!found.ok()) { + frame->Abort(std::move(found).status()); + return; + } + if (*found) { frame->value_stack().Push(std::move(result)); } else { frame->value_stack().Push(value_); } - return absl::OkStatus(); -} - -class DirectShadowableValueStep : public DirectExpressionStep { - public: - DirectShadowableValueStep(std::string identifier, cel::Value value, - int64_t expr_id) - : DirectExpressionStep(expr_id), - identifier_(std::move(identifier)), - value_(std::move(value)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override; - - private: - std::string identifier_; - Value value_; -}; - -// TODO(uncreated-issue/67): Attribute tracking is skipped for the shadowed case. May -// cause problems for users with unknown tracking and variables named like -// 'list' etc, but follows the current behavior of the stack machine version. -absl::Status DirectShadowableValueStep::Evaluate( - ExecutionFrameBase& frame, Value& result, AttributeTrail& attribute) const { - CEL_ASSIGN_OR_RETURN(auto found, - frame.activation().FindVariable( - identifier_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), &result)); - if (!found) { - result = value_; - } - return absl::OkStatus(); } } // namespace -absl::StatusOr> CreateShadowableValueStep( - absl::string_view name, cel::Value value, int64_t expr_id) { - return absl::make_unique(std::string(name), - std::move(value), expr_id); -} - -std::unique_ptr CreateDirectShadowableValueStep( - absl::string_view name, cel::Value value, int64_t expr_id) { - return std::make_unique(std::string(name), - std::move(value), expr_id); +absl::StatusOr> CreateShadowableValueStep( + absl::string_view name, cel::Value value) { + return std::make_unique(std::string(name), + std::move(value)); } } // namespace google::api::expr::runtime diff --git a/eval/eval/shadowable_value_step.h b/eval/eval/shadowable_value_step.h index 9c386f02d..c6f0a370a 100644 --- a/eval/eval/shadowable_value_step.h +++ b/eval/eval/shadowable_value_step.h @@ -1,25 +1,20 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_SHADOWABLE_VALUE_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_SHADOWABLE_VALUE_STEP_H_ -#include #include #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "common/value.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { // Create an identifier resolution step with a default value that may be // shadowed by an identifier of the same name within the runtime-provided // Activation. -absl::StatusOr> CreateShadowableValueStep( - absl::string_view name, cel::Value value, int64_t expr_id); - -std::unique_ptr CreateDirectShadowableValueStep( - absl::string_view name, cel::Value value, int64_t expr_id); +absl::StatusOr> CreateShadowableValueStep( + absl::string_view name, cel::Value value); } // namespace google::api::expr::runtime diff --git a/eval/eval/shadowable_value_step_test.cc b/eval/eval/shadowable_value_step_test.cc index 4a7cabea1..46721f100 100644 --- a/eval/eval/shadowable_value_step_test.cc +++ b/eval/eval/shadowable_value_step_test.cc @@ -6,6 +6,7 @@ #include "absl/base/nullability.h" #include "absl/status/statusor.h" +#include "absl/strings/string_view.h" #include "base/type_provider.h" #include "common/value.h" #include "eval/eval/cel_expression_flat_impl.h" @@ -18,6 +19,7 @@ #include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" #include "runtime/runtime_options.h" +#include "google/protobuf/arena.h" namespace google::api::expr::runtime { @@ -32,18 +34,17 @@ using ::testing::Eq; absl::StatusOr RunShadowableExpression( const absl_nonnull std::shared_ptr& env, - std::string identifier, cel::Value value, const Activation& activation, - Arena* arena) { - CEL_ASSIGN_OR_RETURN( - auto step, - CreateShadowableValueStep(std::move(identifier), std::move(value), 1)); + absl::string_view identifier, cel::Value value, + const Activation& activation, Arena* arena) { + CEL_ASSIGN_OR_RETURN(auto step, + CreateShadowableValueStep(identifier, std::move(value))); ExecutionPath path; - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step), 1)); CelExpressionFlatImpl impl( env, FlatExpression(std::move(path), /*comprehension_slot_count=*/0, env->type_registry.GetComposedTypeProvider(), - cel::RuntimeOptions{})); + cel::RuntimeOptions{}, /*arena=*/nullptr, {})); return impl.Evaluate(activation, arena); } @@ -55,11 +56,10 @@ TEST(ShadowableValueStepTest, TestEvaluateNoShadowing) { Arena arena; auto type_value = CreateTypeValueFromView(&arena, type_name); - auto status = - RunShadowableExpression(env, type_name, type_value, activation, &arena); - ASSERT_OK(status); + ASSERT_OK_AND_ASSIGN( + auto value, + RunShadowableExpression(env, type_name, type_value, activation, &arena)); - auto value = status.value(); ASSERT_TRUE(value.IsCelType()); EXPECT_THAT(value.CelTypeOrDie().value(), Eq(type_name)); } @@ -74,11 +74,10 @@ TEST(ShadowableValueStepTest, TestEvaluateShadowedIdentifier) { Arena arena; auto type_value = CreateTypeValueFromView(&arena, type_name); - auto status = - RunShadowableExpression(env, type_name, type_value, activation, &arena); - ASSERT_OK(status); + ASSERT_OK_AND_ASSIGN( + auto value, + RunShadowableExpression(env, type_name, type_value, activation, &arena)); - auto value = status.value(); ASSERT_TRUE(value.IsInt64()); EXPECT_THAT(value.Int64OrDie(), Eq(1024L)); } diff --git a/eval/eval/ternary_step.cc b/eval/eval/ternary_step.cc index a12d6863e..d1ed0fd1e 100644 --- a/eval/eval/ternary_step.cc +++ b/eval/eval/ternary_step.cc @@ -1,20 +1,16 @@ #include "eval/eval/ternary_step.h" #include -#include #include #include #include "absl/status/status.h" -#include "absl/status/statusor.h" #include "base/builtins.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" #include "eval/internal/errors.h" -#include "internal/status_macros.h" namespace google::api::expr::runtime { @@ -27,112 +23,19 @@ inline constexpr size_t kTernaryStepCondition = 0; inline constexpr size_t kTernaryStepTrue = 1; inline constexpr size_t kTernaryStepFalse = 2; -class ExhaustiveDirectTernaryStep : public DirectExpressionStep { - public: - ExhaustiveDirectTernaryStep(std::unique_ptr condition, - std::unique_ptr left, - std::unique_ptr right, - int64_t expr_id) - : DirectExpressionStep(expr_id), - condition_(std::move(condition)), - left_(std::move(left)), - right_(std::move(right)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& attribute) const override { - cel::Value condition; - cel::Value lhs; - cel::Value rhs; - - AttributeTrail condition_attr; - AttributeTrail lhs_attr; - AttributeTrail rhs_attr; - - CEL_RETURN_IF_ERROR(condition_->Evaluate(frame, condition, condition_attr)); - CEL_RETURN_IF_ERROR(left_->Evaluate(frame, lhs, lhs_attr)); - CEL_RETURN_IF_ERROR(right_->Evaluate(frame, rhs, rhs_attr)); - - if (condition.IsError() || condition.IsUnknown()) { - result = std::move(condition); - attribute = std::move(condition_attr); - return absl::OkStatus(); - } - - if (!condition.IsBool()) { - result = cel::ErrorValue(CreateNoMatchingOverloadError(kTernary)); - return absl::OkStatus(); - } - - if (condition.GetBool().NativeValue()) { - result = std::move(lhs); - attribute = std::move(lhs_attr); - } else { - result = std::move(rhs); - attribute = std::move(rhs_attr); - } - return absl::OkStatus(); - } - - private: - std::unique_ptr condition_; - std::unique_ptr left_; - std::unique_ptr right_; -}; - -class ShortcircuitingDirectTernaryStep : public DirectExpressionStep { - public: - ShortcircuitingDirectTernaryStep( - std::unique_ptr condition, - std::unique_ptr left, - std::unique_ptr right, int64_t expr_id) - : DirectExpressionStep(expr_id), - condition_(std::move(condition)), - left_(std::move(left)), - right_(std::move(right)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& attribute) const override { - cel::Value condition; - - AttributeTrail condition_attr; - - CEL_RETURN_IF_ERROR(condition_->Evaluate(frame, condition, condition_attr)); - - if (condition.IsError() || condition.IsUnknown()) { - result = std::move(condition); - attribute = std::move(condition_attr); - return absl::OkStatus(); - } - - if (!condition.IsBool()) { - result = cel::ErrorValue(CreateNoMatchingOverloadError(kTernary)); - return absl::OkStatus(); - } - - if (condition.GetBool().NativeValue()) { - return left_->Evaluate(frame, result, attribute); - } - return right_->Evaluate(frame, result, attribute); - } - - private: - std::unique_ptr condition_; - std::unique_ptr left_; - std::unique_ptr right_; -}; - class TernaryStep : public ExpressionStepBase { public: // Constructs FunctionStep that uses overloads specified. - explicit TernaryStep(int64_t expr_id) : ExpressionStepBase(expr_id) {} + TernaryStep() : ExpressionStepBase() {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; }; -absl::Status TernaryStep::Evaluate(ExecutionFrame* frame) const { +void TernaryStep::Evaluate(ExecutionFrame* frame) const { // Must have 3 or more values on the stack. if (!frame->value_stack().HasEnough(3)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); + frame->Abort(absl::InternalError("Value stack underflow")); + return; } // Create Span object that contains input arguments to the function. @@ -146,18 +49,19 @@ absl::Status TernaryStep::Evaluate(ExecutionFrame* frame) const { // Check if unknown? if (condition.IsUnknown()) { frame->value_stack().Pop(2); - return absl::OkStatus(); + return; } } if (condition.IsError()) { frame->value_stack().Pop(2); - return absl::OkStatus(); + return; } cel::Value result; if (!condition.IsBool()) { - result = cel::ErrorValue(CreateNoMatchingOverloadError(kTernary)); + result = cel::ErrorValue::From(CreateNoMatchingOverloadError(kTernary), + frame->arena()); } else if (condition.GetBool().NativeValue()) { result = args[kTernaryStepTrue]; } else { @@ -165,30 +69,12 @@ absl::Status TernaryStep::Evaluate(ExecutionFrame* frame) const { } frame->value_stack().PopAndPush(args.size(), std::move(result)); - - return absl::OkStatus(); } } // namespace -// Factory method for ternary (_?_:_) recursive execution step -std::unique_ptr CreateDirectTernaryStep( - std::unique_ptr condition, - std::unique_ptr left, - std::unique_ptr right, int64_t expr_id, - bool shortcircuiting) { - if (shortcircuiting) { - return std::make_unique( - std::move(condition), std::move(left), std::move(right), expr_id); - } - - return std::make_unique( - std::move(condition), std::move(left), std::move(right), expr_id); -} - -absl::StatusOr> CreateTernaryStep( - int64_t expr_id) { - return std::make_unique(expr_id); +std::unique_ptr CreateTernaryStep() { + return std::make_unique(); } } // namespace google::api::expr::runtime diff --git a/eval/eval/ternary_step.h b/eval/eval/ternary_step.h index 2b51e95ea..75cf5d315 100644 --- a/eval/eval/ternary_step.h +++ b/eval/eval/ternary_step.h @@ -1,25 +1,14 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_TERNARY_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_TERNARY_STEP_H_ -#include #include -#include "absl/status/statusor.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { -// Factory method for ternary (_?_:_) recursive execution step -std::unique_ptr CreateDirectTernaryStep( - std::unique_ptr condition, - std::unique_ptr left, - std::unique_ptr right, int64_t expr_id, - bool shortcircuiting = true); - // Factory method for ternary (_?_:_) execution step -absl::StatusOr> CreateTernaryStep( - int64_t expr_id); +std::unique_ptr CreateTernaryStep(); } // namespace google::api::expr::runtime diff --git a/eval/eval/ternary_step_test.cc b/eval/eval/ternary_step_test.cc index ff66c0308..e1d9e6cbe 100644 --- a/eval/eval/ternary_step_test.cc +++ b/eval/eval/ternary_step_test.cc @@ -3,34 +3,18 @@ #include #include #include -#include #include "absl/base/nullability.h" #include "absl/status/status.h" -#include "base/attribute.h" -#include "base/attribute_set.h" #include "base/type_provider.h" -#include "common/casting.h" -#include "common/expr.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/const_value_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/ident_step.h" #include "eval/public/activation.h" #include "eval/public/cel_value.h" -#include "eval/public/unknown_attribute_set.h" -#include "eval/public/unknown_set.h" -#include "internal/status_macros.h" #include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" #include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" -#include "runtime/internal/runtime_type_provider.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -38,23 +22,12 @@ namespace google::api::expr::runtime { namespace { -using ::absl_testing::StatusIs; -using ::cel::BoolValue; -using ::cel::Cast; -using ::cel::ErrorValue; -using ::cel::Expr; -using ::cel::InstanceOf; -using ::cel::IntValue; +using ::absl_testing::IsOk; using ::cel::RuntimeOptions; using ::cel::TypeProvider; -using ::cel::UnknownValue; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::runtime_internal::RuntimeEnv; using ::google::protobuf::Arena; -using ::testing::ElementsAre; -using ::testing::Eq; -using ::testing::HasSubstr; -using ::testing::Truly; class LogicStepTest : public testing::TestWithParam { public: @@ -64,17 +37,10 @@ class LogicStepTest : public testing::TestWithParam { CelValue* result, bool enable_unknown) { ExecutionPath path; - CEL_ASSIGN_OR_RETURN(auto step, CreateIdentStep("name0", /*expr_id=*/-1)); - path.push_back(std::move(step)); - - CEL_ASSIGN_OR_RETURN(step, CreateIdentStep("name1", /*expr_id=*/-1)); - path.push_back(std::move(step)); - - CEL_ASSIGN_OR_RETURN(step, CreateIdentStep("name2", /*expr_id=*/-1)); - path.push_back(std::move(step)); - - CEL_ASSIGN_OR_RETURN(step, CreateTernaryStep(4)); - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeIdentifierStep(0)); + path.push_back(ExpressionStep::MakeIdentifierStep(1)); + path.push_back(ExpressionStep::MakeIdentifierStep(2)); + path.push_back(ExpressionStep::MakeGenericStep(CreateTernaryStep(), 4)); cel::RuntimeOptions options; if (enable_unknown) { @@ -82,9 +48,11 @@ class LogicStepTest : public testing::TestWithParam { cel::UnknownProcessingOptions::kAttributeOnly; } CelExpressionFlatImpl impl( - env_, - FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env_->type_registry.GetComposedTypeProvider(), options)); + env_, FlatExpression( + std::move(path), /*comprehension_slot_count=*/3, + env_->type_registry.GetComposedTypeProvider(), options, + /*arena=*/nullptr, + IdentifierTable{{"name0", 0}, {"name1", 1}, {"name2", 2}})); Activation activation; std::string value("test"); @@ -109,14 +77,14 @@ TEST_P(LogicStepTest, TestBoolCond) { absl::Status status = EvaluateLogic(CelValue::CreateBool(true), CelValue::CreateBool(true), CelValue::CreateBool(false), &result, GetParam()); - ASSERT_OK(status); + ASSERT_THAT(status, IsOk()); ASSERT_TRUE(result.IsBool()); ASSERT_TRUE(result.BoolOrDie()); status = EvaluateLogic(CelValue::CreateBool(false), CelValue::CreateBool(true), CelValue::CreateBool(false), &result, GetParam()); - ASSERT_OK(status); + ASSERT_THAT(status, IsOk()); ASSERT_TRUE(result.IsBool()); ASSERT_FALSE(result.BoolOrDie()); } @@ -125,251 +93,25 @@ TEST_P(LogicStepTest, TestErrorHandling) { CelValue result; CelError error = absl::CancelledError(); CelValue error_value = CelValue::CreateError(&error); - ASSERT_OK(EvaluateLogic(error_value, CelValue::CreateBool(true), - CelValue::CreateBool(false), &result, GetParam())); + ASSERT_THAT(EvaluateLogic(error_value, CelValue::CreateBool(true), + CelValue::CreateBool(false), &result, GetParam()), + IsOk()); ASSERT_TRUE(result.IsError()); - ASSERT_OK(EvaluateLogic(CelValue::CreateBool(true), error_value, - CelValue::CreateBool(false), &result, GetParam())); + ASSERT_THAT(EvaluateLogic(CelValue::CreateBool(true), error_value, + CelValue::CreateBool(false), &result, GetParam()), + IsOk()); ASSERT_TRUE(result.IsError()); - ASSERT_OK(EvaluateLogic(CelValue::CreateBool(false), error_value, - CelValue::CreateBool(false), &result, GetParam())); + ASSERT_THAT(EvaluateLogic(CelValue::CreateBool(false), error_value, + CelValue::CreateBool(false), &result, GetParam()), + IsOk()); ASSERT_TRUE(result.IsBool()); ASSERT_FALSE(result.BoolOrDie()); } -TEST_F(LogicStepTest, TestUnknownHandling) { - CelValue result; - UnknownSet unknown_set; - CelError cel_error = absl::CancelledError(); - CelValue unknown_value = CelValue::CreateUnknownSet(&unknown_set); - CelValue error_value = CelValue::CreateError(&cel_error); - ASSERT_OK(EvaluateLogic(unknown_value, CelValue::CreateBool(true), - CelValue::CreateBool(false), &result, true)); - ASSERT_TRUE(result.IsUnknownSet()); - - ASSERT_OK(EvaluateLogic(CelValue::CreateBool(true), unknown_value, - CelValue::CreateBool(false), &result, true)); - ASSERT_TRUE(result.IsUnknownSet()); - - ASSERT_OK(EvaluateLogic(CelValue::CreateBool(false), unknown_value, - CelValue::CreateBool(false), &result, true)); - ASSERT_TRUE(result.IsBool()); - ASSERT_FALSE(result.BoolOrDie()); - - ASSERT_OK(EvaluateLogic(error_value, unknown_value, - CelValue::CreateBool(false), &result, true)); - ASSERT_TRUE(result.IsError()); - - ASSERT_OK(EvaluateLogic(unknown_value, error_value, - CelValue::CreateBool(false), &result, true)); - ASSERT_TRUE(result.IsUnknownSet()); - - Expr expr0; - auto& ident_expr0 = expr0.mutable_ident_expr(); - ident_expr0.set_name("name0"); - - Expr expr1; - auto& ident_expr1 = expr1.mutable_ident_expr(); - ident_expr1.set_name("name1"); - - CelAttribute attr0(expr0.ident_expr().name(), {}), - attr1(expr1.ident_expr().name(), {}); - UnknownAttributeSet unknown_attr_set0({attr0}); - UnknownAttributeSet unknown_attr_set1({attr1}); - UnknownSet unknown_set0(unknown_attr_set0); - UnknownSet unknown_set1(unknown_attr_set1); - - EXPECT_THAT(unknown_attr_set0.size(), Eq(1)); - EXPECT_THAT(unknown_attr_set1.size(), Eq(1)); - - ASSERT_OK(EvaluateLogic(CelValue::CreateUnknownSet(&unknown_set0), - CelValue::CreateUnknownSet(&unknown_set1), - CelValue::CreateBool(false), &result, true)); - ASSERT_TRUE(result.IsUnknownSet()); - const auto& attrs = result.UnknownSetOrDie()->unknown_attributes(); - ASSERT_THAT(attrs, testing::SizeIs(1)); - EXPECT_THAT(attrs.begin()->variable_name(), Eq("name0")); -} - INSTANTIATE_TEST_SUITE_P(LogicStepTest, LogicStepTest, testing::Bool()); -class TernaryStepDirectTest : public testing::TestWithParam { - public: - TernaryStepDirectTest() - : type_provider_(cel::internal::GetTestingDescriptorPool()) {} - - bool Shortcircuiting() { return GetParam(); } - - protected: - Arena arena_; - cel::runtime_internal::RuntimeTypeProvider type_provider_; -}; - -TEST_P(TernaryStepDirectTest, ReturnLhs) { - cel::Activation activation; - RuntimeOptions opts; - ExecutionFrameBase frame(activation, opts, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectTernaryStep( - CreateConstValueDirectStep(BoolValue(true), -1), - CreateConstValueDirectStep(IntValue(1), -1), - CreateConstValueDirectStep(IntValue(2), -1), -1, Shortcircuiting()); - - cel::Value result; - AttributeTrail attr_unused; - - ASSERT_OK(step->Evaluate(frame, result, attr_unused)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_EQ(Cast(result).NativeValue(), 1); -} - -TEST_P(TernaryStepDirectTest, ReturnRhs) { - cel::Activation activation; - RuntimeOptions opts; - ExecutionFrameBase frame(activation, opts, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectTernaryStep( - CreateConstValueDirectStep(BoolValue(false), -1), - CreateConstValueDirectStep(IntValue(1), -1), - CreateConstValueDirectStep(IntValue(2), -1), -1, Shortcircuiting()); - - cel::Value result; - AttributeTrail attr_unused; - - ASSERT_OK(step->Evaluate(frame, result, attr_unused)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_EQ(Cast(result).NativeValue(), 2); -} - -TEST_P(TernaryStepDirectTest, ForwardError) { - cel::Activation activation; - RuntimeOptions opts; - ExecutionFrameBase frame(activation, opts, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - cel::Value error_value = cel::ErrorValue(absl::InternalError("test error")); - - std::unique_ptr step = CreateDirectTernaryStep( - CreateConstValueDirectStep(error_value, -1), - CreateConstValueDirectStep(IntValue(1), -1), - CreateConstValueDirectStep(IntValue(2), -1), -1, Shortcircuiting()); - - cel::Value result; - AttributeTrail attr_unused; - - ASSERT_OK(step->Evaluate(frame, result, attr_unused)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), - StatusIs(absl::StatusCode::kInternal, "test error")); -} - -TEST_P(TernaryStepDirectTest, ForwardUnknown) { - cel::Activation activation; - RuntimeOptions opts; - opts.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - ExecutionFrameBase frame(activation, opts, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::vector attrs{{cel::Attribute("var")}}; - - cel::UnknownValue unknown_value = - cel::UnknownValue(cel::Unknown(cel::AttributeSet(attrs))); - - std::unique_ptr step = CreateDirectTernaryStep( - CreateConstValueDirectStep(unknown_value, -1), - CreateConstValueDirectStep(IntValue(2), -1), - CreateConstValueDirectStep(IntValue(3), -1), -1, Shortcircuiting()); - - cel::Value result; - AttributeTrail attr_unused; - - ASSERT_OK(step->Evaluate(frame, result, attr_unused)); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue().unknown_attributes(), - ElementsAre(Truly([](const cel::Attribute& attr) { - return attr.variable_name() == "var"; - }))); -} - -TEST_P(TernaryStepDirectTest, UnexpectedCondtionKind) { - cel::Activation activation; - RuntimeOptions opts; - ExecutionFrameBase frame(activation, opts, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectTernaryStep( - CreateConstValueDirectStep(IntValue(-1), -1), - CreateConstValueDirectStep(IntValue(1), -1), - CreateConstValueDirectStep(IntValue(2), -1), -1, Shortcircuiting()); - - cel::Value result; - AttributeTrail attr_unused; - - ASSERT_OK(step->Evaluate(frame, result, attr_unused)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), - StatusIs(absl::StatusCode::kUnknown, - HasSubstr("No matching overloads found"))); -} - -TEST_P(TernaryStepDirectTest, Shortcircuiting) { - class RecordCallStep : public DirectExpressionStep { - public: - explicit RecordCallStep(bool& was_called) - : DirectExpressionStep(-1), was_called_(&was_called) {} - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& trail) const override { - *was_called_ = true; - result = IntValue(1); - return absl::OkStatus(); - } - - private: - bool* absl_nonnull was_called_; - }; - - bool lhs_was_called = false; - bool rhs_was_called = false; - - cel::Activation activation; - RuntimeOptions opts; - ExecutionFrameBase frame(activation, opts, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectTernaryStep( - CreateConstValueDirectStep(BoolValue(false), -1), - std::make_unique(lhs_was_called), - std::make_unique(rhs_was_called), -1, Shortcircuiting()); - - cel::Value result; - AttributeTrail attr_unused; - - ASSERT_OK(step->Evaluate(frame, result, attr_unused)); - - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), Eq(1)); - bool expect_eager_eval = !Shortcircuiting(); - EXPECT_EQ(lhs_was_called, expect_eager_eval); - EXPECT_TRUE(rhs_was_called); -} - -INSTANTIATE_TEST_SUITE_P(TernaryStepDirectTest, TernaryStepDirectTest, - testing::Bool()); - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/eval/trace_step.h b/eval/eval/trace_step.h deleted file mode 100644 index cf4240248..000000000 --- a/eval/eval/trace_step.h +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_TRACE_STEP_H_ -#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_TRACE_STEP_H_ - -#include -#include -#include - -#include "absl/status/status.h" -#include "absl/types/optional.h" -#include "common/native_type.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" -#include "internal/status_macros.h" -namespace google::api::expr::runtime { - -// A decorator that implements tracing for recursively evaluated CEL -// expressions. -// -// Allows inspection for extensions to extract the wrapped expression. -class TraceStep : public DirectExpressionStep { - public: - explicit TraceStep(std::unique_ptr expression) - : DirectExpressionStep(-1), expression_(std::move(expression)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& trail) const override { - CEL_RETURN_IF_ERROR(expression_->Evaluate(frame, result, trail)); - if (!frame.callback()) { - return absl::OkStatus(); - } - return frame.callback()(expression_->expr_id(), result, - frame.descriptor_pool(), frame.message_factory(), - frame.arena()); - } - - cel::NativeTypeId GetNativeTypeId() const override { - return cel::NativeTypeId::For(); - } - - absl::optional> GetDependencies() - const override { - return {{expression_.get()}}; - } - - absl::optional>> - ExtractDependencies() override { - std::vector> dependencies; - dependencies.push_back(std::move(expression_)); - return dependencies; - }; - - private: - std::unique_ptr expression_; -}; - -} // namespace google::api::expr::runtime - -#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_TRACE_STEP_H_ diff --git a/eval/internal/BUILD b/eval/internal/BUILD index d6f31493e..614ca48c4 100644 --- a/eval/internal/BUILD +++ b/eval/internal/BUILD @@ -56,11 +56,10 @@ cc_test( "//eval/public/structs:trivial_legacy_type_info", "//eval/testutil:test_message_cc_proto", "//internal:testing", - "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", "@com_google_absl//absl/types:span", - "@com_google_absl//absl/types:variant", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", "@com_google_protobuf//:any_cc_proto", "@com_google_protobuf//:protobuf", @@ -85,20 +84,17 @@ cc_library( hdrs = ["adapter_activation_impl.h"], deps = [ ":interop", - "//base:attributes", + "//base:attribute_matcher", "//common:value", "//eval/public:base_activation", "//eval/public:cel_value", "//internal:status_macros", "//runtime:activation_interface", "//runtime:function_overload_reference", - "//runtime/internal:activation_attribute_matcher_access", - "//runtime/internal:attribute_matcher", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", "@com_google_protobuf//:protobuf", ], ) diff --git a/eval/internal/adapter_activation_impl.cc b/eval/internal/adapter_activation_impl.cc index c88fe8145..a0fcea02d 100644 --- a/eval/internal/adapter_activation_impl.cc +++ b/eval/internal/adapter_activation_impl.cc @@ -25,8 +25,6 @@ #include "eval/public/cel_value.h" #include "internal/status_macros.h" #include "runtime/function_overload_reference.h" -#include "runtime/internal/activation_attribute_matcher_access.h" -#include "runtime/internal/attribute_matcher.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -68,20 +66,4 @@ AdapterActivationImpl::FindFunctionOverloads(absl::string_view name) const return result; } -absl::Span AdapterActivationImpl::GetUnknownAttributes() - const { - return legacy_activation_.unknown_attribute_patterns(); -} - -absl::Span AdapterActivationImpl::GetMissingAttributes() - const { - return legacy_activation_.missing_attribute_patterns(); -} - -const runtime_internal::AttributeMatcher* absl_nullable -AdapterActivationImpl::GetAttributeMatcher() const { - return runtime_internal::ActivationAttributeMatcherAccess:: - GetAttributeMatcher(legacy_activation_); -} - } // namespace cel::interop_internal diff --git a/eval/internal/adapter_activation_impl.h b/eval/internal/adapter_activation_impl.h index ebf3156aa..89014cfd3 100644 --- a/eval/internal/adapter_activation_impl.h +++ b/eval/internal/adapter_activation_impl.h @@ -20,13 +20,11 @@ #include "absl/base/nullability.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "absl/types/span.h" -#include "base/attribute.h" +#include "base/attribute_matcher.h" #include "common/value.h" #include "eval/public/base_activation.h" #include "runtime/activation_interface.h" #include "runtime/function_overload_reference.h" -#include "runtime/internal/attribute_matcher.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -52,14 +50,19 @@ class AdapterActivationImpl : public ActivationInterface { std::vector FindFunctionOverloads( absl::string_view name) const override; - absl::Span GetUnknownAttributes() const override; + const AttributeMatcher& GetUnknownAttributeMatcher() const override { + return legacy_activation_.GetUnknownAttributeMatcher(); + } - absl::Span GetMissingAttributes() const override; + const AttributeMatcher& GetKnownAttributeMatcher() const override { + return legacy_activation_.GetKnownAttributeMatcher(); + } - private: - const runtime_internal::AttributeMatcher* absl_nullable GetAttributeMatcher() - const override; + const AttributeMatcher& GetMissingAttributeMatcher() const override { + return legacy_activation_.GetMissingAttributeMatcher(); + } + private: const google::api::expr::runtime::BaseActivation& legacy_activation_; }; diff --git a/eval/internal/cel_value_equal.cc b/eval/internal/cel_value_equal.cc index f61f93ca4..b9f1c9cc7 100644 --- a/eval/internal/cel_value_equal.cc +++ b/eval/internal/cel_value_equal.cc @@ -101,7 +101,7 @@ absl::optional MapEqual(const CelMap* t1, const CelMap* t2) { google::protobuf::Arena arena; auto list_keys = t1->ListKeys(&arena); if (!list_keys.ok()) { - return absl::nullopt; + return std::nullopt; } const CelList* keys = *list_keys; for (int i = 0; i < keys->size(); i++) { @@ -166,7 +166,7 @@ template absl::optional HomogenousCelValueEqual(const CelValue& t1, const CelValue& t2) { if (t1.type() != t2.type()) { - return absl::nullopt; + return std::nullopt; } switch (t1.type()) { case Kind::kNullType: @@ -198,7 +198,7 @@ absl::optional HomogenousCelValueEqual(const CelValue& t1, default: break; } - return absl::nullopt; + return std::nullopt; } absl::optional HeterogeneousEqualProvider::operator()( @@ -233,7 +233,7 @@ absl::optional CelValueEqualImpl(const CelValue& v1, const CelValue& v2) { // map containing an Error. Return no matching overload to propagate an error // instead of a false result. if (v1.IsError() || v1.IsUnknownSet() || v2.IsError() || v2.IsUnknownSet()) { - return absl::nullopt; + return std::nullopt; } return false; diff --git a/eval/internal/cel_value_equal_test.cc b/eval/internal/cel_value_equal_test.cc index f52f38916..13787e973 100644 --- a/eval/internal/cel_value_equal_test.cc +++ b/eval/internal/cel_value_equal_test.cc @@ -18,20 +18,21 @@ #include #include #include +#include #include #include #include +#include #include #include "google/protobuf/any.pb.h" #include "google/rpc/context/attribute_context.pb.h" #include "google/protobuf/descriptor.pb.h" -#include "absl/status/statusor.h" +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" #include "absl/types/span.h" -#include "absl/types/variant.h" #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_list_impl.h" #include "eval/public/containers/container_backed_map_impl.h" @@ -67,7 +68,7 @@ using ::testing::ValuesIn; struct EqualityTestCase { enum class ErrorKind { kMissingOverload, kMissingIdentifier }; absl::string_view expr; - absl::variant result; + std::variant result; CelValue lhs = CelValue::CreateNull(); CelValue rhs = CelValue::CreateNull(); }; @@ -131,6 +132,8 @@ const std::vector& ValueExamples1() { result->push_back(CelValue::CreateMap(&CelMapExample1())); result->push_back(CelValue::CreateCelTypeView("type")); + ABSL_CHECK_EQ(arena.SpaceUsed(), 0) << "Arena should not be used."; + return result.release(); }(); return *examples; @@ -185,7 +188,7 @@ std::string CelValueEqualTestName( } TEST_P(CelValueEqualImplTypesTest, Basic) { - absl::optional result = CelValueEqualImpl(lhs(), rhs()); + std::optional result = CelValueEqualImpl(lhs(), rhs()); if (lhs().IsNull() || rhs().IsNull()) { if (lhs().IsNull() && rhs().IsNull()) { @@ -267,7 +270,7 @@ const std::vector& NumericValuesNotEqualExample() { using NumericInequalityTest = testing::TestWithParam; TEST_P(NumericInequalityTest, NumericValues) { NumericInequalityTestCase test_case = GetParam(); - absl::optional result = CelValueEqualImpl(test_case.a, test_case.b); + std::optional result = CelValueEqualImpl(test_case.a, test_case.b); EXPECT_TRUE(result.has_value()); EXPECT_EQ(*result, false); } @@ -280,7 +283,7 @@ INSTANTIATE_TEST_SUITE_P( }); TEST(CelValueEqualImplTest, LossyNumericEquality) { - absl::optional result = CelValueEqualImpl( + std::optional result = CelValueEqualImpl( CelValue::CreateDouble( static_cast(std::numeric_limits::max()) - 1), CelValue::CreateInt64(std::numeric_limits::max())); diff --git a/eval/public/BUILD b/eval/public/BUILD index 31ad2d480..d3340c893 100644 --- a/eval/public/BUILD +++ b/eval/public/BUILD @@ -29,6 +29,13 @@ licenses(["notice"]) exports_files(["LICENSE"]) +# No new users should depend on MessageWrapper using legacy APIs. +# This was added as a temporary workaround for implementing custom reflection operations +# on messages that were compiled with the LITE_RUNTIME (MessageLite) option. +# Internally, we now assume legacy Message types are always normal (Message) protobuf messages with +# standard reflection operations. +# Custom structs (including MessageLite protos) should use the modern API (see +# `cel::CustomStructValue`). cc_library( name = "message_wrapper", hdrs = [ @@ -51,7 +58,6 @@ cc_test( ":message_wrapper", "//eval/public/structs:trivial_legacy_type_info", "//eval/testutil:test_message_cc_proto", - "//internal:casts", "//internal:testing", "@com_google_protobuf//:protobuf", ], @@ -64,9 +70,7 @@ cc_library( ], deps = [ ":message_wrapper", - "//internal:casts", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/numeric:bits", "@com_google_absl//absl/types:variant", "@com_google_protobuf//:protobuf", ], @@ -89,7 +93,6 @@ cc_library( "//common:native_type", "//eval/internal:errors", "//eval/public/structs:legacy_type_info_apis", - "//extensions/protobuf:memory_manager", "//internal:casts", "//internal:status_macros", "//internal:utf8", @@ -107,6 +110,28 @@ cc_library( ], ) +cc_library( + name = "cel_value_clone", + srcs = ["cel_value_clone.cc"], + hdrs = ["cel_value_clone.h"], + deps = [ + ":cel_value", + ":message_wrapper", + ":unknown_set", + "//eval/public/containers:container_backed_list_impl", + "//eval/public/containers:container_backed_map_impl", + "//internal:status_macros", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:cord", + "@com_google_absl//absl/time", + "@com_google_absl//absl/types:optional", + "@com_google_protobuf//:protobuf", + ], +) + cc_library( name = "cel_attribute", srcs = [ @@ -156,12 +181,15 @@ cc_library( ":cel_function", ":cel_value", ":cel_value_producer", - "//runtime/internal:attribute_matcher", - "@com_google_absl//absl/base:nullability", + "//base:attribute_matcher", + "//internal:status_macros", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", "@com_google_protobuf//:protobuf", ], ) @@ -494,11 +522,12 @@ cc_library( ":cel_function_registry", ":cel_type_registry", ":cel_value", - "//common:legacy_value", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", + "@com_google_protobuf//:protobuf", ], ) @@ -698,14 +727,14 @@ cc_test( ], deps = [ ":activation", - ":cel_attribute", ":cel_function", - "//eval/eval:attribute_trail", - "//eval/eval:ident_step", - "//extensions/protobuf:memory_manager", - "//internal:status_macros", + ":cel_value", + ":cel_value_producer", "//internal:testing", - "//parser", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + "@com_google_protobuf//:protobuf", ], ) @@ -813,6 +842,7 @@ cc_library( "//eval/public/structs:legacy_type_provider", "//eval/public/structs:protobuf_descriptor_type_provider", "//runtime:type_registry", + "@com_google_absl//absl/base", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -1027,6 +1057,7 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", "@com_google_cel_spec//proto/cel/expr:value_cc_proto", "@com_google_protobuf//:any_cc_proto", "@com_google_protobuf//:differencer", @@ -1049,8 +1080,8 @@ cc_library( ":cel_attribute", ":cel_function", ":cel_value", - "//runtime/internal:attribute_matcher", - "@com_google_absl//absl/base:nullability", + "//base:attribute_matcher", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/strings", "@com_google_protobuf//:field_mask_cc_proto", ], diff --git a/eval/public/activation.h b/eval/public/activation.h index 6f2bb59c1..183580bf3 100644 --- a/eval/public/activation.h +++ b/eval/public/activation.h @@ -6,16 +6,20 @@ #include #include -#include "absl/base/nullability.h" +#include "absl/base/attributes.h" #include "absl/container/flat_hash_map.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "absl/types/span.h" +#include "base/attribute_matcher.h" #include "eval/public/base_activation.h" #include "eval/public/cel_attribute.h" #include "eval/public/cel_function.h" #include "eval/public/cel_value.h" #include "eval/public/cel_value_producer.h" -#include "runtime/internal/attribute_matcher.h" +#include "internal/status_macros.h" #include "google/protobuf/arena.h" namespace cel::runtime_internal { @@ -77,28 +81,66 @@ class Activation : public BaseActivation { // // If a field access is found to match any of the provided patterns, the // result is treated as a missing attribute error. - void set_missing_attribute_patterns( - std::vector missing_attribute_patterns) { - missing_attribute_patterns_ = std::move(missing_attribute_patterns); + absl::Status SetMissingAttributePatterns( + absl::Span patterns) { + cel::AttributeMatcher matcher; + for (const auto& pattern : patterns) { + // This function used to return void, thus they could not fail. We use + // upsert because it just works most of the time, especially if the caller + // provided overlapped attribute patterns. Ideally we would use insert. + CEL_RETURN_IF_ERROR(matcher.UpsertAttribute(pattern)); + } + missing_attributes_ = std::move(matcher); + return absl::OkStatus(); } - const std::vector& missing_attribute_patterns() - const override { - return missing_attribute_patterns_; + ABSL_DEPRECATED("Use SetUnknownAttributePatterns") + void set_unknown_attribute_patterns( + absl::Span patterns) { + unknown_attributes_.ClearAttributes(); + absl::Status status = SetUnknownAttributePatterns(patterns); + ABSL_DCHECK_OK(status); + status.IgnoreError(); } // Sets the collection of attribute patterns that will be recognized as // "unknown" values during expression evaluation. - void set_unknown_attribute_patterns( - std::vector unknown_attribute_patterns) { - unknown_attribute_patterns_ = std::move(unknown_attribute_patterns); + absl::Status SetUnknownAttributePatterns( + absl::Span patterns) { + cel::AttributeMatcher matcher; + for (const auto& pattern : patterns) { + // This function used to return void, thus they could not fail. We use + // upsert because it just works most of the time, especially if the caller + // provided overlapped attribute patterns. Ideally we would use insert. + CEL_RETURN_IF_ERROR(matcher.UpsertAttribute(pattern)); + } + unknown_attributes_ = std::move(matcher); + return absl::OkStatus(); + } + + absl::Status SetKnownAttributePatterns( + absl::Span patterns) { + cel::AttributeMatcher matcher; + for (const auto& pattern : patterns) { + // This function used to return void, thus they could not fail. We use + // upsert because it just works most of the time, especially if the caller + // provided overlapped attribute patterns. Ideally we would use insert. + CEL_RETURN_IF_ERROR(matcher.UpsertAttribute(pattern)); + } + known_attributes_ = std::move(matcher); + return absl::OkStatus(); + } + + const cel::AttributeMatcher& GetUnknownAttributeMatcher() const final { + return unknown_attributes_; } - // Return the collection of attribute patterns that determine "unknown" - // values. - const std::vector& unknown_attribute_patterns() - const override { - return unknown_attribute_patterns_; + const cel::AttributeMatcher& GetKnownAttributeMatcher() const final { + return known_attributes_; + } + + const cel::AttributeMatcher& GetMissingAttributeMatcher() const final { + return missing_attributes_; } private: @@ -137,32 +179,13 @@ class Activation : public BaseActivation { friend class cel::runtime_internal::ActivationAttributeMatcherAccess; - void SetAttributeMatcher( - const cel::runtime_internal::AttributeMatcher* matcher) { - attribute_matcher_ = matcher; - } - - void SetAttributeMatcher( - std::unique_ptr matcher) { - owned_attribute_matcher_ = std::move(matcher); - attribute_matcher_ = owned_attribute_matcher_.get(); - } - - const cel::runtime_internal::AttributeMatcher* absl_nullable - GetAttributeMatcher() const override { - return attribute_matcher_; - } - absl::flat_hash_map value_map_; absl::flat_hash_map>> function_map_; - std::vector missing_attribute_patterns_; - std::vector unknown_attribute_patterns_; - - const cel::runtime_internal::AttributeMatcher* attribute_matcher_ = nullptr; - std::unique_ptr - owned_attribute_matcher_; + cel::AttributeMatcher unknown_attributes_; + cel::AttributeMatcher known_attributes_; + cel::AttributeMatcher missing_attributes_; }; } // namespace google::api::expr::runtime diff --git a/eval/public/activation_test.cc b/eval/public/activation_test.cc index 238caf45e..d628d5fa8 100644 --- a/eval/public/activation_test.cc +++ b/eval/public/activation_test.cc @@ -4,14 +4,14 @@ #include #include -#include "eval/eval/attribute_trail.h" -#include "eval/eval/ident_step.h" -#include "eval/public/cel_attribute.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" #include "eval/public/cel_function.h" -#include "extensions/protobuf/memory_manager.h" -#include "internal/status_macros.h" +#include "eval/public/cel_value.h" +#include "eval/public/cel_value_producer.h" #include "internal/testing.h" -#include "parser/parser.h" +#include "google/protobuf/arena.h" namespace google { namespace api { @@ -21,8 +21,6 @@ namespace runtime { namespace { using ::absl_testing::StatusIs; -using ::cel::extensions::ProtoMemoryManager; -using ::cel::expr::Expr; using ::google::protobuf::Arena; using ::testing::ElementsAre; using ::testing::Eq; @@ -204,34 +202,6 @@ TEST(ActivationTest, CheckValueProducerClear) { EXPECT_EQ(1, activation.ClearCachedValues()); } -TEST(ActivationTest, ErrorPathTest) { - Activation activation; - - Expr expr; - auto* select_expr = expr.mutable_select_expr(); - select_expr->set_field("ip"); - - Expr* ident_expr = select_expr->mutable_operand(); - ident_expr->mutable_ident_expr()->set_name("destination"); - - const CelAttributePattern destination_ip_pattern( - "destination", - {CreateCelAttributeQualifierPattern(CelValue::CreateStringView("ip"))}); - - AttributeTrail trail("destination"); - trail = - trail.Step(CreateCelAttributeQualifier(CelValue::CreateStringView("ip"))); - - ASSERT_EQ(destination_ip_pattern.IsMatch(trail.attribute()), - CelAttributePattern::MatchType::FULL); - EXPECT_TRUE(activation.missing_attribute_patterns().empty()); - - activation.set_missing_attribute_patterns({destination_ip_pattern}); - EXPECT_EQ( - activation.missing_attribute_patterns()[0].IsMatch(trail.attribute()), - CelAttributePattern::MatchType::FULL); -} - } // namespace } // namespace runtime diff --git a/eval/public/ast_rewrite.cc b/eval/public/ast_rewrite.cc index 3c210e607..86f5ee4d1 100644 --- a/eval/public/ast_rewrite.cc +++ b/eval/public/ast_rewrite.cc @@ -68,11 +68,11 @@ struct ExprRecord { }; using StackRecordKind = - absl::variant; + std::variant; struct StackRecord { public: - ABSL_ATTRIBUTE_UNUSED static constexpr int kNotCallArg = -1; + [[maybe_unused]] static constexpr int kNotCallArg = -1; static constexpr int kTarget = -2; StackRecord(Expr* e, const SourceInfo* info) { diff --git a/eval/public/ast_rewrite_test.cc b/eval/public/ast_rewrite_test.cc index b2ee8d13c..c7297b5b1 100644 --- a/eval/public/ast_rewrite_test.cc +++ b/eval/public/ast_rewrite_test.cc @@ -501,7 +501,7 @@ TEST(AstCrawlerTest, CheckExprRewriteHandlers) { // Simple rewrite that replaces a select path with a dot-qualified identifier. class RewriterExample : public AstRewriterBase { public: - RewriterExample() {} + RewriterExample() = default; bool PostVisitRewrite(Expr* expr, const SourcePosition* info) override { if (target_.has_value() && expr->id() == *target_) { expr->mutable_ident_expr()->set_name("com.google.Identifier"); @@ -552,7 +552,7 @@ TEST(AstRewrite, SelectRewriteExample) { // both passes. class PreRewriterExample : public AstRewriterBase { public: - PreRewriterExample() {} + PreRewriterExample() = default; bool PreVisitRewrite(Expr* expr, const SourcePosition* info) override { if (expr->ident_expr().name() == "x") { expr->mutable_ident_expr()->set_name("y"); diff --git a/eval/public/ast_traverse.cc b/eval/public/ast_traverse.cc index a86923c67..46639da71 100644 --- a/eval/public/ast_traverse.cc +++ b/eval/public/ast_traverse.cc @@ -14,6 +14,7 @@ #include "eval/public/ast_traverse.h" +#include #include #include "cel/expr/syntax.pb.h" @@ -67,11 +68,11 @@ struct ExprRecord { }; using StackRecordKind = - absl::variant; + std::variant; struct StackRecord { public: - ABSL_ATTRIBUTE_UNUSED static constexpr int kNotCallArg = -1; + [[maybe_unused]] static constexpr int kNotCallArg = -1; static constexpr int kTarget = -2; StackRecord(const Expr* e, const SourceInfo* info) { @@ -196,6 +197,8 @@ struct PostVisitor { visitor->PostVisitComprehension(&expr->comprehension_expr(), expr, &position); break; + case Expr::EXPR_KIND_NOT_SET: + break; default: ABSL_LOG(ERROR) << "Unsupported Expr kind: " << expr->expr_kind_case(); } @@ -344,6 +347,47 @@ void PushDependencies(const StackRecord& record, std::stack& stack, } // namespace +namespace internal { +struct AstTraversalState { + std::stack stack; +}; +} // namespace internal + +AstTraversal AstTraversal::Create(const Expr* expr, + const SourceInfo* source_info, + TraversalOptions options) { + AstTraversal instance(options); + instance.state_ = std::make_unique(); + instance.state_->stack.push(StackRecord(expr, source_info)); + return instance; +} + +AstTraversal::AstTraversal(TraversalOptions options) : options_(options) {} + +AstTraversal::~AstTraversal() = default; + +bool AstTraversal::Step(AstVisitor* visitor) { + if (IsDone()) { + return false; + } + auto& stack = state_->stack; + StackRecord& record = stack.top(); + if (!record.visited) { + PreVisit(record, visitor); + PushDependencies(record, stack, options_); + record.visited = true; + } else { + PostVisit(record, visitor); + stack.pop(); + } + + return !stack.empty(); +} + +bool AstTraversal::IsDone() { + return state_ == nullptr || state_->stack.empty(); +} + void AstTraverse(const Expr* expr, const SourceInfo* source_info, AstVisitor* visitor, TraversalOptions options) { std::stack stack; diff --git a/eval/public/ast_traverse.h b/eval/public/ast_traverse.h index f81c6f47a..a330bd28d 100644 --- a/eval/public/ast_traverse.h +++ b/eval/public/ast_traverse.h @@ -17,17 +17,60 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_AST_TRAVERSE_H_ #define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_AST_TRAVERSE_H_ +#include + #include "cel/expr/syntax.pb.h" #include "eval/public/ast_visitor.h" namespace google::api::expr::runtime { +namespace internal { +struct AstTraversalState; +} // namespace internal + struct TraversalOptions { bool use_comprehension_callbacks; TraversalOptions() : use_comprehension_callbacks(false) {} }; +// Helper class for managing the traversal of the AST. +// Allows caller to step through the traversal. +// +// Usage: +// +// AstTraversal traversal = AstTraversal::Create(expr, source_info); +// +// MyVisitor visitor(); +// while (!traversal.IsDone()) { +// traversal.Step(&visitor); +// } +class AstTraversal { + public: + static AstTraversal Create(const cel::expr::Expr* expr, + const cel::expr::SourceInfo* source_info, + TraversalOptions options = TraversalOptions()); + + ~AstTraversal(); + + AstTraversal(const AstTraversal&) = delete; + AstTraversal& operator=(const AstTraversal&) = delete; + AstTraversal(AstTraversal&&) = default; + AstTraversal& operator=(AstTraversal&&) = default; + + // Advances the traversal. Returns true if there is more work to do. This is a + // no-op if the traversal is done and IsDone() is true. + bool Step(AstVisitor* visitor); + + // Returns true if there is no work left to do. + bool IsDone(); + + private: + explicit AstTraversal(TraversalOptions options); + TraversalOptions options_; + std::unique_ptr state_; +}; + // Traverses the AST representation in an expr proto. // // expr: root node of the tree. diff --git a/eval/public/ast_traverse_test.cc b/eval/public/ast_traverse_test.cc index ca6d81b72..dfc4a7700 100644 --- a/eval/public/ast_traverse_test.cc +++ b/eval/public/ast_traverse_test.cc @@ -465,6 +465,51 @@ TEST(AstCrawlerTest, CheckExprHandlers) { AstTraverse(&expr, &source_info, &handler); } +TEST(AstTraversal, Interrupt) { + SourceInfo source_info; + MockAstVisitor handler; + + Expr expr; + auto* select_expr = expr.mutable_select_expr(); + auto* operand = select_expr->mutable_operand(); + auto* ident_expr = operand->mutable_ident_expr(); + + testing::InSequence seq; + + auto traversal = AstTraversal::Create(&expr, &source_info); + + EXPECT_CALL(handler, PreVisitExpr(_, _)).Times(2); + + EXPECT_CALL(handler, PostVisitIdent(ident_expr, operand, _)).Times(1); + EXPECT_CALL(handler, PostVisitSelect(select_expr, &expr, _)).Times(0); + + EXPECT_TRUE(traversal.Step(&handler)); + EXPECT_TRUE(traversal.Step(&handler)); + EXPECT_TRUE(traversal.Step(&handler)); + + EXPECT_FALSE(traversal.IsDone()); +} + +TEST(AstTraversal, NoInterrupt) { + SourceInfo source_info; + MockAstVisitor handler; + + Expr expr; + auto* select_expr = expr.mutable_select_expr(); + auto* operand = select_expr->mutable_operand(); + auto* ident_expr = operand->mutable_ident_expr(); + + testing::InSequence seq; + + auto traversal = AstTraversal::Create(&expr, &source_info); + + EXPECT_CALL(handler, PostVisitIdent(ident_expr, operand, _)).Times(1); + EXPECT_CALL(handler, PostVisitSelect(select_expr, &expr, _)).Times(1); + + while (traversal.Step(&handler)) continue; + EXPECT_TRUE(traversal.IsDone()); +} + } // namespace } // namespace google::api::expr::runtime diff --git a/eval/public/base_activation.h b/eval/public/base_activation.h index 7d9e0a51c..7a0c6b208 100644 --- a/eval/public/base_activation.h +++ b/eval/public/base_activation.h @@ -4,12 +4,12 @@ #include #include "google/protobuf/field_mask.pb.h" -#include "absl/base/nullability.h" +#include "absl/base/macros.h" #include "absl/strings/string_view.h" +#include "base/attribute_matcher.h" #include "eval/public/cel_attribute.h" #include "eval/public/cel_function.h" #include "eval/public/cel_value.h" -#include "runtime/internal/attribute_matcher.h" namespace cel::runtime_internal { class ActivationAttributeMatcherAccess; @@ -42,32 +42,31 @@ class BaseActivation { // Return the collection of attribute patterns that determine missing // attributes. - virtual const std::vector& missing_attribute_patterns() - const { - static const std::vector* empty = - new std::vector({}); - return *empty; + ABSL_DEPRECATE_AND_INLINE() + std::vector missing_attribute_patterns() const { + return GetMissingAttributeMatcher().GetAttributes(); } // Return the collection of attribute patterns that determine "unknown" // values. - virtual const std::vector& unknown_attribute_patterns() - const { - static const std::vector* empty = - new std::vector({}); - return *empty; + ABSL_DEPRECATE_AND_INLINE() + std::vector unknown_attribute_patterns() const { + return GetUnknownAttributeMatcher().GetAttributes(); } - virtual ~BaseActivation() = default; + virtual const cel::AttributeMatcher& GetUnknownAttributeMatcher() const { + return cel::EmptyAttributeMatcher(); + } - private: - friend class cel::runtime_internal::ActivationAttributeMatcherAccess; + virtual const cel::AttributeMatcher& GetKnownAttributeMatcher() const { + return cel::EmptyAttributeMatcher(); + } - // Internal getter for overriding the attribute matching behavior. - virtual const cel::runtime_internal::AttributeMatcher* absl_nullable - GetAttributeMatcher() const { - return nullptr; + virtual const cel::AttributeMatcher& GetMissingAttributeMatcher() const { + return cel::EmptyAttributeMatcher(); } + + virtual ~BaseActivation() = default; }; } // namespace google::api::expr::runtime diff --git a/eval/public/builtin_func_test.cc b/eval/public/builtin_func_test.cc index b73a2dc55..6355bd033 100644 --- a/eval/public/builtin_func_test.cc +++ b/eval/public/builtin_func_test.cc @@ -55,7 +55,7 @@ using ::testing::Eq; class BuiltinsTest : public ::testing::Test { protected: - BuiltinsTest() {} + BuiltinsTest() = default; // Helper method. Looks up in registry and tests comparison operation. void PerformRun(absl::string_view operation, absl::optional target, @@ -1123,7 +1123,7 @@ class FakeList : public CelList { class FakeErrorMap : public CelMap { public: - FakeErrorMap() {} + FakeErrorMap() = default; int size() const override { return 0; } @@ -1132,7 +1132,7 @@ class FakeErrorMap : public CelMap { } absl::optional operator[](CelValue key) const override { - return absl::nullopt; + return std::nullopt; } absl::StatusOr ListKeys() const override { @@ -1160,11 +1160,11 @@ class FakeMap : public CelMap { absl::optional operator[](CelValue key) const override { absl::optional raw_value = get_cel_value_(key); if (!raw_value) { - return absl::nullopt; + return std::nullopt; } auto it = data_.find(*raw_value); if (it == data_.end()) { - return absl::nullopt; + return std::nullopt; } return it->second; } @@ -1185,7 +1185,7 @@ class FakeBoolMap : public FakeMap { : FakeMap(data, CelValue::CreateBool, [](CelValue v) -> absl::optional { if (!v.IsBool()) { - return absl::nullopt; + return std::nullopt; } return v.BoolOrDie(); }) {} @@ -1197,7 +1197,7 @@ class FakeInt64Map : public FakeMap { : FakeMap(data, CelValue::CreateInt64, [](CelValue v) -> absl::optional { if (!v.IsInt64()) { - return absl::nullopt; + return std::nullopt; } return v.Int64OrDie(); }) {} @@ -1209,7 +1209,7 @@ class FakeUint64Map : public FakeMap { : FakeMap(data, CelValue::CreateUint64, [](CelValue v) -> absl::optional { if (!v.IsUint64()) { - return absl::nullopt; + return std::nullopt; } return v.Uint64OrDie(); }) {} @@ -1223,7 +1223,7 @@ class FakeStringMap : public FakeMap { [](CelValue::StringHolder v) { return CelValue::CreateString(v); }, [](CelValue v) -> absl::optional { if (!v.IsString()) { - return absl::nullopt; + return std::nullopt; } return v.StringOrDie(); }) {} diff --git a/eval/public/cel_attribute.cc b/eval/public/cel_attribute.cc index 015289bed..70525a04d 100644 --- a/eval/public/cel_attribute.cc +++ b/eval/public/cel_attribute.cc @@ -76,8 +76,8 @@ CelAttributeQualifier CreateCelAttributeQualifier(const CelValue& value) { CelAttributePattern CreateCelAttributePattern( absl::string_view variable, - std::initializer_list> + std::initializer_list> path_spec) { std::vector path; path.reserve(path_spec.size()); diff --git a/eval/public/cel_attribute_test.cc b/eval/public/cel_attribute_test.cc index b72189332..a2d08d819 100644 --- a/eval/public/cel_attribute_test.cc +++ b/eval/public/cel_attribute_test.cc @@ -17,7 +17,6 @@ namespace { using cel::expr::Expr; -using ::absl_testing::IsOkAndHolds; using ::absl_testing::StatusIs; using ::testing::Eq; using ::testing::IsEmpty; @@ -52,7 +51,7 @@ TEST(CelAttributeQualifierTest, TestBoolAccess) { EXPECT_FALSE(qualifier.GetUint64Key().has_value()); EXPECT_TRUE(qualifier.GetBoolKey().has_value()); EXPECT_THAT(qualifier.GetBoolKey().value(), Eq(true)); - EXPECT_THAT(qualifier.AsString(), IsOkAndHolds("true")); + EXPECT_THAT(qualifier.ToString(), "true"); } TEST(CelAttributeQualifierTest, TestInt64Access) { @@ -64,7 +63,7 @@ TEST(CelAttributeQualifierTest, TestInt64Access) { EXPECT_TRUE(qualifier.GetInt64Key().has_value()); EXPECT_THAT(qualifier.GetInt64Key().value(), Eq(-1)); - EXPECT_THAT(qualifier.AsString(), IsOkAndHolds("-1")); + EXPECT_THAT(qualifier.ToString(), "-1"); } TEST(CelAttributeQualifierTest, TestUint64Access) { @@ -76,7 +75,7 @@ TEST(CelAttributeQualifierTest, TestUint64Access) { EXPECT_TRUE(qualifier.GetUint64Key().has_value()); EXPECT_THAT(qualifier.GetUint64Key().value(), Eq(1UL)); - EXPECT_THAT(qualifier.AsString(), IsOkAndHolds("1")); + EXPECT_THAT(qualifier.ToString(), "1u"); } TEST(CelAttributeQualifierTest, TestStringAccess) { @@ -89,7 +88,7 @@ TEST(CelAttributeQualifierTest, TestStringAccess) { EXPECT_TRUE(qualifier.GetStringKey().has_value()); EXPECT_THAT(qualifier.GetStringKey().value(), Eq("test")); - EXPECT_THAT(qualifier.AsString(), IsOkAndHolds("test")); + EXPECT_THAT(qualifier.ToString(), "test"); } void TestAllInequalities(const CelAttributeQualifier& qualifier) { diff --git a/eval/public/cel_expr_builder_factory.cc b/eval/public/cel_expr_builder_factory.cc index a56c450b0..14f2b4111 100644 --- a/eval/public/cel_expr_builder_factory.cc +++ b/eval/public/cel_expr_builder_factory.cc @@ -122,19 +122,23 @@ std::unique_ptr CreateCelExpressionBuilder( // Add overloads for select optimization signature. // These are never bound, only used to prevent the builder from failing on // the overloads check. - absl::Status status = - builder->GetRegistry()->RegisterLazyFunction(CelFunctionDescriptor( - kCelAttribute, false, {cel::Kind::kAny, cel::Kind::kList})); - if (!status.ok()) { - ABSL_LOG(ERROR) << "Failed to register " << kCelAttribute << ": " - << status; - } - status = builder->GetRegistry()->RegisterLazyFunction(CelFunctionDescriptor( - kCelHasField, false, {cel::Kind::kAny, cel::Kind::kList})); - if (!status.ok()) { - ABSL_LOG(ERROR) << "Failed to register " << kCelHasField << ": " - << status; + for (const auto& desc : { + CelFunctionDescriptor(kCelAttribute, false, + {cel::Kind::kAny, cel::Kind::kList}), + CelFunctionDescriptor( + kCelAttribute, false, + {cel::Kind::kAny, cel::Kind::kList, cel::Kind::kAny}), + CelFunctionDescriptor(kCelHasField, false, + {cel::Kind::kAny, cel::Kind::kList}), + }) { + if (absl::Status status = + builder->GetRegistry()->RegisterLazyFunction(desc); + !status.ok()) { + ABSL_LOG(ERROR) << "Failed to register " << desc.name() << ": " + << status; + } } + flat_expr_builder.enable_select_optimization(); // Add runtime implementation. flat_expr_builder.AddProgramOptimizer( CreateSelectOptimizationProgramOptimizer()); diff --git a/eval/public/cel_expression.h b/eval/public/cel_expression.h index 4cf029e89..3f215207a 100644 --- a/eval/public/cel_expression.h +++ b/eval/public/cel_expression.h @@ -5,15 +5,18 @@ #include #include #include +#include #include "cel/expr/checked.pb.h" #include "cel/expr/syntax.pb.h" +#include "absl/base/attributes.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "eval/public/base_activation.h" #include "eval/public/cel_function_registry.h" #include "eval/public/cel_type_registry.h" #include "eval/public/cel_value.h" +#include "google/protobuf/arena.h" namespace google::api::expr::runtime { @@ -24,6 +27,13 @@ namespace google::api::expr::runtime { // then the order of the callback invocations is guaranteed to correspond // the order of variable sub-elements (e.g. the order of elements returned // by Comprehension.iter_range). +// +// Expression IDs outside of the range [0, INT32_MAX] are not supported and +// will not invoke the listener. While the AST allows any int64, supported +// parser implementations should use a dense range starting at 1. In practice, +// no AST should contain more than ~ 1e9 nodes. +// +// ID 0 should not be considered valid, but is supported for legacy reasons. using CelEvaluationListener = std::function; @@ -39,34 +49,59 @@ class CelExpression { virtual ~CelExpression() = default; // Initializes the state + ABSL_DEPRECATED("Use CreateState and pass arena on each evaluation") virtual std::unique_ptr InitializeState( google::protobuf::Arena* arena) const = 0; + // Initializes the state + virtual std::unique_ptr CreateState() const = 0; + // Evaluates expression and returns value. // activation contains bindings from parameter names to values // arena parameter specifies Arena object where output result and // internal data will be allocated. virtual absl::StatusOr Evaluate(const BaseActivation& activation, - google::protobuf::Arena* arena) const = 0; + google::protobuf::Arena* arena) const { + return Evaluate(activation, arena, nullptr); + } // Evaluates expression and returns value. // activation contains bindings from parameter names to values // state must be non-null and created prior to calling Evaluate by // InitializeState. - virtual absl::StatusOr Evaluate( - const BaseActivation& activation, CelEvaluationState* state) const = 0; + ABSL_DEPRECATED( + "Use CreateState and the overloads which take both an arena and state") + virtual absl::StatusOr Evaluate(const BaseActivation& activation, + CelEvaluationState* state) const { + return Evaluate(activation, nullptr, state); + } + virtual absl::StatusOr Evaluate(const BaseActivation& activation, + google::protobuf::Arena* arena, + CelEvaluationState* state) const { + return Trace(activation, arena, nullptr, state); + } // Trace evaluates expression calling the callback on each sub-tree. - virtual absl::StatusOr Trace( - const BaseActivation& activation, google::protobuf::Arena* arena, - CelEvaluationListener callback) const = 0; + virtual absl::StatusOr Trace(const BaseActivation& activation, + google::protobuf::Arena* arena, + CelEvaluationListener callback) const { + return Trace(activation, arena, std::move(callback), nullptr); + } // Trace evaluates expression calling the callback on each sub-tree. // state must be non-null and created prior to calling Evaluate by // InitializeState. - virtual absl::StatusOr Trace( - const BaseActivation& activation, CelEvaluationState* state, - CelEvaluationListener callback) const = 0; + ABSL_DEPRECATED( + "Use CreateState and the overloads which take both an arena and state") + virtual absl::StatusOr Trace(const BaseActivation& activation, + CelEvaluationState* state, + CelEvaluationListener callback) const { + return Trace(activation, nullptr, std::move(callback), state); + } + virtual absl::StatusOr Trace(const BaseActivation& activation, + google::protobuf::Arena* arena, + CelEvaluationListener callback, + CelEvaluationState* state) const = 0; }; // Base class for Expression Builder implementations diff --git a/eval/public/cel_function_adapter_impl.h b/eval/public/cel_function_adapter_impl.h index 6cd661c10..d4ecd54bd 100644 --- a/eval/public/cel_function_adapter_impl.h +++ b/eval/public/cel_function_adapter_impl.h @@ -346,9 +346,9 @@ class FunctionAdapterImpl { #else inline absl::Status RunWrap( std::function func, - ABSL_ATTRIBUTE_UNUSED const absl::Span argset, + [[maybe_unused]] const absl::Span argset, ::google::protobuf::Arena* arena, CelValue* result, - ABSL_ATTRIBUTE_UNUSED int arg_index) const { + [[maybe_unused]] int arg_index) const { return ValueConverter().NativeToValue(func(), arena, result); } diff --git a/eval/public/cel_function_registry_test.cc b/eval/public/cel_function_registry_test.cc index 75963cda7..7920d794f 100644 --- a/eval/public/cel_function_registry_test.cc +++ b/eval/public/cel_function_registry_test.cc @@ -151,7 +151,7 @@ TEST(CelFunctionRegistryTest, DefaultLazyProviderNoOverloadFound) { activation); ASSERT_OK(func.status()); - EXPECT_EQ(*func, absl::nullopt); + EXPECT_EQ(*func, std::nullopt); } TEST(CelFunctionRegistryTest, DefaultLazyProviderAmbiguousLookup) { diff --git a/eval/public/cel_number.cc b/eval/public/cel_number.cc index e08afb6a3..052a4f4e6 100644 --- a/eval/public/cel_number.cc +++ b/eval/public/cel_number.cc @@ -26,6 +26,6 @@ absl::optional GetNumberFromCelValue(const CelValue& value) { } else if (double val; value.GetValue(&val)) { return CelNumber(val); } - return absl::nullopt; + return std::nullopt; } } // namespace google::api::expr::runtime diff --git a/eval/public/cel_number_test.cc b/eval/public/cel_number_test.cc index 3c6f36e9b..bdddeaf20 100644 --- a/eval/public/cel_number_test.cc +++ b/eval/public/cel_number_test.cc @@ -36,7 +36,7 @@ TEST(CelNumber, GetNumberFromCelValue) { Optional(CelNumber::FromDouble(1.0))); EXPECT_EQ(GetNumberFromCelValue(CelValue::CreateDuration(absl::Seconds(1))), - absl::nullopt); + std::nullopt); } diff --git a/eval/public/cel_options.cc b/eval/public/cel_options.cc index 938b5e96f..c3aef0167 100644 --- a/eval/public/cel_options.cc +++ b/eval/public/cel_options.cc @@ -19,29 +19,34 @@ namespace google::api::expr::runtime { cel::RuntimeOptions ConvertToRuntimeOptions(const InterpreterOptions& options) { - return cel::RuntimeOptions{/*.container=*/"", - options.unknown_processing, - options.enable_missing_attribute_errors, - options.enable_timestamp_duration_overflow_errors, - options.short_circuiting, - options.enable_comprehension, - options.comprehension_max_iterations, - options.enable_comprehension_list_append, - options.enable_comprehension_mutable_map, - options.enable_regex, - options.regex_max_program_size, - options.enable_string_conversion, - options.enable_string_concat, - options.enable_list_concat, - options.enable_list_contains, - options.fail_on_warnings, - options.enable_qualified_type_identifiers, - options.enable_heterogeneous_equality, - options.enable_empty_wrapper_null_unboxing, - options.enable_lazy_bind_initialization, - options.max_recursion_depth, - options.enable_recursive_tracing, - options.enable_fast_builtins}; + return cel::RuntimeOptions{ + /*.container=*/"", + options.unknown_processing, + options.enable_missing_attribute_errors, + options.enable_timestamp_duration_overflow_errors, + options.short_circuiting, + options.enable_comprehension, + options.comprehension_max_iterations, + options.enable_comprehension_list_append, + options.enable_comprehension_mutable_map, + options.enable_regex, + options.regex_max_program_size, + options.enable_string_conversion, + options.enable_string_concat, + options.enable_list_concat, + options.enable_list_contains, + options.fail_on_warnings, + options.enable_qualified_type_identifiers, + options.enable_heterogeneous_equality, + options.enable_empty_wrapper_null_unboxing, + options.enable_lazy_bind_initialization, + options.max_recursion_depth, + options.enable_recursive_tracing, + options.enable_fast_builtins, + options.enable_precision_preserving_double_format, + options.enable_typed_field_access, + options.enable_subexpression_inlining, + }; } } // namespace google::api::expr::runtime diff --git a/eval/public/cel_options.h b/eval/public/cel_options.h index 4d81eb8a7..0bb6d81ed 100644 --- a/eval/public/cel_options.h +++ b/eval/public/cel_options.h @@ -188,12 +188,14 @@ struct InterpreterOptions { // -1 means unbounded. // 0 means disabled (using a heap-based stack machine instead), which is the // default. + ABSL_DEPRECATED("Recursive evaluation has been removed") int max_recursion_depth = 0; // Enable tracing support for recursively planned programs. // // Unlike the stack machine implementation, supporting tracing can affect // performance whether or not tracing is requested for a given evaluation. + ABSL_DEPRECATED("Recursive evaluation has been removed") bool enable_recursive_tracing = false; // Enable fast implementations for some CEL standard functions. @@ -218,6 +220,18 @@ struct InterpreterOptions { // Otherwise, will fall back to formatting with the worst-case required // precision. bool enable_precision_preserving_double_format = true; + + // When enabled, the planner will attempt to use a more performant execution + // path for field access when the type is known at plan time, instead of using + // the generic field access implementation. + bool enable_typed_field_access = true; + + // When enabled, trivial lazy subexpressions (such as constants or + // identifiers) bound via cel.bind or cel.@block will be inlined directly + // at reference sites during planning. + // + // Note: Enabling this option may change observed expression tracing behavior. + bool enable_subexpression_inlining = false; }; // LINT.ThenChange(//depot/google3/runtime/runtime_options.h) diff --git a/eval/public/cel_type_registry.cc b/eval/public/cel_type_registry.cc index 639a348dd..d43fddd16 100644 --- a/eval/public/cel_type_registry.cc +++ b/eval/public/cel_type_registry.cc @@ -58,7 +58,7 @@ absl::optional CelTypeRegistry::FindTypeAdapter( if (maybe_adapter.has_value()) { return maybe_adapter; } - return absl::nullopt; + return std::nullopt; } } // namespace google::api::expr::runtime diff --git a/eval/public/cel_type_registry.h b/eval/public/cel_type_registry.h index 3fb80bcea..b54b5bb99 100644 --- a/eval/public/cel_type_registry.h +++ b/eval/public/cel_type_registry.h @@ -20,6 +20,7 @@ #include #include +#include "absl/base/call_once.h" #include "absl/base/nullability.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" @@ -28,6 +29,7 @@ #include "base/type_provider.h" #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_provider.h" +#include "eval/public/structs/protobuf_descriptor_type_provider.h" #include "runtime/type_registry.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -60,7 +62,9 @@ class CelTypeRegistry { CelTypeRegistry(const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nullable message_factory) - : modern_type_registry_(descriptor_pool, message_factory) {} + : descriptor_pool_(descriptor_pool), + message_factory_(message_factory), + modern_type_registry_(descriptor_pool, message_factory) {} ~CelTypeRegistry() = default; @@ -77,8 +81,12 @@ class CelTypeRegistry { // Get the first registered type provider. std::shared_ptr GetFirstTypeProvider() const { - return cel::runtime_internal::GetLegacyRuntimeTypeProvider( - modern_type_registry_); + absl::call_once(legacy_type_provider_once_, [&]() { + this->legacy_type_provider_ = std::make_shared< + google::api::expr::runtime::ProtobufDescriptorProvider>( + descriptor_pool_, message_factory_); + }); + return legacy_type_provider_; } // Returns the effective type provider that has been configured with the @@ -136,6 +144,13 @@ class CelTypeRegistry { } private: + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool_; + google::protobuf::MessageFactory* absl_nullable message_factory_; + + // Legacy type provider. This is now disconnected from the actual type + // resolution, but preserved for legacy clients that used it directly. + mutable absl::once_flag legacy_type_provider_once_; + mutable std::shared_ptr legacy_type_provider_; // Internal modern registry. cel::TypeRegistry modern_type_registry_; }; diff --git a/eval/public/cel_type_registry_test.cc b/eval/public/cel_type_registry_test.cc index 9f3fde9be..9dc8d946e 100644 --- a/eval/public/cel_type_registry_test.cc +++ b/eval/public/cel_type_registry_test.cc @@ -36,10 +36,10 @@ class TestTypeProvider : public LegacyTypeProvider { absl::string_view name) const override { for (const auto& type : types_) { if (name == type) { - return LegacyTypeAdapter(/*access=*/nullptr, /*mutation=*/nullptr); + return LegacyTypeAdapter(/*access=*/nullptr); } } - return absl::nullopt; + return std::nullopt; } private: diff --git a/eval/public/cel_value.cc b/eval/public/cel_value.cc index 25da7fe75..33b78c1c9 100644 --- a/eval/public/cel_value.cc +++ b/eval/public/cel_value.cc @@ -14,10 +14,8 @@ #include "absl/strings/str_join.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/memory.h" #include "eval/internal/errors.h" #include "eval/public/structs/legacy_type_info_apis.h" -#include "extensions/protobuf/memory_manager.h" #include "google/protobuf/arena.h" namespace google::api::expr::runtime { @@ -263,7 +261,7 @@ class EmptyCelMap final : public CelMap { } absl::optional operator[](CelValue key) const override { - return absl::nullopt; + return std::nullopt; } absl::StatusOr Has(const CelValue& key) const override { @@ -286,23 +284,6 @@ CelValue CelValue::CreateList() { return CreateList(EmptyCelList::Get()); } CelValue CelValue::CreateMap() { return CreateMap(EmptyCelMap::Get()); } -CelValue CreateErrorValue(cel::MemoryManagerRef manager, - absl::string_view message, - absl::StatusCode error_code) { - // TODO(uncreated-issue/1): assume arena-style allocator while migrating to new - // value type. - Arena* arena = cel::extensions::ProtoMemoryManagerArena(manager); - return CreateErrorValue(arena, message, error_code); -} - -CelValue CreateErrorValue(cel::MemoryManagerRef manager, - const absl::Status& status) { - // TODO(uncreated-issue/1): assume arena-style allocator while migrating to new - // value type. - Arena* arena = cel::extensions::ProtoMemoryManagerArena(manager); - return CreateErrorValue(arena, status); -} - CelValue CreateErrorValue(Arena* arena, absl::string_view message, absl::StatusCode error_code) { CelError* error = Arena::Create(arena, error_code, message); @@ -314,12 +295,6 @@ CelValue CreateErrorValue(Arena* arena, const absl::Status& status) { return CelValue::CreateError(error); } -CelValue CreateNoMatchingOverloadError(cel::MemoryManagerRef manager, - absl::string_view fn) { - return CelValue::CreateError(interop::CreateNoMatchingOverloadError( - cel::extensions::ProtoMemoryManagerArena(manager), fn)); -} - CelValue CreateNoMatchingOverloadError(google::protobuf::Arena* arena, absl::string_view fn) { return CelValue::CreateError( @@ -333,22 +308,10 @@ bool CheckNoMatchingOverloadError(CelValue value) { cel::runtime_internal::kErrNoMatchingOverload); } -CelValue CreateNoSuchFieldError(cel::MemoryManagerRef manager, - absl::string_view field) { - return CelValue::CreateError(interop::CreateNoSuchFieldError( - cel::extensions::ProtoMemoryManagerArena(manager), field)); -} - CelValue CreateNoSuchFieldError(google::protobuf::Arena* arena, absl::string_view field) { return CelValue::CreateError(interop::CreateNoSuchFieldError(arena, field)); } -CelValue CreateNoSuchKeyError(cel::MemoryManagerRef manager, - absl::string_view key) { - return CelValue::CreateError(interop::CreateNoSuchKeyError( - cel::extensions::ProtoMemoryManagerArena(manager), key)); -} - CelValue CreateNoSuchKeyError(google::protobuf::Arena* arena, absl::string_view key) { return CelValue::CreateError(interop::CreateNoSuchKeyError(arena, key)); } @@ -365,15 +328,6 @@ CelValue CreateMissingAttributeError(google::protobuf::Arena* arena, interop::CreateMissingAttributeError(arena, missing_attribute_path)); } -CelValue CreateMissingAttributeError(cel::MemoryManagerRef manager, - absl::string_view missing_attribute_path) { - // TODO(uncreated-issue/1): assume arena-style allocator while migrating - // to new value type. - return CelValue::CreateError(interop::CreateMissingAttributeError( - cel::extensions::ProtoMemoryManagerArena(manager), - missing_attribute_path)); -} - bool IsMissingAttributeError(const CelValue& value) { const CelError* error; if (!value.GetValue(&error)) return false; @@ -385,12 +339,6 @@ bool IsMissingAttributeError(const CelValue& value) { return false; } -CelValue CreateUnknownFunctionResultError(cel::MemoryManagerRef manager, - absl::string_view help_message) { - return CelValue::CreateError(interop::CreateUnknownFunctionResultError( - cel::extensions::ProtoMemoryManagerArena(manager), help_message)); -} - CelValue CreateUnknownFunctionResultError(google::protobuf::Arena* arena, absl::string_view help_message) { return CelValue::CreateError( diff --git a/eval/public/cel_value.h b/eval/public/cel_value.h index 76b4d09bb..2a98414f3 100644 --- a/eval/public/cel_value.h +++ b/eval/public/cel_value.h @@ -494,7 +494,7 @@ class CelValue { explicit CelValue(T value) : value_(value) {} // Crashes with a null pointer error. - static void CrashNullPointer(Type type) ABSL_ATTRIBUTE_COLD { + ABSL_ATTRIBUTE_COLD static void CrashNullPointer(Type type) { ABSL_LOG(FATAL) << "Null pointer supplied for " << TypeName(type); // Crash ok } @@ -507,8 +507,8 @@ class CelValue { } // Crashes with a type mismatch error. - static void CrashTypeMismatch(Type requested_type, - Type actual_type) ABSL_ATTRIBUTE_COLD { + ABSL_ATTRIBUTE_COLD static void CrashTypeMismatch(Type requested_type, + Type actual_type) { ABSL_LOG(FATAL) << "Type mismatch" // Crash ok << ": expected " << TypeName(requested_type) // Crash ok << ", encountered " << TypeName(actual_type); // Crash ok @@ -653,44 +653,54 @@ class CelMap { // Utility method that generates CelValue containing CelError. // message an error message // error_code error code -CelValue CreateErrorValue( - cel::MemoryManagerRef manager ABSL_ATTRIBUTE_LIFETIME_BOUND, - absl::string_view message, - absl::StatusCode error_code = absl::StatusCode::kUnknown); CelValue CreateErrorValue( google::protobuf::Arena* arena, absl::string_view message, absl::StatusCode error_code = absl::StatusCode::kUnknown); - -// Utility method for generating a CelValue from an absl::Status. -CelValue CreateErrorValue(cel::MemoryManagerRef manager - ABSL_ATTRIBUTE_LIFETIME_BOUND, - const absl::Status& status); +ABSL_DEPRECATE_AND_INLINE() +inline CelValue CreateErrorValue( + cel::MemoryManagerRef manager ABSL_ATTRIBUTE_LIFETIME_BOUND, + absl::string_view message, + absl::StatusCode error_code = absl::StatusCode::kUnknown) { + return CreateErrorValue(manager.arena(), message, error_code); +} // Utility method for generating a CelValue from an absl::Status. CelValue CreateErrorValue(google::protobuf::Arena* arena, const absl::Status& status); +ABSL_DEPRECATE_AND_INLINE() +inline CelValue CreateErrorValue(cel::MemoryManagerRef manager + ABSL_ATTRIBUTE_LIFETIME_BOUND, + const absl::Status& status) { + return CreateErrorValue(manager.arena(), status); +} // Create an error for failed overload resolution, optionally including the name // of the function. -CelValue CreateNoMatchingOverloadError(cel::MemoryManagerRef manager - ABSL_ATTRIBUTE_LIFETIME_BOUND, - absl::string_view fn = ""); -ABSL_DEPRECATED("Prefer using the generic MemoryManager overload") CelValue CreateNoMatchingOverloadError(google::protobuf::Arena* arena, absl::string_view fn = ""); +ABSL_DEPRECATE_AND_INLINE() +inline CelValue CreateNoMatchingOverloadError(cel::MemoryManagerRef manager + ABSL_ATTRIBUTE_LIFETIME_BOUND, + absl::string_view fn = "") { + return CreateNoMatchingOverloadError(manager.arena(), fn); +} bool CheckNoMatchingOverloadError(CelValue value); -CelValue CreateNoSuchFieldError(cel::MemoryManagerRef manager - ABSL_ATTRIBUTE_LIFETIME_BOUND, - absl::string_view field = ""); -ABSL_DEPRECATED("Prefer using the generic MemoryManager overload") CelValue CreateNoSuchFieldError(google::protobuf::Arena* arena, absl::string_view field = ""); +ABSL_DEPRECATE_AND_INLINE() +inline CelValue CreateNoSuchFieldError(cel::MemoryManagerRef manager + ABSL_ATTRIBUTE_LIFETIME_BOUND, + absl::string_view field = "") { + return CreateNoSuchFieldError(manager.arena(), field); +} -CelValue CreateNoSuchKeyError(cel::MemoryManagerRef manager - ABSL_ATTRIBUTE_LIFETIME_BOUND, - absl::string_view key); -ABSL_DEPRECATED("Prefer using the generic MemoryManager overload") CelValue CreateNoSuchKeyError(google::protobuf::Arena* arena, absl::string_view key); +ABSL_DEPRECATE_AND_INLINE() +inline CelValue CreateNoSuchKeyError(cel::MemoryManagerRef manager + ABSL_ATTRIBUTE_LIFETIME_BOUND, + absl::string_view key) { + return CreateNoSuchKeyError(manager.arena(), key); +} bool CheckNoSuchKeyError(CelValue value); @@ -698,12 +708,14 @@ bool CheckNoSuchKeyError(CelValue value); // value is undefined. For example, this may represent a field in a proto // message bound to the activation whose value can't be determined by the // hosting application. -CelValue CreateMissingAttributeError(cel::MemoryManagerRef manager - ABSL_ATTRIBUTE_LIFETIME_BOUND, - absl::string_view missing_attribute_path); -ABSL_DEPRECATED("Prefer using the generic MemoryManager overload") CelValue CreateMissingAttributeError(google::protobuf::Arena* arena, absl::string_view missing_attribute_path); +ABSL_DEPRECATE_AND_INLINE() +inline CelValue CreateMissingAttributeError( + cel::MemoryManagerRef manager ABSL_ATTRIBUTE_LIFETIME_BOUND, + absl::string_view missing_attribute_path) { + return CreateMissingAttributeError(manager.arena(), missing_attribute_path); +} ABSL_CONST_INIT extern const absl::string_view kPayloadUrlMissingAttributePath; bool IsMissingAttributeError(const CelValue& value); @@ -711,12 +723,14 @@ bool IsMissingAttributeError(const CelValue& value); // Returns error indicating the result of the function is unknown. This is used // as a signal to create an unknown set if unknown function handling is opted // into. -CelValue CreateUnknownFunctionResultError(cel::MemoryManagerRef manager - ABSL_ATTRIBUTE_LIFETIME_BOUND, - absl::string_view help_message); -ABSL_DEPRECATED("Prefer using the generic MemoryManager overload") CelValue CreateUnknownFunctionResultError(google::protobuf::Arena* arena, absl::string_view help_message); +ABSL_DEPRECATE_AND_INLINE() +inline CelValue CreateUnknownFunctionResultError( + cel::MemoryManagerRef manager ABSL_ATTRIBUTE_LIFETIME_BOUND, + absl::string_view help_message) { + return CreateUnknownFunctionResultError(manager.arena(), help_message); +} // Returns true if this is unknown value error indicating that evaluation // called an extension function whose value is unknown for the given args. diff --git a/eval/public/cel_value_clone.cc b/eval/public/cel_value_clone.cc new file mode 100644 index 000000000..8a07033df --- /dev/null +++ b/eval/public/cel_value_clone.cc @@ -0,0 +1,166 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "eval/public/cel_value_clone.h" + +#include +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/cord.h" +#include "absl/strings/str_cat.h" +#include "absl/time/time.h" +#include "absl/types/optional.h" +#include "eval/public/cel_value.h" +#include "eval/public/containers/container_backed_list_impl.h" +#include "eval/public/containers/container_backed_map_impl.h" +#include "eval/public/message_wrapper.h" +#include "eval/public/unknown_set.h" +#include "internal/status_macros.h" +#include "google/protobuf/arena.h" + +namespace google::api::expr::runtime { + +namespace { + +struct CloneVisitor { + google::protobuf::Arena* const arena; + + absl::StatusOr operator()(bool arg) { + return CelValue::CreateBool(arg); + } + absl::StatusOr operator()(int64_t arg) { + return CelValue::CreateInt64(arg); + } + absl::StatusOr operator()(uint64_t arg) { + return CelValue::CreateUint64(arg); + } + absl::StatusOr operator()(double arg) { + return CelValue::CreateDouble(arg); + } + absl::StatusOr operator()(CelValue::NullType) { + return CelValue::CreateNull(); + } + + absl::StatusOr operator()(CelValue::StringHolder arg) { + if (arg.value().empty()) { + return CelValue::CreateStringView(""); + } + return CelValue::CreateString(CelValue::StringHolder( + google::protobuf::Arena::Create(arena, arg.value()))); + } + + absl::StatusOr operator()(CelValue::BytesHolder arg) { + if (arg.value().empty()) { + return CelValue::CreateBytesView(""); + } + return CelValue::CreateBytes(CelValue::BytesHolder( + google::protobuf::Arena::Create(arena, arg.value()))); + } + + absl::StatusOr operator()(const MessageWrapper& arg) { + if (arg.HasFullProto()) { + google::protobuf::Message* dup = + static_cast(arg.message_ptr())->New(arena); + dup->CopyFrom(*static_cast(arg.message_ptr())); + return CelValue::CreateMessageWrapper( + MessageWrapper(dup, arg.legacy_type_info())); + } + google::protobuf::MessageLite* dup = arg.message_ptr()->New(arena); + absl::Cord serialized; + if (!arg.message_ptr()->SerializePartialToCord(&serialized)) { + return absl::UnknownError( + absl::StrCat("failed to serialize message: ", dup->GetTypeName())); + } + if (!dup->ParsePartialFromString(serialized)) { + return absl::UnknownError( + absl::StrCat("failed to parse message: ", dup->GetTypeName())); + } + return CelValue::CreateMessageWrapper( + MessageWrapper(dup, arg.legacy_type_info())); + } + + absl::StatusOr operator()(absl::Duration arg) { + return CelValue::CreateUncheckedDuration(arg); + } + + absl::StatusOr operator()(absl::Time arg) { + return CelValue::CreateTimestamp(arg); + } + + absl::StatusOr operator()(const CelList* arg) { + if (arg->empty()) { + return CelValue::CreateList(); + } + std::vector elements; + elements.reserve(arg->size()); + for (int i = 0; i < arg->size(); i++) { + CEL_ASSIGN_OR_RETURN(auto element, + CelValueClone(arena, arg->Get(arena, i))); + elements.push_back(element); + } + return CelValue::CreateList(google::protobuf::Arena::Create( + arena, std::move(elements))); + } + + absl::StatusOr operator()(const CelMap* arg) { + if (arg->empty()) { + return CelValue::CreateMap(); + } + CEL_ASSIGN_OR_RETURN(auto keys, arg->ListKeys(arena)); + CelMapBuilder* builder = google::protobuf::Arena::Create(arena); + for (int i = 0; i < keys->size(); ++i) { + CelValue key = keys->Get(arena, i); + absl::optional value = arg->Get(arena, key); + if (!value.has_value()) { + return absl::UnknownError( + "CelMap::ListKeys returned key but CelMap::Get did not find the " + "entry"); + } + CEL_ASSIGN_OR_RETURN(auto key_dup, CelValueClone(arena, key)); + CEL_ASSIGN_OR_RETURN(auto value_dup, CelValueClone(arena, *value)); + CEL_RETURN_IF_ERROR(builder->Add(key_dup, value_dup)); + } + return CelValue::CreateMap(builder); + } + + absl::StatusOr operator()(const UnknownSet* arg) { + return CelValue::CreateUnknownSet( + google::protobuf::Arena::Create(arena, *arg)); + } + + absl::StatusOr operator()(CelValue::CelTypeHolder arg) { + return CelValue::CreateCelType(CelValue::CelTypeHolder( + google::protobuf::Arena::Create(arena, arg.value()))); + } + + absl::StatusOr operator()(const CelError* arg) { + return CelValue::CreateError(google::protobuf::Arena::Create(arena, *arg)); + } +}; + +} // namespace + +absl::StatusOr CelValueClone(google::protobuf::Arena* arena, CelValue in) { + ABSL_DCHECK(arena != nullptr); + return in.Visit>(CloneVisitor{.arena = arena}); +} + +} // namespace google::api::expr::runtime diff --git a/eval/public/cel_value_clone.h b/eval/public/cel_value_clone.h new file mode 100644 index 000000000..2ffe47894 --- /dev/null +++ b/eval/public/cel_value_clone.h @@ -0,0 +1,40 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_CEL_VALUE_CLONE_H_ +#define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_CEL_VALUE_CLONE_H_ + +#include "absl/status/statusor.h" +#include "eval/public/cel_value.h" +#include "google/protobuf/arena.h" + +namespace google::api::expr::runtime { + +// Clones (deep copy) a CelValue such that the resulting CelValue has a lifetime +// tied to the given arena. The container implementations for the output are not +// guaranteed to be the same as the input. +absl::StatusOr CelValueClone(google::protobuf::Arena* arena, CelValue in); +inline absl::StatusOr CelValueClone(google::protobuf::Arena* arena, + absl::StatusOr in) { + if (!in.ok()) { + return in; + } + return CelValueClone(arena, *in); +} + +} // namespace google::api::expr::runtime + +#endif // THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_CEL_VALUE_CLONE_H_ diff --git a/eval/public/cel_value_test.cc b/eval/public/cel_value_test.cc index 0af6eb9e7..985bb08a9 100644 --- a/eval/public/cel_value_test.cc +++ b/eval/public/cel_value_test.cc @@ -294,7 +294,7 @@ TEST(CelValueTest, TestEmptyMap) { EXPECT_EQ(value2->size(), 0); EXPECT_THAT(value2->Has(CelValue::CreateBool(false)), IsOkAndHolds(false)); EXPECT_THAT(value2->Get(&arena, CelValue::CreateBool(false)), - Eq(absl::nullopt)); + Eq(std::nullopt)); EXPECT_THAT(value2->ListKeys(&arena), IsOkAndHolds(NotNull())); } diff --git a/eval/public/containers/BUILD b/eval/public/containers/BUILD index 18ad48734..54dbf6189 100644 --- a/eval/public/containers/BUILD +++ b/eval/public/containers/BUILD @@ -113,6 +113,7 @@ cc_test( "//eval/public:cel_value", "//internal:testing", "@com_google_absl//absl/status", + "@com_google_absl//absl/types:span", ], ) diff --git a/eval/public/containers/container_backed_map_impl.cc b/eval/public/containers/container_backed_map_impl.cc index 5ac08af92..99d0cdeba 100644 --- a/eval/public/containers/container_backed_map_impl.cc +++ b/eval/public/containers/container_backed_map_impl.cc @@ -88,7 +88,7 @@ class CelValueEq { absl::optional CelMapBuilder::operator[](CelValue cel_key) const { auto item = values_map_.find(cel_key); if (item == values_map_.end()) { - return absl::nullopt; + return std::nullopt; } return item->second; } diff --git a/eval/public/containers/container_backed_map_impl_test.cc b/eval/public/containers/container_backed_map_impl_test.cc index 59d38d235..a88f0884d 100644 --- a/eval/public/containers/container_backed_map_impl_test.cc +++ b/eval/public/containers/container_backed_map_impl_test.cc @@ -5,6 +5,7 @@ #include #include "absl/status/status.h" +#include "absl/types/span.h" #include "eval/public/cel_value.h" #include "internal/testing.h" @@ -22,10 +23,7 @@ TEST(ContainerBackedMapImplTest, TestMapInt64) { {CelValue::CreateInt64(1), CelValue::CreateInt64(2)}, {CelValue::CreateInt64(2), CelValue::CreateInt64(3)}}; - auto cel_map = - CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); + auto cel_map = CreateContainerBackedMap(absl::MakeSpan(args)).value(); ASSERT_THAT(cel_map, Not(IsNull())); @@ -56,10 +54,7 @@ TEST(ContainerBackedMapImplTest, TestMapUint64) { std::vector> args = { {CelValue::CreateUint64(1), CelValue::CreateInt64(2)}, {CelValue::CreateUint64(2), CelValue::CreateInt64(3)}}; - auto cel_map = - CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); + auto cel_map = CreateContainerBackedMap(absl::MakeSpan(args)).value(); ASSERT_THAT(cel_map, Not(IsNull())); @@ -94,10 +89,7 @@ TEST(ContainerBackedMapImplTest, TestMapString) { std::vector> args = { {CelValue::CreateString(&kKey1), CelValue::CreateInt64(2)}, {CelValue::CreateString(&kKey2), CelValue::CreateInt64(3)}}; - auto cel_map = - CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); + auto cel_map = CreateContainerBackedMap(absl::MakeSpan(args)).value(); ASSERT_THAT(cel_map, Not(IsNull())); diff --git a/eval/public/containers/internal_field_backed_map_impl.cc b/eval/public/containers/internal_field_backed_map_impl.cc index a879955d1..e4569aa82 100644 --- a/eval/public/containers/internal_field_backed_map_impl.cc +++ b/eval/public/containers/internal_field_backed_map_impl.cc @@ -154,7 +154,7 @@ absl::optional FieldBackedMapImpl::operator[](CelValue key) const { return CreateErrorValue(arena_, lookup_result.status()); } if (!*lookup_result) { - return absl::nullopt; + return std::nullopt; } // Get value descriptor treating it as a repeated field. @@ -174,7 +174,6 @@ absl::StatusOr FieldBackedMapImpl::LookupMapValue( return InvalidMapKeyType(key_desc_->cpp_type_name()); } - std::string map_key_string; google::protobuf::MapKey proto_key; switch (key_desc_->cpp_type()) { case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { @@ -199,8 +198,7 @@ absl::StatusOr FieldBackedMapImpl::LookupMapValue( case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { CelValue::StringHolder key_value; key.GetValue(&key_value); - map_key_string.assign(key_value.value().data(), key_value.value().size()); - proto_key.SetStringValue(map_key_string); + proto_key.SetStringValue(key_value.value()); } break; case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { uint64_t key_value; diff --git a/eval/public/equality_function_registrar_test.cc b/eval/public/equality_function_registrar_test.cc index 772ddfeba..a77a92734 100644 --- a/eval/public/equality_function_registrar_test.cc +++ b/eval/public/equality_function_registrar_test.cc @@ -86,7 +86,7 @@ MATCHER_P2(DefinesHomogenousOverload, name, argument_type, struct EqualityTestCase { enum class ErrorKind { kMissingOverload, kMissingIdentifier }; absl::string_view expr; - absl::variant result; + std::variant result; CelValue lhs = CelValue::CreateNull(); CelValue rhs = CelValue::CreateNull(); }; @@ -204,7 +204,7 @@ std::string CelValueEqualTestName( } TEST_P(CelValueEqualImplTypesTest, Basic) { - absl::optional result = CelValueEqualImpl(lhs(), rhs()); + std::optional result = CelValueEqualImpl(lhs(), rhs()); if (lhs().IsNull() || rhs().IsNull()) { if (lhs().IsNull() && rhs().IsNull()) { @@ -286,7 +286,7 @@ const std::vector& NumericValuesNotEqualExample() { using NumericInequalityTest = testing::TestWithParam; TEST_P(NumericInequalityTest, NumericValues) { NumericInequalityTestCase test_case = GetParam(); - absl::optional result = CelValueEqualImpl(test_case.a, test_case.b); + std::optional result = CelValueEqualImpl(test_case.a, test_case.b); EXPECT_TRUE(result.has_value()); EXPECT_EQ(*result, false); } @@ -299,7 +299,7 @@ INSTANTIATE_TEST_SUITE_P( }); TEST(CelValueEqualImplTest, LossyNumericEquality) { - absl::optional result = CelValueEqualImpl( + std::optional result = CelValueEqualImpl( CelValue::CreateDouble( static_cast(std::numeric_limits::max()) - 1), CelValue::CreateInt64(std::numeric_limits::max())); diff --git a/eval/public/extension_func_test.cc b/eval/public/extension_func_test.cc index 2e2497d7d..c9f5fb4fa 100644 --- a/eval/public/extension_func_test.cc +++ b/eval/public/extension_func_test.cc @@ -32,7 +32,7 @@ static const int kNanosPerSecond = 1000000000; class ExtensionTest : public ::testing::Test { protected: - ExtensionTest() {} + ExtensionTest() = default; void SetUp() override { ASSERT_OK(RegisterBuiltinFunctions(®istry_)); diff --git a/eval/public/message_wrapper_test.cc b/eval/public/message_wrapper_test.cc index ff0e691ab..15e5e88da 100644 --- a/eval/public/message_wrapper_test.cc +++ b/eval/public/message_wrapper_test.cc @@ -18,7 +18,6 @@ #include "eval/public/structs/trivial_legacy_type_info.h" #include "eval/testutil/test_message.pb.h" -#include "internal/casts.h" #include "internal/testing.h" #include "google/protobuf/message.h" #include "google/protobuf/message_lite.h" @@ -60,7 +59,7 @@ TEST(MessageWrapperBuilder, Builder) { static_cast(&test_message)); auto mutable_message = - cel::internal::down_cast(builder.message_ptr()); + google::protobuf::DownCastMessage(builder.message_ptr()); mutable_message->set_int64_value(20); mutable_message->set_double_value(12.3); diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index d301ff0ca..107451e01 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -28,12 +28,19 @@ cc_library( "cel_proto_wrapper.h", ], deps = [ - ":cel_proto_wrap_util", + ":cel_proto_wrap_value_to_message", ":proto_message_type_adapter", + ":trivial_legacy_type_info_internal", + "//common:value", "//eval/public:cel_value", "//eval/public:message_wrapper", "//internal:proto_time_encoding", - "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_protobuf//:duration_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:timestamp_cc_proto", @@ -62,10 +69,8 @@ cc_library( deps = [ ":protobuf_value_factory", "//eval/public:cel_value", - "//internal:overflow", "//internal:proto_time_encoding", "//internal:status_macros", - "//internal:time", "//internal:well_known_types", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", @@ -96,26 +101,80 @@ cc_test( ], deps = [ ":cel_proto_wrap_util", - ":protobuf_value_factory", ":trivial_legacy_type_info", "//eval/public:cel_value", - "//eval/public:message_wrapper", - "//eval/public/containers:container_backed_list_impl", - "//eval/public/containers:container_backed_map_impl", "//eval/testutil:test_message_cc_proto", "//internal:proto_time_encoding", + "//internal:testing", + "//testutil:util", + "@com_google_absl//absl/status", + "@com_google_protobuf//:any_cc_proto", + "@com_google_protobuf//:duration_cc_proto", + "@com_google_protobuf//:empty_cc_proto", + "@com_google_protobuf//:field_mask_cc_proto", + "@com_google_protobuf//:protobuf", + "@com_google_protobuf//:struct_cc_proto", + "@com_google_protobuf//:wrappers_cc_proto", + ], +) + +cc_library( + name = "cel_proto_wrap_value_to_message", + srcs = [ + "cel_proto_wrap_value_to_message.cc", + ], + hdrs = [ + "cel_proto_wrap_value_to_message.h", + ], + deps = [ + "//eval/public:cel_value", + "//internal:overflow", + "//internal:proto_time_encoding", "//internal:status_macros", + "//internal:time", + "//internal:well_known_types", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:cord", + "@com_google_absl//absl/time", + "@com_google_protobuf//:any_cc_proto", + "@com_google_protobuf//:duration_cc_proto", + "@com_google_protobuf//:protobuf", + "@com_google_protobuf//:struct_cc_proto", + "@com_google_protobuf//:timestamp_cc_proto", + "@com_google_protobuf//:wrappers_cc_proto", + ], +) + +cc_test( + name = "cel_proto_wrap_value_to_message_test", + size = "small", + srcs = [ + "cel_proto_wrap_value_to_message_test.cc", + ], + deps = [ + ":cel_proto_wrap_value_to_message", + ":trivial_legacy_type_info", + "//eval/public:cel_value", + "//eval/public/containers:container_backed_list_impl", + "//eval/public/containers:container_backed_map_impl", + "//eval/testutil:test_message_cc_proto", "//internal:testing", "//testutil:util", - "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", + "@com_google_absl//absl/types:span", "@com_google_protobuf//:any_cc_proto", "@com_google_protobuf//:duration_cc_proto", "@com_google_protobuf//:empty_cc_proto", + "@com_google_protobuf//:field_mask_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:struct_cc_proto", + "@com_google_protobuf//:timestamp_cc_proto", "@com_google_protobuf//:wrappers_cc_proto", ], ) @@ -130,6 +189,7 @@ cc_library( ], deps = [ ":cel_proto_wrap_util", + ":cel_proto_wrap_value_to_message", ":protobuf_value_factory", "//eval/public:cel_options", "//eval/public:cel_value", @@ -218,6 +278,7 @@ cc_test( "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", + "@com_google_absl//absl/types:span", "@com_google_protobuf//:any_cc_proto", "@com_google_protobuf//:duration_cc_proto", "@com_google_protobuf//:empty_cc_proto", @@ -229,26 +290,12 @@ cc_test( cc_library( name = "legacy_type_provider", - srcs = ["legacy_type_provider.cc"], hdrs = ["legacy_type_provider.h"], deps = [ ":legacy_type_adapter", ":legacy_type_info_apis", - "//common:legacy_value", - "//common:memory", - "//common:type", - "//common:value", - "//eval/public:message_wrapper", - "//extensions/protobuf:memory_manager", - "//internal:status_macros", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", - "@com_google_absl//absl/types:optional", - "@com_google_protobuf//:protobuf", ], ) @@ -272,14 +319,9 @@ cc_test( srcs = ["legacy_type_adapter_test.cc"], deps = [ ":legacy_type_adapter", - ":trivial_legacy_type_info", - "//eval/public:cel_value", - "//eval/public/testing:matchers", + ":proto_message_type_adapter", "//eval/testutil:test_message_cc_proto", - "//extensions/protobuf:memory_manager", - "//internal:status_macros", "//internal:testing", - "@com_google_protobuf//:protobuf", ], ) @@ -288,7 +330,6 @@ cc_library( srcs = ["proto_message_type_adapter.cc"], hdrs = ["proto_message_type_adapter.h"], deps = [ - ":cel_proto_wrap_util", ":field_access_impl", ":legacy_type_adapter", ":legacy_type_info_apis", @@ -301,7 +342,6 @@ cc_library( "//eval/public/containers:internal_field_backed_map_impl", "//extensions/protobuf:memory_manager", "//extensions/protobuf/internal:qualify", - "//internal:casts", "//internal:status_macros", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", @@ -309,7 +349,6 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_protobuf//:differencer", "@com_google_protobuf//:protobuf", @@ -323,16 +362,15 @@ cc_test( ":legacy_type_adapter", ":legacy_type_info_apis", ":proto_message_type_adapter", + ":trivial_legacy_type_info", "//base:attributes", "//common:value", + "//common:value_testing", "//eval/public:cel_value", "//eval/public:message_wrapper", - "//eval/public/containers:container_backed_list_impl", - "//eval/public/containers:container_backed_map_impl", "//eval/public/testing:matchers", "//eval/testutil:test_message_cc_proto", "//extensions/protobuf:memory_manager", - "//internal:proto_matchers", "//internal:testing", "//runtime:runtime_options", "@com_google_absl//absl/status", @@ -369,6 +407,7 @@ cc_test( "//eval/public/testing:matchers", "//extensions/protobuf:memory_manager", "//internal:testing", + "@com_google_absl//absl/status:status_matchers", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:wrappers_cc_proto", ], @@ -380,24 +419,33 @@ cc_library( deps = [ "//eval/public:message_wrapper", "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/status", "@com_google_absl//absl/strings:string_view", "@com_google_protobuf//:protobuf", ], ) cc_library( - name = "trivial_legacy_type_info", - testonly = True, - hdrs = ["trivial_legacy_type_info.h"], + name = "trivial_legacy_type_info_internal", + hdrs = ["trivial_legacy_type_info_internal.h"], deps = [ ":legacy_type_info_apis", "//eval/public:message_wrapper", "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/strings:string_view", ], ) +cc_library( + name = "trivial_legacy_type_info", + testonly = True, + hdrs = ["trivial_legacy_type_info.h"], + deps = [ + ":legacy_type_info_apis", + ":trivial_legacy_type_info_internal", + ], +) + cc_test( name = "trivial_legacy_type_info_test", srcs = ["trivial_legacy_type_info_test.cc"], @@ -412,10 +460,13 @@ cc_test( name = "legacy_type_provider_test", srcs = ["legacy_type_provider_test.cc"], deps = [ + ":legacy_type_adapter", ":legacy_type_info_apis", ":legacy_type_provider", + ":trivial_legacy_type_info", "//internal:testing", "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:optional", ], ) @@ -442,3 +493,23 @@ cc_test( "@com_google_protobuf//:protobuf", ], ) + +cc_test( + name = "field_access_impl_benchmark_test", + srcs = ["field_access_impl_benchmark_test.cc"], + tags = [ + "benchmark", + "manual", + ], + deps = [ + ":cel_proto_wrapper", + ":field_access_impl", + "//eval/public:cel_options", + "//eval/public:cel_value", + "//extensions/protobuf/internal:map_reflection", + "//internal:benchmark", + "//internal:testing", + "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", + "@com_google_protobuf//:protobuf", + ], +) diff --git a/eval/public/structs/cel_proto_wrap_util.cc b/eval/public/structs/cel_proto_wrap_util.cc index d0f80171f..08d716d46 100644 --- a/eval/public/structs/cel_proto_wrap_util.cc +++ b/eval/public/structs/cel_proto_wrap_util.cc @@ -16,7 +16,7 @@ #include #include -#include +#include #include #include #include @@ -34,7 +34,6 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/cord.h" -#include "absl/strings/escaping.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" @@ -43,15 +42,12 @@ #include "absl/types/variant.h" #include "eval/public/cel_value.h" #include "eval/public/structs/protobuf_value_factory.h" -#include "internal/overflow.h" #include "internal/proto_time_encoding.h" #include "internal/status_macros.h" -#include "internal/time.h" #include "internal/well_known_types.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" -#include "google/protobuf/message_lite.h" namespace google::api::expr::runtime::internal { @@ -80,24 +76,6 @@ using google::protobuf::DescriptorPool; using google::protobuf::Message; using google::protobuf::MessageFactory; -// kMaxIntJSON is defined as the Number.MAX_SAFE_INTEGER value per EcmaScript 6. -constexpr int64_t kMaxIntJSON = (1ll << 53) - 1; - -// kMinIntJSON is defined as the Number.MIN_SAFE_INTEGER value per EcmaScript 6. -constexpr int64_t kMinIntJSON = -kMaxIntJSON; - -// IsJSONSafe indicates whether the int is safely representable as a floating -// point value in JSON. -static bool IsJSONSafe(int64_t i) { - return i >= kMinIntJSON && i <= kMaxIntJSON; -} - -// IsJSONSafe indicates whether the uint is safely representable as a floating -// point value in JSON. -static bool IsJSONSafe(uint64_t i) { - return i <= static_cast(kMaxIntJSON); -} - // Map implementation wrapping google.protobuf.ListValue class DynamicList : public CelList { public: @@ -133,7 +111,7 @@ class DynamicMap : public CelMap { "Invalid map key type: '", CelValue::TypeName(key.type()), "'")); } - return values_->fields().contains(std::string(str_key.value())); + return values_->fields().contains(str_key.value()); } absl::optional operator[](CelValue key) const override; @@ -691,7 +669,7 @@ class ValueFromMessageMaker { return CreateWellknownTypeValue(message, factory, arena); // WELLKNOWNTYPE_FIELDMASK has no special CelValue type default: - return absl::nullopt; + return std::nullopt; } } @@ -714,742 +692,14 @@ absl::optional DynamicMap::operator[](CelValue key) const { CelValue::TypeName(key.type()), "'"))); } - auto it = values_->fields().find(std::string(str_key.value())); + auto it = values_->fields().find(str_key.value()); if (it == values_->fields().end()) { - return absl::nullopt; + return std::nullopt; } return ValueManager(factory_, arena_).ValueFromMessage(&it->second); } -google::protobuf::Message* DurationFromValue(const google::protobuf::Message* prototype, - const CelValue& value, - google::protobuf::Arena* arena) { - absl::Duration val; - if (!value.GetValue(&val)) { - return nullptr; - } - if (!cel::internal::ValidateDuration(val).ok()) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetDurationReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.UnsafeSetFromAbslDuration(message, val); - return message; -} - -google::protobuf::Message* BoolFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - bool val; - if (!value.GetValue(&val)) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetBoolValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, val); - return message; -} - -google::protobuf::Message* BytesFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - CelValue::BytesHolder view_val; - if (!value.GetValue(&view_val)) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetBytesValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, view_val.value()); - return message; -} - -google::protobuf::Message* DoubleFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - double val; - if (!value.GetValue(&val)) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetDoubleValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, val); - return message; -} - -google::protobuf::Message* FloatFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - double val; - if (!value.GetValue(&val)) { - return nullptr; - } - float fval = val; - // Abort the conversion if the value is outside the float range. - if (val > std::numeric_limits::max()) { - fval = std::numeric_limits::infinity(); - } else if (val < std::numeric_limits::lowest()) { - fval = -std::numeric_limits::infinity(); - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetFloatValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, static_cast(fval)); - return message; -} - -google::protobuf::Message* Int32FromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - int64_t val; - if (!value.GetValue(&val)) { - return nullptr; - } - if (!cel::internal::CheckedInt64ToInt32(val).ok()) { - return nullptr; - } - int32_t ival = static_cast(val); - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetInt32ValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, ival); - return message; -} - -google::protobuf::Message* Int64FromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - int64_t val; - if (!value.GetValue(&val)) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetInt64ValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, val); - return message; -} - -google::protobuf::Message* StringFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - CelValue::StringHolder view_val; - if (!value.GetValue(&view_val)) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetStringValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, view_val.value()); - return message; -} - -google::protobuf::Message* TimestampFromValue(const google::protobuf::Message* prototype, - const CelValue& value, - google::protobuf::Arena* arena) { - absl::Time val; - if (!value.GetValue(&val)) { - return nullptr; - } - if (!cel::internal::ValidateTimestamp(val).ok()) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetTimestampReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.UnsafeSetFromAbslTime(message, val); - return message; -} - -google::protobuf::Message* UInt32FromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - uint64_t val; - if (!value.GetValue(&val)) { - return nullptr; - } - if (!cel::internal::CheckedUint64ToUint32(val).ok()) { - return nullptr; - } - uint32_t ival = static_cast(val); - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetUInt32ValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, ival); - return message; -} - -google::protobuf::Message* UInt64FromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - uint64_t val; - if (!value.GetValue(&val)) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetUInt64ValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, val); - return message; -} - -google::protobuf::Message* ValueFromValue(google::protobuf::Message* message, const CelValue& value, - google::protobuf::Arena* arena); - -google::protobuf::Message* ValueFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - return ValueFromValue(prototype->New(arena), value, arena); -} - -google::protobuf::Message* ListFromValue(google::protobuf::Message* message, const CelValue& value, - google::protobuf::Arena* arena) { - if (!value.IsList()) { - return nullptr; - } - const CelList& list = *value.ListOrDie(); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetListValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - for (int i = 0; i < list.size(); i++) { - auto e = list.Get(arena, i); - auto* elem = reflection.AddValues(message); - if (ValueFromValue(elem, e, arena) == nullptr) { - return nullptr; - } - } - return message; -} - -google::protobuf::Message* ListFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - if (!value.IsList()) { - return nullptr; - } - return ListFromValue(prototype->New(arena), value, arena); -} - -google::protobuf::Message* StructFromValue(google::protobuf::Message* message, - const CelValue& value, google::protobuf::Arena* arena) { - if (!value.IsMap()) { - return nullptr; - } - const CelMap& map = *value.MapOrDie(); - absl::StatusOr keys_or = map.ListKeys(arena); - if (!keys_or.ok()) { - // If map doesn't support listing keys, it can't pack into a Struct value. - // This will surface as a CEL error when the object creation expression - // fails. - return nullptr; - } - const CelList& keys = **keys_or; - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetStructReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - for (int i = 0; i < keys.size(); i++) { - auto k = keys.Get(arena, i); - // If the key is not a string type, abort the conversion. - if (!k.IsString()) { - return nullptr; - } - absl::string_view key = k.StringOrDie().value(); - - auto v = map.Get(arena, k); - if (!v.has_value()) { - return nullptr; - } - auto* field = reflection.InsertField(message, key); - if (ValueFromValue(field, *v, arena) == nullptr) { - return nullptr; - } - } - return message; -} - -google::protobuf::Message* StructFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - if (!value.IsMap()) { - return nullptr; - } - return StructFromValue(prototype->New(arena), value, arena); -} - -google::protobuf::Message* ValueFromValue(google::protobuf::Message* message, const CelValue& value, - google::protobuf::Arena* arena) { - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - switch (value.type()) { - case CelValue::Type::kBool: { - bool val; - if (value.GetValue(&val)) { - reflection.SetBoolValue(message, val); - return message; - } - } break; - case CelValue::Type::kBytes: { - // Base64 encode byte strings to ensure they can safely be transported - // in a JSON string. - CelValue::BytesHolder val; - if (value.GetValue(&val)) { - reflection.SetStringValueFromBytes(message, val.value()); - return message; - } - } break; - case CelValue::Type::kDouble: { - double val; - if (value.GetValue(&val)) { - reflection.SetNumberValue(message, val); - return message; - } - } break; - case CelValue::Type::kDuration: { - // Convert duration values to a protobuf JSON format. - absl::Duration val; - if (value.GetValue(&val)) { - CEL_RETURN_IF_ERROR(cel::internal::ValidateDuration(val)) - .With(IgnoreErrorAndReturnNullptr()); - reflection.SetStringValueFromDuration(message, val); - return message; - } - } break; - case CelValue::Type::kInt64: { - int64_t val; - // Convert int64_t values within the int53 range to doubles, otherwise - // serialize the value to a string. - if (value.GetValue(&val)) { - reflection.SetNumberValue(message, val); - return message; - } - } break; - case CelValue::Type::kString: { - CelValue::StringHolder val; - if (value.GetValue(&val)) { - reflection.SetStringValue(message, val.value()); - return message; - } - } break; - case CelValue::Type::kTimestamp: { - // Convert timestamp values to a protobuf JSON format. - absl::Time val; - if (value.GetValue(&val)) { - CEL_RETURN_IF_ERROR(cel::internal::ValidateTimestamp(val)) - .With(IgnoreErrorAndReturnNullptr()); - reflection.SetStringValueFromTimestamp(message, val); - return message; - } - } break; - case CelValue::Type::kUint64: { - uint64_t val; - // Convert uint64_t values within the int53 range to doubles, otherwise - // serialize the value to a string. - if (value.GetValue(&val)) { - reflection.SetNumberValue(message, val); - return message; - } - } break; - case CelValue::Type::kList: { - if (ListFromValue(reflection.MutableListValue(message), value, arena) != - nullptr) { - return message; - } - } break; - case CelValue::Type::kMap: { - if (StructFromValue(reflection.MutableStructValue(message), value, - arena) != nullptr) { - return message; - } - } break; - case CelValue::Type::kNullType: - reflection.SetNullValue(message); - return message; - break; - default: - return nullptr; - } - return nullptr; -} - -bool ValueFromValue(Value* json, const CelValue& value, google::protobuf::Arena* arena); - -bool ListFromValue(ListValue* json_list, const CelValue& value, - google::protobuf::Arena* arena) { - if (!value.IsList()) { - return false; - } - const CelList& list = *value.ListOrDie(); - for (int i = 0; i < list.size(); i++) { - auto e = list.Get(arena, i); - Value* elem = json_list->add_values(); - if (!ValueFromValue(elem, e, arena)) { - return false; - } - } - return true; -} - -bool StructFromValue(Struct* json_struct, const CelValue& value, - google::protobuf::Arena* arena) { - if (!value.IsMap()) { - return false; - } - const CelMap& map = *value.MapOrDie(); - absl::StatusOr keys_or = map.ListKeys(arena); - if (!keys_or.ok()) { - // If map doesn't support listing keys, it can't pack into a Struct value. - // This will surface as a CEL error when the object creation expression - // fails. - return false; - } - const CelList& keys = **keys_or; - auto fields = json_struct->mutable_fields(); - for (int i = 0; i < keys.size(); i++) { - auto k = keys.Get(arena, i); - // If the key is not a string type, abort the conversion. - if (!k.IsString()) { - return false; - } - absl::string_view key = k.StringOrDie().value(); - - auto v = map.Get(arena, k); - if (!v.has_value()) { - return false; - } - Value field_value; - if (!ValueFromValue(&field_value, *v, arena)) { - return false; - } - (*fields)[std::string(key)] = field_value; - } - return true; -} - -bool ValueFromValue(Value* json, const CelValue& value, google::protobuf::Arena* arena) { - switch (value.type()) { - case CelValue::Type::kBool: { - bool val; - if (value.GetValue(&val)) { - json->set_bool_value(val); - return true; - } - } break; - case CelValue::Type::kBytes: { - // Base64 encode byte strings to ensure they can safely be transported - // in a JSON string. - CelValue::BytesHolder val; - if (value.GetValue(&val)) { - json->set_string_value(absl::Base64Escape(val.value())); - return true; - } - } break; - case CelValue::Type::kDouble: { - double val; - if (value.GetValue(&val)) { - json->set_number_value(val); - return true; - } - } break; - case CelValue::Type::kDuration: { - // Convert duration values to a protobuf JSON format. - absl::Duration val; - if (value.GetValue(&val)) { - auto encode = cel::internal::EncodeDurationToString(val); - if (!encode.ok()) { - return false; - } - json->set_string_value(*encode); - return true; - } - } break; - case CelValue::Type::kInt64: { - int64_t val; - // Convert int64_t values within the int53 range to doubles, otherwise - // serialize the value to a string. - if (value.GetValue(&val)) { - if (IsJSONSafe(val)) { - json->set_number_value(val); - } else { - json->set_string_value(absl::StrCat(val)); - } - return true; - } - } break; - case CelValue::Type::kString: { - CelValue::StringHolder val; - if (value.GetValue(&val)) { - json->set_string_value(val.value()); - return true; - } - } break; - case CelValue::Type::kTimestamp: { - // Convert timestamp values to a protobuf JSON format. - absl::Time val; - if (value.GetValue(&val)) { - auto encode = cel::internal::EncodeTimeToString(val); - if (!encode.ok()) { - return false; - } - json->set_string_value(*encode); - return true; - } - } break; - case CelValue::Type::kUint64: { - uint64_t val; - // Convert uint64_t values within the int53 range to doubles, otherwise - // serialize the value to a string. - if (value.GetValue(&val)) { - if (IsJSONSafe(val)) { - json->set_number_value(val); - } else { - json->set_string_value(absl::StrCat(val)); - } - return true; - } - } break; - case CelValue::Type::kList: - return ListFromValue(json->mutable_list_value(), value, arena); - case CelValue::Type::kMap: - return StructFromValue(json->mutable_struct_value(), value, arena); - case CelValue::Type::kNullType: - json->set_null_value(protobuf::NULL_VALUE); - return true; - default: - return false; - } - return false; -} - -google::protobuf::Message* AnyFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - std::string type_name; - absl::Cord payload; - - // In open source, any->PackFrom() returns void rather than boolean. - switch (value.type()) { - case CelValue::Type::kBool: { - BoolValue v; - type_name = v.GetTypeName(); - v.set_value(value.BoolOrDie()); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kBytes: { - BytesValue v; - type_name = v.GetTypeName(); - v.set_value(std::string(value.BytesOrDie().value())); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kDouble: { - DoubleValue v; - type_name = v.GetTypeName(); - v.set_value(value.DoubleOrDie()); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kDuration: { - Duration v; - if (!cel::internal::EncodeDuration(value.DurationOrDie(), &v).ok()) { - return nullptr; - } - type_name = v.GetTypeName(); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kInt64: { - Int64Value v; - type_name = v.GetTypeName(); - v.set_value(value.Int64OrDie()); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kString: { - StringValue v; - type_name = v.GetTypeName(); - v.set_value(std::string(value.StringOrDie().value())); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kTimestamp: { - Timestamp v; - if (!cel::internal::EncodeTime(value.TimestampOrDie(), &v).ok()) { - return nullptr; - } - type_name = v.GetTypeName(); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kUint64: { - UInt64Value v; - type_name = v.GetTypeName(); - v.set_value(value.Uint64OrDie()); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kList: { - ListValue v; - if (!ListFromValue(&v, value, arena)) { - return nullptr; - } - type_name = v.GetTypeName(); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kMap: { - Struct v; - if (!StructFromValue(&v, value, arena)) { - return nullptr; - } - type_name = v.GetTypeName(); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kNullType: { - Value v; - type_name = v.GetTypeName(); - v.set_null_value(google::protobuf::NULL_VALUE); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kMessage: { - type_name = value.MessageWrapperOrDie().message_ptr()->GetTypeName(); - payload = value.MessageWrapperOrDie().message_ptr()->SerializeAsCord(); - } break; - default: - return nullptr; - } - - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetAnyReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetTypeUrl(message, - absl::StrCat("type.googleapis.com/", type_name)); - reflection.SetValue(message, payload); - return message; -} - -bool IsAlreadyWrapped(google::protobuf::Descriptor::WellKnownType wkt, - const CelValue& value) { - if (value.IsMessage()) { - const auto* msg = value.MessageOrDie(); - if (wkt == msg->GetDescriptor()->well_known_type()) { - return true; - } - } - return false; -} - -// MessageFromValueMaker makes a specific protobuf Message instance based on -// the desired protobuf type name and an input CelValue. -// -// It holds a registry of CelValue factories for specific subtypes of Message. -// If message does not match any of types stored in registry, an the factory -// returns an absent value. -class MessageFromValueMaker { - public: - // Non-copyable, non-assignable - MessageFromValueMaker(const MessageFromValueMaker&) = delete; - MessageFromValueMaker& operator=(const MessageFromValueMaker&) = delete; - - static google::protobuf::Message* MaybeWrapMessage(const google::protobuf::Descriptor* descriptor, - google::protobuf::MessageFactory* factory, - const CelValue& value, - Arena* arena) { - switch (descriptor->well_known_type()) { - case google::protobuf::Descriptor::WELLKNOWNTYPE_DOUBLEVALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return DoubleFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_FLOATVALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return FloatFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_INT64VALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return Int64FromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT64VALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return UInt64FromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_INT32VALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return Int32FromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT32VALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return UInt32FromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_STRINGVALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return StringFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_BYTESVALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return BytesFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_BOOLVALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return BoolFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_ANY: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return AnyFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_DURATION: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return DurationFromValue(factory->GetPrototype(descriptor), value, - arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_TIMESTAMP: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return TimestampFromValue(factory->GetPrototype(descriptor), value, - arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return ValueFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return ListFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return StructFromValue(factory->GetPrototype(descriptor), value, arena); - // WELLKNOWNTYPE_FIELDMASK has no special CelValue type - default: - return nullptr; - } - } -}; - } // namespace CelValue UnwrapMessageToValue(const google::protobuf::Message* value, @@ -1468,12 +718,4 @@ CelValue UnwrapMessageToValue(const google::protobuf::Message* value, return factory(value); } -const google::protobuf::Message* MaybeWrapValueToMessage( - const google::protobuf::Descriptor* descriptor, google::protobuf::MessageFactory* factory, - const CelValue& value, Arena* arena) { - google::protobuf::Message* msg = MessageFromValueMaker::MaybeWrapMessage( - descriptor, factory, value, arena); - return msg; -} - } // namespace google::api::expr::runtime::internal diff --git a/eval/public/structs/cel_proto_wrap_util.h b/eval/public/structs/cel_proto_wrap_util.h index 508985209..1d15c9ead 100644 --- a/eval/public/structs/cel_proto_wrap_util.h +++ b/eval/public/structs/cel_proto_wrap_util.h @@ -28,18 +28,6 @@ CelValue UnwrapMessageToValue(const google::protobuf::Message* value, const ProtobufValueFactory& factory, google::protobuf::Arena* arena); -// MaybeWrapValue attempts to wrap the input value in a proto message with -// the given type_name. If the value can be wrapped, it is returned as a -// protobuf message. Otherwise, the result will be nullptr. -// -// This method is the complement to MaybeUnwrapValue which may unwrap a protobuf -// message to native CelValue representation during a protobuf field read. -// Just as CreateMessage should only be used when reading protobuf values, -// MaybeWrapValue should only be used when assigning protobuf fields. -const google::protobuf::Message* MaybeWrapValueToMessage( - const google::protobuf::Descriptor* descriptor, google::protobuf::MessageFactory* factory, - const CelValue& value, google::protobuf::Arena* arena); - } // namespace google::api::expr::runtime::internal #endif // THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_CEL_PROTO_WRAP_UTIL_H_ diff --git a/eval/public/structs/cel_proto_wrap_util_test.cc b/eval/public/structs/cel_proto_wrap_util_test.cc index 59597fe8f..bbcb64060 100644 --- a/eval/public/structs/cel_proto_wrap_util_test.cc +++ b/eval/public/structs/cel_proto_wrap_util_test.cc @@ -15,32 +15,25 @@ #include "eval/public/structs/cel_proto_wrap_util.h" #include -#include +#include #include #include -#include #include #include "google/protobuf/any.pb.h" #include "google/protobuf/duration.pb.h" #include "google/protobuf/empty.pb.h" +#include "google/protobuf/field_mask.pb.h" #include "google/protobuf/struct.pb.h" #include "google/protobuf/wrappers.pb.h" -#include "absl/base/no_destructor.h" #include "absl/status/status.h" -#include "absl/strings/str_cat.h" -#include "absl/time/time.h" #include "eval/public/cel_value.h" -#include "eval/public/containers/container_backed_list_impl.h" -#include "eval/public/containers/container_backed_map_impl.h" -#include "eval/public/message_wrapper.h" -#include "eval/public/structs/protobuf_value_factory.h" #include "eval/public/structs/trivial_legacy_type_info.h" #include "eval/testutil/test_message.pb.h" #include "internal/proto_time_encoding.h" -#include "internal/status_macros.h" #include "internal/testing.h" #include "testutil/util.h" +#include "google/protobuf/arena.h" #include "google/protobuf/dynamic_message.h" #include "google/protobuf/message.h" @@ -77,40 +70,9 @@ CelValue ProtobufValueFactoryImpl(const google::protobuf::Message* m) { class CelProtoWrapperTest : public ::testing::Test { protected: - CelProtoWrapperTest() {} - - void ExpectWrappedMessage(const CelValue& value, - const google::protobuf::Message& message) { - // Test the input value wraps to the destination message type. - auto* result = MaybeWrapValueToMessage( - message.GetDescriptor(), message.GetReflection()->GetMessageFactory(), - value, arena()); - EXPECT_TRUE(result != nullptr); - EXPECT_THAT(result, testutil::EqualsProto(message)); - - // Ensure that double wrapping results in the object being wrapped once. - auto* identity = MaybeWrapValueToMessage( - message.GetDescriptor(), message.GetReflection()->GetMessageFactory(), - ProtobufValueFactoryImpl(result), arena()); - EXPECT_TRUE(identity == nullptr); - - // Check to make sure that even dynamic messages can be used as input to - // the wrapping call. - result = MaybeWrapValueToMessage( - ReflectedCopy(message)->GetDescriptor(), - ReflectedCopy(message)->GetReflection()->GetMessageFactory(), value, - arena()); - EXPECT_TRUE(result != nullptr); - EXPECT_THAT(result, testutil::EqualsProto(message)); - } + CelProtoWrapperTest() = default; + - void ExpectNotWrapped(const CelValue& value, const google::protobuf::Message& message) { - // Test the input value does not wrap by asserting value == result. - auto result = MaybeWrapValueToMessage( - message.GetDescriptor(), message.GetReflection()->GetMessageFactory(), - value, arena()); - EXPECT_TRUE(result == nullptr); - } template void ExpectUnwrappedPrimitive(const google::protobuf::Message& message, T result) { @@ -299,7 +261,7 @@ TEST_F(CelProtoWrapperTest, UnwrapMessageToValueStruct) { ASSERT_OK(missing_field_presence); EXPECT_FALSE(*missing_field_presence); - const CelList* key_list = cel_map->ListKeys().value(); + ASSERT_OK_AND_ASSIGN(const CelList* key_list, cel_map->ListKeys()); ASSERT_EQ(key_list->size(), kFields.size()); std::vector result_keys; @@ -511,376 +473,6 @@ TEST_F(CelProtoWrapperTest, UnwrapBytesWrapper) { ExpectUnwrappedPrimitive(wrapper, value); } -TEST_F(CelProtoWrapperTest, WrapNull) { - auto cel_value = CelValue::CreateNull(); - - Value json; - json.set_null_value(protobuf::NULL_VALUE); - ExpectWrappedMessage(cel_value, json); - - Any any; - any.PackFrom(json); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapBool) { - auto cel_value = CelValue::CreateBool(true); - - Value json; - json.set_bool_value(true); - ExpectWrappedMessage(cel_value, json); - - BoolValue wrapper; - wrapper.set_value(true); - ExpectWrappedMessage(cel_value, wrapper); - - Any any; - any.PackFrom(wrapper); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapBytes) { - std::string str = "hello world"; - auto cel_value = CelValue::CreateBytes(CelValue::BytesHolder(&str)); - - BytesValue wrapper; - wrapper.set_value(str); - ExpectWrappedMessage(cel_value, wrapper); - - Any any; - any.PackFrom(wrapper); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapBytesToValue) { - std::string str = "hello world"; - auto cel_value = CelValue::CreateBytes(CelValue::BytesHolder(&str)); - - Value json; - json.set_string_value("aGVsbG8gd29ybGQ="); - ExpectWrappedMessage(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapDuration) { - auto cel_value = CelValue::CreateDuration(absl::Seconds(300)); - - Duration d; - d.set_seconds(300); - ExpectWrappedMessage(cel_value, d); - - Any any; - any.PackFrom(d); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapDurationToValue) { - auto cel_value = CelValue::CreateDuration(absl::Seconds(300)); - - Value json; - json.set_string_value("300s"); - ExpectWrappedMessage(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapDouble) { - double num = 1.5; - auto cel_value = CelValue::CreateDouble(num); - - Value json; - json.set_number_value(num); - ExpectWrappedMessage(cel_value, json); - - DoubleValue wrapper; - wrapper.set_value(num); - ExpectWrappedMessage(cel_value, wrapper); - - Any any; - any.PackFrom(wrapper); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapDoubleToFloatValue) { - double num = 1.5; - auto cel_value = CelValue::CreateDouble(num); - - FloatValue wrapper; - wrapper.set_value(num); - ExpectWrappedMessage(cel_value, wrapper); - - // Imprecise double -> float representation results in truncation. - double small_num = -9.9e-100; - wrapper.set_value(small_num); - cel_value = CelValue::CreateDouble(small_num); - ExpectWrappedMessage(cel_value, wrapper); -} - -TEST_F(CelProtoWrapperTest, WrapDoubleOverflow) { - double lowest_double = std::numeric_limits::lowest(); - auto cel_value = CelValue::CreateDouble(lowest_double); - - // Double exceeds float precision, overflow to -infinity. - FloatValue wrapper; - wrapper.set_value(-std::numeric_limits::infinity()); - ExpectWrappedMessage(cel_value, wrapper); - - double max_double = std::numeric_limits::max(); - cel_value = CelValue::CreateDouble(max_double); - - wrapper.set_value(std::numeric_limits::infinity()); - ExpectWrappedMessage(cel_value, wrapper); -} - -TEST_F(CelProtoWrapperTest, WrapInt64) { - int32_t num = std::numeric_limits::lowest(); - auto cel_value = CelValue::CreateInt64(num); - - Value json; - json.set_number_value(static_cast(num)); - ExpectWrappedMessage(cel_value, json); - - Int64Value wrapper; - wrapper.set_value(num); - ExpectWrappedMessage(cel_value, wrapper); - - Any any; - any.PackFrom(wrapper); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapInt64ToInt32Value) { - int32_t num = std::numeric_limits::lowest(); - auto cel_value = CelValue::CreateInt64(num); - - Int32Value wrapper; - wrapper.set_value(num); - ExpectWrappedMessage(cel_value, wrapper); -} - -TEST_F(CelProtoWrapperTest, WrapFailureInt64ToInt32Value) { - int64_t num = std::numeric_limits::lowest(); - auto cel_value = CelValue::CreateInt64(num); - - Int32Value wrapper; - ExpectNotWrapped(cel_value, wrapper); -} - -TEST_F(CelProtoWrapperTest, WrapInt64ToValue) { - int64_t max = std::numeric_limits::max(); - auto cel_value = CelValue::CreateInt64(max); - - Value json; - json.set_string_value(absl::StrCat(max)); - ExpectWrappedMessage(cel_value, json); - - int64_t min = std::numeric_limits::min(); - cel_value = CelValue::CreateInt64(min); - - json.set_string_value(absl::StrCat(min)); - ExpectWrappedMessage(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapUint64) { - uint32_t num = std::numeric_limits::max(); - auto cel_value = CelValue::CreateUint64(num); - - Value json; - json.set_number_value(static_cast(num)); - ExpectWrappedMessage(cel_value, json); - - UInt64Value wrapper; - wrapper.set_value(num); - ExpectWrappedMessage(cel_value, wrapper); - - Any any; - any.PackFrom(wrapper); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapUint64ToUint32Value) { - uint32_t num = std::numeric_limits::max(); - auto cel_value = CelValue::CreateUint64(num); - - UInt32Value wrapper; - wrapper.set_value(num); - ExpectWrappedMessage(cel_value, wrapper); -} - -TEST_F(CelProtoWrapperTest, WrapUint64ToValue) { - uint64_t num = std::numeric_limits::max(); - auto cel_value = CelValue::CreateUint64(num); - - Value json; - json.set_string_value(absl::StrCat(num)); - ExpectWrappedMessage(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapFailureUint64ToUint32Value) { - uint64_t num = std::numeric_limits::max(); - auto cel_value = CelValue::CreateUint64(num); - - UInt32Value wrapper; - ExpectNotWrapped(cel_value, wrapper); -} - -TEST_F(CelProtoWrapperTest, WrapString) { - std::string str = "test"; - auto cel_value = CelValue::CreateString(CelValue::StringHolder(&str)); - - Value json; - json.set_string_value(str); - ExpectWrappedMessage(cel_value, json); - - StringValue wrapper; - wrapper.set_value(str); - ExpectWrappedMessage(cel_value, wrapper); - - Any any; - any.PackFrom(wrapper); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapTimestamp) { - absl::Time ts = absl::FromUnixSeconds(1615852799); - auto cel_value = CelValue::CreateTimestamp(ts); - - Timestamp t; - t.set_seconds(1615852799); - ExpectWrappedMessage(cel_value, t); - - Any any; - any.PackFrom(t); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapTimestampToValue) { - absl::Time ts = absl::FromUnixSeconds(1615852799); - auto cel_value = CelValue::CreateTimestamp(ts); - - Value json; - json.set_string_value("2021-03-15T23:59:59Z"); - ExpectWrappedMessage(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapList) { - std::vector list_elems = { - CelValue::CreateDouble(1.5), - CelValue::CreateInt64(-2L), - }; - ContainerBackedListImpl list(std::move(list_elems)); - auto cel_value = CelValue::CreateList(&list); - - Value json; - json.mutable_list_value()->add_values()->set_number_value(1.5); - json.mutable_list_value()->add_values()->set_number_value(-2.); - ExpectWrappedMessage(cel_value, json); - ExpectWrappedMessage(cel_value, json.list_value()); - - Any any; - any.PackFrom(json.list_value()); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapFailureListValueBadJSON) { - TestMessage message; - std::vector list_elems = { - CelValue::CreateDouble(1.5), - UnwrapMessageToValue(&message, &ProtobufValueFactoryImpl, arena()), - }; - ContainerBackedListImpl list(std::move(list_elems)); - auto cel_value = CelValue::CreateList(&list); - - Value json; - ExpectNotWrapped(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapStruct) { - const std::string kField1 = "field1"; - std::vector> args = { - {CelValue::CreateString(CelValue::StringHolder(&kField1)), - CelValue::CreateBool(true)}}; - auto cel_map = - CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); - auto cel_value = CelValue::CreateMap(cel_map.get()); - - Value json; - (*json.mutable_struct_value()->mutable_fields())[kField1].set_bool_value( - true); - ExpectWrappedMessage(cel_value, json); - ExpectWrappedMessage(cel_value, json.struct_value()); - - Any any; - any.PackFrom(json.struct_value()); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapFailureStructBadKeyType) { - std::vector> args = { - {CelValue::CreateInt64(1L), CelValue::CreateBool(true)}}; - auto cel_map = - CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); - auto cel_value = CelValue::CreateMap(cel_map.get()); - - Value json; - ExpectNotWrapped(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapFailureStructBadValueType) { - const std::string kField1 = "field1"; - TestMessage bad_value; - std::vector> args = { - {CelValue::CreateString(CelValue::StringHolder(&kField1)), - UnwrapMessageToValue(&bad_value, &ProtobufValueFactoryImpl, arena())}}; - auto cel_map = - CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); - auto cel_value = CelValue::CreateMap(cel_map.get()); - Value json; - ExpectNotWrapped(cel_value, json); -} - -class TestMap : public CelMapBuilder { - public: - absl::StatusOr ListKeys() const override { - return absl::UnimplementedError("test"); - } -}; - -TEST_F(CelProtoWrapperTest, WrapFailureStructListKeysUnimplemented) { - const std::string kField1 = "field1"; - TestMap map; - ASSERT_OK(map.Add(CelValue::CreateString(CelValue::StringHolder(&kField1)), - CelValue::CreateString(CelValue::StringHolder(&kField1)))); - - auto cel_value = CelValue::CreateMap(&map); - Value json; - ExpectNotWrapped(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapFailureWrongType) { - auto cel_value = CelValue::CreateNull(); - std::vector wrong_types = { - &BoolValue::default_instance(), &BytesValue::default_instance(), - &DoubleValue::default_instance(), &Duration::default_instance(), - &FloatValue::default_instance(), &Int32Value::default_instance(), - &Int64Value::default_instance(), &ListValue::default_instance(), - &StringValue::default_instance(), &Struct::default_instance(), - &Timestamp::default_instance(), &UInt32Value::default_instance(), - &UInt64Value::default_instance(), - }; - for (const auto* wrong_type : wrong_types) { - ExpectNotWrapped(cel_value, *wrong_type); - } -} - -TEST_F(CelProtoWrapperTest, WrapFailureErrorToAny) { - auto cel_value = CreateNoSuchFieldError(arena(), "error_field"); - ExpectNotWrapped(cel_value, Any::default_instance()); -} - TEST_F(CelProtoWrapperTest, DebugString) { google::protobuf::Empty e; // Note: the value factory is trivial so the debug string for a message-typed diff --git a/eval/public/structs/cel_proto_wrap_value_to_message.cc b/eval/public/structs/cel_proto_wrap_value_to_message.cc new file mode 100644 index 000000000..0b04e93d2 --- /dev/null +++ b/eval/public/structs/cel_proto_wrap_value_to_message.cc @@ -0,0 +1,895 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "eval/public/structs/cel_proto_wrap_value_to_message.h" + +#include +#include +#include +#include +#include + +#include "google/protobuf/any.pb.h" +#include "google/protobuf/duration.pb.h" +#include "google/protobuf/struct.pb.h" +#include "google/protobuf/timestamp.pb.h" +#include "google/protobuf/wrappers.pb.h" +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/cord.h" +#include "absl/strings/escaping.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/time/time.h" +#include "eval/public/cel_value.h" +#include "internal/overflow.h" +#include "internal/proto_time_encoding.h" +#include "internal/status_macros.h" +#include "internal/time.h" +#include "internal/well_known_types.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/json/json.h" +#include "google/protobuf/message.h" + +namespace google::api::expr::runtime::internal { + +namespace { + +using google::protobuf::BoolValue; +using google::protobuf::BytesValue; +using google::protobuf::DoubleValue; +using google::protobuf::Duration; +using google::protobuf::Int64Value; +using google::protobuf::ListValue; +using google::protobuf::StringValue; +using google::protobuf::Struct; +using google::protobuf::Timestamp; +using google::protobuf::UInt64Value; +using google::protobuf::Value; +using google::protobuf::Arena; +using google::protobuf::Descriptor; +using google::protobuf::Message; +using google::protobuf::MessageFactory; +using google::protobuf::json::MessageToJsonString; +using google::protobuf::json::PrintOptions; + +// kMaxIntJSON is defined as the Number.MAX_SAFE_INTEGER value per EcmaScript 6. +constexpr int64_t kMaxIntJSON = (1ll << 53) - 1; + +// kMinIntJSON is defined as the Number.MIN_SAFE_INTEGER value per EcmaScript 6. +constexpr int64_t kMinIntJSON = -kMaxIntJSON; + +// IsJSONSafe indicates whether the int is safely representable as a floating +// point value in JSON. +static bool IsJSONSafe(int64_t i) { + return i >= kMinIntJSON && i <= kMaxIntJSON; +} + +// IsJSONSafe indicates whether the uint is safely representable as a floating +// point value in JSON. +static bool IsJSONSafe(uint64_t i) { + return i <= static_cast(kMaxIntJSON); +} + +static bool IsEmptyProto(const google::protobuf::Descriptor* descriptor) { + return descriptor->full_name() == "google.protobuf.Empty"; +} + +static bool IsFieldMaskProto(const google::protobuf::Descriptor* descriptor) { + return descriptor->full_name() == "google.protobuf.FieldMask"; +} + +static std::optional GetFieldMaskJsonString( + const google::protobuf::Message& message) { + // TODO(b/540507668): Refactor to pipe descriptor_pool through + // ValueFromValue to use internal::MessageToJson. + PrintOptions json_options; + std::string json_str; + auto status = MessageToJsonString(message, &json_str, json_options); + if (!status.ok()) { + ABSL_LOG(ERROR) << "Failed to convert FieldMask to JSON: " << status; + return std::nullopt; + } + // If JSON marshalling is correct, we know we'll always get a plain + // JSON string value and it shouldn't contain any escapes that we need + // to interpret. + if (json_str.size() >= 2 && json_str.front() == '"' && + json_str.back() == '"') { + return json_str.substr(1, json_str.size() - 2); + } + return json_str; +} + +struct IgnoreErrorAndReturnNullptr { + std::nullptr_t operator()(const absl::Status& status) const { + status.IgnoreError(); + return nullptr; + } +}; + +google::protobuf::Message* DurationFromValue(const google::protobuf::Message* prototype, + const CelValue& value, + google::protobuf::Arena* arena) { + absl::Duration val; + if (!value.GetValue(&val)) { + return nullptr; + } + if (!cel::internal::ValidateDuration(val).ok()) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetDurationReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.UnsafeSetFromAbslDuration(message, val); + return message; +} + +google::protobuf::Message* BoolFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + bool val; + if (!value.GetValue(&val)) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetBoolValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, val); + return message; +} + +google::protobuf::Message* BytesFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + CelValue::BytesHolder view_val; + if (!value.GetValue(&view_val)) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetBytesValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, view_val.value()); + return message; +} + +google::protobuf::Message* DoubleFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + double val; + if (!value.GetValue(&val)) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetDoubleValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, val); + return message; +} + +google::protobuf::Message* FloatFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + double val; + if (!value.GetValue(&val)) { + return nullptr; + } + float fval = val; + // Abort the conversion if the value is outside the float range. + if (val > std::numeric_limits::max()) { + fval = std::numeric_limits::infinity(); + } else if (val < std::numeric_limits::lowest()) { + fval = -std::numeric_limits::infinity(); + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetFloatValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, static_cast(fval)); + return message; +} + +google::protobuf::Message* Int32FromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + int64_t val; + if (!value.GetValue(&val)) { + return nullptr; + } + if (!cel::internal::CheckedInt64ToInt32(val).ok()) { + return nullptr; + } + int32_t ival = static_cast(val); + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetInt32ValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, ival); + return message; +} + +google::protobuf::Message* Int64FromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + int64_t val; + if (!value.GetValue(&val)) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetInt64ValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, val); + return message; +} + +google::protobuf::Message* StringFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + CelValue::StringHolder view_val; + if (!value.GetValue(&view_val)) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetStringValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, view_val.value()); + return message; +} + +google::protobuf::Message* TimestampFromValue(const google::protobuf::Message* prototype, + const CelValue& value, + google::protobuf::Arena* arena) { + absl::Time val; + if (!value.GetValue(&val)) { + return nullptr; + } + if (!cel::internal::ValidateTimestamp(val).ok()) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetTimestampReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.UnsafeSetFromAbslTime(message, val); + return message; +} + +google::protobuf::Message* UInt32FromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + uint64_t val; + if (!value.GetValue(&val)) { + return nullptr; + } + if (!cel::internal::CheckedUint64ToUint32(val).ok()) { + return nullptr; + } + uint32_t ival = static_cast(val); + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetUInt32ValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, ival); + return message; +} + +google::protobuf::Message* UInt64FromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + uint64_t val; + if (!value.GetValue(&val)) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetUInt64ValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, val); + return message; +} + +google::protobuf::Message* ValueFromValue(google::protobuf::Message* message, const CelValue& value, + google::protobuf::Arena* arena); + +google::protobuf::Message* ValueFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + return ValueFromValue(prototype->New(arena), value, arena); +} + +google::protobuf::Message* ListFromValue(google::protobuf::Message* message, const CelValue& value, + google::protobuf::Arena* arena) { + if (!value.IsList()) { + return nullptr; + } + const CelList& list = *value.ListOrDie(); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetListValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + for (int i = 0; i < list.size(); i++) { + auto e = list.Get(arena, i); + auto* elem = reflection.AddValues(message); + if (ValueFromValue(elem, e, arena) == nullptr) { + return nullptr; + } + } + return message; +} + +google::protobuf::Message* ListFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + if (!value.IsList()) { + return nullptr; + } + return ListFromValue(prototype->New(arena), value, arena); +} + +google::protobuf::Message* StructFromValue(google::protobuf::Message* message, + const CelValue& value, google::protobuf::Arena* arena) { + if (!value.IsMap()) { + return nullptr; + } + const CelMap& map = *value.MapOrDie(); + absl::StatusOr keys_or = map.ListKeys(arena); + if (!keys_or.ok()) { + // If map doesn't support listing keys, it can't pack into a Struct value. + // This will surface as a CEL error when the object creation expression + // fails. + return nullptr; + } + const CelList& keys = **keys_or; + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetStructReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + for (int i = 0; i < keys.size(); i++) { + auto k = keys.Get(arena, i); + // If the key is not a string type, abort the conversion. + if (!k.IsString()) { + return nullptr; + } + absl::string_view key = k.StringOrDie().value(); + + auto v = map.Get(arena, k); + if (!v.has_value()) { + return nullptr; + } + auto* field = reflection.InsertField(message, key); + if (ValueFromValue(field, *v, arena) == nullptr) { + return nullptr; + } + } + return message; +} + +google::protobuf::Message* StructFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + if (!value.IsMap()) { + return nullptr; + } + return StructFromValue(prototype->New(arena), value, arena); +} + +google::protobuf::Message* ValueFromValue(google::protobuf::Message* message, const CelValue& value, + google::protobuf::Arena* arena) { + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + switch (value.type()) { + case CelValue::Type::kBool: { + bool val; + if (value.GetValue(&val)) { + reflection.SetBoolValue(message, val); + return message; + } + } break; + case CelValue::Type::kBytes: { + // Base64 encode byte strings to ensure they can safely be transported + // in a JSON string. + CelValue::BytesHolder val; + if (value.GetValue(&val)) { + reflection.SetStringValueFromBytes(message, val.value()); + return message; + } + } break; + case CelValue::Type::kDouble: { + double val; + if (value.GetValue(&val)) { + reflection.SetNumberValue(message, val); + return message; + } + } break; + case CelValue::Type::kDuration: { + // Convert duration values to a protobuf JSON format. + absl::Duration val; + if (value.GetValue(&val)) { + CEL_RETURN_IF_ERROR(cel::internal::ValidateDuration(val)) + .With(IgnoreErrorAndReturnNullptr()); + reflection.SetStringValueFromDuration(message, val); + return message; + } + } break; + case CelValue::Type::kInt64: { + int64_t val; + // Convert int64_t values within the int53 range to doubles, otherwise + // serialize the value to a string. + if (value.GetValue(&val)) { + reflection.SetNumberValue(message, val); + return message; + } + } break; + case CelValue::Type::kString: { + CelValue::StringHolder val; + if (value.GetValue(&val)) { + reflection.SetStringValue(message, val.value()); + return message; + } + } break; + case CelValue::Type::kTimestamp: { + // Convert timestamp values to a protobuf JSON format. + absl::Time val; + if (value.GetValue(&val)) { + CEL_RETURN_IF_ERROR(cel::internal::ValidateTimestamp(val)) + .With(IgnoreErrorAndReturnNullptr()); + reflection.SetStringValueFromTimestamp(message, val); + return message; + } + } break; + case CelValue::Type::kUint64: { + uint64_t val; + // Convert uint64_t values within the int53 range to doubles, otherwise + // serialize the value to a string. + if (value.GetValue(&val)) { + reflection.SetNumberValue(message, val); + return message; + } + } break; + case CelValue::Type::kList: { + if (ListFromValue(reflection.MutableListValue(message), value, arena) != + nullptr) { + return message; + } + } break; + case CelValue::Type::kMap: { + if (StructFromValue(reflection.MutableStructValue(message), value, + arena) != nullptr) { + return message; + } + } break; + case CelValue::Type::kMessage: { + const google::protobuf::Message* message_ptr = value.MessageOrDie(); + if (IsEmptyProto(message_ptr->GetDescriptor())) { + reflection.MutableStructValue(message); + return message; + } + if (IsFieldMaskProto(message_ptr->GetDescriptor())) { + std::optional fm_str = + GetFieldMaskJsonString(*message_ptr); + if (fm_str.has_value()) { + reflection.SetStringValue(message, *fm_str); + return message; + } + return nullptr; + } + return nullptr; + } break; + case CelValue::Type::kNullType: + reflection.SetNullValue(message); + return message; + break; + default: + return nullptr; + } + return nullptr; +} + +bool ValueFromValue(Value* json, const CelValue& value, google::protobuf::Arena* arena); + +bool ListFromValue(ListValue* json_list, const CelValue& value, + google::protobuf::Arena* arena) { + if (!value.IsList()) { + return false; + } + const CelList& list = *value.ListOrDie(); + for (int i = 0; i < list.size(); i++) { + auto e = list.Get(arena, i); + Value* elem = json_list->add_values(); + if (!ValueFromValue(elem, e, arena)) { + return false; + } + } + return true; +} + +bool StructFromValue(Struct* json_struct, const CelValue& value, + google::protobuf::Arena* arena) { + if (!value.IsMap()) { + return false; + } + const CelMap& map = *value.MapOrDie(); + absl::StatusOr keys_or = map.ListKeys(arena); + if (!keys_or.ok()) { + // If map doesn't support listing keys, it can't pack into a Struct value. + // This will surface as a CEL error when the object creation expression + // fails. + return false; + } + const CelList& keys = **keys_or; + auto fields = json_struct->mutable_fields(); + for (int i = 0; i < keys.size(); i++) { + auto k = keys.Get(arena, i); + // If the key is not a string type, abort the conversion. + if (!k.IsString()) { + return false; + } + absl::string_view key = k.StringOrDie().value(); + + auto v = map.Get(arena, k); + if (!v.has_value()) { + return false; + } + Value field_value; + if (!ValueFromValue(&field_value, *v, arena)) { + return false; + } + (*fields)[std::string(key)] = field_value; + } + return true; +} + +bool ValueFromValue(Value* json, const CelValue& value, google::protobuf::Arena* arena) { + switch (value.type()) { + case CelValue::Type::kBool: { + bool val; + if (value.GetValue(&val)) { + json->set_bool_value(val); + return true; + } + } break; + case CelValue::Type::kBytes: { + // Base64 encode byte strings to ensure they can safely be transported + // in a JSON string. + CelValue::BytesHolder val; + if (value.GetValue(&val)) { + json->set_string_value(absl::Base64Escape(val.value())); + return true; + } + } break; + case CelValue::Type::kDouble: { + double val; + if (value.GetValue(&val)) { + json->set_number_value(val); + return true; + } + } break; + case CelValue::Type::kDuration: { + // Convert duration values to a protobuf JSON format. + absl::Duration val; + if (value.GetValue(&val)) { + auto encode = cel::internal::EncodeDurationToString(val); + if (!encode.ok()) { + return false; + } + json->set_string_value(*encode); + return true; + } + } break; + case CelValue::Type::kInt64: { + int64_t val; + // Convert int64_t values within the int53 range to doubles, otherwise + // serialize the value to a string. + if (value.GetValue(&val)) { + if (IsJSONSafe(val)) { + json->set_number_value(val); + } else { + json->set_string_value(absl::StrCat(val)); + } + return true; + } + } break; + case CelValue::Type::kString: { + CelValue::StringHolder val; + if (value.GetValue(&val)) { + json->set_string_value(val.value()); + return true; + } + } break; + case CelValue::Type::kTimestamp: { + // Convert timestamp values to a protobuf JSON format. + absl::Time val; + if (value.GetValue(&val)) { + auto encode = cel::internal::EncodeTimeToString(val); + if (!encode.ok()) { + return false; + } + json->set_string_value(*encode); + return true; + } + } break; + case CelValue::Type::kUint64: { + uint64_t val; + // Convert uint64_t values within the int53 range to doubles, otherwise + // serialize the value to a string. + if (value.GetValue(&val)) { + if (IsJSONSafe(val)) { + json->set_number_value(val); + } else { + json->set_string_value(absl::StrCat(val)); + } + return true; + } + } break; + case CelValue::Type::kList: + return ListFromValue(json->mutable_list_value(), value, arena); + case CelValue::Type::kMap: + return StructFromValue(json->mutable_struct_value(), value, arena); + case CelValue::Type::kMessage: { + const google::protobuf::Message* message_ptr = value.MessageOrDie(); + if (IsEmptyProto(message_ptr->GetDescriptor())) { + json->mutable_struct_value(); + return true; + } + if (IsFieldMaskProto(message_ptr->GetDescriptor())) { + std::optional fm_str = + GetFieldMaskJsonString(*message_ptr); + if (fm_str.has_value()) { + json->set_string_value(*fm_str); + return true; + } + return false; + } + return false; + } + case CelValue::Type::kNullType: + json->set_null_value(protobuf::NULL_VALUE); + return true; + default: + return false; + } + return false; +} + +google::protobuf::Message* AnyFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + std::string type_name; + absl::Cord payload; + + // In open source, any->PackFrom() returns void rather than boolean. + switch (value.type()) { + case CelValue::Type::kBool: { + BoolValue v; + type_name = v.GetTypeName(); + v.set_value(value.BoolOrDie()); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kBytes: { + BytesValue v; + type_name = v.GetTypeName(); + v.set_value(value.BytesOrDie().value()); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kDouble: { + DoubleValue v; + type_name = v.GetTypeName(); + v.set_value(value.DoubleOrDie()); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kDuration: { + Duration v; + if (!cel::internal::EncodeDuration(value.DurationOrDie(), &v).ok()) { + return nullptr; + } + type_name = v.GetTypeName(); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kInt64: { + Int64Value v; + type_name = v.GetTypeName(); + v.set_value(value.Int64OrDie()); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kString: { + StringValue v; + type_name = v.GetTypeName(); + v.set_value(std::string(value.StringOrDie().value())); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kTimestamp: { + Timestamp v; + if (!cel::internal::EncodeTime(value.TimestampOrDie(), &v).ok()) { + return nullptr; + } + type_name = v.GetTypeName(); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kUint64: { + UInt64Value v; + type_name = v.GetTypeName(); + v.set_value(value.Uint64OrDie()); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kList: { + ListValue v; + if (!ListFromValue(&v, value, arena)) { + return nullptr; + } + type_name = v.GetTypeName(); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kMap: { + Struct v; + if (!StructFromValue(&v, value, arena)) { + return nullptr; + } + type_name = v.GetTypeName(); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kNullType: { + Value v; + type_name = v.GetTypeName(); + v.set_null_value(google::protobuf::NULL_VALUE); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kMessage: { + type_name = value.MessageWrapperOrDie().message_ptr()->GetTypeName(); + payload = value.MessageWrapperOrDie().message_ptr()->SerializeAsCord(); + } break; + default: + return nullptr; + } + + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetAnyReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetTypeUrl(message, + absl::StrCat("type.googleapis.com/", type_name)); + reflection.SetValue(message, payload); + return message; +} + +bool IsAlreadyWrapped(google::protobuf::Descriptor::WellKnownType wkt, + const CelValue& value) { + if (value.IsMessage()) { + const auto* msg = value.MessageOrDie(); + if (wkt == msg->GetDescriptor()->well_known_type()) { + return true; + } + } + return false; +} + +// MessageFromValueMaker makes a specific protobuf Message instance based on +// the desired protobuf type name and an input CelValue. +// +// It holds a registry of CelValue factories for specific subtypes of Message. +// If message does not match any of types stored in registry, an the factory +// returns an absent value. +class MessageFromValueMaker { + public: + // Non-copyable, non-assignable + MessageFromValueMaker(const MessageFromValueMaker&) = delete; + MessageFromValueMaker& operator=(const MessageFromValueMaker&) = delete; + + static google::protobuf::Message* MaybeWrapMessage(const google::protobuf::Descriptor* descriptor, + google::protobuf::MessageFactory* factory, + const CelValue& value, + Arena* arena) { + switch (descriptor->well_known_type()) { + case google::protobuf::Descriptor::WELLKNOWNTYPE_DOUBLEVALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return DoubleFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_FLOATVALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return FloatFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_INT64VALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return Int64FromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT64VALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return UInt64FromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_INT32VALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return Int32FromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT32VALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return UInt32FromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_STRINGVALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return StringFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_BYTESVALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return BytesFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_BOOLVALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return BoolFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_ANY: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return AnyFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_DURATION: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return DurationFromValue(factory->GetPrototype(descriptor), value, + arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_TIMESTAMP: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return TimestampFromValue(factory->GetPrototype(descriptor), value, + arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return ValueFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return ListFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return StructFromValue(factory->GetPrototype(descriptor), value, arena); + // WELLKNOWNTYPE_FIELDMASK has no special CelValue type + default: + return nullptr; + } + } +}; + +} // namespace + +const google::protobuf::Message* MaybeWrapValueToMessage( + const google::protobuf::Descriptor* descriptor, google::protobuf::MessageFactory* factory, + const CelValue& value, Arena* arena) { + google::protobuf::Message* msg = MessageFromValueMaker::MaybeWrapMessage( + descriptor, factory, value, arena); + return msg; +} + +} // namespace google::api::expr::runtime::internal diff --git a/eval/public/structs/cel_proto_wrap_value_to_message.h b/eval/public/structs/cel_proto_wrap_value_to_message.h new file mode 100644 index 000000000..9bc8581e1 --- /dev/null +++ b/eval/public/structs/cel_proto_wrap_value_to_message.h @@ -0,0 +1,40 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_CEL_PROTO_WRAP_VALUE_TO_MESSAGE_H_ +#define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_CEL_PROTO_WRAP_VALUE_TO_MESSAGE_H_ + +#include "eval/public/cel_value.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" + +namespace google::api::expr::runtime::internal { + +// MaybeWrapValue attempts to wrap the input value in a proto message with +// the given type_name. If the value can be wrapped, it is returned as a +// protobuf message. Otherwise, the result will be nullptr. +// +// This method is the complement to UnwrapMessageToValue which may unwrap a +// protobuf message to native CelValue representation during a protobuf field +// read. +// Just as CreateMessage should only be used when reading protobuf values, +// MaybeWrapValueToMessage should only be used when assigning protobuf fields. +const google::protobuf::Message* MaybeWrapValueToMessage( + const google::protobuf::Descriptor* descriptor, google::protobuf::MessageFactory* factory, + const CelValue& value, google::protobuf::Arena* arena); + +} // namespace google::api::expr::runtime::internal + +#endif // THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_CEL_PROTO_WRAP_VALUE_TO_MESSAGE_H_ diff --git a/eval/public/structs/cel_proto_wrap_value_to_message_test.cc b/eval/public/structs/cel_proto_wrap_value_to_message_test.cc new file mode 100644 index 000000000..40b9398e2 --- /dev/null +++ b/eval/public/structs/cel_proto_wrap_value_to_message_test.cc @@ -0,0 +1,606 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "eval/public/structs/cel_proto_wrap_value_to_message.h" + +#include +#include +#include +#include +#include +#include + +#include "google/protobuf/any.pb.h" +#include "google/protobuf/duration.pb.h" +#include "google/protobuf/empty.pb.h" +#include "google/protobuf/field_mask.pb.h" +#include "google/protobuf/struct.pb.h" +#include "google/protobuf/timestamp.pb.h" +#include "google/protobuf/wrappers.pb.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/strings/str_cat.h" +#include "absl/time/time.h" +#include "absl/types/span.h" +#include "eval/public/cel_value.h" +#include "eval/public/containers/container_backed_list_impl.h" +#include "eval/public/containers/container_backed_map_impl.h" +#include "eval/public/structs/trivial_legacy_type_info.h" +#include "eval/testutil/test_message.pb.h" +#include "internal/testing.h" +#include "testutil/util.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/dynamic_message.h" +#include "google/protobuf/message.h" +#include "google/protobuf/text_format.h" + +namespace google::api::expr::runtime::internal { + +namespace { + +using google::protobuf::Any; +using google::protobuf::BoolValue; +using google::protobuf::BytesValue; +using google::protobuf::DoubleValue; +using google::protobuf::Duration; +using google::protobuf::FieldMask; +using google::protobuf::FloatValue; +using google::protobuf::Int32Value; +using google::protobuf::Int64Value; +using google::protobuf::ListValue; +using google::protobuf::StringValue; +using google::protobuf::Struct; +using google::protobuf::Timestamp; +using google::protobuf::UInt32Value; +using google::protobuf::UInt64Value; +using google::protobuf::Value; +using google::protobuf::Arena; +using ::google::protobuf::TextFormat; + +CelValue MessageToCelValue(const google::protobuf::Message* m) { + return CelValue::CreateMessageWrapper( + CelValue::MessageWrapper(m, TrivialTypeInfo::GetInstance())); +} + +class CelProtoWrapValueToMessageTest : public ::testing::Test { + protected: + CelProtoWrapValueToMessageTest() = default; + + void ExpectWrappedMessage(const CelValue& value, + const google::protobuf::Message& message) { + // Test the input value wraps to the destination message type. + auto* result = MaybeWrapValueToMessage( + message.GetDescriptor(), message.GetReflection()->GetMessageFactory(), + value, arena()); + EXPECT_TRUE(result != nullptr); + EXPECT_THAT(result, testutil::EqualsProto(message)); + + // Ensure that double wrapping results in the object being wrapped once. + auto* identity = MaybeWrapValueToMessage( + message.GetDescriptor(), message.GetReflection()->GetMessageFactory(), + MessageToCelValue(result), arena()); + EXPECT_TRUE(identity == nullptr); + + // Check to make sure that even dynamic messages can be used as input to + // the wrapping call. + result = MaybeWrapValueToMessage( + ReflectedCopy(message)->GetDescriptor(), + ReflectedCopy(message)->GetReflection()->GetMessageFactory(), value, + arena()); + EXPECT_TRUE(result != nullptr); + EXPECT_THAT(result, testutil::EqualsProto(message)); + } + + void ExpectNotWrapped(const CelValue& value, const google::protobuf::Message& message) { + // Test the input value does not wrap by asserting value == result. + auto result = MaybeWrapValueToMessage( + message.GetDescriptor(), message.GetReflection()->GetMessageFactory(), + value, arena()); + EXPECT_TRUE(result == nullptr); + } + + std::unique_ptr ReflectedCopy( + const google::protobuf::Message& message) { + std::unique_ptr dynamic_value( + factory_.GetPrototype(message.GetDescriptor())->New()); + dynamic_value->CopyFrom(message); + return dynamic_value; + } + + Arena* arena() { return &arena_; } + + private: + Arena arena_; + google::protobuf::DynamicMessageFactory factory_; +}; + +TEST_F(CelProtoWrapValueToMessageTest, WrapNull) { + auto cel_value = CelValue::CreateNull(); + + Value json; + json.set_null_value(protobuf::NULL_VALUE); + ExpectWrappedMessage(cel_value, json); + + Any any; + any.PackFrom(json); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapBool) { + auto cel_value = CelValue::CreateBool(true); + + Value json; + json.set_bool_value(true); + ExpectWrappedMessage(cel_value, json); + + BoolValue wrapper; + wrapper.set_value(true); + ExpectWrappedMessage(cel_value, wrapper); + + Any any; + any.PackFrom(wrapper); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapBytes) { + std::string str = "hello world"; + auto cel_value = CelValue::CreateBytes(CelValue::BytesHolder(&str)); + + BytesValue wrapper; + wrapper.set_value(str); + ExpectWrappedMessage(cel_value, wrapper); + + Any any; + any.PackFrom(wrapper); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapBytesToValue) { + std::string str = "hello world"; + auto cel_value = CelValue::CreateBytes(CelValue::BytesHolder(&str)); + + Value json; + json.set_string_value("aGVsbG8gd29ybGQ="); + ExpectWrappedMessage(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapDuration) { + auto cel_value = CelValue::CreateDuration(absl::Seconds(300)); + + Duration d; + d.set_seconds(300); + ExpectWrappedMessage(cel_value, d); + + Any any; + any.PackFrom(d); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapDurationToValue) { + auto cel_value = CelValue::CreateDuration(absl::Seconds(300)); + + Value json; + json.set_string_value("300s"); + ExpectWrappedMessage(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapDouble) { + double num = 1.5; + auto cel_value = CelValue::CreateDouble(num); + + Value json; + json.set_number_value(num); + ExpectWrappedMessage(cel_value, json); + + DoubleValue wrapper; + wrapper.set_value(num); + ExpectWrappedMessage(cel_value, wrapper); + + Any any; + any.PackFrom(wrapper); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapDoubleToFloatValue) { + double num = 1.5; + auto cel_value = CelValue::CreateDouble(num); + + FloatValue wrapper; + wrapper.set_value(num); + ExpectWrappedMessage(cel_value, wrapper); + + // Imprecise double -> float representation results in truncation. + double small_num = -9.9e-100; + wrapper.set_value(small_num); + cel_value = CelValue::CreateDouble(small_num); + ExpectWrappedMessage(cel_value, wrapper); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapDoubleOverflow) { + double lowest_double = std::numeric_limits::lowest(); + auto cel_value = CelValue::CreateDouble(lowest_double); + + // Double exceeds float precision, overflow to -infinity. + FloatValue wrapper; + wrapper.set_value(-std::numeric_limits::infinity()); + ExpectWrappedMessage(cel_value, wrapper); + + double max_double = std::numeric_limits::max(); + cel_value = CelValue::CreateDouble(max_double); + + wrapper.set_value(std::numeric_limits::infinity()); + ExpectWrappedMessage(cel_value, wrapper); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapInt64) { + int32_t num = std::numeric_limits::lowest(); + auto cel_value = CelValue::CreateInt64(num); + + Value json; + json.set_number_value(static_cast(num)); + ExpectWrappedMessage(cel_value, json); + + Int64Value wrapper; + wrapper.set_value(num); + ExpectWrappedMessage(cel_value, wrapper); + + Any any; + any.PackFrom(wrapper); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapInt64ToInt32Value) { + int32_t num = std::numeric_limits::lowest(); + auto cel_value = CelValue::CreateInt64(num); + + Int32Value wrapper; + wrapper.set_value(num); + ExpectWrappedMessage(cel_value, wrapper); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureInt64ToInt32Value) { + int64_t num = std::numeric_limits::lowest(); + auto cel_value = CelValue::CreateInt64(num); + + Int32Value wrapper; + ExpectNotWrapped(cel_value, wrapper); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapInt64ToValue) { + int64_t max = std::numeric_limits::max(); + auto cel_value = CelValue::CreateInt64(max); + + Value json; + json.set_string_value(absl::StrCat(max)); + ExpectWrappedMessage(cel_value, json); + + int64_t min = std::numeric_limits::min(); + cel_value = CelValue::CreateInt64(min); + + json.set_string_value(absl::StrCat(min)); + ExpectWrappedMessage(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapUint64) { + uint32_t num = std::numeric_limits::max(); + auto cel_value = CelValue::CreateUint64(num); + + Value json; + json.set_number_value(static_cast(num)); + ExpectWrappedMessage(cel_value, json); + + UInt64Value wrapper; + wrapper.set_value(num); + ExpectWrappedMessage(cel_value, wrapper); + + Any any; + any.PackFrom(wrapper); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapUint64ToUint32Value) { + uint32_t num = std::numeric_limits::max(); + auto cel_value = CelValue::CreateUint64(num); + + UInt32Value wrapper; + wrapper.set_value(num); + ExpectWrappedMessage(cel_value, wrapper); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapUint64ToValue) { + uint64_t num = std::numeric_limits::max(); + auto cel_value = CelValue::CreateUint64(num); + + Value json; + json.set_string_value(absl::StrCat(num)); + ExpectWrappedMessage(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureUint64ToUint32Value) { + uint64_t num = std::numeric_limits::max(); + auto cel_value = CelValue::CreateUint64(num); + + UInt32Value wrapper; + ExpectNotWrapped(cel_value, wrapper); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapString) { + std::string str = "test"; + auto cel_value = CelValue::CreateString(CelValue::StringHolder(&str)); + + Value json; + json.set_string_value(str); + ExpectWrappedMessage(cel_value, json); + + StringValue wrapper; + wrapper.set_value(str); + ExpectWrappedMessage(cel_value, wrapper); + + Any any; + any.PackFrom(wrapper); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapTimestamp) { + absl::Time ts = absl::FromUnixSeconds(1615852799); + auto cel_value = CelValue::CreateTimestamp(ts); + + Timestamp t; + t.set_seconds(1615852799); + ExpectWrappedMessage(cel_value, t); + + Any any; + any.PackFrom(t); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapTimestampToValue) { + absl::Time ts = absl::FromUnixSeconds(1615852799); + auto cel_value = CelValue::CreateTimestamp(ts); + + Value json; + json.set_string_value("2021-03-15T23:59:59Z"); + ExpectWrappedMessage(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapList) { + std::vector list_elems = { + CelValue::CreateDouble(1.5), + CelValue::CreateInt64(-2L), + }; + ContainerBackedListImpl list(std::move(list_elems)); + auto cel_value = CelValue::CreateList(&list); + + Value json; + json.mutable_list_value()->add_values()->set_number_value(1.5); + json.mutable_list_value()->add_values()->set_number_value(-2.); + ExpectWrappedMessage(cel_value, json); + ExpectWrappedMessage(cel_value, json.list_value()); + + Any any; + any.PackFrom(json.list_value()); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureListValueBadJSON) { + TestMessage message; + std::vector list_elems = { + CelValue::CreateDouble(1.5), + MessageToCelValue(&message), + }; + ContainerBackedListImpl list(std::move(list_elems)); + auto cel_value = CelValue::CreateList(&list); + + Value json; + ExpectNotWrapped(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapStruct) { + const std::string kField1 = "field1"; + std::vector> args = { + {CelValue::CreateString(CelValue::StringHolder(&kField1)), + CelValue::CreateBool(true)}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_map, + CreateContainerBackedMap(absl::MakeSpan(args))); + auto cel_value = CelValue::CreateMap(cel_map.get()); + + Value json; + (*json.mutable_struct_value()->mutable_fields())[kField1].set_bool_value( + true); + ExpectWrappedMessage(cel_value, json); + ExpectWrappedMessage(cel_value, json.struct_value()); + + Any any; + any.PackFrom(json.struct_value()); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureStructBadKeyType) { + std::vector> args = { + {CelValue::CreateInt64(1L), CelValue::CreateBool(true)}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_map, + CreateContainerBackedMap(absl::MakeSpan(args))); + auto cel_value = CelValue::CreateMap(cel_map.get()); + + Value json; + ExpectNotWrapped(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureStructBadValueType) { + const std::string kField1 = "field1"; + TestMessage bad_value; + std::vector> args = { + {CelValue::CreateString(CelValue::StringHolder(&kField1)), + MessageToCelValue(&bad_value)}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_map, + CreateContainerBackedMap(absl::MakeSpan(args))); + auto cel_value = CelValue::CreateMap(cel_map.get()); + Value json; + ExpectNotWrapped(cel_value, json); +} + +class TestMap : public CelMapBuilder { + public: + absl::StatusOr ListKeys() const override { + return absl::UnimplementedError("test"); + } +}; + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureStructListKeysUnimplemented) { + const std::string kField1 = "field1"; + TestMap map; + ASSERT_THAT(map.Add(CelValue::CreateString(CelValue::StringHolder(&kField1)), + CelValue::CreateString(CelValue::StringHolder(&kField1))), + absl_testing::IsOk()); + + auto cel_value = CelValue::CreateMap(&map); + Value json; + ExpectNotWrapped(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureWrongType) { + auto cel_value = CelValue::CreateNull(); + std::vector wrong_types = { + &BoolValue::default_instance(), &BytesValue::default_instance(), + &DoubleValue::default_instance(), &Duration::default_instance(), + &FloatValue::default_instance(), &Int32Value::default_instance(), + &Int64Value::default_instance(), &ListValue::default_instance(), + &StringValue::default_instance(), &Struct::default_instance(), + &Timestamp::default_instance(), &UInt32Value::default_instance(), + &UInt64Value::default_instance(), + }; + for (const auto* wrong_type : wrong_types) { + ExpectNotWrapped(cel_value, *wrong_type); + } +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureErrorToAny) { + auto cel_value = CreateNoSuchFieldError(arena(), "error_field"); + ExpectNotWrapped(cel_value, Any::default_instance()); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFieldMaskToValue) { + FieldMask field_mask; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb( + paths: "foo.bar" paths: "baz" + )pb", + &field_mask)); + CelValue value = MessageToCelValue(&field_mask); + + Value expected_message; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb(string_value: "foo.bar,baz")pb", + &expected_message)); + + ExpectWrappedMessage(value, expected_message); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapMapWithFieldMaskToAny) { + const std::string kField = "field_mask"; + FieldMask field_mask; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb( + paths: "foo.bar" paths: "baz" + )pb", + &field_mask)); + CelValue value = MessageToCelValue(&field_mask); + + std::vector> args = { + {CelValue::CreateString(CelValue::StringHolder(&kField)), value}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_map, + CreateContainerBackedMap(absl::MakeSpan(args))); + CelValue cel_value = CelValue::CreateMap(cel_map.get()); + + Struct expected_struct; + ASSERT_TRUE( + TextFormat::ParseFromString(R"pb( + fields { + key: "field_mask" + value { string_value: "foo.bar,baz" } + } + )pb", + &expected_struct)); + Any expected_message; + ASSERT_TRUE(expected_message.PackFrom(expected_struct)); + + ExpectWrappedMessage(cel_value, expected_message); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapListWithFieldMaskToAny) { + FieldMask field_mask; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb( + paths: "foo.bar" paths: "baz" + )pb", + &field_mask)); + CelValue value = MessageToCelValue(&field_mask); + + std::vector list_entries = {value}; + ContainerBackedListImpl cel_list(list_entries); + CelValue list_value = CelValue::CreateList(&cel_list); + + ListValue expected_list; + ASSERT_TRUE(TextFormat::ParseFromString( + R"pb(values { string_value: "foo.bar,baz" })pb", &expected_list)); + Any expected_message; + ASSERT_TRUE(expected_message.PackFrom(expected_list)); + + ExpectWrappedMessage(list_value, expected_message); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapEmptyToValue) { + google::protobuf::Empty empty; + CelValue value = MessageToCelValue(&empty); + + Value expected_message; + expected_message.mutable_struct_value(); + + ExpectWrappedMessage(value, expected_message); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapMapWithEmptyToAny) { + const std::string kField = "empty"; + google::protobuf::Empty empty; + CelValue value = MessageToCelValue(&empty); + + std::vector> args = { + {CelValue::CreateString(CelValue::StringHolder(&kField)), value}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_map, + CreateContainerBackedMap(absl::MakeSpan(args))); + auto cel_value = CelValue::CreateMap(cel_map.get()); + + Struct expected_struct; + (*expected_struct.mutable_fields())[kField].mutable_struct_value(); + Any expected_message; + ASSERT_TRUE(expected_message.PackFrom(expected_struct)); + + ExpectWrappedMessage(cel_value, expected_message); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapListWithEmptyToAny) { + google::protobuf::Empty empty; + CelValue value = MessageToCelValue(&empty); + + std::vector list_entries = {value}; + ContainerBackedListImpl cel_list(list_entries); + CelValue list_value = CelValue::CreateList(&cel_list); + + ListValue expected_list; + expected_list.add_values()->mutable_struct_value(); + Any expected_message; + ASSERT_TRUE(expected_message.PackFrom(expected_list)); + + ExpectWrappedMessage(list_value, expected_message); +} + +} // namespace + +} // namespace google::api::expr::runtime::internal diff --git a/eval/public/structs/cel_proto_wrapper.cc b/eval/public/structs/cel_proto_wrapper.cc index a1dc83ade..8b85aafc2 100644 --- a/eval/public/structs/cel_proto_wrapper.cc +++ b/eval/public/structs/cel_proto_wrapper.cc @@ -14,11 +14,21 @@ #include "eval/public/structs/cel_proto_wrapper.h" -#include "absl/types/optional.h" +#include + +#include "absl/base/no_destructor.h" +#include "absl/base/nullability.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "common/legacy_value.h" +#include "common/value.h" #include "eval/public/cel_value.h" #include "eval/public/message_wrapper.h" -#include "eval/public/structs/cel_proto_wrap_util.h" +#include "eval/public/structs/cel_proto_wrap_value_to_message.h" #include "eval/public/structs/proto_message_type_adapter.h" +#include "eval/public/structs/trivial_legacy_type_info_internal.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -27,9 +37,36 @@ namespace google::api::expr::runtime { namespace { +using ::cel::interop_internal::TrivialTypeInfo; using ::google::protobuf::Arena; using ::google::protobuf::Descriptor; +using ::google::protobuf::DescriptorPool; using ::google::protobuf::Message; +using ::google::protobuf::MessageFactory; + +// Returns the arena for the given message, or the fallback arena if the +// message does not have an arena. +// +// A global fallback arena is used to avoid allocation when the arena is not +// specified. This is effectively a memory leak, but won't trigger leak +// check analyzers. +// +// This emulates the old behavior of tolerating a nullptr arena without +// triggering a crash. +google::protobuf::Arena* GetArena(const Message* absl_nonnull message, + google::protobuf::Arena* absl_nullable arena) { + if (arena != nullptr) { + return arena; + } + if (message->GetArena() != nullptr) { + return message->GetArena(); + } + static absl::NoDestructor fallback_arena; + ABSL_LOG(WARNING) + << "CelValue: using fallback global arena for wrapping message: " + << message->GetTypeName(); + return fallback_arena.get(); +} } // namespace @@ -38,14 +75,50 @@ CelValue CelProtoWrapper::InternalWrapMessage(const Message* message) { MessageWrapper(message, &GetGenericProtoTypeInfoInstance())); } -// CreateMessage creates CelValue from google::protobuf::Message. -// As some of CEL basic types are subclassing google::protobuf::Message, -// this method contains type checking and downcasts. -CelValue CelProtoWrapper::CreateMessage(const Message* value, Arena* arena) { - return internal::UnwrapMessageToValue(value, &InternalWrapMessage, arena); +CelValue CelProtoWrapper::CreateMessage( + const Message* absl_nonnull value, + const google::protobuf::DescriptorPool* absl_nonnull pool, + MessageFactory* absl_nonnull factory, Arena* absl_nonnull arena) { + ABSL_DCHECK(value != nullptr); + if (value->GetDescriptor() == nullptr || value->GetReflection() == nullptr) { + // This only happens for custom google::protobuf::Message subclasses that CEL can't + // support. + return CelValue::CreateMessageWrapper( + MessageWrapper(value, TrivialTypeInfo::GetInstance())); + } + + auto modern_value = + cel::Value::WrapMessageUnsafe(value, pool, factory, arena); + + absl::StatusOr cel_value = cel::LegacyValue(arena, modern_value); + if (!cel_value.ok()) { + // This only happens for custom google::protobuf::Message subclasses that CEL can't + // support. + auto* status = + google::protobuf::Arena::Create(arena, cel_value.status()); + return CelValue::CreateError(status); + } + return *cel_value; +} + +CelValue CelProtoWrapper::CreateMessage(const Message* absl_nullable value, + Arena* absl_nullable arena) { + if (value == nullptr) { + return CelValue::CreateNull(); + } + + if (value->GetDescriptor() == nullptr || value->GetReflection() == nullptr) { + // This only happens for custom messages subclasses that CEL can't support. + return CelValue::CreateMessageWrapper( + MessageWrapper(value, TrivialTypeInfo::GetInstance())); + } + const auto* pool = value->GetDescriptor()->file()->pool(); + auto* factory = value->GetReflection()->GetMessageFactory(); + arena = GetArena(value, arena); + return CreateMessage(value, pool, factory, arena); } -absl::optional CelProtoWrapper::MaybeWrapValue( +std::optional CelProtoWrapper::MaybeWrapValue( const Descriptor* descriptor, google::protobuf::MessageFactory* factory, const CelValue& value, Arena* arena) { const Message* msg = @@ -53,7 +126,7 @@ absl::optional CelProtoWrapper::MaybeWrapValue( if (msg != nullptr) { return InternalWrapMessage(msg); } else { - return absl::nullopt; + return std::nullopt; } } diff --git a/eval/public/structs/cel_proto_wrapper.h b/eval/public/structs/cel_proto_wrapper.h index 73942c253..8d334961c 100644 --- a/eval/public/structs/cel_proto_wrapper.h +++ b/eval/public/structs/cel_proto_wrapper.h @@ -1,9 +1,11 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_CEL_PROTO_WRAPPER_H_ #define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_CEL_PROTO_WRAPPER_H_ +#include + #include "google/protobuf/duration.pb.h" #include "google/protobuf/timestamp.pb.h" -#include "absl/types/optional.h" +#include "absl/base/nullability.h" #include "eval/public/cel_value.h" #include "internal/proto_time_encoding.h" #include "google/protobuf/arena.h" @@ -17,8 +19,19 @@ class CelProtoWrapper { // CreateMessage creates CelValue from google::protobuf::Message. // As some of CEL basic types are subclassing google::protobuf::Message, // this method contains type checking and downcasts. - static CelValue CreateMessage(const google::protobuf::Message* value, - google::protobuf::Arena* arena); + static CelValue CreateMessage(const google::protobuf::Message* absl_nonnull value, + const google::protobuf::DescriptorPool* absl_nonnull pool, + google::protobuf::MessageFactory* absl_nonnull factory, + google::protobuf::Arena* absl_nonnull arena); + + // Prefer using the overload that takes an explicit descriptor pool and + // message factory instead. This overload will use the ones associated with + // the value. + // + // For backward compatibility, nullptr message is allowed and will result in + // the CEL null_type value. + static CelValue CreateMessage(const google::protobuf::Message* absl_nullable value, + google::protobuf::Arena* absl_nullable arena); // Internal utility for creating a CelValue wrapping a user defined type. // Assumes that the message has been properly unpacked. @@ -43,7 +56,7 @@ class CelProtoWrapper { // message to native CelValue representation during a protobuf field read. // Just as CreateMessage should only be used when reading protobuf values, // MaybeWrapValue should only be used when assigning protobuf fields. - static absl::optional MaybeWrapValue( + static std::optional MaybeWrapValue( const google::protobuf::Descriptor* descriptor, google::protobuf::MessageFactory* factory, const CelValue& value, google::protobuf::Arena* arena); }; diff --git a/eval/public/structs/cel_proto_wrapper_test.cc b/eval/public/structs/cel_proto_wrapper_test.cc index b9fcd6b51..e38dcaa55 100644 --- a/eval/public/structs/cel_proto_wrapper_test.cc +++ b/eval/public/structs/cel_proto_wrapper_test.cc @@ -1,6 +1,7 @@ #include "eval/public/structs/cel_proto_wrapper.h" #include +#include #include #include #include @@ -16,6 +17,7 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/time/time.h" +#include "absl/types/span.h" #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_list_impl.h" #include "eval/public/containers/container_backed_map_impl.h" @@ -24,6 +26,7 @@ #include "internal/status_macros.h" #include "internal/testing.h" #include "testutil/util.h" +#include "google/protobuf/arena.h" #include "google/protobuf/dynamic_message.h" #include "google/protobuf/message.h" @@ -55,7 +58,7 @@ using google::protobuf::Arena; class CelProtoWrapperTest : public ::testing::Test { protected: - CelProtoWrapperTest() {} + CelProtoWrapperTest() = default; void ExpectWrappedMessage(const CelValue& value, const google::protobuf::Message& message) { @@ -101,7 +104,7 @@ class CelProtoWrapperTest : public ::testing::Test { T dyn_value; CelValue cel_dyn_value = - CelProtoWrapper::CreateMessage(ReflectedCopy(message).get(), arena()); + CelProtoWrapper::CreateMessage(ReflectedCopy(message), arena()); EXPECT_THAT(cel_dyn_value.type(), Eq(cel_value.type())); EXPECT_TRUE(cel_dyn_value.GetValue(&dyn_value)); EXPECT_THAT(value, Eq(dyn_value)); @@ -118,10 +121,9 @@ class CelProtoWrapperTest : public ::testing::Test { EXPECT_THAT(cel_value.MessageOrDie(), testutil::EqualsProto(*result)); } - std::unique_ptr ReflectedCopy( - const google::protobuf::Message& message) { - std::unique_ptr dynamic_value( - factory_.GetPrototype(message.GetDescriptor())->New()); + google::protobuf::Message* ReflectedCopy(const google::protobuf::Message& message) { + google::protobuf::Message* dynamic_value = + factory_.GetPrototype(message.GetDescriptor())->New(&arena_); dynamic_value->CopyFrom(message); return dynamic_value; } @@ -210,7 +212,7 @@ TEST_F(CelProtoWrapperTest, UnwrapDynamicValueNull) { value_msg.set_null_value(protobuf::NULL_VALUE); CelValue value = - CelProtoWrapper::CreateMessage(ReflectedCopy(value_msg).get(), arena()); + CelProtoWrapper::CreateMessage(ReflectedCopy(value_msg), arena()); EXPECT_TRUE(value.IsNull()); } @@ -290,7 +292,7 @@ TEST_F(CelProtoWrapperTest, UnwrapValueStruct) { ASSERT_OK(missing_field_presence); EXPECT_FALSE(*missing_field_presence); - const CelList* key_list = cel_map->ListKeys().value(); + ASSERT_OK_AND_ASSIGN(const CelList* key_list, cel_map->ListKeys()); ASSERT_EQ(key_list->size(), kFields.size()); std::vector result_keys; @@ -311,8 +313,8 @@ TEST_F(CelProtoWrapperTest, UnwrapDynamicStruct) { const std::string kFieldBool = "field_bool"; (*struct_msg.mutable_fields())[kFieldInt].set_number_value(1.); (*struct_msg.mutable_fields())[kFieldBool].set_bool_value(true); - CelValue value = - CelProtoWrapper::CreateMessage(ReflectedCopy(struct_msg).get(), arena()); + auto reflected_copy = ReflectedCopy(struct_msg); + CelValue value = CelProtoWrapper::CreateMessage(reflected_copy, arena()); EXPECT_TRUE(value.IsMap()); const CelMap* cel_map = value.MapOrDie(); ASSERT_TRUE(cel_map != nullptr); @@ -352,7 +354,7 @@ TEST_F(CelProtoWrapperTest, UnwrapDynamicValueStruct) { .set_number_value(2); CelValue value = - CelProtoWrapper::CreateMessage(ReflectedCopy(value_msg).get(), arena()); + CelProtoWrapper::CreateMessage(ReflectedCopy(value_msg), arena()); EXPECT_TRUE(value.IsMap()); EXPECT_TRUE( (*value.MapOrDie())[CelValue::CreateString(&kField1)].has_value()); @@ -395,7 +397,7 @@ TEST_F(CelProtoWrapperTest, UnwrapDynamicValueListValue) { value_msg.mutable_list_value()->add_values()->set_number_value(2.); CelValue value = - CelProtoWrapper::CreateMessage(ReflectedCopy(value_msg).get(), arena()); + CelProtoWrapper::CreateMessage(ReflectedCopy(value_msg), arena()); EXPECT_TRUE(value.IsList()); EXPECT_THAT((*value.ListOrDie())[0].DoubleOrDie(), testing::DoubleEq(1)); EXPECT_THAT((*value.ListOrDie())[1].DoubleOrDie(), testing::DoubleEq(2)); @@ -423,6 +425,47 @@ TEST_F(CelProtoWrapperTest, UnwrapInvalidAny) { ASSERT_TRUE(CelProtoWrapper::CreateMessage(&any, arena()).IsError()); } +TEST_F(CelProtoWrapperTest, CreateMessageExplicitPoolAndFactory) { + TestMessage test_message; + test_message.set_string_value("test"); + + CelValue value = CelProtoWrapper::CreateMessage( + &test_message, google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), arena()); + ASSERT_TRUE(value.IsMessage()); + EXPECT_THAT(value.MessageOrDie(), testutil::EqualsProto(test_message)); +} + +TEST_F(CelProtoWrapperTest, CreateMessageExplicitPoolAndFactoryUnpackAny) { + TestMessage test_message; + test_message.set_string_value("test"); + + Any any; + any.PackFrom(test_message); + + google::protobuf::DynamicMessageFactory factory( + google::protobuf::DescriptorPool::generated_pool()); + CelValue value = CelProtoWrapper::CreateMessage( + &any, google::protobuf::DescriptorPool::generated_pool(), &factory, arena()); + ASSERT_TRUE(value.IsMessage()); + EXPECT_THAT(value.MessageOrDie(), testutil::EqualsProto(test_message)); +} + +TEST_F(CelProtoWrapperTest, + CreateMessageExplicitPoolAndFactoryUnpackAnyNotFound) { + TestMessage test_message; + test_message.set_string_value("test"); + + Any any; + any.PackFrom(test_message); + + google::protobuf::DescriptorPool empty_pool; + google::protobuf::DynamicMessageFactory factory(&empty_pool); + CelValue value = + CelProtoWrapper::CreateMessage(&any, &empty_pool, &factory, arena()); + EXPECT_TRUE(value.IsError()); +} + // Test support of google.protobuf.Value wrappers in CelValue. TEST_F(CelProtoWrapperTest, UnwrapBoolWrapper) { bool value = true; @@ -784,10 +827,8 @@ TEST_F(CelProtoWrapperTest, WrapStruct) { std::vector> args = { {CelValue::CreateString(CelValue::StringHolder(&kField1)), CelValue::CreateBool(true)}}; - auto cel_map = - CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_map, + CreateContainerBackedMap(absl::MakeSpan(args))); auto cel_value = CelValue::CreateMap(cel_map.get()); Value json; @@ -801,13 +842,56 @@ TEST_F(CelProtoWrapperTest, WrapStruct) { ExpectWrappedMessage(cel_value, any); } +TEST_F(CelProtoWrapperTest, WrapEmptyToValue) { + google::protobuf::Empty empty; + CelValue value = CelProtoWrapper::CreateMessage(&empty, arena()); + + Value expected_message; + expected_message.mutable_struct_value(); + + ExpectWrappedMessage(value, expected_message); +} + +TEST_F(CelProtoWrapperTest, WrapMapWithEmptyToAny) { + const std::string kField = "empty"; + google::protobuf::Empty empty; + CelValue value = CelProtoWrapper::CreateMessage(&empty, arena()); + + std::vector> args = { + {CelValue::CreateString(CelValue::StringHolder(&kField)), value}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_map, + CreateContainerBackedMap(absl::MakeSpan(args))); + auto cel_value = CelValue::CreateMap(cel_map.get()); + + Struct expected_struct; + (*expected_struct.mutable_fields())[kField].mutable_struct_value(); + Any expected_message; + expected_message.PackFrom(expected_struct); + + ExpectWrappedMessage(cel_value, expected_message); +} + +TEST_F(CelProtoWrapperTest, WrapListWithEmptyToAny) { + google::protobuf::Empty empty; + CelValue value = CelProtoWrapper::CreateMessage(&empty, arena()); + + std::vector list_entries = {value}; + ContainerBackedListImpl cel_list(list_entries); + CelValue list_value = CelValue::CreateList(&cel_list); + + ListValue expected_list; + expected_list.add_values()->mutable_struct_value(); + Any expected_message; + expected_message.PackFrom(expected_list); + + ExpectWrappedMessage(list_value, expected_message); +} + TEST_F(CelProtoWrapperTest, WrapFailureStructBadKeyType) { std::vector> args = { {CelValue::CreateInt64(1L), CelValue::CreateBool(true)}}; - auto cel_map = - CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_map, + CreateContainerBackedMap(absl::MakeSpan(args))); auto cel_value = CelValue::CreateMap(cel_map.get()); Value json; @@ -820,10 +904,8 @@ TEST_F(CelProtoWrapperTest, WrapFailureStructBadValueType) { std::vector> args = { {CelValue::CreateString(CelValue::StringHolder(&kField1)), CelProtoWrapper::CreateMessage(&bad_value, arena())}}; - auto cel_map = - CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_map, + CreateContainerBackedMap(absl::MakeSpan(args))); auto cel_value = CelValue::CreateMap(cel_map.get()); Value json; ExpectNotWrapped(cel_value, json); diff --git a/eval/public/structs/field_access_impl.cc b/eval/public/structs/field_access_impl.cc index 2bd9fff9d..652bcf778 100644 --- a/eval/public/structs/field_access_impl.cc +++ b/eval/public/structs/field_access_impl.cc @@ -29,6 +29,7 @@ #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" #include "eval/public/structs/cel_proto_wrap_util.h" +#include "eval/public/structs/cel_proto_wrap_value_to_message.h" #include "internal/casts.h" #include "internal/overflow.h" #include "google/protobuf/arena.h" diff --git a/eval/public/structs/field_access_impl_benchmark_test.cc b/eval/public/structs/field_access_impl_benchmark_test.cc new file mode 100644 index 000000000..888e424b1 --- /dev/null +++ b/eval/public/structs/field_access_impl_benchmark_test.cc @@ -0,0 +1,239 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "eval/public/cel_options.h" +#include "eval/public/cel_value.h" +#include "eval/public/structs/cel_proto_wrapper.h" +#include "eval/public/structs/field_access_impl.h" +#include "extensions/protobuf/internal/map_reflection.h" +#include "internal/benchmark.h" +#include "cel/expr/conformance/proto3/test_all_types.pb.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/map_field.h" +#include "google/protobuf/message.h" + +namespace google::api::expr::runtime::internal { +namespace { + +using ::cel::expr::conformance::proto3::TestAllTypes; + +void BM_CreateValueFromSingleField_Int64(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + msg.set_single_int64(42); + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("single_int64"); + + for (auto _ : state) { + auto value = CreateValueFromSingleField( + &msg, desc, ProtoWrapperTypeOptions::kUnsetProtoDefault, + &CelProtoWrapper::InternalWrapMessage, &arena); + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_CreateValueFromSingleField_Int64); + +void BM_CreateValueFromSingleField_String(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + msg.set_single_string("hello world"); + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("single_string"); + + for (auto _ : state) { + auto value = CreateValueFromSingleField( + &msg, desc, ProtoWrapperTypeOptions::kUnsetProtoDefault, + &CelProtoWrapper::InternalWrapMessage, &arena); + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_CreateValueFromSingleField_String); + +void BM_CreateValueFromSingleField_Message(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + msg.mutable_standalone_message()->set_bb(123); + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("standalone_message"); + + for (auto _ : state) { + auto value = CreateValueFromSingleField( + &msg, desc, ProtoWrapperTypeOptions::kUnsetProtoDefault, + &CelProtoWrapper::InternalWrapMessage, &arena); + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_CreateValueFromSingleField_Message); + +void BM_CreateValueFromRepeatedField_Int64(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + msg.add_repeated_int64(42); + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("repeated_int64"); + + for (auto _ : state) { + auto value = CreateValueFromRepeatedField( + &msg, desc, 0, &CelProtoWrapper::InternalWrapMessage, &arena); + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_CreateValueFromRepeatedField_Int64); + +void BM_CreateValueFromRepeatedField_String(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + msg.add_repeated_string("hello world"); + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("repeated_string"); + + for (auto _ : state) { + auto value = CreateValueFromRepeatedField( + &msg, desc, 0, &CelProtoWrapper::InternalWrapMessage, &arena); + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_CreateValueFromRepeatedField_String); + +void BM_CreateValueFromMapValue_Int64(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + (*msg.mutable_map_int64_int64())[42] = 100; + const google::protobuf::FieldDescriptor* map_desc = + TestAllTypes::descriptor()->FindFieldByName("map_int64_int64"); + const google::protobuf::FieldDescriptor* value_desc = + map_desc->message_type()->FindFieldByName("value"); + + google::protobuf::ConstMapIterator iter = + cel::extensions::protobuf_internal::ConstMapBegin(*msg.GetReflection(), + msg, *map_desc); + google::protobuf::MapValueConstRef value_ref = iter.GetValueRef(); + + for (auto _ : state) { + auto value = + CreateValueFromMapValue(&msg, value_desc, &value_ref, + &CelProtoWrapper::InternalWrapMessage, &arena); + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_CreateValueFromMapValue_Int64); + +void BM_SetValueToSingleField_Int64(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("single_int64"); + CelValue val = CelValue::CreateInt64(42); + + for (auto _ : state) { + auto status = SetValueToSingleField(val, desc, &msg, &arena); + benchmark::DoNotOptimize(status); + } +} +BENCHMARK(BM_SetValueToSingleField_Int64); + +void BM_SetValueToSingleField_String(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("single_string"); + CelValue val = CelValue::CreateStringView("hello world"); + + for (auto _ : state) { + auto status = SetValueToSingleField(val, desc, &msg, &arena); + benchmark::DoNotOptimize(status); + } +} +BENCHMARK(BM_SetValueToSingleField_String); + +void BM_SetValueToSingleField_Message(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("standalone_message"); + + TestAllTypes::NestedMessage nested_msg; + nested_msg.set_bb(123); + CelValue val = CelProtoWrapper::CreateMessage(&nested_msg, &arena); + + for (auto _ : state) { + auto status = SetValueToSingleField(val, desc, &msg, &arena); + benchmark::DoNotOptimize(status); + } +} +BENCHMARK(BM_SetValueToSingleField_Message); + +void BM_AddValueToRepeatedField_Int64(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("repeated_int64"); + CelValue val = CelValue::CreateInt64(42); + + for (auto _ : state) { + msg.clear_repeated_int64(); + auto status = AddValueToRepeatedField(val, desc, &msg, &arena); + benchmark::DoNotOptimize(status); + } +} +BENCHMARK(BM_AddValueToRepeatedField_Int64); + +void BM_AddValueToRepeatedField_String(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("repeated_string"); + CelValue val = CelValue::CreateStringView("hello world"); + + for (auto _ : state) { + msg.clear_repeated_string(); + auto status = AddValueToRepeatedField(val, desc, &msg, &arena); + benchmark::DoNotOptimize(status); + } +} +BENCHMARK(BM_AddValueToRepeatedField_String); + +void BM_CreateValueFromRepeatedField_StringPiece(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + msg.add_repeated_string_piece("hello world"); + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("repeated_string_piece"); + + for (auto _ : state) { + auto value = CreateValueFromRepeatedField( + &msg, desc, 0, &CelProtoWrapper::InternalWrapMessage, &arena); + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_CreateValueFromRepeatedField_StringPiece); + +void BM_AddValueToRepeatedField_StringPiece(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("repeated_string_piece"); + CelValue val = CelValue::CreateStringView("hello world"); + + for (auto _ : state) { + msg.clear_repeated_string_piece(); + auto status = AddValueToRepeatedField(val, desc, &msg, &arena); + benchmark::DoNotOptimize(status); + } +} +BENCHMARK(BM_AddValueToRepeatedField_StringPiece); + +} // namespace +} // namespace google::api::expr::runtime::internal diff --git a/eval/public/structs/legacy_type_adapter.h b/eval/public/structs/legacy_type_adapter.h index dc7a3ab1b..f526460d7 100644 --- a/eval/public/structs/legacy_type_adapter.h +++ b/eval/public/structs/legacy_type_adapter.h @@ -18,7 +18,7 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_LEGACY_TYPE_ADPATER_H_ #define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_LEGACY_TYPE_ADPATER_H_ -#include +#include #include #include "absl/status/status.h" @@ -32,46 +32,9 @@ namespace google::api::expr::runtime { -// Interface for mutation apis. -// Note: in the new type system, a type provider represents this by returning -// a cel::Type and cel::ValueManager for the type. -class LegacyTypeMutationApis { - public: - virtual ~LegacyTypeMutationApis() = default; - - // Return whether the type defines the given field. - // TODO(uncreated-issue/3): This is only used to eagerly fail during the planning - // phase. Check if it's safe to remove this behavior and fail at runtime. - virtual bool DefinesField(absl::string_view field_name) const = 0; - - // Create a new empty instance of the type. - // May return a status if the type is not possible to create. - virtual absl::StatusOr NewInstance( - cel::MemoryManagerRef memory_manager) const = 0; - - // Normalize special types to a native CEL value after building. - // The interpreter guarantees that instance is uniquely owned by the - // interpreter, and can be safely mutated. - virtual absl::StatusOr AdaptFromWellKnownType( - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder instance) const = 0; - - // Set field on instance to value. - // The interpreter guarantees that instance is uniquely owned by the - // interpreter, and can be safely mutated. - virtual absl::Status SetField( - absl::string_view field_name, const CelValue& value, - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder& instance) const = 0; - - virtual absl::Status SetFieldByNumber( - int64_t field_number [[maybe_unused]], - const CelValue& value [[maybe_unused]], - cel::MemoryManagerRef memory_manager [[maybe_unused]], - CelValue::MessageWrapper::Builder& instance [[maybe_unused]]) const { - return absl::UnimplementedError("SetFieldByNumber is not yet implemented"); - } -}; +// Forward declare permitted subclasses. +class DucktypedMessageAdapter; +class ProtoMessageTypeAdapter; // Interface for access apis. // Note: in new type system this is integrated into the StructValue (via @@ -138,14 +101,19 @@ class LegacyTypeAccessApis { virtual std::vector ListFields( const CelValue::MessageWrapper& instance) const = 0; + + private: + // This class should only be implemented by CEL. Custom structs are only + // supported using the cel::Value APIs. + friend class DucktypedMessageAdapter; + friend class ProtoMessageTypeAdapter; + + LegacyTypeAccessApis() = default; }; // Type information about a legacy Struct type. // Provides methods to the interpreter for interacting with a custom type. // -// mutation_apis() provide equivalent behavior to a cel::Type and -// cel::ValueManager (resolved from a type name). -// // access_apis() provide equivalent behavior to cel::StructValue accessors // (virtual dispatch to a concrete implementation for accessing underlying // values). @@ -155,21 +123,18 @@ class LegacyTypeAccessApis { // the type provider that returned this object. class LegacyTypeAdapter { public: - LegacyTypeAdapter(const LegacyTypeAccessApis* access, - const LegacyTypeMutationApis* mutation) - : access_apis_(access), mutation_apis_(mutation) {} + explicit LegacyTypeAdapter(const LegacyTypeAccessApis* access) + : access_apis_(access) {} + // Temporary constructor to support fakes in client tests. + LegacyTypeAdapter(const LegacyTypeAccessApis* access, std::nullptr_t) + : access_apis_(access) {} // Apis for access for the represented type. // If null, access is not supported (this is an opaque type). - const LegacyTypeAccessApis* access_apis() { return access_apis_; } - - // Apis for mutation for the represented type. - // If null, mutation is not supported (this type cannot be created). - const LegacyTypeMutationApis* mutation_apis() { return mutation_apis_; } + const LegacyTypeAccessApis* access_apis() const { return access_apis_; } private: const LegacyTypeAccessApis* access_apis_; - const LegacyTypeMutationApis* mutation_apis_; }; } // namespace google::api::expr::runtime diff --git a/eval/public/structs/legacy_type_adapter_test.cc b/eval/public/structs/legacy_type_adapter_test.cc index 4c16a59ad..603087b6b 100644 --- a/eval/public/structs/legacy_type_adapter_test.cc +++ b/eval/public/structs/legacy_type_adapter_test.cc @@ -14,49 +14,18 @@ #include "eval/public/structs/legacy_type_adapter.h" -#include - -#include "eval/public/cel_value.h" -#include "eval/public/structs/trivial_legacy_type_info.h" -#include "eval/public/testing/matchers.h" +#include "eval/public/structs/proto_message_type_adapter.h" #include "eval/testutil/test_message.pb.h" -#include "extensions/protobuf/memory_manager.h" -#include "internal/status_macros.h" #include "internal/testing.h" namespace google::api::expr::runtime { namespace { -class TestAccessApiImpl : public LegacyTypeAccessApis { - public: - TestAccessApiImpl() {} - absl::StatusOr HasField( - absl::string_view field_name, - const CelValue::MessageWrapper& value) const override { - return absl::UnimplementedError("Not implemented"); - } - - absl::StatusOr GetField( - absl::string_view field_name, const CelValue::MessageWrapper& instance, - ProtoWrapperTypeOptions unboxing_option, - cel::MemoryManagerRef memory_manager) const override { - return absl::UnimplementedError("Not implemented"); - } - - std::vector ListFields( - const CelValue::MessageWrapper& instance) const override { - return std::vector(); - } -}; - -TEST(LegacyTypeAdapterAccessApis, DefaultAlwaysInequal) { - TestMessage message; - MessageWrapper wrapper(&message, nullptr); - MessageWrapper wrapper2(&message, nullptr); - - TestAccessApiImpl impl; +TEST(LegacyTypeAdapter, Basic) { + ProtoMessageTypeAdapter adapter(TestMessage::descriptor(), nullptr); + LegacyTypeAdapter type_adapter(&adapter); - EXPECT_FALSE(impl.IsEqualTo(wrapper, wrapper2)); + EXPECT_EQ(type_adapter.access_apis(), &adapter); } } // namespace diff --git a/eval/public/structs/legacy_type_info_apis.h b/eval/public/structs/legacy_type_info_apis.h index 4f07470a1..999f12945 100644 --- a/eval/public/structs/legacy_type_info_apis.h +++ b/eval/public/structs/legacy_type_info_apis.h @@ -15,19 +15,26 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_LEGACY_TYPE_INFO_APIS_H_ #define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_LEGACY_TYPE_INFO_APIS_H_ +#include #include #include "absl/base/nullability.h" -#include "absl/status/status.h" #include "absl/strings/string_view.h" #include "eval/public/message_wrapper.h" #include "google/protobuf/descriptor.h" +namespace cel::interop_internal { +struct TrivialTypeInfo; +} // namespace cel::interop_internal + namespace google::api::expr::runtime { // Forward declared to resolve cyclic dependency. class LegacyTypeAccessApis; -class LegacyTypeMutationApis; + +// Forward declare permitted subclasses. +class DucktypedMessageAdapter; +class ProtoMessageTypeAdapter; // Interface for providing type info from a user defined type (represented as a // message). @@ -35,8 +42,8 @@ class LegacyTypeMutationApis; // Provides ability to obtain field access apis, type info, and debug // representation of a message. // -// The message parameter may wrap a nullptr to request generic accessors / -// mutators for the TypeInfo instance if it is available. +// The message parameter may wrap a nullptr to request generic accessors for +// the TypeInfo instance if it is available. // // This is implemented as a separate class from LegacyTypeAccessApis to resolve // cyclic dependency between CelValue (which needs to access these apis to @@ -61,6 +68,11 @@ class LegacyTypeInfoApis { virtual absl::string_view GetTypename( const MessageWrapper& wrapped_message) const = 0; + // Return a pointer to the descriptor for the wrapped message's type. + // + // Should only be defined for messages with standard behavior (i.e. normal + // duck-typed behavior of resolving fields by associated descriptor is + // correct). virtual const google::protobuf::Descriptor* absl_nullable GetDescriptor( const MessageWrapper& wrapped_message [[maybe_unused]]) const { return nullptr; @@ -77,25 +89,23 @@ class LegacyTypeInfoApis { virtual const LegacyTypeAccessApis* GetAccessApis( const MessageWrapper& wrapped_message) const = 0; - // Return a pointer to the wrapped message's mutation api implementation. - // - // The CEL interpreter assumes that the returned pointer is owned externally - // and will outlive any CelValues created by the interpreter. - // - // Nullptr signals that the value does not provide mutation apis. - virtual const LegacyTypeMutationApis* GetMutationApis( - const MessageWrapper& wrapped_message [[maybe_unused]]) const { - return nullptr; - } - // Return a description of the underlying field if defined. // // The underlying string is expected to remain valid as long as the // LegacyTypeInfoApis instance. - virtual absl::optional FindFieldByName( + virtual std::optional FindFieldByName( absl::string_view name [[maybe_unused]]) const { - return absl::nullopt; + return std::nullopt; } + + private: + // This class should only be implemented by CEL. Custom structs are only + // supported using the cel::Value APIs. + friend class DucktypedMessageAdapter; + friend class ProtoMessageTypeAdapter; + friend class cel::interop_internal::TrivialTypeInfo; + + LegacyTypeInfoApis() = default; }; } // namespace google::api::expr::runtime diff --git a/eval/public/structs/legacy_type_provider.cc b/eval/public/structs/legacy_type_provider.cc deleted file mode 100644 index f87ab9645..000000000 --- a/eval/public/structs/legacy_type_provider.cc +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "eval/public/structs/legacy_type_provider.h" - -#include -#include -#include - -#include "absl/base/nullability.h" -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "common/legacy_value.h" -#include "common/memory.h" -#include "common/type.h" -#include "common/type_introspector.h" -#include "common/value.h" -#include "eval/public/message_wrapper.h" -#include "eval/public/structs/legacy_type_adapter.h" -#include "eval/public/structs/legacy_type_info_apis.h" -#include "extensions/protobuf/memory_manager.h" -#include "internal/status_macros.h" -#include "google/protobuf/arena.h" -#include "google/protobuf/message.h" - -namespace google::api::expr::runtime { - -namespace { - -using google::api::expr::runtime::LegacyTypeAdapter; -using google::api::expr::runtime::MessageWrapper; - -class LegacyStructValueBuilder final : public cel::StructValueBuilder { - public: - LegacyStructValueBuilder(cel::MemoryManagerRef memory_manager, - LegacyTypeAdapter adapter, - MessageWrapper::Builder builder) - : memory_manager_(memory_manager), - adapter_(adapter), - builder_(std::move(builder)) {} - - absl::StatusOr> SetFieldByName( - absl::string_view name, cel::Value value) override { - CEL_ASSIGN_OR_RETURN( - auto legacy_value, - LegacyValue(cel::extensions::ProtoMemoryManagerArena(memory_manager_), - value), - _.With(cel::ErrorValueReturn())); - CEL_RETURN_IF_ERROR(adapter_.mutation_apis()->SetField( - name, legacy_value, memory_manager_, builder_)) - .With(cel::ErrorValueReturn()); - return absl::nullopt; - } - - absl::StatusOr> SetFieldByNumber( - int64_t number, cel::Value value) override { - CEL_ASSIGN_OR_RETURN( - auto legacy_value, - LegacyValue(cel::extensions::ProtoMemoryManagerArena(memory_manager_), - value), - _.With(cel::ErrorValueReturn())); - CEL_RETURN_IF_ERROR(adapter_.mutation_apis()->SetFieldByNumber( - number, legacy_value, memory_manager_, builder_)) - .With(cel::ErrorValueReturn()); - return absl::nullopt; - } - - absl::StatusOr Build() && override { - CEL_ASSIGN_OR_RETURN(auto message, - adapter_.mutation_apis()->AdaptFromWellKnownType( - memory_manager_, std::move(builder_))); - if (!message.IsMessage()) { - return absl::FailedPreconditionError("expected MessageWrapper"); - } - auto message_wrapper = message.MessageWrapperOrDie(); - return cel::common_internal::LegacyStructValue( - google::protobuf::DownCastMessage(message_wrapper.message_ptr()), - message_wrapper.legacy_type_info()); - } - - private: - cel::MemoryManagerRef memory_manager_; - LegacyTypeAdapter adapter_; - MessageWrapper::Builder builder_; -}; - -class LegacyValueBuilder final : public cel::ValueBuilder { - public: - LegacyValueBuilder(cel::MemoryManagerRef memory_manager, - LegacyTypeAdapter adapter, MessageWrapper::Builder builder) - : memory_manager_(memory_manager), - adapter_(adapter), - builder_(std::move(builder)) {} - - absl::StatusOr> SetFieldByName( - absl::string_view name, cel::Value value) override { - CEL_ASSIGN_OR_RETURN( - auto legacy_value, - LegacyValue(cel::extensions::ProtoMemoryManagerArena(memory_manager_), - value), - _.With(cel::ErrorValueReturn())); - CEL_RETURN_IF_ERROR(adapter_.mutation_apis()->SetField( - name, legacy_value, memory_manager_, builder_)) - .With(cel::ErrorValueReturn()); - return absl::nullopt; - } - - absl::StatusOr> SetFieldByNumber( - int64_t number, cel::Value value) override { - CEL_ASSIGN_OR_RETURN( - auto legacy_value, - LegacyValue(cel::extensions::ProtoMemoryManagerArena(memory_manager_), - value), - _.With(cel::ErrorValueReturn())); - CEL_RETURN_IF_ERROR(adapter_.mutation_apis()->SetFieldByNumber( - number, legacy_value, memory_manager_, builder_)) - .With(cel::ErrorValueReturn()); - return absl::nullopt; - } - - absl::StatusOr Build() && override { - CEL_ASSIGN_OR_RETURN(auto value, - adapter_.mutation_apis()->AdaptFromWellKnownType( - memory_manager_, std::move(builder_)), - _.With(cel::ErrorValueReturn())); - CEL_ASSIGN_OR_RETURN( - auto result, - cel::ModernValue( - cel::extensions::ProtoMemoryManagerArena(memory_manager_), value), - _.With(cel::ErrorValueReturn())); - return result; - } - - private: - cel::MemoryManagerRef memory_manager_; - LegacyTypeAdapter adapter_; - MessageWrapper::Builder builder_; -}; - -} // namespace - -absl::StatusOr -LegacyTypeProvider::NewValueBuilder( - absl::string_view name, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena) const { - if (auto type_adapter = ProvideLegacyType(name); type_adapter.has_value()) { - const auto* mutation_apis = type_adapter->mutation_apis(); - if (mutation_apis == nullptr) { - return absl::FailedPreconditionError( - absl::StrCat("LegacyTypeMutationApis missing for type: ", name)); - } - CEL_ASSIGN_OR_RETURN( - auto builder, - mutation_apis->NewInstance(cel::MemoryManagerRef::Pooling(arena))); - return std::make_unique( - cel::MemoryManagerRef::Pooling(arena), *type_adapter, - std::move(builder)); - } - return nullptr; -} - -absl::StatusOr> LegacyTypeProvider::FindTypeImpl( - absl::string_view name) const { - if (auto type = cel::FindWellKnownType(name); type.has_value()) { - return type; - } - if (auto type_info = ProvideLegacyTypeInfo(name); type_info.has_value()) { - const auto* descriptor = (*type_info)->GetDescriptor(MessageWrapper()); - if (descriptor != nullptr) { - return cel::MessageType(descriptor); - } - return cel::common_internal::MakeBasicStructType( - (*type_info)->GetTypename(MessageWrapper())); - } - return absl::nullopt; -} - -absl::StatusOr> -LegacyTypeProvider::FindStructTypeFieldByNameImpl( - absl::string_view type, absl::string_view name) const { - if (auto result = cel::FindWellKnownTypeFieldByName(type, name); - result.has_value()) { - return result; - } - if (auto type_info = ProvideLegacyTypeInfo(type); type_info.has_value()) { - if (auto field_desc = (*type_info)->FindFieldByName(name); - field_desc.has_value()) { - return cel::common_internal::BasicStructTypeField( - field_desc->name, field_desc->number, cel::DynType{}); - } else { - const auto* mutation_apis = - (*type_info)->GetMutationApis(MessageWrapper()); - if (mutation_apis == nullptr || !mutation_apis->DefinesField(name)) { - return absl::nullopt; - } - return cel::common_internal::BasicStructTypeField(name, 0, - cel::DynType{}); - } - } - return absl::nullopt; -} - -} // namespace google::api::expr::runtime diff --git a/eval/public/structs/legacy_type_provider.h b/eval/public/structs/legacy_type_provider.h index e2e67411c..b04f7af36 100644 --- a/eval/public/structs/legacy_type_provider.h +++ b/eval/public/structs/legacy_type_provider.h @@ -15,26 +15,23 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TYPE_PROVIDER_H_ #define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TYPE_PROVIDER_H_ +#include + #include "absl/base/attributes.h" -#include "absl/base/nullability.h" -#include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "common/type.h" -#include "common/type_reflector.h" -#include "common/value.h" #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" -#include "google/protobuf/arena.h" -#include "google/protobuf/message.h" namespace google::api::expr::runtime { -// An internal extension of cel::TypeProvider that also deals with legacy types. +// Provides legacy type adapters for google::api::expr::runtime::CelValue +// wrapping messages. +// +// Note: internal use of this type is limited to interop with the Legacy +// (CelValue) runtime with an internal implementation. // -// Note: This API is not finalized. Consult the CEL team before introducing new -// implementations. -class LegacyTypeProvider : public cel::TypeReflector { +// New implementations cannot be registered with the evaluator. +class LegacyTypeProvider { public: virtual ~LegacyTypeProvider() = default; @@ -45,7 +42,7 @@ class LegacyTypeProvider : public cel::TypeReflector { // Returned non-null pointers from the adapter implemententation must remain // valid as long as the type provider. // TODO(uncreated-issue/3): add alternative for new type system. - virtual absl::optional ProvideLegacyType( + virtual std::optional ProvideLegacyType( absl::string_view name) const = 0; // Return LegacyTypeInfoApis for the fully qualified type name if available. @@ -55,23 +52,10 @@ class LegacyTypeProvider : public cel::TypeReflector { // Since custom type providers should create values compatible with evaluator // created ones, the TypeInfoApis returned from this method should be the same // as the ones used in value creation. - virtual absl::optional ProvideLegacyTypeInfo( - ABSL_ATTRIBUTE_UNUSED absl::string_view name) const { - return absl::nullopt; + virtual std::optional ProvideLegacyTypeInfo( + [[maybe_unused]] absl::string_view name) const { + return std::nullopt; } - - absl::StatusOr NewValueBuilder( - absl::string_view name, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena) const final; - - protected: - absl::StatusOr> FindTypeImpl( - absl::string_view name) const final; - - absl::StatusOr> - FindStructTypeFieldByNameImpl(absl::string_view type, - absl::string_view name) const final; }; } // namespace google::api::expr::runtime diff --git a/eval/public/structs/legacy_type_provider_test.cc b/eval/public/structs/legacy_type_provider_test.cc index 160ac49f3..59246fed5 100644 --- a/eval/public/structs/legacy_type_provider_test.cc +++ b/eval/public/structs/legacy_type_provider_test.cc @@ -15,10 +15,12 @@ #include "eval/public/structs/legacy_type_provider.h" #include -#include #include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" +#include "eval/public/structs/trivial_legacy_type_info.h" #include "internal/testing.h" namespace google::api::expr::runtime { @@ -28,29 +30,10 @@ class LegacyTypeProviderTestEmpty : public LegacyTypeProvider { public: absl::optional ProvideLegacyType( absl::string_view name) const override { - return absl::nullopt; + return std::nullopt; } }; -class LegacyTypeInfoApisEmpty : public LegacyTypeInfoApis { - public: - std::string DebugString( - const MessageWrapper& wrapped_message) const override { - return ""; - } - absl::string_view GetTypename( - const MessageWrapper& wrapped_message) const override { - return test_string_; - } - const LegacyTypeAccessApis* GetAccessApis( - const MessageWrapper& wrapped_message) const override { - return nullptr; - } - - private: - const std::string test_string_ = "test"; -}; - class LegacyTypeProviderTestImpl : public LegacyTypeProvider { public: explicit LegacyTypeProviderTestImpl(const LegacyTypeInfoApis* test_type_info) @@ -58,16 +41,16 @@ class LegacyTypeProviderTestImpl : public LegacyTypeProvider { absl::optional ProvideLegacyType( absl::string_view name) const override { if (name == "test") { - return LegacyTypeAdapter(nullptr, nullptr); + return LegacyTypeAdapter(nullptr); } - return absl::nullopt; + return std::nullopt; } absl::optional ProvideLegacyTypeInfo( absl::string_view name) const override { if (name == "test") { return test_type_info_; } - return absl::nullopt; + return std::nullopt; } private: @@ -76,17 +59,16 @@ class LegacyTypeProviderTestImpl : public LegacyTypeProvider { TEST(LegacyTypeProviderTest, EmptyTypeProviderHasProvideTypeInfo) { LegacyTypeProviderTestEmpty provider; - EXPECT_EQ(provider.ProvideLegacyType("test"), absl::nullopt); - EXPECT_EQ(provider.ProvideLegacyTypeInfo("test"), absl::nullopt); + EXPECT_EQ(provider.ProvideLegacyType("test"), std::nullopt); + EXPECT_EQ(provider.ProvideLegacyTypeInfo("test"), std::nullopt); } TEST(LegacyTypeProviderTest, NonEmptyTypeProviderProvidesSomeTypes) { - LegacyTypeInfoApisEmpty test_type_info; - LegacyTypeProviderTestImpl provider(&test_type_info); + LegacyTypeProviderTestImpl provider(TrivialTypeInfo::GetInstance()); EXPECT_TRUE(provider.ProvideLegacyType("test").has_value()); EXPECT_TRUE(provider.ProvideLegacyTypeInfo("test").has_value()); - EXPECT_EQ(provider.ProvideLegacyType("other"), absl::nullopt); - EXPECT_EQ(provider.ProvideLegacyTypeInfo("other"), absl::nullopt); + EXPECT_EQ(provider.ProvideLegacyType("other"), std::nullopt); + EXPECT_EQ(provider.ProvideLegacyTypeInfo("other"), std::nullopt); } } // namespace diff --git a/eval/public/structs/proto_message_type_adapter.cc b/eval/public/structs/proto_message_type_adapter.cc index a351890c2..95127c77e 100644 --- a/eval/public/structs/proto_message_type_adapter.cc +++ b/eval/public/structs/proto_message_type_adapter.cc @@ -15,7 +15,7 @@ #include "eval/public/structs/proto_message_type_adapter.h" #include -#include +#include #include #include #include @@ -26,8 +26,6 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "absl/strings/substitute.h" -#include "absl/types/optional.h" #include "absl/types/span.h" #include "base/attribute.h" #include "common/memory.h" @@ -36,13 +34,11 @@ #include "eval/public/containers/internal_field_backed_list_impl.h" #include "eval/public/containers/internal_field_backed_map_impl.h" #include "eval/public/message_wrapper.h" -#include "eval/public/structs/cel_proto_wrap_util.h" #include "eval/public/structs/field_access_impl.h" #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" #include "extensions/protobuf/internal/qualify.h" #include "extensions/protobuf/memory_manager.h" -#include "internal/casts.h" #include "internal/status_macros.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -142,28 +138,6 @@ absl::StatusOr HasFieldImpl(const google::protobuf::Message* message, return CelFieldIsPresent(message, field_desc, reflection); } -absl::StatusOr CreateCelValueFromField( - const google::protobuf::Message* message, const google::protobuf::FieldDescriptor* field_desc, - ProtoWrapperTypeOptions unboxing_option, google::protobuf::Arena* arena) { - if (field_desc->is_map()) { - auto* map = google::protobuf::Arena::Create( - arena, message, field_desc, &MessageCelValueFactory, arena); - - return CelValue::CreateMap(map); - } - if (field_desc->is_repeated()) { - auto* list = google::protobuf::Arena::Create( - arena, message, field_desc, &MessageCelValueFactory, arena); - return CelValue::CreateList(list); - } - - CEL_ASSIGN_OR_RETURN( - CelValue result, - internal::CreateValueFromSingleField(message, field_desc, unboxing_option, - &MessageCelValueFactory, arena)); - return result; -} - // Shared implementation for GetField. // Handles list or map specific behavior before calling reflection helpers. absl::StatusOr GetFieldImpl(const google::protobuf::Message* message, @@ -200,7 +174,7 @@ class LegacyQualifyState final LegacyQualifyState(const LegacyQualifyState&) = delete; LegacyQualifyState& operator=(const LegacyQualifyState&) = delete; - absl::optional& result() { return result_; } + std::optional& result() { return result_; } private: void SetResultFromError(absl::Status status, @@ -243,7 +217,7 @@ class LegacyQualifyState final return absl::OkStatus(); } - absl::optional result_; + std::optional result_; }; absl::StatusOr QualifyImpl( @@ -301,8 +275,9 @@ std::vector ListFieldsImpl( return field_names; } +} // namespace + class DucktypedMessageAdapter : public LegacyTypeAccessApis, - public LegacyTypeMutationApis, public LegacyTypeInfoApis { public: // Implement field access APIs. @@ -373,46 +348,6 @@ class DucktypedMessageAdapter : public LegacyTypeAccessApis, return message->ShortDebugString(); } - bool DefinesField(absl::string_view field_name) const override { - // Pretend all our fields exist. Real errors will be returned from field - // getters and setters. - return true; - } - - absl::StatusOr NewInstance( - cel::MemoryManagerRef memory_manager) const override { - return absl::UnimplementedError("NewInstance is not implemented"); - } - - absl::StatusOr AdaptFromWellKnownType( - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder instance) const override { - if (!instance.HasFullProto() || instance.message_ptr() == nullptr) { - return absl::UnimplementedError( - "MessageLite is not supported, descriptor is required"); - } - return ProtoMessageTypeAdapter( - static_cast(instance.message_ptr()) - ->GetDescriptor(), - nullptr) - .AdaptFromWellKnownType(memory_manager, instance); - } - - absl::Status SetField( - absl::string_view field_name, const CelValue& value, - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder& instance) const override { - if (!instance.HasFullProto() || instance.message_ptr() == nullptr) { - return absl::UnimplementedError( - "MessageLite is not supported, descriptor is required"); - } - return ProtoMessageTypeAdapter( - static_cast(instance.message_ptr()) - ->GetDescriptor(), - nullptr) - .SetField(field_name, value, memory_manager, instance); - } - std::vector ListFields( const CelValue::MessageWrapper& instance) const override { return ListFieldsImpl(instance); @@ -423,17 +358,18 @@ class DucktypedMessageAdapter : public LegacyTypeAccessApis, return this; } - const LegacyTypeMutationApis* GetMutationApis( - const MessageWrapper& wrapped_message) const override { - return this; - } - static const DucktypedMessageAdapter& GetSingleton() { static absl::NoDestructor instance; return *instance; } + + private: + friend class absl::NoDestructor; + DucktypedMessageAdapter() = default; }; +namespace { + CelValue MessageCelValueFactory(const google::protobuf::Message* message) { return CelValue::CreateMessageWrapper( MessageWrapper(message, &DucktypedMessageAdapter::GetSingleton())); @@ -441,6 +377,28 @@ CelValue MessageCelValueFactory(const google::protobuf::Message* message) { } // namespace +absl::StatusOr CreateCelValueFromField( + const google::protobuf::Message* message, const google::protobuf::FieldDescriptor* field_desc, + ProtoWrapperTypeOptions unboxing_option, google::protobuf::Arena* arena) { + if (field_desc->is_map()) { + auto* map = google::protobuf::Arena::Create( + arena, message, field_desc, &MessageCelValueFactory, arena); + + return CelValue::CreateMap(map); + } + if (field_desc->is_repeated()) { + auto* list = google::protobuf::Arena::Create( + arena, message, field_desc, &MessageCelValueFactory, arena); + return CelValue::CreateList(list); + } + + CEL_ASSIGN_OR_RETURN( + CelValue result, + internal::CreateValueFromSingleField(message, field_desc, unboxing_option, + &MessageCelValueFactory, arena)); + return result; +} + std::string ProtoMessageTypeAdapter::DebugString( const MessageWrapper& wrapped_message) const { if (!wrapped_message.HasFullProto() || @@ -457,70 +415,29 @@ absl::string_view ProtoMessageTypeAdapter::GetTypename( return descriptor_->full_name(); } -const LegacyTypeMutationApis* ProtoMessageTypeAdapter::GetMutationApis( - const MessageWrapper& wrapped_message) const { - // Defer checks for misuse on wrong message kind in the accessor calls. - return this; -} - const LegacyTypeAccessApis* ProtoMessageTypeAdapter::GetAccessApis( const MessageWrapper& wrapped_message) const { // Defer checks for misuse on wrong message kind in the builder calls. return this; } -absl::optional +std::optional ProtoMessageTypeAdapter::FindFieldByName(absl::string_view field_name) const { if (descriptor_ == nullptr) { - return absl::nullopt; + return std::nullopt; } const google::protobuf::FieldDescriptor* field_descriptor = descriptor_->FindFieldByName(field_name); if (field_descriptor == nullptr) { - return absl::nullopt; + return std::nullopt; } return LegacyTypeInfoApis::FieldDescription{field_descriptor->number(), field_descriptor->name()}; } -absl::Status ProtoMessageTypeAdapter::ValidateSetFieldOp( - bool assertion, absl::string_view field, absl::string_view detail) const { - if (!assertion) { - return absl::InvalidArgumentError( - absl::Substitute("SetField failed on message $0, field '$1': $2", - descriptor_->full_name(), field, detail)); - } - return absl::OkStatus(); -} - -absl::StatusOr -ProtoMessageTypeAdapter::NewInstance( - cel::MemoryManagerRef memory_manager) const { - if (message_factory_ == nullptr) { - return absl::UnimplementedError( - absl::StrCat("Cannot create message ", descriptor_->name())); - } - - // This implementation requires arena-backed memory manager. - google::protobuf::Arena* arena = ProtoMemoryManagerArena(memory_manager); - const Message* prototype = message_factory_->GetPrototype(descriptor_); - - Message* msg = (prototype != nullptr) ? prototype->New(arena) : nullptr; - - if (msg == nullptr) { - return absl::InvalidArgumentError( - absl::StrCat("Failed to create message ", descriptor_->name())); - } - return MessageWrapper::Builder(msg); -} - -bool ProtoMessageTypeAdapter::DefinesField(absl::string_view field_name) const { - return descriptor_->FindFieldByName(field_name) != nullptr; -} - absl::StatusOr ProtoMessageTypeAdapter::HasField( absl::string_view field_name, const CelValue::MessageWrapper& value) const { CEL_ASSIGN_OR_RETURN(const google::protobuf::Message* message, @@ -551,120 +468,6 @@ ProtoMessageTypeAdapter::Qualify( memory_manager); } -absl::Status ProtoMessageTypeAdapter::SetField( - const google::protobuf::FieldDescriptor* field, const CelValue& value, - google::protobuf::Arena* arena, google::protobuf::Message* message) const { - if (field->is_map()) { - constexpr int kKeyField = 1; - constexpr int kValueField = 2; - - const CelMap* cel_map; - CEL_RETURN_IF_ERROR(ValidateSetFieldOp( - value.GetValue(&cel_map) && cel_map != nullptr, - field->name(), - absl::StrCat("value is not CelMap - value is ", - CelValue::TypeName(value.type())))); - - auto entry_descriptor = field->message_type(); - - CEL_RETURN_IF_ERROR( - ValidateSetFieldOp(entry_descriptor != nullptr, field->name(), - "failed to find map entry descriptor")); - auto key_field_descriptor = entry_descriptor->FindFieldByNumber(kKeyField); - auto value_field_descriptor = - entry_descriptor->FindFieldByNumber(kValueField); - - CEL_RETURN_IF_ERROR( - ValidateSetFieldOp(key_field_descriptor != nullptr, field->name(), - "failed to find key field descriptor")); - - CEL_RETURN_IF_ERROR( - ValidateSetFieldOp(value_field_descriptor != nullptr, field->name(), - "failed to find value field descriptor")); - - CEL_ASSIGN_OR_RETURN(const CelList* key_list, cel_map->ListKeys(arena)); - for (int i = 0; i < key_list->size(); i++) { - CelValue key = (*key_list).Get(arena, i); - - auto value = (*cel_map).Get(arena, key); - CEL_RETURN_IF_ERROR(ValidateSetFieldOp(value.has_value(), field->name(), - "error serializing CelMap")); - Message* entry_msg = message->GetReflection()->AddMessage(message, field); - CEL_RETURN_IF_ERROR(internal::SetValueToSingleField( - key, key_field_descriptor, entry_msg, arena)); - CEL_RETURN_IF_ERROR(internal::SetValueToSingleField( - value.value(), value_field_descriptor, entry_msg, arena)); - } - - } else if (field->is_repeated()) { - const CelList* cel_list; - CEL_RETURN_IF_ERROR(ValidateSetFieldOp( - value.GetValue(&cel_list) && cel_list != nullptr, - field->name(), - absl::StrCat("expected CelList value - value is", - CelValue::TypeName(value.type())))); - - for (int i = 0; i < cel_list->size(); i++) { - CEL_RETURN_IF_ERROR(internal::AddValueToRepeatedField( - (*cel_list).Get(arena, i), field, message, arena)); - } - } else { - CEL_RETURN_IF_ERROR( - internal::SetValueToSingleField(value, field, message, arena)); - } - return absl::OkStatus(); -} - -absl::Status ProtoMessageTypeAdapter::SetField( - absl::string_view field_name, const CelValue& value, - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder& instance) const { - // Assume proto arena implementation if this provider is used. - google::protobuf::Arena* arena = - cel::extensions::ProtoMemoryManagerArena(memory_manager); - - CEL_ASSIGN_OR_RETURN(google::protobuf::Message * mutable_message, - UnwrapMessage(instance, "SetField")); - - const google::protobuf::FieldDescriptor* field_descriptor = - descriptor_->FindFieldByName(field_name); - CEL_RETURN_IF_ERROR( - ValidateSetFieldOp(field_descriptor != nullptr, field_name, "not found")); - - return SetField(field_descriptor, value, arena, mutable_message); -} - -absl::Status ProtoMessageTypeAdapter::SetFieldByNumber( - int64_t field_number, const CelValue& value, - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder& instance) const { - // Assume proto arena implementation if this provider is used. - google::protobuf::Arena* arena = - cel::extensions::ProtoMemoryManagerArena(memory_manager); - - CEL_ASSIGN_OR_RETURN(google::protobuf::Message * mutable_message, - UnwrapMessage(instance, "SetField")); - - const google::protobuf::FieldDescriptor* field_descriptor = - descriptor_->FindFieldByNumber(field_number); - CEL_RETURN_IF_ERROR(ValidateSetFieldOp( - field_descriptor != nullptr, absl::StrCat(field_number), "not found")); - - return SetField(field_descriptor, value, arena, mutable_message); -} - -absl::StatusOr ProtoMessageTypeAdapter::AdaptFromWellKnownType( - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder instance) const { - // Assume proto arena implementation if this provider is used. - google::protobuf::Arena* arena = - cel::extensions::ProtoMemoryManagerArena(memory_manager); - CEL_ASSIGN_OR_RETURN(google::protobuf::Message * message, - UnwrapMessage(instance, "AdaptFromWellKnownType")); - return internal::UnwrapMessageToValue(message, &MessageCelValueFactory, - arena); -} - bool ProtoMessageTypeAdapter::IsEqualTo( const CelValue::MessageWrapper& instance, const CelValue::MessageWrapper& other_instance) const { diff --git a/eval/public/structs/proto_message_type_adapter.h b/eval/public/structs/proto_message_type_adapter.h index f2fc43a8a..aa609017b 100644 --- a/eval/public/structs/proto_message_type_adapter.h +++ b/eval/public/structs/proto_message_type_adapter.h @@ -21,12 +21,16 @@ #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "base/attribute.h" #include "common/memory.h" #include "eval/public/cel_options.h" #include "eval/public/cel_value.h" #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" +#include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" namespace google::api::expr::runtime { @@ -36,12 +40,12 @@ namespace google::api::expr::runtime { // generally the duck-typed instance to support the default behavior of // deferring to the protobuf reflection apis on the message instance. class ProtoMessageTypeAdapter : public LegacyTypeInfoApis, - public LegacyTypeAccessApis, - public LegacyTypeMutationApis { + public LegacyTypeAccessApis { public: - ProtoMessageTypeAdapter(const google::protobuf::Descriptor* descriptor, - google::protobuf::MessageFactory* message_factory) - : message_factory_(message_factory), descriptor_(descriptor) {} + explicit ProtoMessageTypeAdapter( + const google::protobuf::Descriptor* absl_nonnull descriptor, + google::protobuf::MessageFactory* message_factory = nullptr) + : descriptor_(descriptor) {} ~ProtoMessageTypeAdapter() override = default; @@ -59,32 +63,9 @@ class ProtoMessageTypeAdapter : public LegacyTypeInfoApis, const LegacyTypeAccessApis* GetAccessApis( const MessageWrapper& wrapped_message) const override; - const LegacyTypeMutationApis* GetMutationApis( - const MessageWrapper& wrapped_message) const override; - absl::optional FindFieldByName( absl::string_view field_name) const override; - // Implement LegacyTypeMutation APIs. - absl::StatusOr NewInstance( - cel::MemoryManagerRef memory_manager) const override; - - bool DefinesField(absl::string_view field_name) const override; - - absl::Status SetField( - absl::string_view field_name, const CelValue& value, - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder& instance) const override; - - absl::Status SetFieldByNumber( - int64_t field_number, const CelValue& value, - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder& instance) const override; - - absl::StatusOr AdaptFromWellKnownType( - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder instance) const override; - // Implement LegacyTypeAccessAPIs. absl::StatusOr GetField( absl::string_view field_name, const CelValue::MessageWrapper& instance, @@ -106,19 +87,21 @@ class ProtoMessageTypeAdapter : public LegacyTypeInfoApis, std::vector ListFields( const CelValue::MessageWrapper& instance) const override; - private: - // Helper for standardizing error messages for SetField operation. - absl::Status ValidateSetFieldOp(bool assertion, absl::string_view field, - absl::string_view detail) const; - - absl::Status SetField(const google::protobuf::FieldDescriptor* field, - const CelValue& value, google::protobuf::Arena* arena, - google::protobuf::Message* message) const; + const google::protobuf::Descriptor* absl_nonnull descriptor() const { + return descriptor_; + } - google::protobuf::MessageFactory* message_factory_; - const google::protobuf::Descriptor* descriptor_; + private: + const google::protobuf::Descriptor* absl_nonnull descriptor_; }; +// Creates a CelValue from the given field on the proto message. This is the +// shared implementation for ProtoMessageTypeAdapter and +// DucktypedMessageAdapter. +absl::StatusOr CreateCelValueFromField( + const google::protobuf::Message* message, const google::protobuf::FieldDescriptor* field_desc, + ProtoWrapperTypeOptions unboxing_option, google::protobuf::Arena* arena); + // Returns a TypeInfo provider representing an arbitrary message. // This allows for the legacy duck-typed behavior of messages on field access // instead of expecting a particular message type given a TypeInfo. diff --git a/eval/public/structs/proto_message_type_adapter_test.cc b/eval/public/structs/proto_message_type_adapter_test.cc index 32608bc3f..b44c17062 100644 --- a/eval/public/structs/proto_message_type_adapter_test.cc +++ b/eval/public/structs/proto_message_type_adapter_test.cc @@ -14,42 +14,42 @@ #include "eval/public/structs/proto_message_type_adapter.h" +#include #include #include "google/protobuf/wrappers.pb.h" #include "google/protobuf/descriptor.pb.h" #include "absl/status/status.h" #include "base/attribute.h" +#include "common/legacy_value.h" #include "common/value.h" +#include "common/value_testing.h" #include "eval/public/cel_value.h" -#include "eval/public/containers/container_backed_list_impl.h" -#include "eval/public/containers/container_backed_map_impl.h" #include "eval/public/message_wrapper.h" #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" +#include "eval/public/structs/trivial_legacy_type_info.h" #include "eval/public/testing/matchers.h" #include "eval/testutil/test_message.pb.h" #include "extensions/protobuf/memory_manager.h" -#include "internal/proto_matchers.h" #include "internal/testing.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" +#include "google/protobuf/message_lite.h" namespace google::api::expr::runtime { namespace { +using ::absl_testing::IsOk; using ::absl_testing::IsOkAndHolds; using ::absl_testing::StatusIs; using ::cel::ProtoWrapperTypeOptions; using ::cel::extensions::ProtoMemoryManagerRef; -using ::cel::internal::test::EqualsProto; using ::google::protobuf::Int64Value; using ::testing::_; -using ::testing::AllOf; using ::testing::ElementsAre; -using ::testing::Eq; using ::testing::Field; using ::testing::HasSubstr; using ::testing::Optional; @@ -423,247 +423,6 @@ TEST(GetGenericProtoTypeInfoInstance, FallbackForNonMessage) { EXPECT_EQ(info_api.DebugString(null_message), ""); } -TEST(ProtoMessageTypeAdapter, NewInstance) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder result, - adapter.NewInstance(manager)); - EXPECT_EQ(result.message_ptr()->SerializeAsString(), ""); -} - -TEST(ProtoMessageTypeAdapter, NewInstanceUnsupportedDescriptor) { - google::protobuf::Arena arena; - - google::protobuf::DescriptorPool pool; - google::protobuf::FileDescriptorProto faked_file; - faked_file.set_name("faked.proto"); - faked_file.set_syntax("proto3"); - faked_file.set_package("google.api.expr.runtime"); - auto msg_descriptor = faked_file.add_message_type(); - msg_descriptor->set_name("FakeMessage"); - pool.BuildFile(faked_file); - - ProtoMessageTypeAdapter adapter( - pool.FindMessageTypeByName("google.api.expr.runtime.FakeMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - // Message factory doesn't know how to create our custom message, even though - // we provided a descriptor for it. - EXPECT_THAT( - adapter.NewInstance(manager), - StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("FakeMessage"))); -} - -TEST(ProtoMessageTypeAdapter, DefinesField) { - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - - EXPECT_TRUE(adapter.DefinesField("int64_value")); - EXPECT_FALSE(adapter.DefinesField("not_a_field")); -} - -TEST(ProtoMessageTypeAdapter, SetFieldSingular) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder value, - adapter.NewInstance(manager)); - - ASSERT_OK(adapter.SetField("int64_value", CelValue::CreateInt64(10), manager, - value)); - - TestMessage message; - message.set_int64_value(10); - EXPECT_EQ(value.message_ptr()->SerializeAsString(), - message.SerializeAsString()); - - ASSERT_THAT(adapter.SetField("not_a_field", CelValue::CreateInt64(10), - manager, value), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("field 'not_a_field': not found"))); -} - -TEST(ProtoMessageTypeAdapter, SetFieldRepeated) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - ContainerBackedListImpl list( - {CelValue::CreateInt64(1), CelValue::CreateInt64(2)}); - CelValue value_to_set = CelValue::CreateList(&list); - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder instance, - adapter.NewInstance(manager)); - - ASSERT_OK(adapter.SetField("int64_list", value_to_set, manager, instance)); - - TestMessage message; - message.add_int64_list(1); - message.add_int64_list(2); - - EXPECT_EQ(instance.message_ptr()->SerializeAsString(), - message.SerializeAsString()); -} - -TEST(ProtoMessageTypeAdapter, SetFieldNotAField) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder instance, - adapter.NewInstance(manager)); - - ASSERT_THAT(adapter.SetField("not_a_field", CelValue::CreateInt64(10), - manager, instance), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("field 'not_a_field': not found"))); -} - -TEST(ProtoMesssageTypeAdapter, SetFieldWrongType) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - ContainerBackedListImpl list( - {CelValue::CreateInt64(1), CelValue::CreateInt64(2)}); - CelValue list_value = CelValue::CreateList(&list); - - CelMapBuilder builder; - ASSERT_OK(builder.Add(CelValue::CreateInt64(1), CelValue::CreateInt64(2))); - ASSERT_OK(builder.Add(CelValue::CreateInt64(2), CelValue::CreateInt64(4))); - - CelValue map_value = CelValue::CreateMap(&builder); - - CelValue int_value = CelValue::CreateInt64(42); - - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder instance, - adapter.NewInstance(manager)); - - EXPECT_THAT(adapter.SetField("int64_value", map_value, manager, instance), - StatusIs(absl::StatusCode::kInvalidArgument)); - EXPECT_THAT(adapter.SetField("int64_value", list_value, manager, instance), - StatusIs(absl::StatusCode::kInvalidArgument)); - - EXPECT_THAT( - adapter.SetField("int64_int32_map", list_value, manager, instance), - StatusIs(absl::StatusCode::kInvalidArgument)); - EXPECT_THAT(adapter.SetField("int64_int32_map", int_value, manager, instance), - StatusIs(absl::StatusCode::kInvalidArgument)); - - EXPECT_THAT(adapter.SetField("int64_list", int_value, manager, instance), - StatusIs(absl::StatusCode::kInvalidArgument)); - EXPECT_THAT(adapter.SetField("int64_list", map_value, manager, instance), - StatusIs(absl::StatusCode::kInvalidArgument)); -} - -TEST(ProtoMesssageTypeAdapter, SetFieldNotAMessage) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - CelValue int_value = CelValue::CreateInt64(42); - CelValue::MessageWrapper::Builder instance( - static_cast(nullptr)); - - EXPECT_THAT(adapter.SetField("int64_value", int_value, manager, instance), - StatusIs(absl::StatusCode::kInternal)); -} - -TEST(ProtoMesssageTypeAdapter, SetFieldNullMessage) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - CelValue int_value = CelValue::CreateInt64(42); - CelValue::MessageWrapper::Builder instance( - static_cast(nullptr)); - - EXPECT_THAT(adapter.SetField("int64_value", int_value, manager, instance), - StatusIs(absl::StatusCode::kInternal)); -} - -TEST(ProtoMessageTypeAdapter, AdaptFromWellKnownType) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.protobuf.Int64Value"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder instance, - adapter.NewInstance(manager)); - ASSERT_OK( - adapter.SetField("value", CelValue::CreateInt64(42), manager, instance)); - - ASSERT_OK_AND_ASSIGN(CelValue value, - adapter.AdaptFromWellKnownType(manager, instance)); - - EXPECT_THAT(value, test::IsCelInt64(42)); -} - -TEST(ProtoMessageTypeAdapter, AdaptFromWellKnownTypeUnspecial) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder instance, - adapter.NewInstance(manager)); - - ASSERT_OK(adapter.SetField("int64_value", CelValue::CreateInt64(42), manager, - instance)); - ASSERT_OK_AND_ASSIGN(CelValue value, - adapter.AdaptFromWellKnownType(manager, instance)); - - // TestMessage should not be converted to a CEL primitive type. - EXPECT_THAT(value, test::IsCelMessage(EqualsProto("int64_value: 42"))); -} - -TEST(ProtoMessageTypeAdapter, AdaptFromWellKnownTypeNotAMessageError) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - CelValue::MessageWrapper::Builder instance( - static_cast(nullptr)); - - // Interpreter guaranteed to call this with a message type, otherwise, - // something has broken. - EXPECT_THAT(adapter.AdaptFromWellKnownType(manager, instance), - StatusIs(absl::StatusCode::kInternal)); -} - TEST(ProtoMesssageTypeAdapter, TypeInfoDebug) { ProtoMessageTypeAdapter adapter( google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( @@ -709,23 +468,7 @@ TEST(ProtoMesssageTypeAdapter, FindFieldNotFound) { "google.api.expr.runtime.TestMessage"), google::protobuf::MessageFactory::generated_factory()); - EXPECT_EQ(adapter.FindFieldByName("foo_not_a_field"), absl::nullopt); -} - -TEST(ProtoMesssageTypeAdapter, TypeInfoMutator) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - const LegacyTypeMutationApis* api = adapter.GetMutationApis(MessageWrapper()); - ASSERT_NE(api, nullptr); - - ASSERT_OK_AND_ASSIGN(MessageWrapper::Builder builder, - api->NewInstance(manager)); - EXPECT_NE(dynamic_cast(builder.message_ptr()), nullptr); + EXPECT_EQ(adapter.FindFieldByName("foo_not_a_field"), std::nullopt); } TEST(ProtoMesssageTypeAdapter, TypeInfoAccesor) { @@ -1405,5 +1148,55 @@ TEST(ProtoMesssageTypeAdapter, QualifyMapIndexLeafWrongType) { HasSubstr("Invalid map key type")))))); } +TEST(ProtoMesssageTypeAdapter, InteropUnwrappingNotTestInstance) { + google::protobuf::Arena arena; + + TestMessage message; + message.set_string_value("hello"); + auto legacy_value = CelValue::CreateMessageWrapper( + CelValue::MessageWrapper(&message, TrivialTypeInfo::GetInstance())); + cel::Value modern_value; + ASSERT_THAT(cel::ModernValue(&arena, legacy_value, modern_value), IsOk()); + auto unwrapped = cel::interop_internal::GetLegacyMessage(modern_value); + + // Can't unwrap a non-generic MessageWrapper -- we test by identity to + // be sure we're not dropping a custom adapter. + ASSERT_EQ(unwrapped, nullptr); +} + +TEST(ProtoMesssageTypeAdapter, InteropUnwrappingGeneric) { + google::protobuf::Arena arena; + + TestMessage message; + message.set_string_value("hello"); + auto legacy_value = CelValue::CreateMessageWrapper( + CelValue::MessageWrapper(&message, &GetGenericProtoTypeInfoInstance())); + cel::Value modern_value; + ASSERT_THAT(cel::ModernValue(&arena, legacy_value, modern_value), IsOk()); + auto unwrapped = cel::interop_internal::GetLegacyMessage(modern_value); + + ASSERT_EQ(unwrapped, &message); +} + +TEST(ProtoMesssageTypeAdapter, InteropFieldAccess) { + google::protobuf::Arena arena; + + TestMessage message; + message.set_string_value("hello"); + + const google::protobuf::FieldDescriptor* field = + message.GetDescriptor()->FindFieldByName("string_value"); + ASSERT_NE(field, nullptr); + cel::Value field_value; + ASSERT_THAT( + cel::interop_internal::WrapLegacyMessageField( + &message, field, ProtoWrapperTypeOptions::kUnsetNull, + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), &arena, &field_value), + IsOk()); + + EXPECT_THAT(field_value, cel::test::StringValueIs("hello")); +} + } // namespace } // namespace google::api::expr::runtime diff --git a/eval/public/structs/protobuf_descriptor_type_provider.cc b/eval/public/structs/protobuf_descriptor_type_provider.cc index 68b39c643..37fc8ce77 100644 --- a/eval/public/structs/protobuf_descriptor_type_provider.cc +++ b/eval/public/structs/protobuf_descriptor_type_provider.cc @@ -27,10 +27,9 @@ absl::optional ProtobufDescriptorProvider::ProvideLegacyType( absl::string_view name) const { const ProtoMessageTypeAdapter* result = GetTypeAdapter(name); if (result == nullptr) { - return absl::nullopt; + return std::nullopt; } - // ProtoMessageTypeAdapter provides apis for both access and mutation. - return LegacyTypeAdapter(result, result); + return LegacyTypeAdapter(result); } absl::optional @@ -38,7 +37,7 @@ ProtobufDescriptorProvider::ProvideLegacyTypeInfo( absl::string_view name) const { const ProtoMessageTypeAdapter* result = GetTypeAdapter(name); if (result == nullptr) { - return absl::nullopt; + return std::nullopt; } return result; } diff --git a/eval/public/structs/protobuf_descriptor_type_provider_test.cc b/eval/public/structs/protobuf_descriptor_type_provider_test.cc index 3a8fae26b..ed1ddb811 100644 --- a/eval/public/structs/protobuf_descriptor_type_provider_test.cc +++ b/eval/public/structs/protobuf_descriptor_type_provider_test.cc @@ -19,28 +19,23 @@ #include "google/protobuf/wrappers.pb.h" #include "eval/public/cel_value.h" #include "eval/public/structs/legacy_type_info_apis.h" -#include "eval/public/testing/matchers.h" -#include "extensions/protobuf/memory_manager.h" #include "internal/testing.h" -#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" namespace google::api::expr::runtime { namespace { -using ::cel::extensions::ProtoMemoryManager; - TEST(ProtobufDescriptorProvider, Basic) { ProtobufDescriptorProvider provider( google::protobuf::DescriptorPool::generated_pool(), google::protobuf::MessageFactory::generated_factory()); - google::protobuf::Arena arena; - auto manager = ProtoMemoryManager(&arena); auto type_adapter = provider.ProvideLegacyType("google.protobuf.Int64Value"); - absl::optional type_info = + std::optional type_info = provider.ProvideLegacyTypeInfo("google.protobuf.Int64Value"); ASSERT_TRUE(type_adapter.has_value()); - ASSERT_TRUE(type_adapter->mutation_apis() != nullptr); + ASSERT_TRUE(type_adapter->access_apis() != nullptr); ASSERT_TRUE(type_info.has_value()); ASSERT_TRUE(type_info != nullptr); @@ -48,19 +43,6 @@ TEST(ProtobufDescriptorProvider, Basic) { CelValue::MessageWrapper int64_cel_value(&int64_value, *type_info); EXPECT_EQ((*type_info)->GetTypename(int64_cel_value), "google.protobuf.Int64Value"); - - ASSERT_TRUE(type_adapter->mutation_apis()->DefinesField("value")); - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder value, - type_adapter->mutation_apis()->NewInstance(manager)); - - ASSERT_OK(type_adapter->mutation_apis()->SetField( - "value", CelValue::CreateInt64(10), manager, value)); - - ASSERT_OK_AND_ASSIGN( - CelValue adapted, - type_adapter->mutation_apis()->AdaptFromWellKnownType(manager, value)); - - EXPECT_THAT(adapted, test::IsCelInt64(10)); } // This is an implementation detail, but testing for coverage. @@ -71,12 +53,11 @@ TEST(ProtobufDescriptorProvider, MemoizesAdapters) { auto type_adapter = provider.ProvideLegacyType("google.protobuf.Int64Value"); ASSERT_TRUE(type_adapter.has_value()); - ASSERT_TRUE(type_adapter->mutation_apis() != nullptr); + ASSERT_TRUE(type_adapter->access_apis() != nullptr); auto type_adapter2 = provider.ProvideLegacyType("google.protobuf.Int64Value"); ASSERT_TRUE(type_adapter2.has_value()); - EXPECT_EQ(type_adapter->mutation_apis(), type_adapter2->mutation_apis()); EXPECT_EQ(type_adapter->access_apis(), type_adapter2->access_apis()); } diff --git a/eval/public/structs/trivial_legacy_type_info.h b/eval/public/structs/trivial_legacy_type_info.h index 2189bd478..2a1bf346b 100644 --- a/eval/public/structs/trivial_legacy_type_info.h +++ b/eval/public/structs/trivial_legacy_type_info.h @@ -15,39 +15,14 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TRIVIAL_LEGACY_TYPE_INFO_H_ #define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TRIVIAL_LEGACY_TYPE_INFO_H_ -#include - -#include "absl/base/no_destructor.h" -#include "absl/strings/string_view.h" -#include "eval/public/message_wrapper.h" -#include "eval/public/structs/legacy_type_info_apis.h" +#include "eval/public/structs/legacy_type_info_apis.h" // IWYU pragma: keep +#include "eval/public/structs/trivial_legacy_type_info_internal.h" namespace google::api::expr::runtime { // Implementation of type info APIs suitable for testing where no message // operations need to be supported. -class TrivialTypeInfo : public LegacyTypeInfoApis { - public: - absl::string_view GetTypename(const MessageWrapper& wrapper) const override { - return "opaque"; - } - - std::string DebugString(const MessageWrapper& wrapper) const override { - return "opaque"; - } - - const LegacyTypeAccessApis* GetAccessApis( - const MessageWrapper& wrapper) const override { - // Accessors unsupported -- caller should treat this as an opaque type (no - // fields defined, field access always results in a CEL error). - return nullptr; - } - - static const TrivialTypeInfo* GetInstance() { - static absl::NoDestructor kInstance; - return &*kInstance; - } -}; +using TrivialTypeInfo = cel::interop_internal::TrivialTypeInfo; } // namespace google::api::expr::runtime diff --git a/eval/public/structs/trivial_legacy_type_info_internal.h b/eval/public/structs/trivial_legacy_type_info_internal.h new file mode 100644 index 000000000..68408070a --- /dev/null +++ b/eval/public/structs/trivial_legacy_type_info_internal.h @@ -0,0 +1,63 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TRIVIAL_LEGACY_TYPE_INFO_INTERNAL_H_ +#define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TRIVIAL_LEGACY_TYPE_INFO_INTERNAL_H_ + +#include + +#include "absl/base/no_destructor.h" +#include "absl/base/nullability.h" +#include "absl/strings/string_view.h" +#include "eval/public/message_wrapper.h" +#include "eval/public/structs/legacy_type_info_apis.h" + +namespace cel::interop_internal { + +// Implementation details. Exported here for the sake of Legacy->Modern +// Value adaptation. +class TrivialTypeInfo : public google::api::expr::runtime::LegacyTypeInfoApis { + public: + absl::string_view GetTypename( + const google::api::expr::runtime::MessageWrapper& wrapper) + const override { + return "opaque"; + } + + std::string DebugString(const google::api::expr::runtime::MessageWrapper& + wrapper) const override { + return "opaque"; + } + + const google::api::expr::runtime::LegacyTypeAccessApis* GetAccessApis( + const google::api::expr::runtime::MessageWrapper& wrapper) + const override { + // Accessors unsupported -- caller should treat this as an opaque type (no + // fields defined, field access always results in a CEL error). + return nullptr; + } + + static const TrivialTypeInfo* absl_nonnull GetInstance() { + static absl::NoDestructor kInstance; + return &*kInstance; + } + + private: + friend class absl::NoDestructor; + TrivialTypeInfo() = default; +}; + +} // namespace cel::interop_internal + +#endif // THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TRIVIAL_LEGACY_TYPE_INFO_INTERNAL_H_ diff --git a/eval/public/structs/trivial_legacy_type_info_test.cc b/eval/public/structs/trivial_legacy_type_info_test.cc index 9b4840373..5001c85d5 100644 --- a/eval/public/structs/trivial_legacy_type_info_test.cc +++ b/eval/public/structs/trivial_legacy_type_info_test.cc @@ -21,44 +21,29 @@ namespace google::api::expr::runtime { namespace { TEST(TrivialTypeInfo, GetTypename) { - TrivialTypeInfo info; MessageWrapper wrapper; - EXPECT_EQ(info.GetTypename(wrapper), "opaque"); EXPECT_EQ(TrivialTypeInfo::GetInstance()->GetTypename(wrapper), "opaque"); } TEST(TrivialTypeInfo, DebugString) { - TrivialTypeInfo info; MessageWrapper wrapper; - EXPECT_EQ(info.DebugString(wrapper), "opaque"); EXPECT_EQ(TrivialTypeInfo::GetInstance()->DebugString(wrapper), "opaque"); } TEST(TrivialTypeInfo, GetAccessApis) { - TrivialTypeInfo info; MessageWrapper wrapper; - EXPECT_EQ(info.GetAccessApis(wrapper), nullptr); EXPECT_EQ(TrivialTypeInfo::GetInstance()->GetAccessApis(wrapper), nullptr); } -TEST(TrivialTypeInfo, GetMutationApis) { - TrivialTypeInfo info; - MessageWrapper wrapper; - - EXPECT_EQ(info.GetMutationApis(wrapper), nullptr); - EXPECT_EQ(TrivialTypeInfo::GetInstance()->GetMutationApis(wrapper), nullptr); -} TEST(TrivialTypeInfo, FindFieldByName) { - TrivialTypeInfo info; MessageWrapper wrapper; - EXPECT_EQ(info.FindFieldByName("foo"), absl::nullopt); EXPECT_EQ(TrivialTypeInfo::GetInstance()->FindFieldByName("foo"), - absl::nullopt); + std::nullopt); } } // namespace diff --git a/eval/public/testing/matchers.cc b/eval/public/testing/matchers.cc index f79071fce..4f728c730 100644 --- a/eval/public/testing/matchers.cc +++ b/eval/public/testing/matchers.cc @@ -7,7 +7,6 @@ #include "absl/strings/string_view.h" #include "eval/public/cel_value.h" #include "eval/public/set_util.h" -#include "internal/casts.h" #include "internal/testing.h" #include "google/protobuf/message.h" @@ -76,8 +75,7 @@ class CelValueMatcherImpl CelValue::MessageWrapper arg; return v.GetValue(&arg) && arg.HasFullProto() && underlying_type_matcher_.Matches( - cel::internal::down_cast( - arg.message_ptr())); + google::protobuf::DownCastMessage(arg.message_ptr())); } void DescribeTo(std::ostream* os) const override { diff --git a/eval/public/transform_utility.cc b/eval/public/transform_utility.cc index 6cb859c19..897cf45fa 100644 --- a/eval/public/transform_utility.cc +++ b/eval/public/transform_utility.cc @@ -11,6 +11,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" +#include "absl/types/span.h" #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_list_impl.h" #include "eval/public/containers/container_backed_map_impl.h" @@ -157,9 +158,7 @@ absl::StatusOr ValueToCelValue(const Value& value, key_values.push_back(std::pair(map_key, map_value)); } CEL_ASSIGN_OR_RETURN( - auto cel_map, - CreateContainerBackedMap(absl::Span>( - key_values.data(), key_values.size()))); + auto cel_map, CreateContainerBackedMap(absl::MakeSpan(key_values))); auto* cel_map_ptr = cel_map.release(); arena->Own(cel_map_ptr); return CelValue::CreateMap(cel_map_ptr); diff --git a/eval/public/unknown_function_result_set_test.cc b/eval/public/unknown_function_result_set_test.cc index 745b5b9ff..b629f12fe 100644 --- a/eval/public/unknown_function_result_set_test.cc +++ b/eval/public/unknown_function_result_set_test.cc @@ -38,15 +38,15 @@ CelFunctionDescriptor kTwoInt("TwoInt", false, CelFunctionDescriptor kOneInt("OneInt", false, {CelValue::Type::kInt64}); TEST(UnknownFunctionResult, Equals) { - UnknownFunctionResult call1(kTwoInt, /*expr_id=*/0); + UnknownFunctionResult call1(kTwoInt.name()); - UnknownFunctionResult call2(kTwoInt, /*expr_id=*/0); + UnknownFunctionResult call2(kTwoInt.name()); EXPECT_TRUE(call1.IsEqualTo(call2)); - UnknownFunctionResult call3(kOneInt, /*expr_id=*/0); + UnknownFunctionResult call3(kOneInt.name()); - UnknownFunctionResult call4(kOneInt, /*expr_id=*/0); + UnknownFunctionResult call4(kOneInt.name()); EXPECT_TRUE(call3.IsEqualTo(call4)); @@ -57,25 +57,16 @@ TEST(UnknownFunctionResult, Equals) { } TEST(UnknownFunctionResult, InequalDescriptor) { - UnknownFunctionResult call1(kTwoInt, /*expr_id=*/0); + UnknownFunctionResult call1(kTwoInt.name()); - UnknownFunctionResult call2(kOneInt, /*expr_id=*/0); + UnknownFunctionResult call2(kOneInt.name()); EXPECT_FALSE(call1.IsEqualTo(call2)); - CelFunctionDescriptor one_uint("OneInt", false, {CelValue::Type::kUint64}); - - UnknownFunctionResult call3(kOneInt, /*expr_id=*/0); - - UnknownFunctionResult call4(one_uint, /*expr_id=*/0); - - EXPECT_FALSE(call3.IsEqualTo(call4)); - - UnknownFunctionResultSet call_set({call1, call3, call4}); - EXPECT_EQ(call_set.size(), 3); + UnknownFunctionResultSet call_set({call1, call2}); + EXPECT_EQ(call_set.size(), 2); auto it = call_set.begin(); - EXPECT_EQ(*it++, call3); - EXPECT_EQ(*it++, call4); + EXPECT_EQ(*it++, call2); EXPECT_EQ(*it++, call1); } diff --git a/eval/public/unknown_set_test.cc b/eval/public/unknown_set_test.cc index 3a0d151a5..c15030c00 100644 --- a/eval/public/unknown_set_test.cc +++ b/eval/public/unknown_set_test.cc @@ -22,7 +22,7 @@ using ::testing::UnorderedElementsAre; UnknownFunctionResultSet MakeFunctionResult(Arena* arena, int64_t id) { CelFunctionDescriptor desc("OneInt", false, {CelValue::Type::kInt64}); - return UnknownFunctionResultSet(UnknownFunctionResult(desc, /*expr_id=*/0)); + return UnknownFunctionResultSet(UnknownFunctionResult(desc.name())); } UnknownAttributeSet MakeAttribute(Arena* arena, int64_t id) { diff --git a/eval/tests/BUILD b/eval/tests/BUILD index 9163548d1..c193a2360 100644 --- a/eval/tests/BUILD +++ b/eval/tests/BUILD @@ -24,6 +24,12 @@ cc_test( ], deps = [ ":request_context_cc_proto", + "//common:ast_proto", + "//common:decl", + "//common:type", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", "//eval/public:activation", "//eval/public:builtin_func_registrar", "//eval/public:cel_expr_builder_factory", @@ -42,7 +48,9 @@ cc_test( "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/container:node_hash_set", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", "@com_google_protobuf//:protobuf", @@ -61,22 +69,23 @@ cc_test( ], deps = [ ":request_context_cc_proto", - "//common:allocator", - "//common:casting", + "//base:attributes", + "//checker:validation_result", + "//common:decl", "//common:legacy_value", - "//common:memory", "//common:native_type", + "//common:type", "//common:value", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", "//extensions:comprehensions_v2_functions", "//extensions:comprehensions_v2_macros", "//extensions/protobuf:runtime_adapter", - "//extensions/protobuf:value", "//internal:benchmark", "//internal:testing", "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", "//parser", - "//parser:macro", "//parser:macro_registry", "//runtime", "//runtime:activation", @@ -94,7 +103,6 @@ cc_test( "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", "@com_google_protobuf//:protobuf", @@ -102,6 +110,46 @@ cc_test( ], ) +cc_test( + name = "modern_unknown_test", + srcs = [ + "modern_unknown_test.cc", + ], + deps = [ + "//base:attributes", + "//base:function_result", + "//common:function_descriptor", + "//common:kind", + "//common:value", + "//extensions:bindings_ext", + "//extensions:comprehensions_v2_macros", + "//internal:testing", + "//internal:testing_descriptor_pool", + "//internal:testing_message_factory", + "//parser", + "//parser:options", + "//parser:parser_interface", + "//runtime", + "//runtime:activation", + "//runtime:constant_folding", + "//runtime:function", + "//runtime:optional_types", + "//runtime:reference_resolver", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", + "//runtime/internal:errors", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + "@com_google_protobuf//:protobuf", + "@com_google_protobuf//:struct_cc_proto", + ], +) + cc_test( name = "allocation_benchmark_test", size = "small", @@ -232,8 +280,6 @@ cc_test( "//internal:status_macros", "//internal:testing", "//parser", - "//runtime/internal:activation_attribute_matcher_access", - "//runtime/internal:attribute_matcher", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", diff --git a/eval/tests/benchmark_test.cc b/eval/tests/benchmark_test.cc index fc0c39294..67ab9cd8b 100644 --- a/eval/tests/benchmark_test.cc +++ b/eval/tests/benchmark_test.cc @@ -12,7 +12,15 @@ #include "absl/container/flat_hash_set.h" #include "absl/container/node_hash_set.h" #include "absl/flags/flag.h" +#include "absl/status/status_matchers.h" #include "absl/strings/match.h" +#include "absl/types/span.h" +#include "common/ast_proto.h" +#include "common/decl.h" +#include "common/type.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" #include "eval/public/activation.h" #include "eval/public/builtin_func_registrar.h" #include "eval/public/cel_expr_builder_factory.h" @@ -31,6 +39,7 @@ ABSL_FLAG(bool, enable_optimizations, false, "enable const folding opt"); ABSL_FLAG(bool, enable_recursive_planning, false, "enable recursive planning"); +ABSL_FLAG(bool, enable_typed_field_access, false, "enable typed field access"); namespace google { namespace api { @@ -39,6 +48,7 @@ namespace runtime { namespace { +using ::absl_testing::IsOk; using ::cel::expr::Expr; using ::cel::expr::ParsedExpr; using ::cel::expr::SourceInfo; @@ -56,6 +66,10 @@ InterpreterOptions GetOptions(google::protobuf::Arena& arena) { options.max_recursion_depth = -1; } + if (absl::GetFlag(FLAGS_enable_typed_field_access)) { + options.enable_typed_field_access = true; + } + return options; } @@ -67,7 +81,8 @@ static void BM_Eval(benchmark::State& state) { InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); int len = state.range(0); @@ -113,7 +128,8 @@ static void BM_Eval_Trace(benchmark::State& state) { options.enable_recursive_tracing = true; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); int len = state.range(0); @@ -155,7 +171,8 @@ static void BM_EvalString(benchmark::State& state) { InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); int len = state.range(0); @@ -198,7 +215,8 @@ static void BM_EvalString_Trace(benchmark::State& state) { options.enable_recursive_tracing = true; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); int len = state.range(0); @@ -295,7 +313,8 @@ void BM_PolicySymbolic(benchmark::State& state) { options.constant_arena = &arena; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); SourceInfo source_info; ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression( @@ -317,7 +336,7 @@ BENCHMARK(BM_PolicySymbolic); class RequestMap : public CelMap { public: - absl::optional operator[](CelValue key) const override { + std::optional operator[](CelValue key) const override { if (!key.IsString()) { return {}; } @@ -351,7 +370,8 @@ void BM_PolicySymbolicMap(benchmark::State& state) { InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); SourceInfo source_info; ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression( @@ -373,22 +393,37 @@ BENCHMARK(BM_PolicySymbolicMap); // Uses a protobuf container for "ip", "path", and "token". void BM_PolicySymbolicProto(benchmark::State& state) { google::protobuf::Arena arena; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto validation_result, compiler->Compile(R"cel( !(request.ip in ["10.0.1.4", "10.0.1.5", "10.0.1.6"]) && ((request.path.startsWith("v1") && request.token in ["v1", "v2", "admin"]) || (request.path.startsWith("v2") && request.token in ["v2", "admin"]) || (request.path.startsWith("/admin") && request.token == "admin" && request.ip in ["10.0.1.1", "10.0.1.2", "10.0.1.3"]) ))cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); - SourceInfo source_info; - ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression( - &parsed_expr.expr(), &source_info)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); Activation activation; RequestContext request; @@ -475,7 +510,8 @@ void BM_Comprehension(benchmark::State& state) { InterpreterOptions options = GetOptions(arena); options.comprehension_max_iterations = 10000000; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&expr, nullptr)); @@ -509,7 +545,8 @@ void BM_Comprehension_Trace(benchmark::State& state) { options.comprehension_max_iterations = 10000000; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&expr, nullptr)); @@ -531,16 +568,15 @@ void BM_HasMap(benchmark::State& state) { InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&parsed_expr.expr(), nullptr)); std::vector> map_pairs{ {CelValue::CreateStringView("path"), CelValue::CreateStringView("path")}}; - auto cel_map = - CreateContainerBackedMap(absl::Span>( - map_pairs.data(), map_pairs.size())); + auto cel_map = CreateContainerBackedMap(absl::MakeSpan(map_pairs)); activation.InsertValue("request", CelValue::CreateMap((*cel_map).get())); for (auto _ : state) { @@ -555,16 +591,34 @@ BENCHMARK(BM_HasMap); void BM_HasProto(benchmark::State& state) { google::protobuf::Arena arena; - Activation activation; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, - parser::Parse("has(request.path) && !has(request.ip)")); + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN( + auto validation_result, + compiler->Compile("has(request.path) && !has(request.ip)")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); + InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - auto reg_status = RegisterBuiltinFunctions(builder->GetRegistry(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); - ASSERT_OK_AND_ASSIGN(auto cel_expr, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); + Activation activation; RequestContext request; request.set_path(kPath); request.set_token(kToken); @@ -583,17 +637,34 @@ BENCHMARK(BM_HasProto); void BM_HasProtoMap(benchmark::State& state) { google::protobuf::Arena arena; - Activation activation; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, - parser::Parse("has(request.headers.create_time) && " - "!has(request.headers.update_time)")); + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto validation_result, + compiler->Compile("has(request.headers.create_time) && " + "!has(request.headers.update_time)")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); + InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - auto reg_status = RegisterBuiltinFunctions(builder->GetRegistry(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); - ASSERT_OK_AND_ASSIGN(auto cel_expr, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); + Activation activation; RequestContext request; request.mutable_headers()->insert({"create_time", "2021-01-01"}); activation.InsertValue("request", @@ -611,17 +682,34 @@ BENCHMARK(BM_HasProtoMap); void BM_ReadProtoMap(benchmark::State& state) { google::protobuf::Arena arena; - Activation activation; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto validation_result, compiler->Compile(R"cel( request.headers.create_time == "2021-01-01" )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); + InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - auto reg_status = RegisterBuiltinFunctions(builder->GetRegistry(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); - ASSERT_OK_AND_ASSIGN(auto cel_expr, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); + Activation activation; RequestContext request; request.mutable_headers()->insert({"create_time", "2021-01-01"}); activation.InsertValue("request", @@ -639,17 +727,34 @@ BENCHMARK(BM_ReadProtoMap); void BM_NestedProtoFieldRead(benchmark::State& state) { google::protobuf::Arena arena; - Activation activation; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto validation_result, compiler->Compile(R"cel( !request.a.b.c.d.e )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); + InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - auto reg_status = RegisterBuiltinFunctions(builder->GetRegistry(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); - ASSERT_OK_AND_ASSIGN(auto cel_expr, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); + Activation activation; RequestContext request; request.mutable_a()->mutable_b()->mutable_c()->mutable_d()->set_e(false); activation.InsertValue("request", @@ -667,17 +772,34 @@ BENCHMARK(BM_NestedProtoFieldRead); void BM_NestedProtoFieldReadDefaults(benchmark::State& state) { google::protobuf::Arena arena; - Activation activation; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto validation_result, compiler->Compile(R"cel( !request.a.b.c.d.e )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); + InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - auto reg_status = RegisterBuiltinFunctions(builder->GetRegistry(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); - ASSERT_OK_AND_ASSIGN(auto cel_expr, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); + Activation activation; RequestContext request; activation.InsertValue("request", CelProtoWrapper::CreateMessage(&request, &arena)); @@ -694,17 +816,35 @@ BENCHMARK(BM_NestedProtoFieldReadDefaults); void BM_ProtoStructAccess(benchmark::State& state) { google::protobuf::Arena arena; - Activation activation; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", + cel::MessageType(AttributeContext::Request::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto validation_result, compiler->Compile(R"cel( has(request.auth.claims.iss) && request.auth.claims.iss == 'accounts.google.com' )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); + InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); - ASSERT_OK_AND_ASSIGN(auto cel_expr, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); + Activation activation; AttributeContext::Request request; auto* auth = request.mutable_auth(); (*auth->mutable_claims()->mutable_fields())["iss"].set_string_value( @@ -724,17 +864,35 @@ BENCHMARK(BM_ProtoStructAccess); void BM_ProtoListAccess(benchmark::State& state) { google::protobuf::Arena arena; - Activation activation; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", + cel::MessageType(AttributeContext::Request::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto validation_result, compiler->Compile(R"cel( "//.../accessLevels/MY_LEVEL_4" in request.auth.access_levels )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); + InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); - ASSERT_OK_AND_ASSIGN(auto cel_expr, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); + Activation activation; AttributeContext::Request request; auto* auth = request.mutable_auth(); auth->add_access_levels("//.../accessLevels/MY_LEVEL_0"); @@ -867,7 +1025,8 @@ void BM_NestedComprehension(benchmark::State& state) { InterpreterOptions options = GetOptions(arena); options.comprehension_max_iterations = 10000000; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&expr, nullptr)); @@ -903,7 +1062,8 @@ void BM_NestedComprehension_Trace(benchmark::State& state) { options.enable_recursive_tracing = true; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&expr, nullptr)); @@ -936,7 +1096,8 @@ void BM_ListComprehension(benchmark::State& state) { options.comprehension_max_iterations = 10000000; options.enable_comprehension_list_append = true; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN( auto cel_expr, builder->CreateExpression(&(parsed_expr.expr()), nullptr)); @@ -971,7 +1132,8 @@ void BM_ListComprehension_Trace(benchmark::State& state) { options.enable_recursive_tracing = true; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN( auto cel_expr, builder->CreateExpression(&(parsed_expr.expr()), nullptr)); @@ -1006,7 +1168,7 @@ void BM_ListComprehension_Opt(benchmark::State& state) { options.comprehension_max_iterations = 10000000; options.enable_comprehension_list_append = true; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry())); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry()), IsOk()); ASSERT_OK_AND_ASSIGN( auto cel_expr, builder->CreateExpression(&(parsed_expr.expr()), nullptr)); diff --git a/eval/tests/memory_safety_test.cc b/eval/tests/memory_safety_test.cc index 9c0a683e4..a88844fed 100644 --- a/eval/tests/memory_safety_test.cc +++ b/eval/tests/memory_safety_test.cc @@ -51,7 +51,12 @@ struct TestCase { bool reference_resolver_enabled = false; }; -enum Options { kDefault, kExhaustive, kFoldConstants }; +enum Options { + kDefault, + kExhaustive, + kFoldConstants, + kFoldConstantsManagedArena +}; using ParamType = std::tuple; @@ -68,6 +73,9 @@ std::string TestCaseName(const testing::TestParamInfo& param_info) { case Options::kFoldConstants: opt = "opt"; break; + case Options::kFoldConstantsManagedArena: + opt = "opt_managed_arena"; + break; } return absl::StrCat(std::get<0>(param).name, "_", opt); @@ -110,6 +118,14 @@ class EvaluatorMemorySafetyTest : public testing::TestWithParam { options.enable_comprehension_vulnerability_check = false; options.short_circuiting = true; break; + case Options::kFoldConstantsManagedArena: + options.enable_regex_precompilation = true; + options.constant_folding = true; + options.enable_comprehension_list_append = true; + options.enable_comprehension_vulnerability_check = false; + options.short_circuiting = true; + options.constant_arena = nullptr; + break; } options.enable_qualified_identifier_rewrites = @@ -295,7 +311,8 @@ INSTANTIATE_TEST_SUITE_P( test::IsCelBool(true), }}), testing::Values(Options::kDefault, Options::kExhaustive, - Options::kFoldConstants)), + Options::kFoldConstants, + Options::kFoldConstantsManagedArena)), &TestCaseName); } // namespace diff --git a/eval/tests/mock_cel_expression.h b/eval/tests/mock_cel_expression.h index 07b32b29f..4fd9c8642 100644 --- a/eval/tests/mock_cel_expression.h +++ b/eval/tests/mock_cel_expression.h @@ -15,6 +15,9 @@ class MockCelExpression : public CelExpression { MOCK_METHOD(std::unique_ptr, InitializeState, (google::protobuf::Arena * arena), (const, override)); + MOCK_METHOD(std::unique_ptr, CreateState, (), + (const, override)); + MOCK_METHOD(absl::StatusOr, Evaluate, (const BaseActivation& activation, google::protobuf::Arena* arena), (const, override)); @@ -23,6 +26,11 @@ class MockCelExpression : public CelExpression { (const BaseActivation& activation, CelEvaluationState* state), (const, override)); + MOCK_METHOD(absl::StatusOr, Evaluate, + (const BaseActivation& activation, google::protobuf::Arena* arena, + CelEvaluationState* state), + (const, override)); + MOCK_METHOD(absl::StatusOr, Trace, (const BaseActivation& activation, google::protobuf::Arena* arena, CelEvaluationListener callback), @@ -32,6 +40,11 @@ class MockCelExpression : public CelExpression { (const BaseActivation& activation, CelEvaluationState* state, CelEvaluationListener callback), (const, override)); + + MOCK_METHOD(absl::StatusOr, Trace, + (const BaseActivation& activation, google::protobuf::Arena* arena, + CelEvaluationListener callback, CelEvaluationState* state), + (const, override)); }; } // namespace google::api::expr::runtime diff --git a/eval/tests/modern_benchmark_test.cc b/eval/tests/modern_benchmark_test.cc index 005f93aa5..cdbb2f3f1 100644 --- a/eval/tests/modern_benchmark_test.cc +++ b/eval/tests/modern_benchmark_test.cc @@ -14,7 +14,6 @@ // // General benchmarks for CEL evaluator. -#include #include #include #include @@ -36,19 +35,23 @@ #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" -#include "common/allocator.h" -#include "common/casting.h" +#include "absl/strings/string_view.h" +#include "base/attribute.h" +#include "checker/validation_result.h" +#include "common/decl.h" #include "common/native_type.h" +#include "common/type.h" #include "common/value.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" #include "eval/tests/request_context.pb.h" #include "extensions/comprehensions_v2_functions.h" #include "extensions/comprehensions_v2_macros.h" #include "extensions/protobuf/runtime_adapter.h" -#include "extensions/protobuf/value.h" #include "internal/benchmark.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" #include "parser/macro_registry.h" #include "parser/parser.h" #include "runtime/activation.h" @@ -62,6 +65,7 @@ #include "google/protobuf/text_format.h" ABSL_FLAG(bool, enable_recursive_planning, false, "enable recursive planning"); +ABSL_FLAG(bool, enable_typed_field_access, false, "enable typed field access"); namespace cel { @@ -77,6 +81,7 @@ using ::google::api::expr::parser::EnrichedParse; using ::google::api::expr::parser::Parse; using ::google::api::expr::runtime::RequestContext; using ::google::rpc::context::AttributeContext; +using ::testing::UnorderedElementsAre; RuntimeOptions GetOptions() { RuntimeOptions options; @@ -85,16 +90,21 @@ RuntimeOptions GetOptions() { options.max_recursion_depth = -1; } + if (absl::GetFlag(FLAGS_enable_typed_field_access)) { + options.enable_typed_field_access = true; + } + return options; } enum class ConstFoldingEnabled { kNo, kYes }; std::unique_ptr StandardRuntimeOrDie( - const cel::RuntimeOptions& options, google::protobuf::Arena* arena = nullptr, + const cel::RuntimeOptions& options, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::Arena* arena = nullptr, ConstFoldingEnabled const_folding = ConstFoldingEnabled::kNo) { - auto builder = CreateStandardRuntimeBuilder( - internal::GetTestingDescriptorPool(), options); + auto builder = CreateStandardRuntimeBuilder(descriptor_pool, options); ABSL_CHECK_OK(builder.status()); switch (const_folding) { @@ -111,13 +121,17 @@ std::unique_ptr StandardRuntimeOrDie( return std::move(runtime).value(); } +std::unique_ptr StandardRuntimeOrDie( + const cel::RuntimeOptions& options, google::protobuf::Arena* arena = nullptr, + ConstFoldingEnabled const_folding = ConstFoldingEnabled::kNo) { + return StandardRuntimeOrDie(options, internal::GetTestingDescriptorPool(), + arena, const_folding); +} + template Value WrapMessageOrDie(const T& message, google::protobuf::Arena* absl_nonnull arena) { - auto value = extensions::ProtoMessageToValue( - message, internal::GetTestingDescriptorPool(), - internal::GetTestingMessageFactory(), arena); - ABSL_CHECK_OK(value.status()); - return std::move(value).value(); + return Value::FromMessage(message, google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), arena); } // Benchmark test @@ -150,8 +164,8 @@ static void BM_Eval(benchmark::State& state) { Activation activation; ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_TRUE(Cast(result) == len + 1); + ASSERT_TRUE(result.Is()); + ASSERT_TRUE(result.Get() == len + 1); } } @@ -196,8 +210,8 @@ static void BM_Eval_Trace(benchmark::State& state) { Activation activation; ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Trace(&arena, activation, EmptyCallback)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_TRUE(Cast(result) == len + 1); + ASSERT_TRUE(result.Is()); + ASSERT_TRUE(result.Get() == len + 1); } } @@ -236,8 +250,8 @@ static void BM_EvalString(benchmark::State& state) { Activation activation; ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_TRUE(Cast(result).Size() == len + 1); + ASSERT_TRUE(result.Is()); + ASSERT_TRUE(result.Get().Size() == len + 1); } } @@ -277,8 +291,8 @@ static void BM_EvalString_Trace(benchmark::State& state) { Activation activation; ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Trace(&arena, activation, EmptyCallback)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_TRUE(Cast(result).Size() == len + 1); + ASSERT_TRUE(result.Is()); + ASSERT_TRUE(result.Get().Size() == len + 1); } } @@ -335,7 +349,23 @@ BENCHMARK(BM_PolicyNative); void BM_PolicySymbolic(benchmark::State& state) { google::protobuf::Arena arena; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(cel::internal::GetSharedTestingDescriptorPool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable( + MakeVariableDecl("ip", StringType())), + IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable( + MakeVariableDecl("path", StringType())), + IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable( + MakeVariableDecl("token", StringType())), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( !(ip in ["10.0.1.4", "10.0.1.5", "10.0.1.6"]) && ((path.startsWith("v1") && token in ["v1", "v2", "admin"]) || (path.startsWith("v2") && token in ["v2", "admin"]) || @@ -343,23 +373,24 @@ void BM_PolicySymbolic(benchmark::State& state) { "10.0.1.1", "10.0.1.2", "10.0.1.3" ]) ))cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); RuntimeOptions options = GetOptions(); auto runtime = StandardRuntimeOrDie(options, &arena, ConstFoldingEnabled::kYes); - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); Activation activation; - activation.InsertOrAssignValue("ip", StringValue(&arena, kIP)); - activation.InsertOrAssignValue("path", StringValue(&arena, kPath)); - activation.InsertOrAssignValue("token", StringValue(&arena, kToken)); + activation.InsertOrAssignValue("ip", StringValue::WrapUnsafe(kIP)); + activation.InsertOrAssignValue("path", StringValue::WrapUnsafe(kPath)); + activation.InsertOrAssignValue("token", StringValue::WrapUnsafe(kToken)); for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - auto result_bool = As(result); + auto result_bool = result.As(); ASSERT_TRUE(result_bool && result_bool->NativeValue()); } } @@ -403,16 +434,16 @@ class RequestMapImpl : public CustomMapValueInterface { google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const override { - auto string_value = As(key); + auto string_value = key.As(); if (!string_value) { return false; } if (string_value->Equals("ip")) { - *result = StringValue(kIP); + *result = StringValue::WrapUnsafe(kIP); } else if (string_value->Equals("path")) { - *result = StringValue(kPath); + *result = StringValue::WrapUnsafe(kPath); } else if (string_value->Equals("token")) { - *result = StringValue(kToken); + *result = StringValue::WrapUnsafe(kToken); } else { return false; } @@ -437,21 +468,31 @@ class RequestMapImpl : public CustomMapValueInterface { // Uses a lazily constructed map container for "ip", "path", and "token". void BM_PolicySymbolicMap(benchmark::State& state) { google::protobuf::Arena arena; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(cel::internal::GetSharedTestingDescriptorPool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", + cel::MapType(&arena, cel::StringType(), cel::StringType()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( !(request.ip in ["10.0.1.4", "10.0.1.5", "10.0.1.6"]) && ((request.path.startsWith("v1") && request.token in ["v1", "v2", "admin"]) || (request.path.startsWith("v2") && request.token in ["v2", "admin"]) || (request.path.startsWith("/admin") && request.token == "admin" && request.ip in ["10.0.1.1", "10.0.1.2", "10.0.1.3"]) ))cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); - SourceInfo source_info; - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); Activation activation; CustomMapValue map_value(google::protobuf::Arena::Create(&arena), @@ -462,8 +503,8 @@ void BM_PolicySymbolicMap(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } @@ -472,21 +513,31 @@ BENCHMARK(BM_PolicySymbolicMap); // Uses a protobuf container for "ip", "path", and "token". void BM_PolicySymbolicProto(benchmark::State& state) { google::protobuf::Arena arena; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( !(request.ip in ["10.0.1.4", "10.0.1.5", "10.0.1.6"]) && ((request.path.startsWith("v1") && request.token in ["v1", "v2", "admin"]) || (request.path.startsWith("v2") && request.token in ["v2", "admin"]) || (request.path.startsWith("/admin") && request.token == "admin" && request.ip in ["10.0.1.1", "10.0.1.2", "10.0.1.3"]) ))cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); RuntimeOptions options = GetOptions(); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); - auto runtime = StandardRuntimeOrDie(options); - - SourceInfo source_info; - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); Activation activation; RequestContext request; @@ -497,8 +548,7 @@ void BM_PolicySymbolicProto(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.IsBool() && result.GetBool().NativeValue()); } } @@ -580,8 +630,8 @@ void BM_Comprehension(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_EQ(Cast(result), len); + ASSERT_TRUE(result.Is()); + ASSERT_EQ(result.Get(), len); } } @@ -613,8 +663,8 @@ void BM_Comprehension_Trace(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Trace(&arena, activation, EmptyCallback)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_EQ(Cast(result), len); + ASSERT_TRUE(result.Is()); + ASSERT_EQ(result.Get(), len); } } @@ -634,17 +684,17 @@ void BM_HasMap(benchmark::State& state) { auto map_builder = cel::NewMapValueBuilder(&arena); - ASSERT_THAT( - map_builder->Put(cel::StringValue("path"), cel::StringValue("path")), - IsOk()); + ASSERT_THAT(map_builder->Put(cel::StringValue::WrapUnsafe("path"), + cel::StringValue::WrapUnsafe("path")), + IsOk()); activation.InsertOrAssignValue("request", std::move(*map_builder).Build()); for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } @@ -652,12 +702,25 @@ BENCHMARK(BM_HasMap); void BM_HasProto(benchmark::State& state) { RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); + + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, - Parse("has(request.path) && !has(request.ip)")); - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN( + ValidationResult validation_result, + compiler->Compile("has(request.path) && !has(request.ip)")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; Activation activation; @@ -670,8 +733,8 @@ void BM_HasProto(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } @@ -679,13 +742,25 @@ BENCHMARK(BM_HasProto); void BM_HasProtoMap(benchmark::State& state) { RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); + + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, - Parse("has(request.headers.create_time) && " - "!has(request.headers.update_time)")); - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile("has(request.headers.create_time) && " + "!has(request.headers.update_time)")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; Activation activation; @@ -697,23 +772,35 @@ void BM_HasProtoMap(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } BENCHMARK(BM_HasProtoMap); void BM_ReadProtoMap(benchmark::State& state) { - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + RuntimeOptions options = GetOptions(); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); + + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( request.headers.create_time == "2021-01-01" )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); - RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); - - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; Activation activation; @@ -725,23 +812,35 @@ void BM_ReadProtoMap(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } BENCHMARK(BM_ReadProtoMap); void BM_NestedProtoFieldRead(benchmark::State& state) { - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + RuntimeOptions options = GetOptions(); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); + + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( !request.a.b.c.d.e )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); - RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); - - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; Activation activation; @@ -753,23 +852,35 @@ void BM_NestedProtoFieldRead(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } BENCHMARK(BM_NestedProtoFieldRead); void BM_NestedProtoFieldReadDefaults(benchmark::State& state) { - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + RuntimeOptions options = GetOptions(); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); + + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( !request.a.b.c.d.e )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); - RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); - - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; Activation activation; @@ -780,23 +891,36 @@ void BM_NestedProtoFieldReadDefaults(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } BENCHMARK(BM_NestedProtoFieldReadDefaults); void BM_ProtoStructAccess(benchmark::State& state) { - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + RuntimeOptions options = GetOptions(); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); + + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", + cel::MessageType(AttributeContext::Request::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( has(request.auth.claims.iss) && request.auth.claims.iss == 'accounts.google.com' )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); - RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); - - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; Activation activation; @@ -810,23 +934,36 @@ void BM_ProtoStructAccess(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } BENCHMARK(BM_ProtoStructAccess); void BM_ProtoListAccess(benchmark::State& state) { - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + RuntimeOptions options = GetOptions(); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); + + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", + cel::MessageType(AttributeContext::Request::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( "//.../accessLevels/MY_LEVEL_4" in request.auth.access_levels )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); - RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); - - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; Activation activation; @@ -843,8 +980,8 @@ void BM_ProtoListAccess(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } @@ -971,8 +1108,8 @@ void BM_NestedComprehension(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_EQ(Cast(result), len * len); + ASSERT_TRUE(result.Is()); + ASSERT_EQ(result.Get(), len * len); } } @@ -1008,8 +1145,8 @@ void BM_NestedComprehension_Trace(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Trace(&arena, activation, &EmptyCallback)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_EQ(Cast(result), len * len); + ASSERT_TRUE(result.Is()); + ASSERT_EQ(result.Get(), len * len); } } @@ -1042,8 +1179,8 @@ void BM_ListComprehension(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_THAT(Cast(result).Size(), IsOkAndHolds(len)); + ASSERT_TRUE(result.Is()); + ASSERT_THAT(result.Get().Size(), IsOkAndHolds(len)); } } @@ -1077,8 +1214,8 @@ void BM_ListComprehension_Trace(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Trace(&arena, activation, EmptyCallback)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_THAT(Cast(result).Size(), IsOkAndHolds(len)); + ASSERT_TRUE(result.Is()); + ASSERT_THAT(result.Get().Size(), IsOkAndHolds(len)); } } @@ -1241,8 +1378,8 @@ void BM_ListComprehension_Opt(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_THAT(Cast(result).Size(), IsOkAndHolds(len)); + ASSERT_TRUE(result.Is()); + ASSERT_THAT(result.Get().Size(), IsOkAndHolds(len)); } } @@ -1262,7 +1399,7 @@ void BM_ComprehensionCpp(benchmark::State& state) { auto op = [&list]() { int sum = 0; for (const auto& value : list) { - sum += Cast(value).NativeValue(); + sum += value.Get().NativeValue(); } return sum; }; @@ -1323,13 +1460,186 @@ void BM_MapTransformComprehension(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_THAT(Cast(result).Size(), IsOkAndHolds(len)); + ASSERT_TRUE(result.Is()); + ASSERT_THAT(result.Get().Size(), IsOkAndHolds(len)); } } BENCHMARK(BM_MapTransformComprehension)->Range(1, 1 << 16); +void BM_SingleUnknownVariable(benchmark::State& state) { + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse("unk")); + + RuntimeOptions options = GetOptions(); + options.unknown_processing = UnknownProcessingOptions::kAttributeOnly; + auto runtime = StandardRuntimeOrDie(options); + + ASSERT_OK_AND_ASSIGN(auto program, ProtobufRuntimeAdapter::CreateProgram( + *runtime, parsed_expr)); + + Activation activation; + ASSERT_THAT(activation.SetUnknownPatterns({AttributePattern("unk")}), IsOk()); + + for (auto _ : state) { + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(cel::Value result, + program->Evaluate(&arena, activation)); + ASSERT_TRUE(result.IsUnknown()); + ASSERT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(Attribute("unk"))); + } +} + +BENCHMARK(BM_SingleUnknownVariable); + +void BM_DeepUnknownQualifier(benchmark::State& state) { + ASSERT_OK_AND_ASSIGN( + ParsedExpr parsed_expr, + Parse("a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z")); + + RuntimeOptions options = GetOptions(); + options.unknown_processing = UnknownProcessingOptions::kAttributeOnly; + auto runtime = StandardRuntimeOrDie(options); + + ASSERT_OK_AND_ASSIGN(auto program, ProtobufRuntimeAdapter::CreateProgram( + *runtime, parsed_expr)); + + google::protobuf::Arena arena; + google::protobuf::Arena::UniquePtr struct_value = + google::protobuf::Arena::MakeUnique(&arena); + google::protobuf::Struct* struct_value_ptr = struct_value.get(); + for (char l = 'b'; l <= 'z'; ++l) { + auto* fields = struct_value_ptr->mutable_fields(); + struct_value_ptr = (*fields)[std::string(1, l)].mutable_struct_value(); + } + + Attribute attribute( + "a", + {AttributeQualifier::OfString("b"), AttributeQualifier::OfString("c"), + AttributeQualifier::OfString("d"), AttributeQualifier::OfString("e"), + AttributeQualifier::OfString("f"), AttributeQualifier::OfString("g"), + AttributeQualifier::OfString("h"), AttributeQualifier::OfString("i"), + AttributeQualifier::OfString("j"), AttributeQualifier::OfString("k"), + AttributeQualifier::OfString("l"), AttributeQualifier::OfString("m"), + AttributeQualifier::OfString("n"), AttributeQualifier::OfString("o"), + AttributeQualifier::OfString("p"), AttributeQualifier::OfString("q"), + AttributeQualifier::OfString("r"), AttributeQualifier::OfString("s"), + AttributeQualifier::OfString("t"), AttributeQualifier::OfString("u"), + AttributeQualifier::OfString("v"), AttributeQualifier::OfString("w"), + AttributeQualifier::OfString("x"), AttributeQualifier::OfString("y"), + AttributeQualifier::OfString("z")}); + + Activation activation; + ASSERT_THAT(activation.SetUnknownPatterns({AttributePattern( + "a", {AttributeQualifierPattern::OfString("b"), + AttributeQualifierPattern::OfString("c"), + AttributeQualifierPattern::OfString("d"), + AttributeQualifierPattern::OfString("e"), + AttributeQualifierPattern::OfString("f"), + AttributeQualifierPattern::OfString("g"), + AttributeQualifierPattern::OfString("h"), + AttributeQualifierPattern::OfString("i"), + AttributeQualifierPattern::OfString("j"), + AttributeQualifierPattern::OfString("k"), + AttributeQualifierPattern::OfString("l"), + AttributeQualifierPattern::OfString("m"), + AttributeQualifierPattern::OfString("n"), + AttributeQualifierPattern::OfString("o"), + AttributeQualifierPattern::OfString("p"), + AttributeQualifierPattern::OfString("q"), + AttributeQualifierPattern::OfString("r"), + AttributeQualifierPattern::OfString("s"), + AttributeQualifierPattern::OfString("t"), + AttributeQualifierPattern::OfString("u"), + AttributeQualifierPattern::OfString("v"), + AttributeQualifierPattern::OfString("w"), + AttributeQualifierPattern::OfString("x"), + AttributeQualifierPattern::OfString("y"), + AttributeQualifierPattern::OfString("z")})}), + IsOk()); + activation.InsertOrAssignValue( + "a", + cel::Value::WrapMessage(struct_value.get(), runtime->GetDescriptorPool(), + runtime->GetMessageFactory(), &arena)); + + for (auto _ : state) { + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(cel::Value result, + program->Evaluate(&arena, activation)); + ASSERT_TRUE(result.IsUnknown()); + ASSERT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(attribute)); + } +} + +BENCHMARK(BM_DeepUnknownQualifier); + +void BM_ObnoxiousUnknownQualifier(benchmark::State& state) { + ASSERT_OK_AND_ASSIGN( + ParsedExpr parsed_expr, + Parse("a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z")); + + RuntimeOptions options = GetOptions(); + options.unknown_processing = UnknownProcessingOptions::kAttributeOnly; + auto runtime = StandardRuntimeOrDie(options); + + ASSERT_OK_AND_ASSIGN(auto program, ProtobufRuntimeAdapter::CreateProgram( + *runtime, parsed_expr)); + + google::protobuf::Arena arena; + google::protobuf::Arena::UniquePtr struct_value = + google::protobuf::Arena::MakeUnique(&arena); + google::protobuf::Struct* struct_value_ptr = struct_value.get(); + std::vector attributes; + for (char v = 'a'; v <= 'z'; ++v) { + std::vector qualifiers; + for (char q = 'a'; q <= 'z'; ++q) { + if (v == q) { + continue; + } + qualifiers.push_back( + AttributeQualifierPattern::OfString(std::string(1, q))); + } + attributes.emplace_back(std::string(1, v), std::move(qualifiers)); + } + for (char l = 'b'; l <= 'z'; ++l) { + auto* fields = struct_value_ptr->mutable_fields(); + struct_value_ptr = (*fields)[std::string(1, l)].mutable_struct_value(); + } + Activation activation; + ASSERT_THAT(activation.SetUnknownPatterns(attributes), IsOk()); + activation.InsertOrAssignValue( + "a", + cel::Value::WrapMessage(struct_value.get(), runtime->GetDescriptorPool(), + runtime->GetMessageFactory(), &arena)); + Attribute attribute( + "a", + {AttributeQualifier::OfString("b"), AttributeQualifier::OfString("c"), + AttributeQualifier::OfString("d"), AttributeQualifier::OfString("e"), + AttributeQualifier::OfString("f"), AttributeQualifier::OfString("g"), + AttributeQualifier::OfString("h"), AttributeQualifier::OfString("i"), + AttributeQualifier::OfString("j"), AttributeQualifier::OfString("k"), + AttributeQualifier::OfString("l"), AttributeQualifier::OfString("m"), + AttributeQualifier::OfString("n"), AttributeQualifier::OfString("o"), + AttributeQualifier::OfString("p"), AttributeQualifier::OfString("q"), + AttributeQualifier::OfString("r"), AttributeQualifier::OfString("s"), + AttributeQualifier::OfString("t"), AttributeQualifier::OfString("u"), + AttributeQualifier::OfString("v"), AttributeQualifier::OfString("w"), + AttributeQualifier::OfString("x"), AttributeQualifier::OfString("y"), + AttributeQualifier::OfString("z")}); + + for (auto _ : state) { + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(cel::Value result, + program->Evaluate(&arena, activation)); + ASSERT_TRUE(result.IsUnknown()); + ASSERT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(attribute)); + } +} + +BENCHMARK(BM_ObnoxiousUnknownQualifier); + } // namespace } // namespace cel diff --git a/eval/tests/modern_unknown_test.cc b/eval/tests/modern_unknown_test.cc new file mode 100644 index 000000000..2a6bdc6ab --- /dev/null +++ b/eval/tests/modern_unknown_test.cc @@ -0,0 +1,891 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "google/protobuf/struct.pb.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "base/attribute.h" +#include "base/function_result.h" +#include "common/function_descriptor.h" +#include "common/kind.h" +#include "common/value.h" +#include "extensions/bindings_ext.h" +#include "extensions/comprehensions_v2_macros.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "internal/testing_message_factory.h" +#include "parser/options.h" +#include "parser/parser.h" +#include "parser/parser_interface.h" +#include "runtime/activation.h" +#include "runtime/constant_folding.h" +#include "runtime/function.h" +#include "runtime/internal/errors.h" +#include "runtime/optional_types.h" +#include "runtime/reference_resolver.h" +#include "runtime/runtime.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/json/json.h" +#include "google/protobuf/text_format.h" + +namespace { + +using absl_testing::IsOk; +using ::testing::UnorderedElementsAre; +using ::testing::UnorderedElementsAreArray; + +// Function which just echos its input, this is to force the evaluator to handle +// partials. +class FullFunction final : public cel::Function { + public: + absl::StatusOr Invoke(absl::Span args, + const InvokeContext&) const override { + return args[0]; + } +}; + +class MergeFunction final : public cel::Function { + public: + absl::StatusOr Invoke( + absl::Span args, + const InvokeContext& context) const override { + return cel::ErrorValue::From(absl::CancelledError(), context.arena()); + } +}; + +class UnknownFunction final : public cel::Function { + public: + absl::StatusOr Invoke( + absl::Span args, + const InvokeContext& context) const override { + return cel::ErrorValue::From( + cel::runtime_internal::CreateUnknownFunctionResultError("unknown"), + context.arena()); + } +}; + +class ModernUnknownTest : public ::testing::Test { + public: + void SetUp() override { + auto parser_builder = cel::NewParserBuilder(cel::ParserOptions{ + .enable_optional_syntax = true, + }); + ABSL_CHECK_OK(cel::extensions::RegisterComprehensionsV2Macros( // Crash OK + *parser_builder)); + for (const auto& macro : cel::extensions::bindings_macros()) { + ABSL_CHECK_OK(parser_builder->AddMacro(macro)); // Crash OK + } + auto status_or_parser = parser_builder->Build(); + ABSL_CHECK_OK(status_or_parser); // Crash OK + parser_ = *std::move(status_or_parser); + auto status_or_runtime_builder = cel::CreateStandardRuntimeBuilder( + cel::internal::GetTestingDescriptorPool(), + cel::RuntimeOptions{ + .unknown_processing = + cel::UnknownProcessingOptions::kAttributeAndFunction, + .enable_missing_attribute_errors = true, + .enable_qualified_type_identifiers = true}); + ABSL_CHECK_OK(status_or_runtime_builder); // Crash OK + auto runtime_builder = *std::move(status_or_runtime_builder); + ABSL_CHECK_OK(cel::EnableReferenceResolver( + runtime_builder, + cel::ReferenceResolverEnabled::kCheckedExpressionOnly)); + ABSL_CHECK_OK(cel::extensions::EnableConstantFolding( + runtime_builder, cel::internal::GetTestingMessageFactory())); + ABSL_CHECK_OK(cel::extensions::EnableOptionalTypes(runtime_builder)); + ABSL_CHECK_OK(runtime_builder.function_registry().Register( + cel::FunctionDescriptor("full", false, {cel::Kind::kAny}), + std::make_unique())); + ABSL_CHECK_OK(runtime_builder.function_registry().Register( + cel::FunctionDescriptor("merge", false, + {cel::Kind::kAny, cel::Kind::kAny}), + std::make_unique())); + ABSL_CHECK_OK(runtime_builder.function_registry().Register( + cel::FunctionDescriptor("merge", false, + {cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny}), + std::make_unique())); + ABSL_CHECK_OK(runtime_builder.function_registry().Register( + cel::FunctionDescriptor( + "merge", false, {cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny, cel::Kind::kAny, cel::Kind::kAny, + cel::Kind::kAny}), + std::make_unique())); + for (char c = 'A'; c <= 'Z'; ++c) { + ABSL_CHECK_OK(runtime_builder.function_registry().Register( + cel::FunctionDescriptor( + absl::StrCat("unknown", absl::string_view(&c, 1)), false, {}), + std::make_unique())); + } + auto status_or_runtime = std::move(runtime_builder).Build(); + ABSL_CHECK_OK(status_or_runtime); // Crash OK + runtime_ = *std::move(status_or_runtime); + } + + std::unique_ptr CreateProgram(absl::string_view text) { + auto status_or_source = parser_->PrepareSource(text); + ABSL_CHECK_OK(status_or_source); // Crash OK + auto source = *std::move(status_or_source); + auto status_or_ast = parser_->Parse(*source); + ABSL_CHECK_OK(status_or_ast); // Crash OK + auto ast = *std::move(status_or_ast); + auto status_or_program = runtime_->CreateProgram(std::move(ast)); + ABSL_CHECK_OK(status_or_program); // Crash OK + return *std::move(status_or_program); + } + + void TearDown() override { + parser_.reset(); + runtime_.reset(); + } + + template + void BindProtoToActivation(absl::string_view variable, + absl::string_view text) { + T message; + ABSL_CHECK( + google::protobuf::TextFormat::ParseFromString(text, &message)); // Crash OK + activation_.InsertOrAssignValue( + variable, + cel::Value::FromMessage( + std::move(message), cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena_)); + } + + void BindJsonToActivation(absl::string_view variable, + absl::string_view text) { + google::protobuf::Value message; + ABSL_CHECK_OK( + google::protobuf::json::JsonStringToMessage(text, &message)); // Crash OK + activation_.InsertOrAssignValue( + variable, + cel::Value::FromMessage( + std::move(message), cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena_)); + } + + absl::StatusOr Evaluate(const cel::Program& program) { + return program.Evaluate(&arena_, cel::internal::GetTestingMessageFactory(), + activation_); + } + + protected: + std::unique_ptr parser_; + std::unique_ptr runtime_; + google::protobuf::Arena arena_; + cel::Activation activation_; +}; + +TEST_F(ModernUnknownTest, And) { + auto program = CreateProgram(R"cel(var1 && var2)cel"); + + { + // unk && true + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue("var2", cel::TrueValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } + { + // true && unk + activation_.InsertOrAssignValue("var1", cel::TrueValue()); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var2"))); + } + { + // false && unk + activation_.InsertOrAssignValue("var1", cel::FalseValue()); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsFalse()); + } + { + // unk && false + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue("var2", cel::FalseValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsFalse()); + } + { + // error && unk + activation_.InsertOrAssignValue( + "var1", cel::ErrorValue::From(absl::CancelledError(), &arena_)); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var2"))); + } + { + // unk && error + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue( + "var2", cel::ErrorValue::From(absl::CancelledError(), &arena_)); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } + { + // unk && unk + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1"), + cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT( + result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"), cel::Attribute("var2"))); + } +} + +TEST_F(ModernUnknownTest, Or) { + auto program = CreateProgram(R"cel(var1 || var2)cel"); + + { + // unk || true + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue("var2", cel::TrueValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsTrue()); + } + { + // true || unk + activation_.InsertOrAssignValue("var1", cel::TrueValue()); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsTrue()); + } + { + // false || unk + activation_.InsertOrAssignValue("var1", cel::FalseValue()); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var2"))); + } + { + // unk || false + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue("var2", cel::FalseValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } + { + // error || unk + activation_.InsertOrAssignValue( + "var1", cel::ErrorValue::From(absl::CancelledError(), &arena_)); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var2"))); + } + { + // unk || error + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue( + "var2", cel::ErrorValue::From(absl::CancelledError(), &arena_)); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } + { + // unk || unk + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1"), + cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT( + result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"), cel::Attribute("var2"))); + } +} + +TEST_F(ModernUnknownTest, Ternary) { + auto program = CreateProgram(R"cel(var1 ? var2 : var3)cel"); + + { + // unk ? true : false + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue("var2", cel::TrueValue()); + activation_.InsertOrAssignValue("var3", cel::FalseValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } + { + // true ? unk : false + activation_.InsertOrAssignValue("var1", cel::TrueValue()); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + activation_.InsertOrAssignValue("var3", cel::FalseValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var2"))); + } + { + // false ? unk : false + activation_.InsertOrAssignValue("var1", cel::FalseValue()); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + activation_.InsertOrAssignValue("var3", cel::FalseValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsFalse()); + } + { + // error ? unk : false + activation_.InsertOrAssignValue( + "var1", cel::ErrorValue::From(absl::CancelledError(), &arena_)); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + activation_.InsertOrAssignValue("var3", cel::FalseValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsError()); + } + { + // unk ? error : false + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue( + "var2", cel::ErrorValue::From(absl::CancelledError(), &arena_)); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + activation_.InsertOrAssignValue("var3", cel::FalseValue()); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } + { + // unk ? unk : false + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1"), + cel::AttributePattern("var2")}), + IsOk()); + activation_.InsertOrAssignValue("var3", cel::FalseValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } + { + // unk ? true : unk + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1"), + cel::AttributePattern("var3")}), + IsOk()); + activation_.InsertOrAssignValue("var2", cel::TrueValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } +} + +TEST_F(ModernUnknownTest, ListUnknown) { + auto program = CreateProgram(R"cel([var1, var2])cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1"), + cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT( + result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"), cel::Attribute("var2"))); +} + +TEST_F(ModernUnknownTest, ListErrorUnknown) { + auto program = CreateProgram(R"cel([var1, 1 / 0])cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + EXPECT_TRUE(result.IsError()); +} + +TEST_F(ModernUnknownTest, MapUnknown) { + auto program = CreateProgram(R"cel({"foo": var1, "bar": var2})cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1"), + cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT( + result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"), cel::Attribute("var2"))); +} + +TEST_F(ModernUnknownTest, MapErrorUnknown) { + auto program = CreateProgram(R"cel({"foo": var1, "bar": 1 / 0})cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + EXPECT_TRUE(result.IsError()); +} + +TEST_F(ModernUnknownTest, StructUnknown) { + auto program = CreateProgram( + R"cel(cel.expr.conformance.proto3.TestAllTypes{single_int64: var1, single_uint64: var2})cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1"), + cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT( + result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"), cel::Attribute("var2"))); +} + +TEST_F(ModernUnknownTest, StructErrorUnknown) { + auto program = CreateProgram( + R"cel(cel.expr.conformance.proto3.TestAllTypes{single_int64: var1, single_uint64: 1u / 0u})cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + EXPECT_TRUE(result.IsError()); +} + +TEST_F(ModernUnknownTest, Variable) { + auto program = CreateProgram(R"cel(var1)cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + BindJsonToActivation("var1", R"json({})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); +} + +TEST_F(ModernUnknownTest, ReturnPartiallyUnknownValue) { + // This is an unfortunate but existing bug that we should eventually fix. We + // return a partial unknown from evaluation. + auto program = CreateProgram(R"cel(var1)cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern( + "var1", {cel::AttributeQualifierPattern::Wildcard()})}), + IsOk()); + BindJsonToActivation("var1", R"json({})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsMap()); +} + +TEST_F(ModernUnknownTest, MapVariableSelectStringQualifier) { + auto program = CreateProgram(R"cel(var1.k)cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern( + "var1", {cel::AttributeQualifierPattern::OfString("k")})}), + IsOk()); + BindJsonToActivation("var1", R"json({})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute( + "var1", {cel::AttributeQualifier::OfString("k")}))); +} + +TEST_F(ModernUnknownTest, MapVariableSelectWildcardQualifier) { + auto program = CreateProgram(R"cel(var1.k)cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern( + "var1", {cel::AttributeQualifierPattern::Wildcard()})}), + IsOk()); + BindJsonToActivation("var1", R"json({})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute( + "var1", {cel::AttributeQualifier::OfString("k")}))); +} + +TEST_F(ModernUnknownTest, ListVariableIndexIntQualifier) { + auto program = CreateProgram(R"cel(var1[0])cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern( + "var1", {cel::AttributeQualifierPattern::OfInt(0)})}), + IsOk()); + BindJsonToActivation("var1", R"json([])json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre( + cel::Attribute("var1", {cel::AttributeQualifier::OfInt(0)}))); +} + +TEST_F(ModernUnknownTest, ListVariableIndexWildcardQualifier) { + auto program = CreateProgram(R"cel(var1[0])cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern( + "var1", {cel::AttributeQualifierPattern::Wildcard()})}), + IsOk()); + BindJsonToActivation("var1", R"json([])json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre( + cel::Attribute("var1", {cel::AttributeQualifier::OfInt(0)}))); +} + +TEST_F(ModernUnknownTest, FullKnownUnknown) { + auto program = CreateProgram(R"cel(full(a.b.c))cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("a")}), + IsOk()); + ASSERT_THAT(activation_.SetKnownPatterns({cel::AttributePattern( + "a", {cel::AttributeQualifierPattern::OfString("b")})}), + IsOk()); + BindJsonToActivation("a", R"json({"b": {"c": true}})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsTrue()); +} + +TEST_F(ModernUnknownTest, FullKnownUnknownWildcard) { + auto program = CreateProgram(R"cel(full(a.b.c))cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern( + "a", {cel::AttributeQualifierPattern::Wildcard()})}), + IsOk()); + ASSERT_THAT(activation_.SetKnownPatterns({cel::AttributePattern( + "a", {cel::AttributeQualifierPattern::OfString("b")})}), + IsOk()); + BindJsonToActivation("a", R"json({"b": {"c": true}})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsTrue()); +} + +TEST_F(ModernUnknownTest, FullKnownWildcardUnknown) { + auto program = CreateProgram(R"cel(full(a.b.c))cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("a")}), + IsOk()); + ASSERT_THAT(activation_.SetKnownPatterns({cel::AttributePattern( + "a", {cel::AttributeQualifierPattern::OfString("b"), + cel::AttributeQualifierPattern::Wildcard()})}), + IsOk()); + BindJsonToActivation("a", R"json({"b": {"c": true}})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsTrue()); +} + +TEST_F(ModernUnknownTest, FullKnownWildcardUnknownWildcard) { + auto program = CreateProgram(R"cel(full(a.b.c))cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern( + "a", {cel::AttributeQualifierPattern::Wildcard()})}), + IsOk()); + ASSERT_THAT(activation_.SetKnownPatterns({cel::AttributePattern( + "a", {cel::AttributeQualifierPattern::OfString("b"), + cel::AttributeQualifierPattern::Wildcard()})}), + IsOk()); + BindJsonToActivation("a", R"json({"b": {"c": true}})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsTrue()); +} + +TEST_F(ModernUnknownTest, PartialKnownUnknown) { + auto program = CreateProgram(R"cel(full(a.b))cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("a")}), + IsOk()); + ASSERT_THAT(activation_.SetKnownPatterns({cel::AttributePattern( + "a", {cel::AttributeQualifierPattern::OfString("b")})}), + IsOk()); + BindJsonToActivation("a", R"json({"b": true})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsTrue()); +} + +TEST_F(ModernUnknownTest, PartialKnownUnknownWildcard) { + auto program = CreateProgram(R"cel(full(a.b))cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern( + "a", {cel::AttributeQualifierPattern::Wildcard()})}), + IsOk()); + ASSERT_THAT(activation_.SetKnownPatterns({cel::AttributePattern( + "a", {cel::AttributeQualifierPattern::OfString("b")})}), + IsOk()); + BindJsonToActivation("a", R"json({"b": true})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsTrue()); +} + +TEST_F(ModernUnknownTest, PartialKnownWildcardUnknown) { + auto program = CreateProgram(R"cel(full(a.b))cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("a")}), + IsOk()); + ASSERT_THAT(activation_.SetKnownPatterns({cel::AttributePattern( + "a", {cel::AttributeQualifierPattern::OfString("b"), + cel::AttributeQualifierPattern::Wildcard()})}), + IsOk()); + BindJsonToActivation("a", R"json({"b": true})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsTrue()); +} + +TEST_F(ModernUnknownTest, PartialKnownWildcardUnknownWildcard) { + auto program = CreateProgram(R"cel(full(a.b))cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern( + "a", {cel::AttributeQualifierPattern::Wildcard()})}), + IsOk()); + ASSERT_THAT(activation_.SetKnownPatterns({cel::AttributePattern( + "a", {cel::AttributeQualifierPattern::OfString("b"), + cel::AttributeQualifierPattern::Wildcard()})}), + IsOk()); + BindJsonToActivation("a", R"json({"b": true})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsTrue()); +} + +TEST_F(ModernUnknownTest, FastMerge1) { + auto program = CreateProgram(R"cel(merge(a, unknownA()))cel"); + std::vector attributes; + std::vector attribute_patterns; + std::vector functions; + for (char c = 'a'; c <= 'a'; ++c) { + attributes.push_back(cel::Attribute(std::string(1, c))); + attribute_patterns.push_back(cel::AttributePattern(std::string(1, c))); + char u = absl::ascii_toupper(c); + functions.push_back( + cel::FunctionResult(absl::StrCat("unknown", absl::string_view(&u, 1)))); + } + ASSERT_THAT(activation_.SetUnknownPatterns(attribute_patterns), IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAreArray(attributes)); + EXPECT_THAT(result.GetUnknown().ToFunctionResultSet(), + UnorderedElementsAreArray(functions)); +} + +TEST_F(ModernUnknownTest, FastMerge2) { + auto program = CreateProgram(R"cel(merge(a, b, unknownA(), unknownB()))cel"); + std::vector attributes; + std::vector attribute_patterns; + std::vector functions; + for (char c = 'a'; c <= 'b'; ++c) { + attributes.push_back(cel::Attribute(std::string(1, c))); + attribute_patterns.push_back(cel::AttributePattern(std::string(1, c))); + char u = absl::ascii_toupper(c); + functions.push_back( + cel::FunctionResult(absl::StrCat("unknown", absl::string_view(&u, 1)))); + } + ASSERT_THAT(activation_.SetUnknownPatterns(attribute_patterns), IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAreArray(attributes)); + EXPECT_THAT(result.GetUnknown().ToFunctionResultSet(), + UnorderedElementsAreArray(functions)); +} + +TEST_F(ModernUnknownTest, LargeMerge) { + auto program = CreateProgram( + R"cel(merge(a, b, c, d, e, f, g, h, i, j, k, l, m, + n, o, p, q, r, s, t, u, v, w, x, y, z, + unknownA(), unknownB(), unknownC(), unknownD(), unknownE(), unknownF(), + unknownG(), unknownH(), unknownI(), unknownJ(), unknownK(), unknownL(), + unknownM(), unknownN(), unknownO(), unknownP(), unknownQ(), unknownR(), + unknownS(), unknownT(), unknownU(), unknownV(), unknownW(), unknownX(), + unknownY(), unknownZ()))cel"); + std::vector attributes; + std::vector attribute_patterns; + std::vector functions; + for (char c = 'a'; c <= 'z'; ++c) { + attributes.push_back(cel::Attribute(std::string(1, c))); + attribute_patterns.push_back(cel::AttributePattern(std::string(1, c))); + char u = absl::ascii_toupper(c); + functions.push_back( + cel::FunctionResult(absl::StrCat("unknown", absl::string_view(&u, 1)))); + } + ASSERT_THAT(activation_.SetUnknownPatterns(attribute_patterns), IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAreArray(attributes)); + EXPECT_THAT(result.GetUnknown().ToFunctionResultSet(), + UnorderedElementsAreArray(functions)); +} + +TEST_F(ModernUnknownTest, ListFastMerge1) { + auto program = CreateProgram(R"cel(full([a, unknownA()]))cel"); + std::vector attributes; + std::vector attribute_patterns; + std::vector functions; + for (char c = 'a'; c <= 'a'; ++c) { + attributes.push_back(cel::Attribute(std::string(1, c))); + attribute_patterns.push_back(cel::AttributePattern(std::string(1, c))); + char u = absl::ascii_toupper(c); + functions.push_back( + cel::FunctionResult(absl::StrCat("unknown", absl::string_view(&u, 1)))); + } + ASSERT_THAT(activation_.SetUnknownPatterns(attribute_patterns), IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAreArray(attributes)); + EXPECT_THAT(result.GetUnknown().ToFunctionResultSet(), + UnorderedElementsAreArray(functions)); +} + +TEST_F(ModernUnknownTest, ListFastMerge2) { + auto program = CreateProgram(R"cel(full([a, b, unknownA(), unknownB()]))cel"); + std::vector attributes; + std::vector attribute_patterns; + std::vector functions; + for (char c = 'a'; c <= 'b'; ++c) { + attributes.push_back(cel::Attribute(std::string(1, c))); + attribute_patterns.push_back(cel::AttributePattern(std::string(1, c))); + char u = absl::ascii_toupper(c); + functions.push_back( + cel::FunctionResult(absl::StrCat("unknown", absl::string_view(&u, 1)))); + } + ASSERT_THAT(activation_.SetUnknownPatterns(attribute_patterns), IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAreArray(attributes)); + EXPECT_THAT(result.GetUnknown().ToFunctionResultSet(), + UnorderedElementsAreArray(functions)); +} + +TEST_F(ModernUnknownTest, ListLargeMerge) { + auto program = CreateProgram( + R"cel(full([a, b, c, d, e, f, g, h, i, j, k, l, m, + n, o, p, q, r, s, t, u, v, w, x, y, z, + unknownA(), unknownB(), unknownC(), unknownD(), unknownE(), unknownF(), + unknownG(), unknownH(), unknownI(), unknownJ(), unknownK(), unknownL(), + unknownM(), unknownN(), unknownO(), unknownP(), unknownQ(), unknownR(), + unknownS(), unknownT(), unknownU(), unknownV(), unknownW(), unknownX(), + unknownY(), unknownZ()]))cel"); + std::vector attributes; + std::vector attribute_patterns; + std::vector functions; + for (char c = 'a'; c <= 'z'; ++c) { + attributes.push_back(cel::Attribute(std::string(1, c))); + attribute_patterns.push_back(cel::AttributePattern(std::string(1, c))); + char u = absl::ascii_toupper(c); + functions.push_back( + cel::FunctionResult(absl::StrCat("unknown", absl::string_view(&u, 1)))); + } + ASSERT_THAT(activation_.SetUnknownPatterns(attribute_patterns), IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAreArray(attributes)); + EXPECT_THAT(result.GetUnknown().ToFunctionResultSet(), + UnorderedElementsAreArray(functions)); +} + +TEST_F(ModernUnknownTest, ListPartialMerge) { + auto program = CreateProgram( + R"cel(full([a, b, c, d, e, f, g, h, i, j, k, l, m, + n, o, p, q, r, s, t, u, v, w, x, y, z, + unknownA(), unknownB(), unknownC(), unknownD(), unknownE(), unknownF(), + unknownG(), unknownH(), unknownI(), unknownJ(), unknownK(), unknownL(), + unknownM(), unknownN(), unknownO(), unknownP(), unknownQ(), unknownR(), + unknownS(), unknownT(), unknownU(), unknownV(), unknownW(), unknownX(), + unknownY(), unknownZ()]))cel"); + std::vector attributes; + std::vector attribute_patterns; + std::vector functions; + for (char c = 'a'; c <= 'z'; ++c) { + attributes.push_back(cel::Attribute(std::string(1, c))); + attribute_patterns.push_back(cel::AttributePattern( + std::string(1, c), {cel::AttributeQualifierPattern::Wildcard()})); + ASSERT_TRUE( + activation_.InsertOrAssignValue(std::string(1, c), cel::MapValue())); + char u = absl::ascii_toupper(c); + functions.push_back( + cel::FunctionResult(absl::StrCat("unknown", absl::string_view(&u, 1)))); + } + ASSERT_THAT(activation_.SetUnknownPatterns(attribute_patterns), IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()) << result; + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAreArray(attributes)); + EXPECT_THAT(result.GetUnknown().ToFunctionResultSet(), + UnorderedElementsAreArray(functions)); +} + +TEST_F(ModernUnknownTest, KnownUnknownPartial) { + auto program = CreateProgram(R"cel(full(a.b))cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("a")}), + IsOk()); + ASSERT_THAT(activation_.SetKnownPatterns({cel::AttributePattern( + "a", {cel::AttributeQualifierPattern::OfString("b"), + cel::AttributeQualifierPattern::OfString("c")})}), + IsOk()); + BindJsonToActivation("a", R"json({"b": {"c": true}})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()) << result; + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute( + "a", {cel::AttributeQualifier::OfString("b")}))); +} + +TEST_F(ModernUnknownTest, KnownUnknownFull) { + auto program = CreateProgram(R"cel(full(a.b.d))cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("a")}), + IsOk()); + ASSERT_THAT(activation_.SetKnownPatterns({cel::AttributePattern( + "a", {cel::AttributeQualifierPattern::OfString("b"), + cel::AttributeQualifierPattern::OfString("c")})}), + IsOk()); + BindJsonToActivation("a", R"json({"b": {"c": true}})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()) << result; + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute( + "a", {cel::AttributeQualifier::OfString("b"), + cel::AttributeQualifier::OfString("d")}))); +} + +} // namespace diff --git a/eval/tests/unknowns_end_to_end_test.cc b/eval/tests/unknowns_end_to_end_test.cc index 71ffe652c..cffeae5c6 100644 --- a/eval/tests/unknowns_end_to_end_test.cc +++ b/eval/tests/unknowns_end_to_end_test.cc @@ -18,6 +18,7 @@ #include "absl/strings/string_view.h" #include "absl/types/span.h" #include "base/attribute.h" +#include "base/attribute_set.h" #include "base/function_result.h" #include "eval/public/activation.h" #include "eval/public/builtin_func_registrar.h" @@ -33,8 +34,6 @@ #include "internal/status_macros.h" #include "internal/testing.h" #include "parser/parser.h" -#include "runtime/internal/activation_attribute_matcher_access.h" -#include "runtime/internal/attribute_matcher.h" #include "google/protobuf/arena.h" #include "google/protobuf/text_format.h" @@ -45,35 +44,11 @@ namespace runtime { namespace { using ::absl_testing::IsOk; -using ::cel::runtime_internal::ActivationAttributeMatcherAccess; using ::cel::expr::Expr; using ::cel::expr::ParsedExpr; using ::google::api::expr::parser::Parse; using ::google::protobuf::Arena; using ::testing::ElementsAre; -using ::testing::UnorderedElementsAre; - -absl::StatusOr MakeCelMap(absl::string_view expr, - google::protobuf::Arena* arena) { - static CelExpressionBuilder* builder = []() { - return CreateCelExpressionBuilder(InterpreterOptions()).release(); - }(); - static absl::NoDestructor activation; - - CEL_ASSIGN_OR_RETURN(ParsedExpr parsed_expr, Parse(expr)); - - CEL_ASSIGN_OR_RETURN(auto plan, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); - absl::StatusOr result = plan->Evaluate(*activation, arena); - if (!result.ok()) { - return result.status(); - } - if (!result->IsMap()) { - return absl::FailedPreconditionError( - absl::StrCat("expression did not evaluate to a map: ", expr)); - } - return result; -} enum class FunctionResponse { kUnknown, kTrue, kFalse }; @@ -133,7 +108,7 @@ class UnknownsTest : public testing::Test { MATCHER_P(FunctionCallIs, fn_name, "") { const cel::FunctionResult& result = arg; - return result.descriptor().name() == fn_name; + return result.name() == fn_name; } MATCHER_P(AttributeIs, attr, "") { @@ -168,7 +143,9 @@ TEST_F(UnknownsTest, NoUnknowns) { TEST_F(UnknownsTest, UnknownAttributes) { PrepareBuilder(UnknownProcessingOptions::kAttributeOnly); - activation_.set_unknown_attribute_patterns({CelAttributePattern("var1", {})}); + ASSERT_THAT(activation_.SetUnknownAttributePatterns( + {CelAttributePattern("var1", {})}), + IsOk()); activation_.InsertValue("var2", CelValue::CreateInt64(3)); ASSERT_THAT(activation_.InsertFunction(std::make_unique( "F1", FunctionResponse::kTrue)), @@ -193,7 +170,9 @@ TEST_F(UnknownsTest, UnknownAttributes) { TEST_F(UnknownsTest, UnknownAttributesPruning) { PrepareBuilder(UnknownProcessingOptions::kAttributeOnly); - activation_.set_unknown_attribute_patterns({CelAttributePattern("var1", {})}); + ASSERT_THAT(activation_.SetUnknownAttributePatterns( + {CelAttributePattern("var1", {})}), + IsOk()); activation_.InsertValue("var2", CelValue::CreateInt64(5)); ASSERT_THAT(activation_.InsertFunction(std::make_unique( "F1", FunctionResponse::kTrue)), @@ -215,49 +194,6 @@ TEST_F(UnknownsTest, UnknownAttributesPruning) { EXPECT_TRUE(response.BoolOrDie()); } -class CustomMatcher : public cel::runtime_internal::AttributeMatcher { - public: - MatchResult CheckForUnknown(const cel::Attribute& attr) const override { - // Rendering to a string just for ease of testing. - std::string name = attr.AsString().value_or(""); - if (name == "var1") { - return MatchResult::PARTIAL; - } else if (name == "var1.foo") { - return MatchResult::FULL; - } - return MatchResult::NONE; - } -}; - -TEST_F(UnknownsTest, UnknownAttributesCustomMatcher) { - PrepareBuilder(UnknownProcessingOptions::kAttributeOnly); - - ASSERT_OK_AND_ASSIGN(auto var1, MakeCelMap("{'bar': 1}", &arena_)); - activation_.InsertValue("var1", var1); - CustomMatcher matcher; - ActivationAttributeMatcherAccess::SetAttributeMatcher(activation_, &matcher); - - ASSERT_THAT(activation_.InsertFunction(std::make_unique( - "F1", FunctionResponse::kTrue, CelValue::Type::kMap)), - IsOk()); - ASSERT_THAT(activation_.InsertFunction(std::make_unique( - "F2", FunctionResponse::kTrue)), - IsOk()); - - ASSERT_OK_AND_ASSIGN(ParsedExpr expr, - Parse("F1(var1) || var1.foo || var1.bar")); - auto plan = builder_->CreateExpression(&expr.expr(), nullptr); - ASSERT_THAT(plan, IsOk()); - - ASSERT_OK_AND_ASSIGN(CelValue response, - plan.value()->Evaluate(activation_, &arena_)); - - ASSERT_TRUE(response.IsUnknownSet()) << response.DebugString(); - EXPECT_THAT( - response.UnknownSetOrDie()->unknown_attributes(), - UnorderedElementsAre(AttributeIs("var1"), AttributeIs("var1.foo"))); -} - TEST_F(UnknownsTest, UnknownFunctionsWithoutOptionError) { PrepareBuilder(UnknownProcessingOptions::kAttributeOnly); activation_.InsertValue("var1", CelValue::CreateInt64(5)); @@ -310,7 +246,9 @@ TEST_F(UnknownsTest, UnknownFunctions) { TEST_F(UnknownsTest, UnknownsMerge) { PrepareBuilder(UnknownProcessingOptions::kAttributeAndFunction); activation_.InsertValue("var1", CelValue::CreateInt64(5)); - activation_.set_unknown_attribute_patterns({CelAttributePattern("var2", {})}); + ASSERT_THAT(activation_.SetUnknownAttributePatterns( + {CelAttributePattern("var2", {})}), + IsOk()); ASSERT_THAT(activation_.InsertFunction(std::make_unique( "F1", FunctionResponse::kUnknown)), @@ -725,12 +663,15 @@ TEST(UnknownsIterAttrTest, IterAttributeTrail) { activation.InsertValue("var", CelProtoWrapper::CreateMessage(&list, &arena)); // var[1]['elem1'] is unknown - activation.set_unknown_attribute_patterns({CelAttributePattern( - "var", { - CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), - CreateCelAttributeQualifierPattern( - CelValue::CreateStringView("elem1")), - })}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern( + "var", + { + CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), + CreateCelAttributeQualifierPattern( + CelValue::CreateStringView("elem1")), + })}), + IsOk()); ASSERT_THAT(activation.InsertFunction(std::make_unique( "Fn", FunctionResponse::kFalse, CelValue::Type::kMap)), @@ -756,7 +697,7 @@ TEST(UnknownsIterAttrTest, IterAttributeTrailMapKeyTypes) { Activation activation; Arena arena; - UnknownSet unknown_set; + UnknownSet unknown_set(cel::AttributeSet({cel::Attribute("foo")})); CelError error = absl::CancelledError(); std::vector> backing; @@ -800,7 +741,7 @@ TEST(UnknownsIterAttrTest, IterAttributeTrailMapKeyTypesShortcutted) { Activation activation; Arena arena; - UnknownSet unknown_set; + UnknownSet unknown_set(cel::AttributeSet({cel::Attribute("foo")})); CelError error = absl::CancelledError(); std::vector> backing; @@ -920,12 +861,15 @@ TEST(UnknownsIterAttrTest, IterAttributeTrailMap) { activation.InsertValue("var", CelProtoWrapper::CreateMessage(&list, &arena)); // var[1]['key'] is unknown - activation.set_unknown_attribute_patterns({CelAttributePattern( - "var", - { - CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), - CreateCelAttributeQualifierPattern(CelValue::CreateStringView("key")), - })}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern( + "var", + { + CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), + CreateCelAttributeQualifierPattern( + CelValue::CreateStringView("key")), + })}), + IsOk()); ASSERT_THAT(activation.InsertFunction(std::make_unique( "Fn", FunctionResponse::kFalse, CelValue::Type::kDouble)), @@ -1041,8 +985,9 @@ TEST(UnknownsIterAttrTest, IterAttributeTrailExact) { unknown_attribute_patterns.push_back(CelAttributePattern( "list_var", {CreateCelAttributeQualifierPattern(CelValue::CreateInt64(0))})); - activation.set_unknown_attribute_patterns( - std::move(unknown_attribute_patterns)); + ASSERT_THAT(activation.SetUnknownAttributePatterns( + std::move(unknown_attribute_patterns)), + IsOk()); ASSERT_OK_AND_ASSIGN( auto plan, builder->CreateExpression(&expr.expr(), &expr.source_info())); @@ -1087,12 +1032,15 @@ TEST(UnknownsIterAttrTest, IterAttributeTrailFilterValues) { activation.InsertValue("var", CelProtoWrapper::CreateMessage(&list, &arena)); // var[1]['value_key'] is unknown - activation.set_unknown_attribute_patterns({CelAttributePattern( - "var", { - CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), - CreateCelAttributeQualifierPattern( - CelValue::CreateStringView("value_key")), - })}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern( + "var", + { + CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), + CreateCelAttributeQualifierPattern( + CelValue::CreateStringView("value_key")), + })}), + IsOk()); auto plan = builder->CreateExpression(&expr, nullptr).value(); CelValue response = plan->Evaluate(activation, &arena).value(); @@ -1136,21 +1084,23 @@ TEST(UnknownsIterAttrTest, IterAttributeTrailFilterConditions) { activation.InsertValue("var", CelProtoWrapper::CreateMessage(&list, &arena)); // var[1]['value_key'] is unknown - activation.set_unknown_attribute_patterns( - {CelAttributePattern( - "var", - { - CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), - CreateCelAttributeQualifierPattern( - CelValue::CreateStringView("filter_key")), - }), - CelAttributePattern( - "var", - { - CreateCelAttributeQualifierPattern(CelValue::CreateInt64(0)), - CreateCelAttributeQualifierPattern( - CelValue::CreateStringView("filter_key")), - })}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns( + {CelAttributePattern( + "var", + { + CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), + CreateCelAttributeQualifierPattern( + CelValue::CreateStringView("filter_key")), + }), + CelAttributePattern( + "var", + { + CreateCelAttributeQualifierPattern(CelValue::CreateInt64(0)), + CreateCelAttributeQualifierPattern( + CelValue::CreateStringView("filter_key")), + })}), + IsOk()); auto plan = builder->CreateExpression(&expr, nullptr).value(); CelValue response = plan->Evaluate(activation, &arena).value(); diff --git a/extensions/BUILD b/extensions/BUILD index ff37e2c3f..178f34ca9 100644 --- a/extensions/BUILD +++ b/extensions/BUILD @@ -76,12 +76,12 @@ cc_library( hdrs = ["math_ext.h"], deps = [ ":math_ext_decls", - "//common:casting", "//common:value", "//eval/public:cel_function_registry", "//eval/public:cel_number", "//eval/public:cel_options", "//internal:status_macros", + "//runtime:function", "//runtime:function_adapter", "//runtime:function_registry", "//runtime:runtime_options", @@ -171,7 +171,6 @@ cc_test( "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", - "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", @@ -215,7 +214,10 @@ cc_library( srcs = ["bindings_ext.cc"], hdrs = ["bindings_ext.h"], deps = [ - "//common:ast", + "//checker:type_checker_builder", + "//common:decl", + "//common:expr", + "//common:type", "//compiler", "//internal:status_macros", "//parser:macro", @@ -323,28 +325,30 @@ cc_library( "//base:builtins", "//common:ast", "//common:ast_rewrite", - "//common:casting", "//common:constant", "//common:expr", "//common:function_descriptor", "//common:kind", + "//common:legacy_value", "//common:native_type", "//common:type", "//common:value", + "//common/internal:attribute_trail", "//eval/compiler:flat_expr_builder", "//eval/compiler:flat_expr_builder_extensions", - "//eval/eval:attribute_trail", - "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", "//eval/eval:expression_step_base", + "//extensions/protobuf/internal:map_reflection", "//internal:casts", "//internal:number", "//internal:status_macros", "//runtime:runtime_builder", + "//runtime:runtime_options", "//runtime/internal:errors", "//runtime/internal:runtime_friend_access", "//runtime/internal:runtime_impl", "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/functional:overload", @@ -352,7 +356,6 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_absl//absl/types:variant", "@com_google_protobuf//:protobuf", @@ -374,7 +377,9 @@ cc_test( "//common:decl_proto", "//common:expr", "//common:kind", - "//common:memory", + "//common:native_type", + "//common:type", + "//common:typeinfo", "//common:value", "//compiler", "//compiler:compiler_factory", @@ -388,22 +393,24 @@ cc_test( "//eval/public:cel_type_registry", "//eval/public:cel_value", "//eval/public/structs:cel_proto_wrapper", - "//eval/public/structs:legacy_type_adapter", - "//eval/public/structs:legacy_type_info_apis", "//extensions/protobuf:ast_converters", "//internal:number", "//internal:status_macros", "//internal:testing", "//parser", + "//runtime", "//runtime:activation", "//runtime:function_adapter", "//runtime:function_registry", + "//runtime:runtime_builder", "//runtime:runtime_issue", "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "//runtime:type_registry", "//runtime/internal:issue_collector", "//runtime/internal:runtime_env", "//runtime/internal:runtime_env_testing", + "//testutil:test_external_extensions_descriptor_set", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_check", @@ -412,7 +419,6 @@ cc_test( "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", - "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", @@ -467,6 +473,7 @@ cc_test( ":lists_functions", "//checker:type_check_issue", "//checker:validation_result", + "//common:ast", "//common:source", "//common:value", "//common:value_testing", @@ -522,11 +529,14 @@ cc_test( name = "sets_functions_test", srcs = ["sets_functions_test.cc"], deps = [ + ":set_membership_optimization", ":sets_functions", "//checker:standard_library", "//checker:validation_result", "//common:ast_proto", "//common:minimal_descriptor_pool", + "//common:value", + "//compiler", "//compiler:compiler_factory", "//eval/public:activation", "//eval/public:builtin_func_registrar", @@ -536,7 +546,11 @@ cc_test( "//eval/public:cel_options", "//eval/public:cel_value", "//internal:testing", + "//runtime", + "//runtime:activation", + "//runtime:runtime_builder", "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/status:status_matchers", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", @@ -548,25 +562,35 @@ cc_test( srcs = ["sets_functions_benchmark_test.cc"], tags = ["benchmark"], deps = [ + ":set_membership_optimization", ":sets_functions", + "//checker:type_checker_builder", + "//checker:validation_result", + "//common:ast", + "//common:decl", + "//common:minimal_descriptor_pool", + "//common:type", "//common:value", - "//eval/internal:interop", - "//eval/public:activation", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_expression", - "//eval/public:cel_options", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", "//eval/public:cel_value", "//eval/public/containers:container_backed_list_impl", "//internal:benchmark", "//internal:status_macros", "//internal:testing", - "//parser", + "//runtime", + "//runtime:activation", + "//runtime:constant_folding", + "//runtime:reference_resolver", + "//runtime:runtime_builder", "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", ], ) @@ -586,6 +610,7 @@ cc_library( "//eval/public:cel_function_registry", "//eval/public:cel_options", "//internal:status_macros", + "//runtime:function", "//runtime:function_adapter", "//runtime:function_registry", "//runtime:runtime_options", @@ -741,7 +766,6 @@ cc_library( "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:btree", - "@com_google_absl//absl/memory", "@com_google_absl//absl/numeric:bits", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -855,3 +879,85 @@ cc_test( "@com_google_protobuf//:protobuf", ], ) + +cc_library( + name = "set_membership_optimization", + srcs = ["set_membership_optimization.cc"], + hdrs = ["set_membership_optimization.h"], + deps = [ + "//base:builtins", + "//common:ast", + "//common:constant", + "//common:expr", + "//common:native_type", + "//common:standard_definitions", + "//common:type", + "//common:value", + "//common/internal:attribute_trail", + "//eval/compiler:flat_expr_builder", + "//eval/compiler:flat_expr_builder_extensions", + "//eval/eval:evaluator_core", + "//eval/eval:expression_step_base", + "//internal:casts", + "//internal:status_macros", + "//runtime", + "//runtime:runtime_builder", + "//runtime/internal:errors", + "//runtime/internal:runtime_friend_access", + "//runtime/internal:runtime_impl", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/hash", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + ], +) + +cc_test( + name = "set_membership_optimization_test", + srcs = ["set_membership_optimization_test.cc"], + deps = [ + ":bindings_ext", + ":set_membership_optimization", + ":sets_functions", + "//base:attributes", + "//checker:type_checker_builder", + "//checker:validation_result", + "//common:ast", + "//common:decl", + "//common:standard_definitions", + "//common:type", + "//common:value", + "//common:value_kind", + "//common:value_testing", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", + "//eval/compiler:constant_folding", + "//eval/compiler:flat_expr_builder", + "//eval/compiler:qualified_reference_resolver", + "//eval/eval:evaluator_core", + "//internal:status_macros", + "//internal:testing", + "//internal:testing_descriptor_pool", + "//runtime", + "//runtime:activation", + "//runtime:constant_folding", + "//runtime:function_adapter", + "//runtime:reference_resolver", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_functions", + "//runtime:standard_runtime_builder_factory", + "//runtime/internal:runtime_env", + "//runtime/internal:runtime_env_testing", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", + "@com_google_protobuf//:protobuf", + ], +) diff --git a/extensions/bindings_ext.cc b/extensions/bindings_ext.cc index f097709ca..b9a80d493 100644 --- a/extensions/bindings_ext.cc +++ b/extensions/bindings_ext.cc @@ -21,7 +21,10 @@ #include "absl/status/statusor.h" #include "absl/types/optional.h" #include "absl/types/span.h" -#include "common/ast.h" +#include "checker/type_checker_builder.h" +#include "common/decl.h" +#include "common/expr.h" +#include "common/type.h" #include "compiler/compiler.h" #include "internal/status_macros.h" #include "parser/macro.h" @@ -34,6 +37,8 @@ namespace { static constexpr char kCelNamespace[] = "cel"; static constexpr char kBind[] = "bind"; +static constexpr char kBlock[] = "cel.@block"; +static constexpr char kBlockOverloadId[] = "cel_block_list"; static constexpr char kUnusedIterVar[] = "#unused"; bool IsTargetNamespace(const Expr& target) { @@ -47,6 +52,19 @@ inline absl::Status ConfigureParser(ParserBuilder& parser_builder) { return absl::OkStatus(); } +absl::Status ConfigureChecker(int version, + TypeCheckerBuilder& type_checker_builder) { + if (version < 1) { + return absl::OkStatus(); + } + static Type kParam(TypeParamType("T")); + CEL_ASSIGN_OR_RETURN( + auto decl, + MakeFunctionDecl(kBlock, MakeOverloadDecl(kBlockOverloadId, kParam, + ListType(), kParam))); + return type_checker_builder.AddFunction(std::move(decl)); +} + } // namespace std::vector bindings_macros() { @@ -55,7 +73,7 @@ std::vector bindings_macros() { [](MacroExprFactory& factory, Expr& target, absl::Span args) -> absl::optional { if (!IsTargetNamespace(target)) { - return absl::nullopt; + return std::nullopt; } if (!args[0].has_ident_expr()) { return factory.ReportErrorAt( @@ -70,8 +88,20 @@ std::vector bindings_macros() { return {*cel_bind}; } -CompilerLibrary BindingsCompilerLibrary() { - return CompilerLibrary("cel.lib.ext.bindings", &ConfigureParser); +CompilerLibrary BindingsCompilerLibrary(int version) { + return CompilerLibrary( + "cel.lib.ext.bindings", &ConfigureParser, + [version](auto& b) { return ConfigureChecker(version, b); }); +} + +CheckerLibrary BindingsCheckerLibrary(int version) { + return CheckerLibrary{"cel.lib.ext.bindings", [version](auto& b) { + return ConfigureChecker(version, b); + }}; +} + +ParserLibrary BindingsParserLibrary() { + return ParserLibrary{"cel.lib.ext.bindings", &ConfigureParser}; } } // namespace cel::extensions diff --git a/extensions/bindings_ext.h b/extensions/bindings_ext.h index a338b24f6..5f54abf25 100644 --- a/extensions/bindings_ext.h +++ b/extensions/bindings_ext.h @@ -22,9 +22,11 @@ #include "parser/macro.h" #include "parser/macro_registry.h" #include "parser/options.h" +#include "parser/parser_interface.h" namespace cel::extensions { +constexpr int kBindingsVersionLatest = 1; // bindings_macros() returns a macro for cel.bind() which can be used to support // local variable bindings within expressions. std::vector bindings_macros(); @@ -35,7 +37,13 @@ inline absl::Status RegisterBindingsMacros(MacroRegistry& registry, } // Declarations for the bindings extension library. -CompilerLibrary BindingsCompilerLibrary(); +CompilerLibrary BindingsCompilerLibrary(int version = kBindingsVersionLatest); + +// Declarations for the bindings extension library. +CheckerLibrary BindingsCheckerLibrary(int version = kBindingsVersionLatest); + +// Macros for the bindings extension library. +ParserLibrary BindingsParserLibrary(); } // namespace cel::extensions diff --git a/extensions/bindings_ext_test.cc b/extensions/bindings_ext_test.cc index c8b12c24a..3c396616c 100644 --- a/extensions/bindings_ext_test.cc +++ b/extensions/bindings_ext_test.cc @@ -624,9 +624,10 @@ TEST_P(BindingsExtInteractionsTest, UnknownAttributesSelectOptimization) { ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&expr)); Arena arena; Activation activation; - activation.set_unknown_attribute_patterns({AttributePattern( - "msg", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("child")})}); + ASSERT_THAT(activation.SetUnknownAttributePatterns({AttributePattern( + "msg", {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("child")})}), + IsOk()); NestedTestAllTypes msg; msg.mutable_child()->mutable_child()->mutable_payload()->set_single_int64(42); @@ -662,11 +663,13 @@ TEST_P(BindingsExtInteractionsTest, ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&expr)); Arena arena; Activation activation; - activation.set_unknown_attribute_patterns({AttributePattern( - "msg", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("payload"), - AttributeQualifierPattern::OfString("single_int64")})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({AttributePattern( + "msg", {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("payload"), + AttributeQualifierPattern::OfString("single_int64")})}), + IsOk()); NestedTestAllTypes msg; msg.mutable_child()->mutable_child()->mutable_payload()->set_single_int64(42); @@ -703,11 +706,13 @@ TEST_P(BindingsExtInteractionsTest, MissingAttributesSelectOptimization) { ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&expr)); Arena arena; Activation activation; - activation.set_missing_attribute_patterns({AttributePattern( - "msg", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("payload"), - AttributeQualifierPattern::OfString("single_int64")})}); + ASSERT_THAT( + activation.SetMissingAttributePatterns({AttributePattern( + "msg", {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("payload"), + AttributeQualifierPattern::OfString("single_int64")})}), + IsOk()); NestedTestAllTypes msg; msg.mutable_child()->mutable_child()->mutable_payload()->set_single_int64(42); @@ -751,10 +756,12 @@ TEST_P(BindingsExtInteractionsTest, UnknownAttribute) { &expr.expr(), &expr.source_info())); Arena arena; Activation activation; - activation.set_unknown_attribute_patterns({AttributePattern( - "msg", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("payload"), - AttributeQualifierPattern::OfString("single_int64")})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({AttributePattern( + "msg", {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("payload"), + AttributeQualifierPattern::OfString("single_int64")})}), + IsOk()); NestedTestAllTypes msg; msg.mutable_child()->mutable_child()->mutable_payload()->set_single_int64(42); @@ -797,10 +804,12 @@ TEST_P(BindingsExtInteractionsTest, UnknownAttributeReturnValue) { &expr.expr(), &expr.source_info())); Arena arena; Activation activation; - activation.set_unknown_attribute_patterns({AttributePattern( - "msg", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("payload"), - AttributeQualifierPattern::OfString("single_int64")})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({AttributePattern( + "msg", {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("payload"), + AttributeQualifierPattern::OfString("single_int64")})}), + IsOk()); NestedTestAllTypes msg; msg.mutable_child()->mutable_child()->mutable_payload()->set_single_int64(42); @@ -847,10 +856,12 @@ TEST_P(BindingsExtInteractionsTest, MissingAttribute) { &expr.expr(), &expr.source_info())); Arena arena; Activation activation; - activation.set_missing_attribute_patterns({AttributePattern( - "msg", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("payload"), - AttributeQualifierPattern::OfString("single_int64")})}); + ASSERT_THAT( + activation.SetMissingAttributePatterns({AttributePattern( + "msg", {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("payload"), + AttributeQualifierPattern::OfString("single_int64")})}), + IsOk()); NestedTestAllTypes msg; msg.mutable_child()->mutable_child()->mutable_payload()->set_single_int64(42); diff --git a/extensions/comprehensions_v2_functions.cc b/extensions/comprehensions_v2_functions.cc index bf23780c0..b6414b5d4 100644 --- a/extensions/comprehensions_v2_functions.cc +++ b/extensions/comprehensions_v2_functions.cc @@ -45,7 +45,7 @@ absl::StatusOr MapInsertKeyValue( // Fast path, runtime has given us a mutable map. We can mutate it directly // and return it. CEL_RETURN_IF_ERROR(mutable_map_value->Put(key, value)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); return map; } // Slow path, we have to make a copy. @@ -63,8 +63,8 @@ absl::StatusOr MapInsertKeyValue( return true; }, descriptor_pool, message_factory, arena)) - .With(ErrorValueReturn()); - CEL_RETURN_IF_ERROR(builder->Put(key, value)).With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); + CEL_RETURN_IF_ERROR(builder->Put(key, value)).With(ErrorValueReturn(arena)); return std::move(*builder).Build(); } @@ -85,7 +85,7 @@ absl::StatusOr MapInsertMap( return true; }, descriptor_pool, message_factory, arena)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); return map; } // Slow path, we have to make a copy. @@ -103,7 +103,7 @@ absl::StatusOr MapInsertMap( return true; }, descriptor_pool, message_factory, arena)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); CEL_RETURN_IF_ERROR( value.ForEach( [&builder](const Value& key, @@ -112,7 +112,7 @@ absl::StatusOr MapInsertMap( return true; }, descriptor_pool, message_factory, arena)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); return std::move(*builder).Build(); } diff --git a/extensions/comprehensions_v2_macros.cc b/extensions/comprehensions_v2_macros.cc index a8de3a103..a054626f9 100644 --- a/extensions/comprehensions_v2_macros.cc +++ b/extensions/comprehensions_v2_macros.cc @@ -14,12 +14,14 @@ #include "extensions/comprehensions_v2_macros.h" +#include #include #include #include "absl/base/no_destructor.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" +#include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/types/optional.h" #include "absl/types/span.h" @@ -38,16 +40,21 @@ namespace { using ::google::api::expr::common::CelOperator; +bool IsSimpleIdentifier(const Expr& expr) { + return expr.has_ident_expr() && !expr.ident_expr().name().empty() && + !absl::StartsWith(expr.ident_expr().name(), "."); +} + absl::optional ExpandAllMacro2(MacroExprFactory& factory, Expr& target, absl::Span args) { if (args.size() != 3) { return factory.ReportError("all() requires 3 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "all() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "all() second variable name must be a simple identifier"); } @@ -56,15 +63,15 @@ absl::optional ExpandAllMacro2(MacroExprFactory& factory, Expr& target, args[0], "all() second variable must be different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("all() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("all() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewBoolConst(true); auto condition = @@ -89,11 +96,11 @@ absl::optional ExpandExistsMacro2(MacroExprFactory& factory, Expr& target, if (args.size() != 3) { return factory.ReportError("exists() requires 3 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "exists() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "exists() second variable name must be a simple identifier"); } @@ -102,15 +109,15 @@ absl::optional ExpandExistsMacro2(MacroExprFactory& factory, Expr& target, args[0], "exists() second variable must be different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("exists() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("exists() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewBoolConst(false); auto condition = factory.NewCall( @@ -138,11 +145,11 @@ absl::optional ExpandExistsOneMacro2(MacroExprFactory& factory, if (args.size() != 3) { return factory.ReportError("existsOne() requires 3 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "existsOne() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "existsOne() second variable name must be a simple identifier"); @@ -153,15 +160,15 @@ absl::optional ExpandExistsOneMacro2(MacroExprFactory& factory, "existsOne() second variable must be different " "from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("existsOne() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("existsOne() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewIntConst(0); auto condition = factory.NewBoolConst(true); @@ -190,12 +197,12 @@ absl::optional ExpandTransformList3Macro(MacroExprFactory& factory, if (args.size() != 3) { return factory.ReportError("transformList() requires 3 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "transformList() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "transformList() second variable name must be a simple identifier"); @@ -205,15 +212,15 @@ absl::optional ExpandTransformList3Macro(MacroExprFactory& factory, "transformList() second variable must be " "different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("transformList() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("transformList() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } std::string iter_var = args[0].ident_expr().name(); std::string iter_var2 = args[1].ident_expr().name(); @@ -239,12 +246,12 @@ absl::optional ExpandTransformList4Macro(MacroExprFactory& factory, if (args.size() != 4) { return factory.ReportError("transformList() requires 4 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "transformList() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "transformList() second variable name must be a simple identifier"); @@ -254,15 +261,15 @@ absl::optional ExpandTransformList4Macro(MacroExprFactory& factory, "transformList() second variable must be " "different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("transformList() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("transformList() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } std::string iter_var = args[0].ident_expr().name(); std::string iter_var2 = args[1].ident_expr().name(); @@ -290,12 +297,12 @@ absl::optional ExpandTransformMap3Macro(MacroExprFactory& factory, if (args.size() != 3) { return factory.ReportError("transformMap() requires 3 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "transformMap() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "transformMap() second variable name must be a simple identifier"); @@ -305,15 +312,15 @@ absl::optional ExpandTransformMap3Macro(MacroExprFactory& factory, "transformMap() second variable must be " "different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("transformMap() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("transformMap() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } std::string iter_var = args[0].ident_expr().name(); std::string iter_var2 = args[1].ident_expr().name(); @@ -338,12 +345,12 @@ absl::optional ExpandTransformMap4Macro(MacroExprFactory& factory, if (args.size() != 4) { return factory.ReportError("transformMap() requires 4 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "transformMap() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "transformMap() second variable name must be a simple identifier"); @@ -353,15 +360,15 @@ absl::optional ExpandTransformMap4Macro(MacroExprFactory& factory, "transformMap() second variable must be " "different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("transformMap() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("transformMap() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } std::string iter_var = args[0].ident_expr().name(); std::string iter_var2 = args[1].ident_expr().name(); @@ -388,12 +395,12 @@ absl::optional ExpandTransformMapEntry3Macro(MacroExprFactory& factory, if (args.size() != 3) { return factory.ReportError("transformMapEntry() requires 3 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "transformMapEntry() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "transformMapEntry() second variable name must be a simple identifier"); @@ -403,17 +410,17 @@ absl::optional ExpandTransformMapEntry3Macro(MacroExprFactory& factory, "transformMapEntry() second variable must be " "different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("transformMapEntry() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("transformMapEntry() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } std::string iter_var = args[0].ident_expr().name(); std::string iter_var2 = args[1].ident_expr().name(); @@ -438,12 +445,12 @@ absl::optional ExpandTransformMapEntry4Macro(MacroExprFactory& factory, if (args.size() != 4) { return factory.ReportError("transformMapEntry() requires 4 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "transformMapEntry() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "transformMapEntry() second variable name must be a simple identifier"); @@ -453,17 +460,17 @@ absl::optional ExpandTransformMapEntry4Macro(MacroExprFactory& factory, "transformMapEntry() second variable must be " "different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("transformMapEntry() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("transformMapEntry() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } std::string iter_var = args[0].ident_expr().name(); std::string iter_var2 = args[1].ident_expr().name(); diff --git a/extensions/encoders.cc b/extensions/encoders.cc index 66431b30b..4aeb00cca 100644 --- a/extensions/encoders.cc +++ b/extensions/encoders.cc @@ -48,9 +48,10 @@ absl::StatusOr Base64Decode( std::string in; std::string out; if (!absl::Base64Unescape(value.NativeString(in), &out)) { - return ErrorValue{absl::InvalidArgumentError("invalid base64 data")}; + return ErrorValue::From(absl::InvalidArgumentError("invalid base64 data"), + arena); } - return BytesValue(arena, std::move(out)); + return BytesValue::From(std::move(out), arena); } absl::StatusOr Base64Encode( @@ -61,7 +62,7 @@ absl::StatusOr Base64Encode( std::string in; std::string out; out = absl::Base64Escape(value.NativeString(in)); - return StringValue(arena, std::move(out)); + return StringValue::From(std::move(out), arena); } absl::Status RegisterEncodersDecls(TypeCheckerBuilder& builder) { diff --git a/extensions/formatting.cc b/extensions/formatting.cc index 935815569..d05fbe7f6 100644 --- a/extensions/formatting.cc +++ b/extensions/formatting.cc @@ -296,7 +296,7 @@ absl::StatusOr FormatBinary( sign_bit = true; // Negating min int is undefined behavior, so we need to use unsigned // arithmetic. - using unsigned_type = std::make_unsigned::type; + using unsigned_type = std::make_unsigned_t; unsigned_value = -static_cast(tmp); } else { unsigned_value = tmp; @@ -354,7 +354,7 @@ absl::StatusOr FormatHex( if (tmp < 0) { // Negating min int is undefined behavior, so we need to use unsigned // arithmetic. - using unsigned_type = std::make_unsigned::type; + using unsigned_type = std::make_unsigned_t; scratch = absl::StrFormat("-%x", -static_cast(tmp)); } else { scratch = absl::StrFormat("%x", tmp); @@ -386,7 +386,7 @@ absl::StatusOr FormatOctal( if (tmp < 0) { // Negating min int is undefined behavior, so we need to use unsigned // arithmetic. - using unsigned_type = std::make_unsigned::type; + using unsigned_type = std::make_unsigned_t; scratch = absl::StrFormat("-%o", -static_cast(tmp)); } else { scratch = absl::StrFormat("%o", tmp); @@ -419,6 +419,12 @@ absl::StatusOr GetDouble(const Value& value, std::string& scratch) { str)); } } + if (value.kind() == ValueKind::kInt) { + return static_cast(value.GetInt().NativeValue()); + } + if (value.kind() == ValueKind::kUint) { + return static_cast(value.GetUint().NativeValue()); + } if (value.kind() != ValueKind::kDouble) { return absl::InvalidArgumentError( absl::StrCat("expected a double but got a ", value.GetTypeName())); @@ -512,16 +518,17 @@ absl::StatusOr Format( } ++i; if (i >= format.size()) { - return ErrorValue( - absl::InvalidArgumentError("unexpected end of format string")); + return ErrorValue::From( + absl::InvalidArgumentError("unexpected end of format string"), arena); } if (format[i] == '%') { result.push_back('%'); continue; } if (arg_index >= args_size) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrFormat("index %d out of range", arg_index))); + return ErrorValue::From(absl::InvalidArgumentError(absl::StrFormat( + "index %d out of range", arg_index)), + arena); } CEL_ASSIGN_OR_RETURN(auto value, args.Get(arg_index++, descriptor_pool, message_factory, arena)); @@ -530,7 +537,7 @@ absl::StatusOr Format( descriptor_pool, message_factory, arena, clause_scratch); if (!clause.ok()) { - return ErrorValue(std::move(clause).status()); + return ErrorValue::From(std::move(clause).status(), arena); } absl::StrAppend(&result, clause->second); i += clause->first; diff --git a/extensions/formatting_test.cc b/extensions/formatting_test.cc index b80fe9bc0..a07dd0bb1 100644 --- a/extensions/formatting_test.cc +++ b/extensions/formatting_test.cc @@ -182,8 +182,8 @@ TEST_P(StringFormatTest, TestStringFormatting) { Activation activation; for (const auto& [name, value] : test_case.dyn_args) { if (std::holds_alternative(value)) { - activation.InsertOrAssignValue(name, - StringValue{std::get(value)}); + activation.InsertOrAssignValue( + name, StringValue::From(std::get(value), &arena)); } else if (std::holds_alternative(value)) { activation.InsertOrAssignValue(name, BoolValue{std::get(value)}); } else if (std::holds_alternative(value)) { @@ -553,6 +553,18 @@ INSTANTIATE_TEST_SUITE_P( .format_args = "2.71828", .expected = "2.718280e+00", }, + { + .name = "FixedPointClauseWithInt", + .format = "%f", + .format_args = "3", + .expected = "3.000000", + }, + { + .name = "ScientificNotationWithUint", + .format = "%e", + .format_args = "uint(3)", + .expected = "3.000000e+00", + }, { .name = "NaNSupportForFixedPoint", .format = "%f", diff --git a/extensions/lists_functions.cc b/extensions/lists_functions.cc index 10bc717ed..2617417d3 100644 --- a/extensions/lists_functions.cc +++ b/extensions/lists_functions.cc @@ -68,6 +68,8 @@ absl::Span SortableTypes() { return kTypes; } +constexpr int64_t kMaxRangeSize = 1000000; + // Slow distinct() implementation that uses Equal() to compare values in O(n^2). absl::Status ListDistinctHeterogeneousImpl( const ListValue& list, @@ -214,8 +216,9 @@ absl::StatusOr ListFlatten( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) { if (depth < 0) { - return ErrorValue( - absl::InvalidArgumentError("flatten(): level must be non-negative")); + return ErrorValue::From( + absl::InvalidArgumentError("flatten(): level must be non-negative"), + arena); } auto builder = NewListValueBuilder(arena); CEL_RETURN_IF_ERROR(ListFlattenImpl(list, depth, descriptor_pool, @@ -223,10 +226,24 @@ absl::StatusOr ListFlatten( return std::move(*builder).Build(); } -absl::StatusOr ListRange( - int64_t end, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, +absl::StatusOr ListRange( + int64_t end, int64_t max_range_size, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) { + if (end < 0) { + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "lists.range: size must be non-negative, got %d", end)), + arena); + } + if (end > max_range_size) { + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrFormat("lists.range: size %d exceeds maximum allowed (%d)", + end, max_range_size)), + arena); + } auto builder = NewListValueBuilder(arena); builder->Reserve(end); for (int64_t i = 0; i < end; ++i) { @@ -257,18 +274,25 @@ absl::StatusOr ListSlice( google::protobuf::Arena* absl_nonnull arena) { CEL_ASSIGN_OR_RETURN(size_t size, list.Size()); if (start < 0 || end < 0) { - return ErrorValue(absl::InvalidArgumentError(absl::StrFormat( - "cannot slice(%d, %d), negative indexes not supported", start, end))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "cannot slice(%d, %d), negative indexes not supported", start, + end)), + arena); } if (start > end) { - return cel::ErrorValue(absl::InvalidArgumentError( - absl::StrFormat("cannot slice(%d, %d), start index must be less than " - "or equal to end index", - start, end))); + return cel::ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "cannot slice(%d, %d), start index must be less than " + "or equal to end index", + start, end)), + arena); } if (size < end) { - return cel::ErrorValue(absl::InvalidArgumentError(absl::StrFormat( - "cannot slice(%d, %d), list is length %d", start, end, size))); + return cel::ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "cannot slice(%d, %d), list is length %d", start, end, size)), + arena); } auto builder = NewListValueBuilder(arena); for (int64_t i = start; i < end; ++i) { @@ -303,7 +327,7 @@ absl::StatusOr ListSortByAssociatedKeysNative( }, descriptor_pool, message_factory, arena); if (!status.ok()) { - return ErrorValue(status); + return ErrorValue::From(status, arena); } ABSL_ASSERT(keys_vec.size() == size); // Already checked by the caller. std::vector sorted_indices(keys_vec.size()); @@ -345,11 +369,13 @@ absl::StatusOr ListSortByAssociatedKeys( CEL_ASSIGN_OR_RETURN(size_t list_size, list.Size()); CEL_ASSIGN_OR_RETURN(size_t keys_size, keys.Size()); if (list_size != keys_size) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrFormat("@sortByAssociatedKeys() expected a list of the same " - "size as the associated keys list, but got %d and %d " - "elements respectively.", - list_size, keys_size))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "@sortByAssociatedKeys() expected a list of the same " + "size as the associated keys list, but got %d and %d " + "elements respectively.", + list_size, keys_size)), + arena); } // Empty lists are already sorted. // We don't check for size == 1 because the list could contain a single @@ -385,8 +411,10 @@ absl::StatusOr ListSortByAssociatedKeys( return ListSortByAssociatedKeysNative( list, keys, descriptor_pool, message_factory, arena); default: - return ErrorValue(absl::InvalidArgumentError( - absl::StrFormat("sort(): unsupported type %s", first.GetTypeName()))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "sort(): unsupported type %s", first.GetTypeName())), + arena); } } @@ -454,7 +482,7 @@ Macro ListSortByMacro() { MakeMapComprehension(factory, factory.Copy(sortby_input_ident), std::move(key_ident), std::move(key_expr)); if (!map_compr.has_value()) { - return absl::nullopt; + return std::nullopt; } // Build the call expression: @@ -512,11 +540,27 @@ absl::Status RegisterListFlattenFunction(FunctionRegistry& registry) { return absl::OkStatus(); } -absl::Status RegisterListRangeFunction(FunctionRegistry& registry) { - return UnaryFunctionAdapter, - int64_t>::RegisterGlobalOverload("lists.range", - &ListRange, - registry); +absl::Status RegisterListRangeFunction( + FunctionRegistry& registry, + const ListsExtensionOptions& extension_options) { + constexpr int64_t kMaxRangeSize = 1000000; + int64_t effective_limit = kMaxRangeSize; + if (extension_options.max_range_size > 0 && + extension_options.max_range_size < effective_limit) { + effective_limit = extension_options.max_range_size; + } + return UnaryFunctionAdapter, int64_t>:: + RegisterGlobalOverload( + "lists.range", + [effective_limit]( + int64_t end, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) -> absl::StatusOr { + return ListRange(end, effective_limit, descriptor_pool, + message_factory, arena); + }, + registry); } absl::Status RegisterListReverseFunction(FunctionRegistry& registry) { @@ -657,23 +701,23 @@ absl::Status ConfigureParser(ParserBuilder& builder, int version) { } // namespace -absl::Status RegisterListsFunctions(FunctionRegistry& registry, - const RuntimeOptions& options, - int version) { +absl::Status RegisterListsFunctions( + FunctionRegistry& registry, const RuntimeOptions& options, + const ListsExtensionOptions& extension_options) { CEL_RETURN_IF_ERROR(RegisterListSliceFunction(registry)); - if (version == 0) { + if (extension_options.version == 0) { return absl::OkStatus(); } // Since version 1 CEL_RETURN_IF_ERROR(RegisterListFlattenFunction(registry)); - if (version == 1) { + if (extension_options.version == 1) { return absl::OkStatus(); } // Since version 2 CEL_RETURN_IF_ERROR(RegisterListDistinctFunction(registry)); - CEL_RETURN_IF_ERROR(RegisterListRangeFunction(registry)); + CEL_RETURN_IF_ERROR(RegisterListRangeFunction(registry, extension_options)); CEL_RETURN_IF_ERROR(RegisterListReverseFunction(registry)); CEL_RETURN_IF_ERROR(RegisterListSortFunction(registry)); return absl::OkStatus(); @@ -684,18 +728,23 @@ absl::Status RegisterListsMacros(MacroRegistry& registry, const ParserOptions&, return registry.RegisterMacros(lists_macros(version)); } -CheckerLibrary ListsCheckerLibrary(int version) { +CheckerLibrary ListsCheckerLibrary( + const ListsExtensionOptions& extension_options) { return {.id = "cel.lib.ext.lists", - .configure = [version](TypeCheckerBuilder& builder) { + .configure = [version = extension_options.version]( + TypeCheckerBuilder& builder) { return RegisterListsCheckerDecls(builder, version); }}; } -CompilerLibrary ListsCompilerLibrary(int version) { - auto lib = CompilerLibrary::FromCheckerLibrary(ListsCheckerLibrary(version)); - lib.configure_parser = [version](ParserBuilder& builder) { - return ConfigureParser(builder, version); - }; +CompilerLibrary ListsCompilerLibrary( + const ListsExtensionOptions& extension_options) { + auto lib = CompilerLibrary::FromCheckerLibrary( + ListsCheckerLibrary(extension_options)); + lib.configure_parser = + [version = extension_options.version](ParserBuilder& builder) { + return ConfigureParser(builder, version); + }; return lib; } diff --git a/extensions/lists_functions.h b/extensions/lists_functions.h index 0b057170f..a1dd514d2 100644 --- a/extensions/lists_functions.h +++ b/extensions/lists_functions.h @@ -15,6 +15,8 @@ #ifndef THIRD_PARTY_CEL_CPP_EXTENSIONS_LISTS_FUNCTIONS_H_ #define THIRD_PARTY_CEL_CPP_EXTENSIONS_LISTS_FUNCTIONS_H_ +#include + #include "absl/status/status.h" #include "checker/type_checker_builder.h" #include "compiler/compiler.h" @@ -27,6 +29,18 @@ namespace cel::extensions { constexpr int kListsExtensionLatestVersion = 2; +struct ListsExtensionOptions { + int version = kListsExtensionLatestVersion; + + // Maximum size allowed for lists.range(). + // Setting a tighter limit (e.g. 100) will restrict the max size further. + // A standard limit of 1,000,000 applies if a tighter limit isn't + // configured. + int64_t max_range_size = 1000000; +}; + +using ListsFunctionsOptions = ListsExtensionOptions; + // Register implementations for list extension functions. // // === Since version 0 === @@ -45,9 +59,17 @@ constexpr int kListsExtensionLatestVersion = 2; // // .sort() -> list(T) // -absl::Status RegisterListsFunctions(FunctionRegistry& registry, - const RuntimeOptions& options, - int version = kListsExtensionLatestVersion); +absl::Status RegisterListsFunctions( + FunctionRegistry& registry, const RuntimeOptions& options, + const ListsExtensionOptions& extension_options = {}); + +inline absl::Status RegisterListsFunctions(FunctionRegistry& registry, + const RuntimeOptions& options, + int version) { + ListsExtensionOptions extension_options; + extension_options.version = version; + return RegisterListsFunctions(registry, options, extension_options); +} // Register list macros. // @@ -76,7 +98,14 @@ absl::Status RegisterListsMacros(MacroRegistry& registry, // .reverse() -> list(T) // // .sort() -> list(T_) where T_ is partially orderable -CheckerLibrary ListsCheckerLibrary(int version = kListsExtensionLatestVersion); +CheckerLibrary ListsCheckerLibrary( + const ListsExtensionOptions& extension_options = {}); + +inline CheckerLibrary ListsCheckerLibrary(int version) { + ListsExtensionOptions extension_options; + extension_options.version = version; + return ListsCheckerLibrary(extension_options); +} // Provides decls for the following functions: // @@ -96,8 +125,14 @@ CheckerLibrary ListsCheckerLibrary(int version = kListsExtensionLatestVersion); // // .sort() -> list(T_) where T_ is partially orderable CompilerLibrary ListsCompilerLibrary( - int version = kListsExtensionLatestVersion); + const ListsExtensionOptions& extension_options = {}); + +inline CompilerLibrary ListsCompilerLibrary(int version) { + ListsExtensionOptions extension_options; + extension_options.version = version; + return ListsCompilerLibrary(extension_options); +} } // namespace cel::extensions -#endif // THIRD_PARTY_CEL_CPP_EXTENSIONS_SETS_FUNCTIONS_H_ +#endif // THIRD_PARTY_CEL_CPP_EXTENSIONS_LISTS_FUNCTIONS_H_ diff --git a/extensions/lists_functions_test.cc b/extensions/lists_functions_test.cc index 8e9a3c3f5..6e5243889 100644 --- a/extensions/lists_functions_test.cc +++ b/extensions/lists_functions_test.cc @@ -26,6 +26,7 @@ #include "absl/strings/string_view.h" #include "checker/type_check_issue.h" #include "checker/validation_result.h" +#include "common/ast.h" #include "common/source.h" #include "common/value.h" #include "common/value_testing.h" @@ -123,6 +124,10 @@ INSTANTIATE_TEST_SUITE_P( // lists.range() {R"cel(lists.range(4) == [0,1,2,3])cel"}, {R"cel(lists.range(0) == [])cel"}, + {R"cel(lists.range(-1))cel", + "lists.range: size must be non-negative, got -1"}, + {R"cel(lists.range(1000001))cel", + "lists.range: size 1000001 exceeds maximum allowed (1000000)"}, // .reverse() {R"cel([5,1,2,3].reverse() == [3,2,1,5])cel"}, @@ -457,5 +462,84 @@ std::vector CreateListsExtensionVersionParams() { INSTANTIATE_TEST_SUITE_P(ListsExtensionVersionTest, ListsExtensionVersionTest, ValuesIn(CreateListsExtensionVersionParams())); +TEST(ListsFunctionsTest, CustomMaxRangeSizeOption) { + ListsExtensionOptions ext_options; + ext_options.max_range_size = 100; + + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + NewCompilerBuilder(internal::GetTestingDescriptorPool())); + ASSERT_THAT(compiler_builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(compiler_builder->AddLibrary(ListsCompilerLibrary(ext_options)), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, std::move(*compiler_builder).Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult result, + compiler->Compile("lists.range(101)", "")); + ASSERT_TRUE(result.IsValid()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, result.ReleaseAst()); + + const auto runtime_options = RuntimeOptions{}; + ASSERT_OK_AND_ASSIGN( + auto runtime_builder, + CreateStandardRuntimeBuilder(internal::GetTestingDescriptorPool(), + runtime_options)); + ASSERT_THAT(RegisterListsFunctions(runtime_builder.function_registry(), + runtime_options, ext_options), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto runtime, std::move(runtime_builder).Build()); + ASSERT_OK_AND_ASSIGN(auto program, runtime->CreateProgram(std::move(ast))); + + google::protobuf::Arena arena; + Activation activation; + ASSERT_OK_AND_ASSIGN(Value eval_result, + program->Evaluate(&arena, activation)); + EXPECT_THAT( + eval_result, + ErrorValueIs(StatusIs( + testing::_, + HasSubstr("lists.range: size 101 exceeds maximum allowed (100)")))); +} + +TEST(ListsFunctionsTest, HardCodedLimitAppliesWhenOptionIsLooser) { + ListsExtensionOptions ext_options; + ext_options.max_range_size = 2000000; + + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + NewCompilerBuilder(internal::GetTestingDescriptorPool())); + ASSERT_THAT(compiler_builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(compiler_builder->AddLibrary(ListsCompilerLibrary(ext_options)), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, std::move(*compiler_builder).Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult result, + compiler->Compile("lists.range(1000001)", "")); + ASSERT_TRUE(result.IsValid()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, result.ReleaseAst()); + + const auto runtime_options = RuntimeOptions{}; + ASSERT_OK_AND_ASSIGN( + auto runtime_builder, + CreateStandardRuntimeBuilder(internal::GetTestingDescriptorPool(), + runtime_options)); + ASSERT_THAT(RegisterListsFunctions(runtime_builder.function_registry(), + runtime_options, ext_options), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto runtime, std::move(runtime_builder).Build()); + ASSERT_OK_AND_ASSIGN(auto program, runtime->CreateProgram(std::move(ast))); + + google::protobuf::Arena arena; + Activation activation; + ASSERT_OK_AND_ASSIGN(Value eval_result, + program->Evaluate(&arena, activation)); + EXPECT_THAT( + eval_result, + ErrorValueIs(StatusIs( + testing::_, + HasSubstr( + "lists.range: size 1000001 exceeds maximum allowed (1000000)")))); +} + } // namespace } // namespace cel::extensions diff --git a/extensions/math_ext.cc b/extensions/math_ext.cc index 4d133d90c..49a0d2b38 100644 --- a/extensions/math_ext.cc +++ b/extensions/math_ext.cc @@ -25,12 +25,12 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "common/casting.h" #include "common/value.h" #include "eval/public/cel_function_registry.h" #include "eval/public/cel_number.h" #include "eval/public/cel_options.h" #include "internal/status_macros.h" +#include "runtime/function.h" #include "runtime/function_adapter.h" #include "runtime/function_registry.h" #include "runtime/runtime_options.h" @@ -61,13 +61,13 @@ Value NumberToValue(CelNumber number) { absl::StatusOr ValueToNumber(const Value& value, absl::string_view function) { - if (auto int_value = As(value); int_value) { + if (auto int_value = value.AsInt(); int_value) { return CelNumber::FromInt64(int_value->NativeValue()); } - if (auto uint_value = As(value); uint_value) { + if (auto uint_value = value.AsUint(); uint_value) { return CelNumber::FromUint64(uint_value->NativeValue()); } - if (auto double_value = As(value); double_value) { + if (auto double_value = value.AsDouble(); double_value) { return CelNumber::FromDouble(double_value->NativeValue()); } return absl::InvalidArgumentError( @@ -102,15 +102,16 @@ absl::StatusOr MinList( google::protobuf::Arena* absl_nonnull arena) { CEL_ASSIGN_OR_RETURN(auto iterator, values.NewIterator()); if (!iterator->HasNext()) { - return ErrorValue( - absl::InvalidArgumentError("math.@min argument must not be empty")); + return ErrorValue::From( + absl::InvalidArgumentError("math.@min argument must not be empty"), + arena); } Value value; CEL_RETURN_IF_ERROR( iterator->Next(descriptor_pool, message_factory, arena, &value)); absl::StatusOr current = ValueToNumber(value, kMathMin); if (!current.ok()) { - return ErrorValue{current.status()}; + return ErrorValue::From(current.status(), arena); } CelNumber min = *current; while (iterator->HasNext()) { @@ -118,7 +119,7 @@ absl::StatusOr MinList( iterator->Next(descriptor_pool, message_factory, arena, &value)); absl::StatusOr other = ValueToNumber(value, kMathMin); if (!other.ok()) { - return ErrorValue{other.status()}; + return ErrorValue::From(other.status(), arena); } min = MinNumber(min, *other); } @@ -148,15 +149,16 @@ absl::StatusOr MaxList( google::protobuf::Arena* absl_nonnull arena) { CEL_ASSIGN_OR_RETURN(auto iterator, values.NewIterator()); if (!iterator->HasNext()) { - return ErrorValue( - absl::InvalidArgumentError("math.@max argument must not be empty")); + return ErrorValue::From( + absl::InvalidArgumentError("math.@max argument must not be empty"), + arena); } Value value; CEL_RETURN_IF_ERROR( iterator->Next(descriptor_pool, message_factory, arena, &value)); absl::StatusOr current = ValueToNumber(value, kMathMax); if (!current.ok()) { - return ErrorValue{current.status()}; + return ErrorValue::From(current.status(), arena); } CelNumber min = *current; while (iterator->HasNext()) { @@ -164,7 +166,7 @@ absl::StatusOr MaxList( iterator->Next(descriptor_pool, message_factory, arena, &value)); absl::StatusOr other = ValueToNumber(value, kMathMax); if (!other.ok()) { - return ErrorValue{other.status()}; + return ErrorValue::From(other.status(), arena); } min = MaxNumber(min, *other); } @@ -219,9 +221,10 @@ bool IsFiniteDouble(double value) { return std::isfinite(value); } double AbsDouble(double value) { return std::fabs(value); } -Value AbsInt(int64_t value) { +Value AbsInt(int64_t value, const Function::InvokeContext& context) { if (ABSL_PREDICT_FALSE(value == std::numeric_limits::min())) { - return ErrorValue(absl::InvalidArgumentError("integer overflow")); + return ErrorValue::From(absl::InvalidArgumentError("integer overflow"), + context.arena()); } return IntValue(value < 0 ? -value : value); } @@ -258,21 +261,31 @@ int64_t BitNotInt(int64_t value) { return ~value; } uint64_t BitNotUint(uint64_t value) { return ~value; } -Value BitShiftLeftInt(int64_t lhs, int64_t rhs) { +Value BitShiftLeftInt(int64_t lhs, int64_t rhs, + const Function::InvokeContext& context) { if (ABSL_PREDICT_FALSE(rhs < 0)) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("math.bitShiftLeft() invalid negative shift: ", rhs))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrCat("math.bitShiftLeft() invalid negative shift: ", rhs)), + context.arena()); } if (rhs > 63) { return IntValue(0); } - return IntValue(lhs << static_cast(rhs)); + // Shift in the unsigned domain to avoid undefined behaviour when lhs is + // negative or the shift moves bits into the sign bit, matching the bit + // pattern semantics already used by bitShiftRight. + return IntValue(absl::bit_cast(absl::bit_cast(lhs) + << static_cast(rhs))); } -Value BitShiftLeftUint(uint64_t lhs, int64_t rhs) { +Value BitShiftLeftUint(uint64_t lhs, int64_t rhs, + const Function::InvokeContext& context) { if (ABSL_PREDICT_FALSE(rhs < 0)) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("math.bitShiftLeft() invalid negative shift: ", rhs))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrCat("math.bitShiftLeft() invalid negative shift: ", rhs)), + context.arena()); } if (rhs > 63) { return UintValue(0); @@ -280,10 +293,13 @@ Value BitShiftLeftUint(uint64_t lhs, int64_t rhs) { return UintValue(lhs << static_cast(rhs)); } -Value BitShiftRightInt(int64_t lhs, int64_t rhs) { +Value BitShiftRightInt(int64_t lhs, int64_t rhs, + const Function::InvokeContext& context) { if (ABSL_PREDICT_FALSE(rhs < 0)) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("math.bitShiftRight() invalid negative shift: ", rhs))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrCat("math.bitShiftRight() invalid negative shift: ", rhs)), + context.arena()); } if (rhs > 63) { return IntValue(0); @@ -294,10 +310,13 @@ Value BitShiftRightInt(int64_t lhs, int64_t rhs) { static_cast(rhs))); } -Value BitShiftRightUint(uint64_t lhs, int64_t rhs) { +Value BitShiftRightUint(uint64_t lhs, int64_t rhs, + const Function::InvokeContext& context) { if (ABSL_PREDICT_FALSE(rhs < 0)) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("math.bitShiftRight() invalid negative shift: ", rhs))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrCat("math.bitShiftRight() invalid negative shift: ", rhs)), + context.arena()); } if (rhs > 63) { return UintValue(0); diff --git a/extensions/math_ext_macros.cc b/extensions/math_ext_macros.cc index a66720a60..08b163132 100644 --- a/extensions/math_ext_macros.cc +++ b/extensions/math_ext_macros.cc @@ -72,7 +72,7 @@ absl::optional CheckInvalidArgs(MacroExprFactory &factory, } } - return absl::nullopt; + return std::nullopt; } bool IsListLiteralWithValidArgs(const Expr &arg) { @@ -99,7 +99,7 @@ std::vector math_macros() { [](MacroExprFactory &factory, Expr &target, absl::Span arguments) -> absl::optional { if (!IsTargetNamespace(target)) { - return absl::nullopt; + return std::nullopt; } switch (arguments.size()) { @@ -143,7 +143,7 @@ std::vector math_macros() { [](MacroExprFactory &factory, Expr &target, absl::Span arguments) -> absl::optional { if (!IsTargetNamespace(target)) { - return absl::nullopt; + return std::nullopt; } switch (arguments.size()) { diff --git a/extensions/math_ext_test.cc b/extensions/math_ext_test.cc index 3088e6fa8..ce05ae6ed 100644 --- a/extensions/math_ext_test.cc +++ b/extensions/math_ext_test.cc @@ -23,7 +23,6 @@ #include "absl/algorithm/container.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" -#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "absl/types/span.h" @@ -94,7 +93,7 @@ TestCase MinCase(CelValue v1, CelValue v2, CelValue result) { } TestCase MinCase(CelValue list, CelValue result) { - return TestCase{kMathMin, list, absl::nullopt, result}; + return TestCase{kMathMin, list, std::nullopt, result}; } TestCase MaxCase(CelValue v1, CelValue v2, CelValue result) { @@ -102,7 +101,7 @@ TestCase MaxCase(CelValue v1, CelValue v2, CelValue result) { } TestCase MaxCase(CelValue list, CelValue result) { - return TestCase{kMathMax, list, absl::nullopt, result}; + return TestCase{kMathMax, list, std::nullopt, result}; } struct MacroTestCase { @@ -110,19 +109,6 @@ struct MacroTestCase { absl::string_view err = ""; }; -std::string FormatIssues(const cel::ValidationResult& result) { - std::string issues; - for (const auto& issue : result.GetIssues()) { - if (!issues.empty()) { - absl::StrAppend(&issues, "\n", - issue.ToDisplayString(*result.GetSource())); - } else { - issues = issue.ToDisplayString(*result.GetSource()); - } - } - return issues; -} - class TestFunction : public CelFunction { public: explicit TestFunction(absl::string_view name) @@ -352,10 +338,11 @@ TEST_P(MathExtMacroParamsTest, ParserTests) { TEST_P(MathExtMacroParamsTest, ParserAndCheckerTests) { const MacroTestCase& test_case = GetParam(); - - ASSERT_OK_AND_ASSIGN( - auto compiler_builder, - cel::NewCompilerBuilder(internal::GetTestingDescriptorPool())); + CompilerOptions compile_opts; + compile_opts.adapt_parser_errors = true; + ASSERT_OK_AND_ASSIGN(auto compiler_builder, + cel::NewCompilerBuilder( + internal::GetTestingDescriptorPool(), compile_opts)); ASSERT_THAT(compiler_builder->AddLibrary(StandardCheckerLibrary()), IsOk()); ASSERT_THAT(compiler_builder->AddLibrary(MathCompilerLibrary()), IsOk()); @@ -381,16 +368,16 @@ TEST_P(MathExtMacroParamsTest, ParserAndCheckerTests) { ASSERT_OK_AND_ASSIGN(auto compiler, std::move(*compiler_builder).Build()); - auto result = compiler->Compile(test_case.expr, ""); + ASSERT_OK_AND_ASSIGN(auto result, + compiler->Compile(test_case.expr, "")); if (!test_case.err.empty()) { - EXPECT_THAT(result.status(), StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr(test_case.err))); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatError(), HasSubstr(test_case.err)); return; } - ASSERT_THAT(result, IsOk()); - ASSERT_TRUE(result->IsValid()) << FormatIssues(*result); + ASSERT_TRUE(result.IsValid()) << result.FormatError(); RuntimeOptions opts; ASSERT_OK_AND_ASSIGN( @@ -411,9 +398,8 @@ TEST_P(MathExtMacroParamsTest, ParserAndCheckerTests) { IsOk()); ASSERT_OK_AND_ASSIGN(auto runtime, std::move(runtime_builder).Build()); - - ASSERT_OK_AND_ASSIGN(auto program, - runtime->CreateProgram(*result->ReleaseAst())); + ASSERT_OK_AND_ASSIGN(auto ast, result.ReleaseAst()); + ASSERT_OK_AND_ASSIGN(auto program, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; cel::Activation activation; @@ -577,6 +563,8 @@ INSTANTIATE_TEST_SUITE_P( {"math.bitNot(2) == -3"}, {"math.bitAnd(math.bitNot(0x3u), 0xFFu) == 0xFCu"}, {"math.bitShiftLeft(1, 1) == 2"}, + {"math.bitShiftLeft(-1, 1) == -2"}, + {"math.bitShiftLeft(-4, 2) == -16"}, {"math.bitShiftLeft(1u, 1) == 2u"}, {"math.bitShiftRight(4, 1) == 2"}, {"math.bitShiftRight(4u, 1) == 2u"}})); diff --git a/extensions/proto_ext.cc b/extensions/proto_ext.cc index f38039002..48618f7ae 100644 --- a/extensions/proto_ext.cc +++ b/extensions/proto_ext.cc @@ -45,11 +45,11 @@ absl::optional ValidateExtensionIdentifier(const Expr& expr) { absl::Overload( [](const SelectExpr& select_expr) -> absl::optional { if (select_expr.test_only()) { - return absl::nullopt; + return std::nullopt; } auto op_name = ValidateExtensionIdentifier(select_expr.operand()); if (!op_name.has_value()) { - return absl::nullopt; + return std::nullopt; } return absl::StrCat(*op_name, ".", select_expr.field()); }, @@ -57,7 +57,7 @@ absl::optional ValidateExtensionIdentifier(const Expr& expr) { return ident_expr.name(); }, [](const auto&) -> absl::optional { - return absl::nullopt; + return std::nullopt; }), expr.kind()); } @@ -68,7 +68,7 @@ absl::optional GetExtensionFieldName(const Expr& expr) { select_expr) { return ValidateExtensionIdentifier(expr); } - return absl::nullopt; + return std::nullopt; } bool IsExtensionCall(const Expr& target) { @@ -95,7 +95,7 @@ std::vector proto_macros() { [](MacroExprFactory& factory, Expr& target, absl::Span arguments) -> absl::optional { if (!IsExtensionCall(target)) { - return absl::nullopt; + return std::nullopt; } auto extFieldName = GetExtensionFieldName(arguments[1]); if (!extFieldName.has_value()) { @@ -109,7 +109,7 @@ std::vector proto_macros() { [](MacroExprFactory& factory, Expr& target, absl::Span arguments) -> absl::optional { if (!IsExtensionCall(target)) { - return absl::nullopt; + return std::nullopt; } auto extFieldName = GetExtensionFieldName(arguments[1]); if (!extFieldName.has_value()) { diff --git a/extensions/protobuf/BUILD b/extensions/protobuf/BUILD index 3f4081b09..eae7a7b68 100644 --- a/extensions/protobuf/BUILD +++ b/extensions/protobuf/BUILD @@ -118,11 +118,11 @@ cc_test( deps = [ ":value", "//base:attributes", - "//common:casting", "//common:value", "//common:value_kind", "//common:value_testing", "//internal:testing", + "//testutil:test_external_extensions_descriptor_set", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", @@ -165,11 +165,11 @@ cc_library( srcs = ["bind_proto_to_activation.cc"], hdrs = ["bind_proto_to_activation.h"], deps = [ - ":value", - "//common:casting", "//common:value", "//internal:status_macros", "//runtime:activation", + "//runtime:bind_proto_to_activation", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -182,14 +182,12 @@ cc_test( srcs = ["bind_proto_to_activation_test.cc"], deps = [ ":bind_proto_to_activation", - "//common:casting", "//common:value", "//common:value_testing", "//internal:testing", "//runtime:activation", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", - "@com_google_absl//absl/types:optional", "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:wrappers_cc_proto", diff --git a/extensions/protobuf/bind_proto_to_activation.cc b/extensions/protobuf/bind_proto_to_activation.cc index aa151cb85..bdebd08eb 100644 --- a/extensions/protobuf/bind_proto_to_activation.cc +++ b/extensions/protobuf/bind_proto_to_activation.cc @@ -13,80 +13,3 @@ // limitations under the License. #include "extensions/protobuf/bind_proto_to_activation.h" - -#include - -#include "absl/base/nullability.h" -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "common/value.h" -#include "internal/status_macros.h" -#include "runtime/activation.h" -#include "google/protobuf/arena.h" -#include "google/protobuf/descriptor.h" -#include "google/protobuf/message.h" - -namespace cel::extensions::protobuf_internal { - -namespace { - -using ::google::protobuf::Descriptor; - -absl::StatusOr ShouldBindField( - const google::protobuf::FieldDescriptor* field_desc, const StructValue& struct_value, - BindProtoUnsetFieldBehavior unset_field_behavior) { - if (unset_field_behavior == BindProtoUnsetFieldBehavior::kBindDefaultValue || - field_desc->is_repeated()) { - return true; - } - return struct_value.HasFieldByNumber(field_desc->number()); -} - -absl::StatusOr GetFieldValue( - const google::protobuf::FieldDescriptor* field_desc, const StructValue& struct_value, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena) { - // Special case unset any. - if (field_desc->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE && - field_desc->message_type()->well_known_type() == - Descriptor::WELLKNOWNTYPE_ANY) { - CEL_ASSIGN_OR_RETURN(bool present, - struct_value.HasFieldByNumber(field_desc->number())); - if (!present) { - return NullValue(); - } - } - - return struct_value.GetFieldByNumber(field_desc->number(), descriptor_pool, - message_factory, arena); -} - -} // namespace - -absl::Status BindProtoToActivation( - const Descriptor& descriptor, const StructValue& struct_value, - BindProtoUnsetFieldBehavior unset_field_behavior, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { - for (int i = 0; i < descriptor.field_count(); i++) { - const google::protobuf::FieldDescriptor* field_desc = descriptor.field(i); - CEL_ASSIGN_OR_RETURN( - bool should_bind, - ShouldBindField(field_desc, struct_value, unset_field_behavior)); - if (!should_bind) { - continue; - } - - CEL_ASSIGN_OR_RETURN( - Value field, GetFieldValue(field_desc, struct_value, descriptor_pool, - message_factory, arena)); - - activation->InsertOrAssignValue(field_desc->name(), std::move(field)); - } - - return absl::OkStatus(); -} - -} // namespace cel::extensions::protobuf_internal diff --git a/extensions/protobuf/bind_proto_to_activation.h b/extensions/protobuf/bind_proto_to_activation.h index 61f43c13d..93025185b 100644 --- a/extensions/protobuf/bind_proto_to_activation.h +++ b/extensions/protobuf/bind_proto_to_activation.h @@ -15,116 +15,24 @@ #ifndef THIRD_PARTY_CEL_CPP_EXTENSIONS_PROTOBUF_BIND_PROTO_TO_ACTIVATION_H_ #define THIRD_PARTY_CEL_CPP_EXTENSIONS_PROTOBUF_BIND_PROTO_TO_ACTIVATION_H_ -#include - -#include "absl/base/nullability.h" -#include "absl/status/status.h" -#include "common/casting.h" -#include "common/value.h" -#include "extensions/protobuf/value.h" -#include "internal/status_macros.h" -#include "runtime/activation.h" -#include "google/protobuf/arena.h" -#include "google/protobuf/descriptor.h" -#include "google/protobuf/message.h" +#include "absl/base/attributes.h" +#include "runtime/bind_proto_to_activation.h" namespace cel::extensions { -// Option for handling unset fields on the context proto. -enum class BindProtoUnsetFieldBehavior { - // Bind the message defined default or zero value. - kBindDefaultValue, - // Skip binding unset fields, no value is bound for the corresponding - // variable. - kSkip -}; +using BindProtoUnsetFieldBehavior ABSL_DEPRECATED( + "Use cel::BindProtoUnsetFieldBehavior instead") = + ::cel::BindProtoUnsetFieldBehavior; + +using ::cel::BindProtoToActivation; +using ::cel::BindProtoViewToActivation; namespace protobuf_internal { -// Implements binding provided the context message has already -// been adapted to a suitable struct value. -absl::Status BindProtoToActivation( - const google::protobuf::Descriptor& descriptor, const StructValue& struct_value, - BindProtoUnsetFieldBehavior unset_field_behavior, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation); +using ::cel::runtime_internal::BindProtoToActivation; } // namespace protobuf_internal -// Utility method, that takes a protobuf Message and interprets it as a -// namespace, binding its fields to Activation. This is often referred to as a -// context message. -// -// Field names and values become respective names and values of parameters -// bound to the Activation object. -// Example: -// Assume we have a protobuf message of type: -// message Person { -// int age = 1; -// string name = 2; -// } -// -// The sample code snippet will look as follows: -// -// Person person; -// person.set_name("John Doe"); -// person.age(42); -// -// CEL_RETURN_IF_ERROR(BindProtoToActivation(person, value_factory, -// activation)); -// -// After this snippet, activation will have two parameters bound: -// "name", with string value of "John Doe" -// "age", with int value of 42. -// -// The default behavior for unset fields is to skip them. E.g. if the name field -// is not set on the Person message, it will not be bound in to the activation. -// BindProtoUnsetFieldBehavior::kBindDefault, will bind the cc proto api default -// for the field (either an explicit default value or a type specific default). -// -// For repeated fields, an unset field is bound as an empty list. -template -absl::Status BindProtoToActivation( - const T& context, BindProtoUnsetFieldBehavior unset_field_behavior, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { - static_assert(std::is_base_of_v); - // TODO(uncreated-issue/68): for simplicity, just convert the whole message to a - // struct value. For performance, may be better to convert members as needed. - CEL_ASSIGN_OR_RETURN( - Value parent, - ProtoMessageToValue(context, descriptor_pool, message_factory, arena)); - - if (!InstanceOf(parent)) { - return absl::InvalidArgumentError( - absl::StrCat("context is a well-known type: ", context.GetTypeName())); - } - const StructValue& struct_value = Cast(parent); - - const google::protobuf::Descriptor* descriptor = context.GetDescriptor(); - - if (descriptor == nullptr) { - return absl::InvalidArgumentError( - absl::StrCat("context missing descriptor: ", context.GetTypeName())); - } - - return protobuf_internal::BindProtoToActivation( - *descriptor, struct_value, unset_field_behavior, descriptor_pool, - message_factory, arena, activation); -} -template -absl::Status BindProtoToActivation( - const T& context, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { - return BindProtoToActivation(context, BindProtoUnsetFieldBehavior::kSkip, - descriptor_pool, message_factory, arena, - activation); -} - } // namespace cel::extensions #endif // THIRD_PARTY_CEL_CPP_EXTENSIONS_PROTOBUF_BIND_PROTO_TO_ACTIVATION_H_ diff --git a/extensions/protobuf/bind_proto_to_activation_test.cc b/extensions/protobuf/bind_proto_to_activation_test.cc index fd79508ac..a0ab69123 100644 --- a/extensions/protobuf/bind_proto_to_activation_test.cc +++ b/extensions/protobuf/bind_proto_to_activation_test.cc @@ -17,8 +17,6 @@ #include "google/protobuf/wrappers.pb.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" -#include "absl/types/optional.h" -#include "common/casting.h" #include "common/value.h" #include "common/value_testing.h" #include "internal/testing.h" @@ -76,10 +74,10 @@ TEST_F(BindProtoToActivationTest, BindProtoToActivationSkip) { EXPECT_THAT(activation.FindVariable("single_int32", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(activation.FindVariable("single_sint32", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(BindProtoToActivationTest, BindProtoToActivationDefault) { @@ -123,7 +121,7 @@ TEST_F(BindProtoToActivationTest, BindProtoToActivationDefaultAny) { MATCHER_P(IsListValueOfSize, size, "") { const Value& v = arg; - auto value = As(v); + auto value = v.AsList(); if (!value) { return false; } @@ -185,7 +183,7 @@ TEST_F(BindProtoToActivationTest, BindProtoToActivationRepeatedComplex) { MATCHER_P(IsMapValueOfSize, size, "") { const Value& v = arg; - auto value = As(v); + auto value = v.AsMap(); if (!value) { return false; } diff --git a/extensions/protobuf/internal/qualify.cc b/extensions/protobuf/internal/qualify.cc index dba4f44ae..49de92621 100644 --- a/extensions/protobuf/internal/qualify.cc +++ b/extensions/protobuf/internal/qualify.cc @@ -14,6 +14,7 @@ #include "extensions/protobuf/internal/qualify.h" +#include #include #include @@ -104,7 +105,6 @@ absl::StatusOr> LookupMapValu key_desc->cpp_type_name()); } - std::string proto_key_string; google::protobuf::MapKey proto_key; switch (key_desc->cpp_type()) { case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: @@ -122,8 +122,7 @@ absl::StatusOr> LookupMapValu proto_key.SetInt64Value(*key.GetInt64Key()); break; case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { - proto_key_string = std::string(*key.GetStringKey()); - proto_key.SetStringValue(proto_key_string); + proto_key.SetStringValue(*key.GetStringKey()); } break; case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { uint64_t key_value = *key.GetUint64Key(); @@ -145,7 +144,7 @@ absl::StatusOr> LookupMapValu bool found = cel::extensions::protobuf_internal::LookupMapValue( *reflection, *message, *field_desc, proto_key, &value_ref); if (!found) { - return absl::nullopt; + return std::nullopt; } return value_ref; } @@ -292,13 +291,13 @@ absl::StatusOr ProtoQualifyState::CheckListIndex( cel::builtin::kIndex); } - int index = *qualifier.GetInt64Key(); + int64_t index = *qualifier.AsInt(); int size = reflection_->FieldSize(*message_, repeated_field_desc_); if (index < 0 || index >= size) { return absl::InvalidArgumentError( absl::StrCat("index out of bounds: index=", index, " size=", size)); } - return index; + return static_cast(index); } absl::Status ProtoQualifyState::ApplyAttributeQualifierList( @@ -339,11 +338,7 @@ absl::StatusOr ProtoQualifyState::CheckMapIn qualifier)); if (!value_ref.has_value()) { - std::string key_string; - absl::StatusOr key_string_or = qualifier.AsString(); - if (key_string_or.ok()) { - key_string = *key_string_or; - } + std::string key_string = qualifier.ToString(); return runtime_internal::CreateNoSuchKeyError(key_string); } return std::move(value_ref).value(); diff --git a/extensions/protobuf/memory_manager.cc b/extensions/protobuf/memory_manager.cc index 5b3e6e74b..0953d0a45 100644 --- a/extensions/protobuf/memory_manager.cc +++ b/extensions/protobuf/memory_manager.cc @@ -23,8 +23,7 @@ namespace cel { namespace extensions { MemoryManagerRef ProtoMemoryManager(google::protobuf::Arena* arena) { - return arena != nullptr ? MemoryManagerRef::Pooling(arena) - : MemoryManagerRef::ReferenceCounting(); + return MemoryManagerRef::Pooling(arena); } google::protobuf::Arena* absl_nullable ProtoMemoryManagerArena( diff --git a/extensions/protobuf/memory_manager_test.cc b/extensions/protobuf/memory_manager_test.cc index ddab4cf32..956842bb9 100644 --- a/extensions/protobuf/memory_manager_test.cc +++ b/extensions/protobuf/memory_manager_test.cc @@ -22,7 +22,6 @@ namespace cel::extensions { namespace { using ::testing::Eq; -using ::testing::IsNull; using ::testing::NotNull; TEST(ProtoMemoryManager, MemoryManagement) { @@ -41,17 +40,12 @@ TEST(ProtoMemoryManagerRef, MemoryManagement) { google::protobuf::Arena arena; auto memory_manager = ProtoMemoryManagerRef(&arena); EXPECT_EQ(memory_manager.memory_management(), MemoryManagement::kPooling); - memory_manager = ProtoMemoryManagerRef(nullptr); - EXPECT_EQ(memory_manager.memory_management(), - MemoryManagement::kReferenceCounting); } TEST(ProtoMemoryManagerRef, Arena) { google::protobuf::Arena arena; auto memory_manager = ProtoMemoryManagerRef(&arena); EXPECT_THAT(ProtoMemoryManagerArena(memory_manager), Eq(&arena)); - memory_manager = ProtoMemoryManagerRef(nullptr); - EXPECT_THAT(ProtoMemoryManagerArena(memory_manager), IsNull()); } } // namespace diff --git a/extensions/protobuf/value.h b/extensions/protobuf/value.h index b7a654064..e965835a0 100644 --- a/extensions/protobuf/value.h +++ b/extensions/protobuf/value.h @@ -56,6 +56,7 @@ ProtoMessageToValue(T&& value, message_factory, arena); } +// Unwraps a protobuf message from a cel::Value. inline absl::Status ProtoMessageFromValue(const Value& value, google::protobuf::Message& dest_message) { const auto* dest_descriptor = dest_message.GetDescriptor(); @@ -88,9 +89,8 @@ inline absl::Status ProtoMessageFromValue(const Value& value, return absl::OkStatus(); } } - return TypeConversionError(value.GetRuntimeType(), - MessageType(dest_descriptor)) - .NativeValue(); + return common_internal::MakeTypeConversionError(value.GetRuntimeType(), + MessageType(dest_descriptor)); } } // namespace cel::extensions diff --git a/extensions/protobuf/value_test.cc b/extensions/protobuf/value_test.cc index 20d9dce2f..712745c36 100644 --- a/extensions/protobuf/value_test.cc +++ b/extensions/protobuf/value_test.cc @@ -32,12 +32,15 @@ #include "absl/strings/string_view.h" #include "absl/time/time.h" #include "base/attribute.h" -#include "common/casting.h" #include "common/value.h" #include "common/value_kind.h" #include "common/value_testing.h" #include "internal/testing.h" +#include "testutil/test_external_extensions_descriptor_set.h" #include "cel/expr/conformance/proto2/test_all_types.pb.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/dynamic_message.h" +#include "google/protobuf/message.h" #include "google/protobuf/text_format.h" namespace cel::extensions { @@ -263,7 +266,7 @@ TEST_F(ProtoValueWrapTest, GetFieldNoSuchField) { descriptor_pool(), message_factory(), arena())); ASSERT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); EXPECT_THAT(struct_value.GetFieldByName("does_not_exist", descriptor_pool(), message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound, @@ -285,7 +288,7 @@ TEST_F(ProtoValueWrapTest, GetFieldByNumber) { single_bytes: "foo")pb"), descriptor_pool(), message_factory(), arena())); EXPECT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); EXPECT_THAT(struct_value.GetFieldByNumber( TestAllTypes::kSingleInt32FieldNumber, descriptor_pool(), @@ -345,7 +348,7 @@ TEST_F(ProtoValueWrapTest, GetFieldByNumberNoSuchField) { single_bytes: "foo")pb"), descriptor_pool(), message_factory(), arena())); EXPECT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); EXPECT_THAT(struct_value.GetFieldByNumber(999, descriptor_pool(), message_factory(), arena()), @@ -366,7 +369,7 @@ TEST_F(ProtoValueWrapTest, HasFieldByNumber) { single_int64: 2)pb"), descriptor_pool(), message_factory(), arena())); EXPECT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); EXPECT_THAT( struct_value.HasFieldByNumber(TestAllTypes::kSingleInt32FieldNumber), @@ -389,7 +392,7 @@ TEST_F(ProtoValueWrapTest, HasFieldByNumberNoSuchField) { single_int64: 2)pb"), descriptor_pool(), message_factory(), arena())); EXPECT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); // Has returns a status directly instead of a CEL error as in Get. EXPECT_THAT( @@ -446,7 +449,7 @@ TEST_F(ProtoValueWrapTest, ProtoMessageForEachField) { )pb"), descriptor_pool(), message_factory(), arena())); EXPECT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); std::vector fields; auto cb = [&fields](absl::string_view field, @@ -469,7 +472,7 @@ TEST_F(ProtoValueWrapTest, ProtoMessageQualify) { )pb"), descriptor_pool(), message_factory(), arena())); EXPECT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); std::vector qualifiers{ FieldSpecifier{TestAllTypes::kStandaloneMessageFieldNumber, @@ -496,7 +499,7 @@ TEST_F(ProtoValueWrapTest, ProtoMessageQualifyHas) { )pb"), descriptor_pool(), message_factory(), arena())); EXPECT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); std::vector qualifiers{ FieldSpecifier{TestAllTypes::kStandaloneMessageFieldNumber, @@ -521,15 +524,15 @@ TEST_F(ProtoValueWrapTest, ProtoInt64MapListKeys) { R"pb( map_int64_int64 { key: 10 value: 20 })pb"), descriptor_pool(), message_factory(), arena())); - ASSERT_OK_AND_ASSIGN(auto map_value, Cast(value).GetFieldByName( + ASSERT_OK_AND_ASSIGN(auto map_value, value.GetStruct().GetFieldByName( "map_int64_int64", descriptor_pool(), message_factory(), arena())); ASSERT_THAT(map_value, MapValueIs(_)); ASSERT_OK_AND_ASSIGN(ListValue key_set, - Cast(map_value).ListKeys( - descriptor_pool(), message_factory(), arena())); + map_value.GetMap().ListKeys(descriptor_pool(), + message_factory(), arena())); EXPECT_THAT(key_set.Size(), IsOkAndHolds(1)); @@ -546,15 +549,15 @@ TEST_F(ProtoValueWrapTest, ProtoInt32MapListKeys) { R"pb( map_int32_int64 { key: 10 value: 20 })pb"), descriptor_pool(), message_factory(), arena())); - ASSERT_OK_AND_ASSIGN(auto map_value, Cast(value).GetFieldByName( + ASSERT_OK_AND_ASSIGN(auto map_value, value.GetStruct().GetFieldByName( "map_int32_int64", descriptor_pool(), message_factory(), arena())); ASSERT_THAT(map_value, MapValueIs(_)); ASSERT_OK_AND_ASSIGN(ListValue key_set, - Cast(map_value).ListKeys( - descriptor_pool(), message_factory(), arena())); + map_value.GetMap().ListKeys(descriptor_pool(), + message_factory(), arena())); EXPECT_THAT(key_set.Size(), IsOkAndHolds(1)); @@ -571,15 +574,15 @@ TEST_F(ProtoValueWrapTest, ProtoBoolMapListKeys) { R"pb( map_bool_int64 { key: false value: 20 })pb"), descriptor_pool(), message_factory(), arena())); - ASSERT_OK_AND_ASSIGN(auto map_value, Cast(value).GetFieldByName( + ASSERT_OK_AND_ASSIGN(auto map_value, value.GetStruct().GetFieldByName( "map_bool_int64", descriptor_pool(), message_factory(), arena())); ASSERT_THAT(map_value, MapValueIs(_)); ASSERT_OK_AND_ASSIGN(ListValue key_set, - Cast(map_value).ListKeys( - descriptor_pool(), message_factory(), arena())); + map_value.GetMap().ListKeys(descriptor_pool(), + message_factory(), arena())); EXPECT_THAT(key_set.Size(), IsOkAndHolds(1)); @@ -598,14 +601,14 @@ TEST_F(ProtoValueWrapTest, ProtoUint32MapListKeys) { descriptor_pool(), message_factory(), arena())); ASSERT_OK_AND_ASSIGN( auto map_value, - Cast(value).GetFieldByName( - "map_uint32_int64", descriptor_pool(), message_factory(), arena())); + value.GetStruct().GetFieldByName("map_uint32_int64", descriptor_pool(), + message_factory(), arena())); ASSERT_THAT(map_value, MapValueIs(_)); ASSERT_OK_AND_ASSIGN(ListValue key_set, - Cast(map_value).ListKeys( - descriptor_pool(), message_factory(), arena())); + map_value.GetMap().ListKeys(descriptor_pool(), + message_factory(), arena())); EXPECT_THAT(key_set.Size(), IsOkAndHolds(1)); @@ -624,14 +627,14 @@ TEST_F(ProtoValueWrapTest, ProtoUint64MapListKeys) { descriptor_pool(), message_factory(), arena())); ASSERT_OK_AND_ASSIGN( auto map_value, - Cast(value).GetFieldByName( - "map_uint64_int64", descriptor_pool(), message_factory(), arena())); + value.GetStruct().GetFieldByName("map_uint64_int64", descriptor_pool(), + message_factory(), arena())); ASSERT_THAT(map_value, MapValueIs(_)); ASSERT_OK_AND_ASSIGN(ListValue key_set, - Cast(map_value).ListKeys( - descriptor_pool(), message_factory(), arena())); + map_value.GetMap().ListKeys(descriptor_pool(), + message_factory(), arena())); EXPECT_THAT(key_set.Size(), IsOkAndHolds(1)); @@ -651,14 +654,14 @@ TEST_F(ProtoValueWrapTest, ProtoStringMapListKeys) { descriptor_pool(), message_factory(), arena())); ASSERT_OK_AND_ASSIGN( auto map_value, - Cast(value).GetFieldByName( - "map_string_int64", descriptor_pool(), message_factory(), arena())); + value.GetStruct().GetFieldByName("map_string_int64", descriptor_pool(), + message_factory(), arena())); ASSERT_THAT(map_value, MapValueIs(_)); ASSERT_OK_AND_ASSIGN(ListValue key_set, - Cast(map_value).ListKeys( - descriptor_pool(), message_factory(), arena())); + map_value.GetMap().ListKeys(descriptor_pool(), + message_factory(), arena())); EXPECT_THAT(key_set.Size(), IsOkAndHolds(1)); @@ -679,12 +682,12 @@ TEST_F(ProtoValueWrapTest, ProtoMapIterator) { descriptor_pool(), message_factory(), arena())); ASSERT_OK_AND_ASSIGN( auto field_value, - Cast(value).GetFieldByName( - "map_int64_int64", descriptor_pool(), message_factory(), arena())); + value.GetStruct().GetFieldByName("map_int64_int64", descriptor_pool(), + message_factory(), arena())); ASSERT_THAT(field_value, MapValueIs(_)); - MapValue map_value = Cast(field_value); + MapValue map_value = field_value.GetMap(); std::vector keys; @@ -710,12 +713,12 @@ TEST_F(ProtoValueWrapTest, ProtoMapForEach) { descriptor_pool(), message_factory(), arena())); ASSERT_OK_AND_ASSIGN( auto field_value, - Cast(value).GetFieldByName( - "map_int64_int64", descriptor_pool(), message_factory(), arena())); + value.GetStruct().GetFieldByName("map_int64_int64", descriptor_pool(), + message_factory(), arena())); ASSERT_THAT(field_value, MapValueIs(_)); - MapValue map_value = Cast(field_value); + MapValue map_value = field_value.GetMap(); std::vector> pairs; @@ -743,12 +746,12 @@ TEST_F(ProtoValueWrapTest, ProtoListIterator) { descriptor_pool(), message_factory(), arena())); ASSERT_OK_AND_ASSIGN( auto field_value, - Cast(value).GetFieldByName( - "repeated_int64", descriptor_pool(), message_factory(), arena())); + value.GetStruct().GetFieldByName("repeated_int64", descriptor_pool(), + message_factory(), arena())); ASSERT_THAT(field_value, ListValueIs(_)); - ListValue list_value = Cast(field_value); + ListValue list_value = field_value.GetList(); std::vector elements; @@ -773,12 +776,12 @@ TEST_F(ProtoValueWrapTest, ProtoListForEachWithIndex) { descriptor_pool(), message_factory(), arena())); ASSERT_OK_AND_ASSIGN( auto field_value, - Cast(value).GetFieldByName( - "repeated_int64", descriptor_pool(), message_factory(), arena())); + value.GetStruct().GetFieldByName("repeated_int64", descriptor_pool(), + message_factory(), arena())); ASSERT_THAT(field_value, ListValueIs(_)); - ListValue list_value = Cast(field_value); + ListValue list_value = field_value.GetList(); std::vector> elements; @@ -796,5 +799,150 @@ TEST_F(ProtoValueWrapTest, ProtoListForEachWithIndex) { ElementsAre(Pair(0, IntValueIs(1)), Pair(1, IntValueIs(2)))); } +class ProtoValueUnwrapTest : public ProtoValueTest {}; + +TEST_F(ProtoValueUnwrapTest, SameDescriptor) { + TestAllTypes src = ParseTextOrDie( + R"pb(single_int32: 1 single_int64: 2 single_string: "hello")pb"); + ASSERT_OK_AND_ASSIGN( + auto value, + ProtoMessageToValue(src, descriptor_pool(), message_factory(), arena())); + TestAllTypes dest; + ASSERT_THAT(ProtoMessageFromValue(value, dest), IsOk()); + EXPECT_EQ(dest.single_int32(), 1); + EXPECT_EQ(dest.single_int64(), 2); + EXPECT_EQ(dest.single_string(), "hello"); +} + +TEST_F(ProtoValueUnwrapTest, DifferentDescriptorSameFullName) { + auto* dynamic_message = + DynamicParseTextProto(R"pb(single_int32: 42 + single_string: "dynamic")pb"); + Value value = Value::WrapMessage(dynamic_message, descriptor_pool(), + message_factory(), arena()); + TestAllTypes dest; + ASSERT_THAT(ProtoMessageFromValue(value, dest), IsOk()); + EXPECT_EQ(dest.single_int32(), 42); + EXPECT_EQ(dest.single_string(), "dynamic"); +} + +TEST_F(ProtoValueUnwrapTest, TypeMismatch) { + TestAllTypes::NestedMessage nested; + nested.set_bb(100); + Value value = Value::WrapMessage(&nested, descriptor_pool(), + message_factory(), arena()); + TestAllTypes dest; + EXPECT_THAT(ProtoMessageFromValue(value, dest), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +TEST_F(ProtoValueUnwrapTest, NonMessageValue) { + TestAllTypes dest; + EXPECT_THAT(ProtoMessageFromValue(IntValue(42), dest), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT( + ProtoMessageFromValue(StringValue::WrapUnsafe("not a message"), dest), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(ProtoMessageFromValue(NullValue(), dest), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +const google::protobuf::DescriptorPool& +GetTestExternalExtensionsDescriptorPoolUnderlay() { + static const google::protobuf::DescriptorPool* pool = []() { + auto pool = std::make_unique( + google::protobuf::DescriptorPool::generated_pool()); + google::protobuf::LinkMessageReflection(); + ABSL_CHECK( + pool->BuildFile(test::GetTestExternalExtensionsFileDescriptor())); + return pool.release(); + }(); + return *pool; +} + +std::unique_ptr MakeTestExtendedMessage( + const google::protobuf::DescriptorPool& pool, google::protobuf::MessageFactory& factory) { + const google::protobuf::Descriptor* desc = + pool.FindMessageTypeByName("cel.expr.conformance.proto2.TestAllTypes"); + ABSL_CHECK(desc != nullptr); + const google::protobuf::FieldDescriptor* ext_msg_field = + pool.FindExtensionByPrintableName( + desc, "cel.cpp.testutil.test_external_extensions_message"); + ABSL_CHECK(ext_msg_field != nullptr); + const google::protobuf::Message* prototype = factory.GetPrototype(desc); + ABSL_CHECK(prototype != nullptr); + std::unique_ptr dynamic_message(prototype->New()); + + const auto* reflection = dynamic_message->GetReflection(); + reflection->SetInt32(dynamic_message.get(), + desc->FindFieldByName("single_int32"), 42); + reflection->SetString(dynamic_message.get(), + desc->FindFieldByName("single_string"), "dynamic"); + google::protobuf::Message* ext = reflection->MutableMessage(dynamic_message.get(), + ext_msg_field, &factory); + ext->GetReflection()->SetString( + ext, ext->GetDescriptor()->FindFieldByName("string_field"), + "external_string"); + return dynamic_message; +} + +TEST_F(ProtoValueUnwrapTest, DynamicMessageFromUnderlayDescriptorPool) { + GTEST_SKIP() << "TODO(b/562935074): Avoid use-after-free when unwrapping " + "messages with extensions from a different factory."; + const auto& pool = GetTestExternalExtensionsDescriptorPoolUnderlay(); + TestAllTypes dest; + { + google::protobuf::DynamicMessageFactory factory(&pool); + factory.SetDelegateToGeneratedFactory(false); + std::unique_ptr dynamic_message = + MakeTestExtendedMessage(pool, factory); + + Value value = + Value::WrapMessage(dynamic_message.get(), &pool, &factory, arena()); + ASSERT_THAT(ProtoMessageFromValue(value, dest), IsOk()); + } + EXPECT_EQ(dest.single_int32(), 42); + EXPECT_EQ(dest.single_string(), "dynamic"); + EXPECT_EQ(dest.unknown_fields().field_count(), 1); +} + +using ProtoValueUnwrapTestDeathTest = ProtoValueUnwrapTest; + +TEST_F(ProtoValueUnwrapTestDeathTest, + DynamicMessageFromUnderlayDescriptorPoolDelegateToGeneratedFactory) { +#ifndef ADDRESS_SANITIZER + GTEST_SKIP() << "Test requires ASAN enabled"; +#else + EXPECT_DEATH_IF_SUPPORTED( + { + const auto& pool = GetTestExternalExtensionsDescriptorPoolUnderlay(); + auto dest = std::make_unique(); + { + // The CEL runtime should not create a factory like this. + // To get this to work, the caller would need to manage the + // message factory, provide it to the runtime, and guarantee that it + // outlives the CEL value or any derived messages. + google::protobuf::DynamicMessageFactory factory(&pool); + factory.SetDelegateToGeneratedFactory(true); + + std::unique_ptr dynamic_message = + MakeTestExtendedMessage(pool, factory); + + Value value = Value::WrapMessage(dynamic_message.get(), &pool, + &factory, arena()); + ASSERT_THAT(ProtoMessageFromValue(value, *dest), IsOk()); + } + + EXPECT_EQ(dest->single_int32(), 42); + EXPECT_EQ(dest->single_string(), "dynamic"); + EXPECT_TRUE(dest->GetReflection()->HasField( + *dest, pool.FindExtensionByName( + "cel.cpp.testutil.test_external_extensions_message"))); + dest.reset(); + }, + "AddressSanitizer: heap-use-after-free"); +#endif +} + } // namespace } // namespace cel::extensions diff --git a/extensions/regex_ext.cc b/extensions/regex_ext.cc index 9c06d90c2..065adc76d 100644 --- a/extensions/regex_ext.cc +++ b/extensions/regex_ext.cc @@ -65,12 +65,14 @@ Value Extract(int regex_max_program_size, const StringValue& target, absl::string_view regex_view = regex.ToStringView(®ex_scratch); RE2 re2(regex_view, cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, regex_max_program_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); const int group_count = re2.NumberOfCapturingGroups(); if (group_count > 1) { - return ErrorValue(absl::InvalidArgumentError(absl::StrFormat( - "regular expression has more than one capturing group: %s", - regex_view))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "regular expression has more than one capturing group: %s", + regex_view)), + arena); } // Space for the full match (\0) and the first capture group (\1). @@ -97,12 +99,14 @@ Value ExtractAll(int regex_max_program_size, const StringValue& target, absl::string_view regex_view = regex.ToStringView(®ex_scratch); RE2 re2(regex_view, cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, regex_max_program_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); const int group_count = re2.NumberOfCapturingGroups(); if (group_count > 1) { - return ErrorValue(absl::InvalidArgumentError(absl::StrFormat( - "regular expression has more than one capturing group: %s", - regex_view))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "regular expression has more than one capturing group: %s", + regex_view)), + arena); } auto builder = NewListValueBuilder(arena); @@ -135,7 +139,7 @@ Value ExtractAll(int regex_max_program_size, const StringValue& target, absl::Status status = builder->Add(StringValue::From(desired_capture, arena)); if (!status.ok()) { - return ErrorValue(status); + return ErrorValue::From(status, arena); } temp_target.remove_prefix(full_match.data() - temp_target.data() + full_match.length()); @@ -158,11 +162,13 @@ Value ReplaceAll(int regex_max_program_size, const StringValue& target, replacement.ToStringView(&replacement_scratch); RE2 re2(regex_view, cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, regex_max_program_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); std::string error_string; if (!re2.CheckRewriteString(replacement_view, &error_string)) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrFormat("invalid replacement string: %s", error_string))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrFormat("invalid replacement string: %s", error_string)), + arena); } std::string output(target_view); @@ -194,11 +200,13 @@ Value ReplaceN(int regex_max_program_size, const StringValue& target, replacement.ToStringView(&replacement_scratch); RE2 re2(regex_view, cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, regex_max_program_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); std::string error_string; if (!re2.CheckRewriteString(replacement_view, &error_string)) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrFormat("invalid replacement string: %s", error_string))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrFormat("invalid replacement string: %s", error_string)), + arena); } std::string output; @@ -217,7 +225,8 @@ Value ReplaceN(int regex_max_program_size, const StringValue& target, if (!re2.Rewrite(&output, replacement_view, match, nmatch)) { // This should ideally not happen given CheckRewriteString passed - return ErrorValue(absl::InternalError("rewrite failed unexpectedly")); + return ErrorValue::From( + absl::InternalError("rewrite failed unexpectedly"), arena); } temp_target.remove_prefix(full_match.data() - temp_target.data() + diff --git a/extensions/regex_functions.cc b/extensions/regex_functions.cc index 005987ae4..3935772a2 100644 --- a/extensions/regex_functions.cc +++ b/extensions/regex_functions.cc @@ -65,12 +65,13 @@ Value ExtractString(int regex_max_program_size, const StringValue& target, RE2 re2(regex_view, cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, regex_max_program_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); std::string output; bool result = RE2::Extract(target_view, re2, rewrite_view, &output); if (!result) { - return ErrorValue(absl::InvalidArgumentError( - "Unable to extract string for the given regex")); + return ErrorValue::From(absl::InvalidArgumentError( + "Unable to extract string for the given regex"), + arena); } return StringValue::From(std::move(output), arena); } @@ -88,12 +89,13 @@ Value CaptureString(int regex_max_program_size, const StringValue& target, absl::string_view target_view = target.ToStringView(&target_scratch); RE2 re2(regex_view, cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, regex_max_program_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); std::string output; bool result = RE2::FullMatch(target_view, re2, &output); if (!result) { - return ErrorValue(absl::InvalidArgumentError( - "Unable to capture groups for the given regex")); + return ErrorValue::From(absl::InvalidArgumentError( + "Unable to capture groups for the given regex"), + arena); } else { return StringValue::From(std::move(output), arena); } @@ -115,12 +117,14 @@ absl::StatusOr CaptureStringN( absl::string_view regex_view = regex.ToStringView(®ex_scratch); RE2 re2(regex_view, cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, regex_max_program_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); const int capturing_groups_count = re2.NumberOfCapturingGroups(); const auto& named_capturing_groups_map = re2.CapturingGroupNames(); if (capturing_groups_count <= 0) { - return ErrorValue(absl::InvalidArgumentError( - "Capturing groups were not found in the given regex.")); + return ErrorValue::From( + absl::InvalidArgumentError( + "Capturing groups were not found in the given regex."), + arena); } std::vector captured_strings(capturing_groups_count); std::vector captured_string_addresses(capturing_groups_count); @@ -132,8 +136,9 @@ absl::StatusOr CaptureStringN( bool result = RE2::FullMatchN(target_view, re2, argv.data(), capturing_groups_count); if (!result) { - return ErrorValue(absl::InvalidArgumentError( - "Unable to capture groups for the given regex")); + return ErrorValue::From(absl::InvalidArgumentError( + "Unable to capture groups for the given regex"), + arena); } auto builder = cel::NewMapValueBuilder(arena); builder->Reserve(capturing_groups_count); diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 0f09773ae..c4bba0d52 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -17,41 +17,45 @@ #include #include #include +#include #include +#include #include #include +#include #include #include "absl/algorithm/container.h" #include "absl/base/nullability.h" +#include "absl/base/optimization.h" #include "absl/container/flat_hash_map.h" #include "absl/functional/overload.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "absl/types/span.h" #include "absl/types/variant.h" #include "base/attribute.h" #include "base/builtins.h" #include "common/ast.h" #include "common/ast_rewrite.h" -#include "common/casting.h" #include "common/constant.h" #include "common/expr.h" #include "common/function_descriptor.h" +#include "common/internal/attribute_trail.h" #include "common/kind.h" +#include "common/legacy_value.h" #include "common/native_type.h" #include "common/type.h" #include "common/value.h" #include "eval/compiler/flat_expr_builder.h" #include "eval/compiler/flat_expr_builder_extensions.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "extensions/protobuf/internal/map_reflection.h" #include "internal/casts.h" #include "internal/number.h" #include "internal/status_macros.h" @@ -59,9 +63,15 @@ #include "runtime/internal/runtime_friend_access.h" #include "runtime/internal/runtime_impl.h" #include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/map_field.h" #include "google/protobuf/message.h" +#include "google/protobuf/reflection.h" + +// undef for Windows +#undef GetMessage namespace cel::extensions { namespace { @@ -73,10 +83,10 @@ using ::cel::ConstantKind; using ::cel::Expr; using ::cel::ExprKind; using ::cel::SelectExpr; -using ::google::api::expr::runtime::AttributeTrail; -using ::google::api::expr::runtime::DirectExpressionStep; +using ::cel::common_internal::AttributeTrail; using ::google::api::expr::runtime::ExecutionFrame; using ::google::api::expr::runtime::ExecutionFrameBase; +using ::google::api::expr::runtime::ExpressionStep; using ::google::api::expr::runtime::ExpressionStepBase; using ::google::api::expr::runtime::PlannerContext; using ::google::api::expr::runtime::ProgramOptimizer; @@ -92,7 +102,7 @@ struct SelectInstruction { // Represents a single qualifier in a traversal path. // TODO(uncreated-issue/51): support variable indexes. using QualifierInstruction = - absl::variant; + std::variant; struct SelectPath { Expr* operand; @@ -153,20 +163,45 @@ Expr MakeSelectPathExpr( // Returns a single select operation based on the inferred type of the operand // and the field name. If the operand type doesn't define the field, returns // nullopt. -absl::optional GetSelectInstruction( +std::optional MakeFieldStepAst( const StructType& runtime_type, PlannerContext& planner_context, absl::string_view field_name) { auto field_or = planner_context.type_reflector() .FindStructTypeFieldByName(runtime_type, field_name) - .value_or(absl::nullopt); + .value_or(std::nullopt); if (field_or.has_value()) { return SelectInstruction{field_or->number(), std::string(field_or->name())}; } - return absl::nullopt; + return std::nullopt; } +// Returns a qualifier instruction derived from a unoptimized ast. +absl::StatusOr MakeIndexStepAst( + const Constant& constant) { + if (constant.has_int_value()) { + return QualifierInstruction(constant.int_value()); + } else if (constant.has_uint_value()) { + return QualifierInstruction(constant.uint_value()); + } else if (constant.has_bool_value()) { + return QualifierInstruction(constant.bool_value()); + } else if (constant.has_string_value()) { + return QualifierInstruction(constant.string_value()); + } else if (constant.has_double_value()) { + cel::internal::Number number(constant.double_value()); + if (number.LosslessConvertibleToInt()) { + return QualifierInstruction(number.AsInt()); + } else if (number.LosslessConvertibleToUint()) { + return QualifierInstruction(number.AsUint()); + } + } + + return absl::InvalidArgumentError("invalid index constant for cel.attribute"); +} + +// Derives select step info from the optimized AST. absl::StatusOr SelectQualifierFromList(const ListExpr& list) { - if (list.elements().size() != 2) { + // Type codes may follow [number, name]; they're unused here. + if (list.elements().size() < 2 || list.elements().size() > 4) { return absl::InvalidArgumentError("Invalid cel.attribute select list"); } @@ -189,29 +224,7 @@ absl::StatusOr SelectQualifierFromList(const ListExpr& list) { field_name.const_expr().string_value()}; } -// Returns a qualifier instruction derived from a unoptimized ast. -absl::StatusOr SelectInstructionFromConstant( - const Constant& constant) { - if (constant.has_int_value()) { - return QualifierInstruction(constant.int_value()); - } else if (constant.has_uint_value()) { - return QualifierInstruction(constant.uint_value()); - } else if (constant.has_bool_value()) { - return QualifierInstruction(constant.bool_value()); - } else if (constant.has_string_value()) { - return QualifierInstruction(constant.string_value()); - } else if (constant.has_double_value()) { - cel::internal::Number number(constant.double_value()); - if (number.LosslessConvertibleToInt()) { - return QualifierInstruction(number.AsInt()); - } else if (number.LosslessConvertibleToUint()) { - return QualifierInstruction(number.AsUint()); - } - } - - return absl::InvalidArgumentError("invalid index constant for cel.attribute"); -} - +// Derives index step info from the optimized AST. absl::StatusOr SelectQualifierFromConstant( const Constant& constant) { if (constant.has_int_value()) { @@ -234,7 +247,7 @@ absl::StatusOr ListIndexFromQualifier(const AttributeQualifier& qual) { int64_t value = -1; switch (qual.kind()) { case Kind::kInt: - value = *qual.GetInt64Key(); + value = *qual.AsInt(); break; default: // TODO(uncreated-issue/51): type-checker will reject an unsigned literal, but @@ -254,13 +267,14 @@ absl::StatusOr MapKeyFromQualifier(const AttributeQualifier& qual, google::protobuf::Arena* absl_nonnull arena) { switch (qual.kind()) { case Kind::kInt: - return cel::IntValue(*qual.GetInt64Key()); + return cel::IntValue(*qual.AsInt()); case Kind::kUint: - return cel::UintValue(*qual.GetUint64Key()); + return cel::UintValue(*qual.AsUint()); case Kind::kBool: - return cel::BoolValue(*qual.GetBoolKey()); + return cel::BoolValue(*qual.AsBool()); case Kind::kString: - return StringValue::From(*qual.GetStringKey(), arena); + // Safe as `qual` is valid for the lifetime of the expression/program. + return StringValue::WrapUnsafe(*qual.GetStringKey()); default: return runtime_internal::CreateNoMatchingOverloadError( cel::builtin::kIndex); @@ -276,9 +290,10 @@ absl::StatusOr ApplyQualifier( absl::Overload( [&](const FieldSpecifier& field_specifier) -> absl::StatusOr { if (!operand.Is()) { - return cel::ErrorValue( + return cel::ErrorValue::From( cel::runtime_internal::CreateNoMatchingOverloadError( - ""), + arena); } return operand.GetStruct().GetFieldByName( field_specifier.name, descriptor_pool, message_factory, arena); @@ -287,21 +302,22 @@ absl::StatusOr ApplyQualifier( if (operand.Is()) { auto index_or = ListIndexFromQualifier(qualifier); if (!index_or.ok()) { - return cel::ErrorValue(index_or.status()); + return cel::ErrorValue::From(index_or.status(), arena); } return operand.GetList().Get(*index_or, descriptor_pool, message_factory, arena); } else if (operand.Is()) { auto key_or = MapKeyFromQualifier(qualifier, arena); if (!key_or.ok()) { - return cel::ErrorValue(key_or.status()); + return cel::ErrorValue::From(key_or.status(), arena); } return operand.GetMap().Get(*key_or, descriptor_pool, message_factory, arena); } - return cel::ErrorValue( + return cel::ErrorValue::From( cel::runtime_internal::CreateNoMatchingOverloadError( - cel::builtin::kIndex)); + cel::builtin::kIndex), + arena); }), qualifier); } @@ -333,9 +349,10 @@ absl::StatusOr FallbackSelect( [&](const FieldSpecifier& field_specifier) -> absl::StatusOr { if (!elem->Is()) { - return cel::ErrorValue( + return cel::ErrorValue::From( cel::runtime_internal::CreateNoMatchingOverloadError( - ""), + arena); } CEL_ASSIGN_OR_RETURN( bool present, @@ -344,13 +361,13 @@ absl::StatusOr FallbackSelect( }, [&](const AttributeQualifier& qualifier) -> absl::StatusOr { if (!elem->Is() || qualifier.kind() != Kind::kString) { - return cel::ErrorValue( - cel::runtime_internal::CreateNoMatchingOverloadError( - "has")); + return cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError("has"), + arena); } return elem->GetMap().Has( - StringValue(arena, *qualifier.GetStringKey()), + StringValue::WrapUnsafe(*qualifier.GetStringKey()), descriptor_pool, message_factory, arena); }), last_instruction); @@ -370,6 +387,9 @@ absl::StatusOr> SelectInstructionsFromCall( instructions.reserve(ast_path.size()); for (const ListExprElement& element : ast_path) { + if (element.optional()) { + return absl::InvalidArgumentError("Invalid cel.attribute call"); + } // Optimized field select. if (element.has_expr()) { const auto& element_expr = element.expr(); @@ -395,8 +415,9 @@ absl::StatusOr> SelectInstructionsFromCall( class RewriterImpl : public AstRewriterBase { public: - RewriterImpl(const Ast& ast, PlannerContext& planner_context) - : ast_(ast), planner_context_(planner_context) {} + RewriterImpl(const Ast& ast, PlannerContext& planner_context, + const SelectOptimizationOptions& options) + : ast_(ast), planner_context_(planner_context), options_(options) {} void PreVisitExpr(const Expr& expr) override { path_.push_back(&expr); } @@ -405,20 +426,20 @@ class RewriterImpl : public AstRewriterBase { const std::string& field_name = select.field(); // Select optimization can generalize to lists and maps, but for now only // support message traversal. - const TypeSpec checker_type = ast_.GetTypeOrDyn(operand.id()); - - absl::optional rt_type = - (checker_type.has_message_type()) - ? GetRuntimeType(checker_type.message_type().type()) - : absl::nullopt; - if (rt_type.has_value() && (*rt_type).Is()) { + const TypeSpec ast_type = ast_.GetTypeOrDyn(operand.id()); + std::optional rt_type = + planner_context_.TryResolveType(operand.id(), ast_type); + if (!rt_type.has_value()) { + return; + } + if (rt_type->IsStruct()) { const StructType& runtime_type = rt_type->GetStruct(); - absl::optional field_or = - GetSelectInstruction(runtime_type, planner_context_, field_name); + std::optional field_or = + MakeFieldStepAst(runtime_type, planner_context_, field_name); if (field_or.has_value()) { candidates_[&expr] = std::move(field_or).value(); } - } else if (checker_type.has_map_type()) { + } else if (rt_type->IsMap()) { candidates_[&expr] = QualifierInstruction(field_name); } // else @@ -433,8 +454,7 @@ class RewriterImpl : public AstRewriterBase { const auto& qualifier_expr = call.args()[1]; if (qualifier_expr.has_const_expr()) { - auto qualifier_or = - SelectInstructionFromConstant(qualifier_expr.const_expr()); + auto qualifier_or = MakeIndexStepAst(qualifier_expr.const_expr()); if (!qualifier_or.ok()) { // TODO(uncreated-issue/54): should warn, but by default warnings fail overall // program planning. @@ -469,6 +489,10 @@ class RewriterImpl : public AstRewriterBase { } SelectPath path = GetSelectPath(&expr); + if (path.select_instructions.size() < + options_.min_field_depth_for_optimization) { + return false; + } // generate the new cel.attribute call. absl::string_view fn = path.test_only ? kCelHasField : kCelAttribute; @@ -538,11 +562,6 @@ class RewriterImpl : public AstRewriterBase { return candidates_.find(operand) != candidates_.end(); } - absl::optional GetRuntimeType(absl::string_view type_name) { - return planner_context_.type_reflector().FindType(type_name).value_or( - absl::nullopt); - } - void SetProgressStatus(const absl::Status& status) { if (progress_status_.ok() && !status.ok()) { progress_status_ = status; @@ -551,18 +570,108 @@ class RewriterImpl : public AstRewriterBase { const Ast& ast_; PlannerContext& planner_context_; + SelectOptimizationOptions options_; // ids of potentially optimizeable expr nodes. absl::flat_hash_map candidates_; std::vector path_; absl::Status progress_status_; }; +struct FieldSelectStep { + const google::protobuf::Descriptor* absl_nonnull containing_type; + const google::protobuf::FieldDescriptor* absl_nonnull field; +}; + +struct MapSelectStep { + const google::protobuf::FieldDescriptor* absl_nonnull map_field; + const google::protobuf::FieldDescriptor* absl_nonnull key_field; + const google::protobuf::FieldDescriptor* absl_nonnull value_field; + // Store the error to propagate on access at runtime. + // The map key might alias string data owned by the SelectOptimizationImpl + // object. + absl::StatusOr key; +}; + +struct ListSelectStep { + const google::protobuf::FieldDescriptor* absl_nonnull repeated_field; +}; + +using TypedSelectStep = + std::variant; + +absl::StatusOr CreateProtoMapKey( + const google::protobuf::FieldDescriptor* absl_nonnull key_desc, + const cel::AttributeQualifier& key) { + google::protobuf::MapKey proto_key; + switch (key_desc->cpp_type()) { + case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: + if (key.kind() != cel::Kind::kBool) { + return runtime_internal::CreateInvalidMapKeyTypeError( + key_desc->cpp_type_name()); + } + proto_key.SetBoolValue(*key.AsBool()); + return proto_key; + case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { + if (key.kind() != cel::Kind::kInt64) { + return runtime_internal::CreateInvalidMapKeyTypeError( + key_desc->cpp_type_name()); + } + int64_t key_value = *key.AsInt(); + if (key_value > std::numeric_limits::max() || + key_value < std::numeric_limits::lowest()) { + return absl::OutOfRangeError("integer overflow"); + } + proto_key.SetInt32Value(static_cast(key_value)); + return proto_key; + } + case google::protobuf::FieldDescriptor::CPPTYPE_INT64: + if (key.kind() != cel::Kind::kInt64) { + return runtime_internal::CreateInvalidMapKeyTypeError( + key_desc->cpp_type_name()); + } + proto_key.SetInt64Value(*key.AsInt()); + return proto_key; + case google::protobuf::FieldDescriptor::CPPTYPE_STRING: + if (key.kind() != cel::Kind::kString) { + return runtime_internal::CreateInvalidMapKeyTypeError( + key_desc->cpp_type_name()); + } + // This is an alias. The underlying string should be stable. + proto_key.SetStringValue(*key.AsString()); + return proto_key; + case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { + if (key.kind() != cel::Kind::kUint64) { + return runtime_internal::CreateInvalidMapKeyTypeError( + key_desc->cpp_type_name()); + } + uint64_t key_value = *key.AsUint(); + if (key_value > std::numeric_limits::max()) { + return absl::OutOfRangeError("unsigned integer overflow"); + } + proto_key.SetUInt32Value(static_cast(key_value)); + return proto_key; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: + if (key.kind() != cel::Kind::kUint64) { + return runtime_internal::CreateInvalidMapKeyTypeError( + key_desc->cpp_type_name()); + } + proto_key.SetUInt64Value(*key.AsUint()); + return proto_key; + default: + return runtime_internal::CreateInvalidMapKeyTypeError( + key_desc->cpp_type_name()); + } +} + class OptimizedSelectImpl { public: OptimizedSelectImpl(std::vector select_path, + std::vector typed_select_path, std::vector qualifiers, bool presence_test, SelectOptimizationOptions options) : select_path_(std::move(select_path)), + typed_select_path_(std::move(typed_select_path)), qualifiers_(std::move(qualifiers)), presence_test_(presence_test), options_(options) @@ -577,218 +686,421 @@ class OptimizedSelectImpl { OptimizedSelectImpl(OptimizedSelectImpl&&) = default; OptimizedSelectImpl& operator=(OptimizedSelectImpl&&) = delete; - absl::StatusOr ApplySelect(ExecutionFrameBase& frame, - const StructValue& struct_value) const; - - AttributeTrail GetAttributeTrail(const AttributeTrail& operand_trail) const; + absl::Status ApplySelect(ExecutionFrameBase& frame, + Value& result_in_out) const; - absl::optional attribute() const { return attribute_; } + std::optional attribute() const { return attribute_; } const std::vector& qualifiers() const { return qualifiers_; } private: - absl::optional attribute_; + absl::Status ContinueFromValue(ExecutionFrameBase& frame, Value step_result, + size_t next_index, Value& result_in_out) const; + + std::optional attribute_; std::vector select_path_; + std::vector typed_select_path_; std::vector qualifiers_; bool presence_test_; SelectOptimizationOptions options_; }; -// Check for unknowns or missing attributes. -absl::StatusOr> CheckForMarkedAttributes( - ExecutionFrameBase& frame, const AttributeTrail& attribute_trail) { - if (attribute_trail.empty()) { - return absl::nullopt; +bool SupportsCachedFieldDescriptor(const google::protobuf::Message* absl_nonnull message, + const TypedSelectStep& step) { + if (const FieldSelectStep* field_step = absl::get_if(&step); + field_step != nullptr) { + ABSL_DCHECK_EQ(field_step->field->containing_type(), + field_step->containing_type); + ABSL_DCHECK_EQ(field_step->containing_type->file()->pool(), + field_step->field->file()->pool()); + return message->GetDescriptor() == field_step->containing_type; } + return false; +} - if (frame.unknown_processing_enabled() && - frame.attribute_utility().CheckForUnknownExact(attribute_trail)) { - // Check if the inferred attribute is marked. Only matches if this attribute - // or a parent is marked unknown (use_partial = false). - // Partial matches (i.e. descendant of this attribute is marked) aren't - // considered yet in case another operation would select an unmarked - // descended attribute. - // - // TODO(uncreated-issue/51): this may return a more specific attribute than the - // declared pattern. Follow up will truncate the returned attribute to match - // the pattern. - return frame.attribute_utility().CreateUnknownSet( - attribute_trail.attribute()); +Value WrapMessageField( + const google::protobuf::Message* absl_nonnull message, + const google::protobuf::FieldDescriptor* absl_nonnull field_descriptor, + bool is_unsafe, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) { + if (is_unsafe) { + return Value::WrapFieldUnsafe(ProtoWrapperTypeOptions::kUnsetNull, message, + field_descriptor, descriptor_pool, + message_factory, arena); } + return Value::WrapField(ProtoWrapperTypeOptions::kUnsetNull, message, + field_descriptor, descriptor_pool, message_factory, + arena); +} - if (frame.missing_attribute_errors_enabled() && - frame.attribute_utility().CheckForMissingAttribute(attribute_trail)) { - return frame.attribute_utility().CreateMissingAttributeError( - attribute_trail.attribute()); +Value WrapMapElement( + const google::protobuf::MapValueConstRef& value_ref, + const google::protobuf::Message* absl_nonnull message, + const google::protobuf::FieldDescriptor* absl_nonnull value_descriptor, + bool is_unsafe, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) { + if (is_unsafe) { + return Value::WrapMapFieldValueUnsafe(value_ref, message, value_descriptor, + descriptor_pool, message_factory, + arena); } + return Value::WrapMapFieldValue(value_ref, message, value_descriptor, + descriptor_pool, message_factory, arena); +} - return absl::nullopt; +Value WrapRepeatedElement( + int index, const google::protobuf::Message* absl_nonnull message, + const google::protobuf::FieldDescriptor* absl_nonnull repeated_descriptor, + bool is_unsafe, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) { + if (is_unsafe) { + return Value::WrapRepeatedFieldUnsafe(index, message, repeated_descriptor, + descriptor_pool, message_factory, + arena); + } + return Value::WrapRepeatedField(index, message, repeated_descriptor, + descriptor_pool, message_factory, arena); } -absl::StatusOr OptimizedSelectImpl::ApplySelect( - ExecutionFrameBase& frame, const StructValue& struct_value) const { - auto value_or = - (options_.force_fallback_implementation) +absl::StatusOr QualifyOrFallback( + ExecutionFrameBase& frame, const StructValue& struct_value, + absl::Span select_path, bool presence_test, + bool force_fallback_implementation) { + absl::StatusOr> value_and_count = + (force_fallback_implementation) ? absl::UnimplementedError("Forced fallback impl") - : struct_value.Qualify(select_path_, presence_test_, + : struct_value.Qualify(select_path, presence_test, frame.descriptor_pool(), frame.message_factory(), frame.arena()); - if (!value_or.ok()) { - if (value_or.status().code() == absl::StatusCode::kUnimplemented) { - return FallbackSelect(struct_value, select_path_, presence_test_, + if (!value_and_count.ok()) { + if (value_and_count.status().code() == absl::StatusCode::kUnimplemented) { + return FallbackSelect(struct_value, select_path, presence_test, frame.descriptor_pool(), frame.message_factory(), frame.arena()); } - return value_or.status(); + return value_and_count.status(); } - if (value_or->second < 0 || value_or->second >= select_path_.size()) { - return std::move(value_or->first); + if (value_and_count->second < 0 || + value_and_count->second >= select_path.size()) { + return std::move(value_and_count->first); } - return FallbackSelect( - value_or->first, - absl::MakeConstSpan(select_path_).subspan(value_or->second), - presence_test_, frame.descriptor_pool(), frame.message_factory(), - frame.arena()); + return FallbackSelect(value_and_count->first, + select_path.subspan(value_and_count->second), + presence_test, frame.descriptor_pool(), + frame.message_factory(), frame.arena()); } -AttributeTrail OptimizedSelectImpl::GetAttributeTrail( - const AttributeTrail& operand_trail) const { - if (operand_trail.empty()) { - return AttributeTrail(); - } - std::vector qualifiers = std::vector( - operand_trail.attribute().qualifier_path().begin(), - operand_trail.attribute().qualifier_path().end()); - qualifiers.reserve(qualifiers_.size() + qualifiers.size()); - absl::c_copy(qualifiers_, std::back_inserter(qualifiers)); - return AttributeTrail( - Attribute(std::string(operand_trail.attribute().variable_name()), - std::move(qualifiers))); +absl::Status OptimizedSelectImpl::ContinueFromValue( + ExecutionFrameBase& frame, Value step_result, size_t next_index, + Value& result_in_out) const { + if (step_result.IsError() || next_index >= select_path_.size()) { + result_in_out = std::move(step_result); + return absl::OkStatus(); + } + absl::Span remaining = + absl::MakeConstSpan(select_path_).subspan(next_index); + if (step_result.IsStruct()) { + CEL_ASSIGN_OR_RETURN( + result_in_out, + QualifyOrFallback(frame, step_result.GetStruct(), remaining, + presence_test_, + options_.force_fallback_implementation)); + return absl::OkStatus(); + } + CEL_ASSIGN_OR_RETURN(result_in_out, + FallbackSelect(step_result, remaining, presence_test_, + frame.descriptor_pool(), + frame.message_factory(), frame.arena())); + return absl::OkStatus(); } -class StackMachineImpl : public ExpressionStepBase { - public: - StackMachineImpl(int expr_id, OptimizedSelectImpl impl) - : ExpressionStepBase(expr_id), impl_(std::move(impl)) {} +absl::Status OptimizedSelectImpl::ApplySelect(ExecutionFrameBase& frame, + Value& result_in_out) const { + if (options_.force_fallback_implementation || typed_select_path_.empty()) { + CEL_ASSIGN_OR_RETURN( + result_in_out, + QualifyOrFallback(frame, result_in_out.GetStruct(), select_path_, + presence_test_, + options_.force_fallback_implementation)); + return absl::OkStatus(); + } + const google::protobuf::Message* current_message = nullptr; + bool is_unsafe = false; + if (result_in_out.IsParsedMessage()) { + ParsedMessageValue parsed_message = result_in_out.GetParsedMessage(); + current_message = &(*parsed_message); + is_unsafe = + cel::interop_internal::IsUnsafeParsedMessageValue(parsed_message); + } else if (const google::protobuf::Message* legacy_message = + cel::interop_internal::GetLegacyMessage(result_in_out); + legacy_message != nullptr) { + current_message = legacy_message; + is_unsafe = true; + } else { + CEL_ASSIGN_OR_RETURN( + result_in_out, + QualifyOrFallback(frame, result_in_out.GetStruct(), select_path_, + presence_test_, + options_.force_fallback_implementation)); + return absl::OkStatus(); + } - absl::Status Evaluate(ExecutionFrame* frame) const override; + if (!SupportsCachedFieldDescriptor(current_message, typed_select_path_[0])) { + CEL_ASSIGN_OR_RETURN( + result_in_out, + QualifyOrFallback(frame, result_in_out.GetStruct(), select_path_, + presence_test_, + options_.force_fallback_implementation)); + return absl::OkStatus(); + } - private: - // Get the effective attribute for the optimized select expression. - // Assumes the operand is the top of stack if the attribute wasn't known at - // plan time. - AttributeTrail GetAttributeTrail(ExecutionFrame* frame) const; + for (size_t i = 0; i < typed_select_path_.size(); ++i) { + const TypedSelectStep& step = typed_select_path_[i]; + + // handle singular + if (const FieldSelectStep* field_step = + absl::get_if(&step); + field_step != nullptr) { + const google::protobuf::FieldDescriptor* field_descriptor = field_step->field; + // We consumed the whole select path. Materialize the leaf. + if (i + 1 == select_path_.size()) { + if (presence_test_) { + result_in_out = + BoolValue{cel::UnsafeParsedMessageValue(current_message) + .HasField(field_descriptor)}; + return absl::OkStatus(); + } + result_in_out = WrapMessageField( + current_message, field_descriptor, is_unsafe, + frame.descriptor_pool(), frame.message_factory(), frame.arena()); + return absl::OkStatus(); + } - OptimizedSelectImpl impl_; -}; + if (i + 1 >= typed_select_path_.size()) { + Value step_result = WrapMessageField( + current_message, field_descriptor, is_unsafe, + frame.descriptor_pool(), frame.message_factory(), frame.arena()); + return ContinueFromValue(frame, std::move(step_result), i + 1, + result_in_out); + } -AttributeTrail StackMachineImpl::GetAttributeTrail( - ExecutionFrame* frame) const { - const auto& attr = frame->value_stack().PeekAttribute(); - return impl_.GetAttributeTrail(attr); -} + if (field_descriptor->is_repeated()) { + // The next step in `typed_select_path_` is a `MapSelectStep` or + // `ListSelectStep` that operates on `current_message`. + continue; + } -absl::Status StackMachineImpl::Evaluate(ExecutionFrame* frame) const { - // Default empty. - AttributeTrail attribute_trail; - // TODO(uncreated-issue/51): add support for variable qualifiers and string literal - // variable names. - constexpr size_t kStackInputs = 1; + const google::protobuf::Message* next_message = + ¤t_message->GetReflection()->GetMessage(*current_message, + field_descriptor); + // Check for messages extended by overlays. This is redundant if all + // messages are from hermetic pools, which we cannot guarantee. + if (!SupportsCachedFieldDescriptor(next_message, + typed_select_path_[i + 1])) { + Value step_result = WrapMessageField( + current_message, field_descriptor, is_unsafe, + frame.descriptor_pool(), frame.message_factory(), frame.arena()); + return ContinueFromValue(frame, std::move(step_result), i + 1, + result_in_out); + } + current_message = next_message; + continue; + } - // For now, we expect the operand to be top of stack. - const Value& operand = frame->value_stack().Peek(); + // Handle map + if (const MapSelectStep* map_step = absl::get_if(&step); + map_step != nullptr) { + if (i + 1 == select_path_.size() && presence_test_) { + const AttributeQualifier& qualifier = + absl::get(select_path_[i]); + if (qualifier.kind() != cel::Kind::kString) { + result_in_out = ErrorValue::From( + runtime_internal::CreateNoMatchingOverloadError("has"), + frame.arena()); + return absl::OkStatus(); + } + if (!map_step->key.ok()) { + result_in_out = + ErrorValue::From(map_step->key.status(), frame.arena()); + return absl::OkStatus(); + } + bool found = protobuf_internal::ContainsMapKey( + *current_message->GetReflection(), *current_message, + *map_step->map_field, *map_step->key); + result_in_out = BoolValue{found}; + return absl::OkStatus(); + } - if (operand->Is() || operand->Is()) { - // Just forward the error which is already top of stack. - return absl::OkStatus(); - } + if (!map_step->key.ok()) { + result_in_out = ErrorValue::From(map_step->key.status(), frame.arena()); + return absl::OkStatus(); + } + google::protobuf::MapValueConstRef value_ref; + bool found = protobuf_internal::LookupMapValue( + *current_message->GetReflection(), *current_message, + *map_step->map_field, *map_step->key, &value_ref); + if (!found) { + const AttributeQualifier& qualifier = + absl::get(select_path_[i]); + result_in_out = ErrorValue::From( + runtime_internal::CreateNoSuchKeyError(qualifier.ToString()), + frame.arena()); + return absl::OkStatus(); + } - if (frame->enable_attribute_tracking()) { - // Compute the attribute trail then check for any marked values. - // When possible, this is computed at plan time based on the optimized - // select arguments. - // TODO(uncreated-issue/51): add support variable qualifiers - attribute_trail = GetAttributeTrail(frame); - CEL_ASSIGN_OR_RETURN(absl::optional value, - CheckForMarkedAttributes(*frame, attribute_trail)); - if (value.has_value()) { - frame->value_stack().Pop(kStackInputs); - frame->value_stack().Push(std::move(value).value(), - std::move(attribute_trail)); - return absl::OkStatus(); + if (i + 1 == select_path_.size()) { + result_in_out = WrapMapElement( + value_ref, current_message, map_step->value_field, is_unsafe, + frame.descriptor_pool(), frame.message_factory(), frame.arena()); + return absl::OkStatus(); + } + + // We don't have a typed select step for the next step, so we fall back. + if (i + 1 >= typed_select_path_.size()) { + Value step_result = WrapMapElement( + value_ref, current_message, map_step->value_field, is_unsafe, + frame.descriptor_pool(), frame.message_factory(), frame.arena()); + return ContinueFromValue(frame, std::move(step_result), i + 1, + result_in_out); + } + + const google::protobuf::Message* next_message = &value_ref.GetMessageValue(); + if (!SupportsCachedFieldDescriptor(next_message, + typed_select_path_[i + 1])) { + Value step_result = WrapMapElement( + value_ref, current_message, map_step->value_field, is_unsafe, + frame.descriptor_pool(), frame.message_factory(), frame.arena()); + return ContinueFromValue(frame, std::move(step_result), i + 1, + result_in_out); + } + current_message = next_message; + continue; } - } - if (!operand->Is()) { - return absl::InvalidArgumentError( - "Expected struct type for select optimization."); - } + // handle list + if (const ListSelectStep* list_step = absl::get_if(&step); + list_step != nullptr) { + if (i + 1 == select_path_.size() && presence_test_) { + result_in_out = ErrorValue::From( + runtime_internal::CreateNoMatchingOverloadError("has"), + frame.arena()); + return absl::OkStatus(); + } + + const AttributeQualifier& qualifier = + absl::get(select_path_[i]); + if (qualifier.kind() != cel::Kind::kInt64) { + result_in_out = + ErrorValue::From(runtime_internal::CreateNoMatchingOverloadError( + cel::builtin::kIndex), + frame.arena()); + return absl::OkStatus(); + } + int64_t index = *qualifier.AsInt(); + const google::protobuf::Reflection* reflection = current_message->GetReflection(); + int size = + reflection->FieldSize(*current_message, list_step->repeated_field); + if (index < 0 || index >= size) { + result_in_out = ErrorValue::From( + absl::InvalidArgumentError(absl::StrCat( + "index out of bounds: index=", index, " size=", size)), + frame.arena()); + return absl::OkStatus(); + } + int int_index = static_cast(index); + + if (i + 1 == select_path_.size()) { + result_in_out = WrapRepeatedElement( + int_index, current_message, list_step->repeated_field, is_unsafe, + frame.descriptor_pool(), frame.message_factory(), frame.arena()); + return absl::OkStatus(); + } + + // We don't have a typed select step for the next step, so we fall back. + if (i + 1 >= typed_select_path_.size()) { + Value step_result = WrapRepeatedElement( + int_index, current_message, list_step->repeated_field, is_unsafe, + frame.descriptor_pool(), frame.message_factory(), frame.arena()); + return ContinueFromValue(frame, std::move(step_result), i + 1, + result_in_out); + } + + const google::protobuf::Message* next_message = &reflection->GetRepeatedMessage( + *current_message, list_step->repeated_field, int_index); + if (!SupportsCachedFieldDescriptor(next_message, + typed_select_path_[i + 1])) { + Value step_result = WrapRepeatedElement( + int_index, current_message, list_step->repeated_field, is_unsafe, + frame.descriptor_pool(), frame.message_factory(), frame.arena()); + return ContinueFromValue(frame, std::move(step_result), i + 1, + result_in_out); + } + current_message = next_message; + continue; + } - CEL_ASSIGN_OR_RETURN(Value result, - impl_.ApplySelect(*frame, operand.GetStruct())); + ABSL_UNREACHABLE(); + return absl::InternalError("Unknown select step type"); + } - frame->value_stack().Pop(kStackInputs); - frame->value_stack().Push(std::move(result), std::move(attribute_trail)); return absl::OkStatus(); } -class RecursiveImpl : public DirectExpressionStep { +class StackMachineImpl : public ExpressionStepBase { public: - RecursiveImpl(int64_t expr_id, std::unique_ptr operand, - OptimizedSelectImpl impl) - : DirectExpressionStep(expr_id), - operand_(std::move(operand)), - impl_(std::move(impl)) {} + StackMachineImpl(int expr_id, OptimizedSelectImpl impl) + : ExpressionStepBase(expr_id), impl_(std::move(impl)) {} - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override; + void Evaluate(ExecutionFrame* frame) const override; private: - // Get the effective attribute for the optimized select expression. - // Assumes the operand is the top of stack if the attribute wasn't known at - // plan time. - AttributeTrail GetAttributeTrail(const AttributeTrail& operand_trail) const; - std::unique_ptr operand_; OptimizedSelectImpl impl_; }; -AttributeTrail RecursiveImpl::GetAttributeTrail( - const AttributeTrail& operand_trail) const { - return impl_.GetAttributeTrail(operand_trail); -} - -absl::Status RecursiveImpl::Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const { - CEL_RETURN_IF_ERROR(operand_->Evaluate(frame, result, attribute)); +void StackMachineImpl::Evaluate(ExecutionFrame* frame) const { + // TODO(uncreated-issue/51): add support for variable qualifiers and string literal + // variable names. + // For now, we expect the operand to be top of stack. + Value& operand = frame->value_stack().Peek(); + AttributeTrail& trail = frame->value_stack().PeekAttribute(); - if (InstanceOf(result) || InstanceOf(result)) { - // Just forward. - return absl::OkStatus(); + if (operand.Is() || operand.Is()) { + // Just forward the error which is already top of stack. + return; } - if (frame.attribute_tracking_enabled()) { - attribute = impl_.GetAttributeTrail(attribute); - CEL_ASSIGN_OR_RETURN(auto value, - CheckForMarkedAttributes(frame, attribute)); - if (value.has_value()) { - result = std::move(value).value(); - return absl::OkStatus(); + if (frame->enable_attribute_tracking()) { + // Compute the attribute trail then check for any marked values. + // When possible, this is computed at plan time based on the optimized + // select arguments. + // TODO(uncreated-issue/51): add support variable qualifiers + for (const auto& qualifier : impl_.qualifiers()) { + if (trail.Match(qualifier, operand, + frame->unknown_tree())) { + return; + } } } - if (!InstanceOf(result)) { - return absl::InvalidArgumentError( - "Expected struct type for select optimization"); + if (!operand.Is()) { + frame->Abort(absl::InvalidArgumentError( + "Expected struct type for select optimization.")); + return; + } + + absl::Status status = impl_.ApplySelect(*frame, operand); + if (!status.ok()) { + frame->Abort(std::move(status)); } - CEL_ASSIGN_OR_RETURN(result, - impl_.ApplySelect(frame, Cast(result))); - return absl::OkStatus(); } class SelectOptimizer : public ProgramOptimizer { @@ -806,6 +1118,74 @@ class SelectOptimizer : public ProgramOptimizer { SelectOptimizationOptions options_; }; +std::vector BuildTypedSelectPath( + PlannerContext& context, const Expr& operand, + absl::Span instructions) { + std::vector typed_select_path; + std::optional current_type = context.GetResolvedType(operand.id()); + for (const SelectQualifier& instruction : instructions) { + if (!current_type.has_value()) { + break; + } + if (const FieldSpecifier* field_specifier = + absl::get_if(&instruction); + field_specifier != nullptr) { + if (!current_type->IsMessage()) { + break; + } + cel::MessageType message_type = current_type->GetMessage(); + absl::StatusOr> field = + context.type_reflector().FindStructTypeFieldByName( + message_type, field_specifier->name); + if (!field.ok() || !field->has_value() || !(*field)->IsMessage()) { + break; + } + const google::protobuf::Descriptor* containing_descriptor = + message_type.descriptor(); + const google::protobuf::FieldDescriptor* field_descriptor = + (*field)->GetMessage().descriptor(); + if (containing_descriptor->file()->pool() != + field_descriptor->file()->pool()) { + break; + } + typed_select_path.push_back( + FieldSelectStep{containing_descriptor, field_descriptor}); + current_type = (*field)->GetType(); + } else if (const AttributeQualifier* attr_qualifier = + absl::get_if(&instruction); + attr_qualifier != nullptr) { + if (typed_select_path.empty() || + !absl::holds_alternative(typed_select_path.back())) { + break; + } + const google::protobuf::FieldDescriptor* prev_field = + absl::get(typed_select_path.back()).field; + if (prev_field->is_map() && current_type->IsMap()) { + const google::protobuf::Descriptor* entry_descriptor = prev_field->message_type(); + const google::protobuf::FieldDescriptor* key_field = entry_descriptor->map_key(); + const google::protobuf::FieldDescriptor* value_field = + entry_descriptor->map_value(); + if (key_field == nullptr || value_field == nullptr) { + break; + } + typed_select_path.push_back( + MapSelectStep{prev_field, key_field, value_field, + CreateProtoMapKey(key_field, *attr_qualifier)}); + current_type = current_type->GetMap().GetValue(); + } else if (prev_field->is_repeated() && !prev_field->is_map() && + current_type->IsList()) { + typed_select_path.push_back(ListSelectStep{prev_field}); + current_type = current_type->GetList().GetElement(); + } else { + break; + } + } else { + break; + } + } + return typed_select_path; +} + absl::Status SelectOptimizer::OnPostVisit(PlannerContext& context, const Expr& node) { if (!node.has_call_expr()) { @@ -816,16 +1196,15 @@ absl::Status SelectOptimizer::OnPostVisit(PlannerContext& context, if (fn != kCelHasField && fn != kCelAttribute) { return absl::OkStatus(); } + const bool presence_test = fn == kCelHasField; - if (node.call_expr().args().size() < 2 || - node.call_expr().args().size() > 3) { + // cel.@attribute may take a 3rd arg that only types the result; it's unused. + const size_t max_args = presence_test ? 2 : 3; + const size_t num_args = node.call_expr().args().size(); + if (num_args < 2 || num_args > max_args) { return absl::InvalidArgumentError("Invalid cel.attribute call"); } - if (node.call_expr().args().size() == 3) { - return absl::UnimplementedError("Optionals not yet supported"); - } - CEL_ASSIGN_OR_RETURN(std::vector instructions, SelectInstructionsFromCall(node.call_expr())); @@ -833,12 +1212,6 @@ absl::Status SelectOptimizer::OnPostVisit(PlannerContext& context, return absl::InvalidArgumentError("Invalid cel.attribute no select steps."); } - bool presence_test = false; - - if (fn == kCelHasField) { - presence_test = true; - } - const Expr& operand = node.call_expr().args()[0]; absl::string_view identifier; if (operand.has_ident_expr()) { @@ -869,22 +1242,16 @@ absl::Status SelectOptimizer::OnPostVisit(PlannerContext& context, return absl::OkStatus(); } - OptimizedSelectImpl impl(std::move(instructions), std::move(qualifiers), - presence_test, options_); - - if (subexpression->IsRecursive()) { - auto program = subexpression->ExtractRecursiveProgram(); - auto deps = program.step->ExtractDependencies(); - if (!deps.has_value() || deps->empty()) { - return absl::InvalidArgumentError("Unexpected cel.@attribute call"); - } - subexpression->set_recursive_program( - std::make_unique(node.id(), std::move(deps->at(0)), - std::move(impl)), - program.depth); - return absl::OkStatus(); + std::vector typed_select_path; + if (options_.enable_field_descriptor_caching && + context.options().enable_typed_field_access) { + typed_select_path = BuildTypedSelectPath(context, operand, instructions); } + OptimizedSelectImpl impl(std::move(instructions), + std::move(typed_select_path), std::move(qualifiers), + presence_test, options_); + google::api::expr::runtime::ExecutionPath path; // else, we need to preserve the original plan for the first argument. @@ -895,8 +1262,9 @@ absl::Status SelectOptimizer::OnPostVisit(PlannerContext& context, CEL_ASSIGN_OR_RETURN(auto operand_subplan, context.ExtractSubplan(operand)); absl::c_move(operand_subplan, std::back_inserter(path)); - path.push_back( - std::make_unique(node.id(), std::move(impl))); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(node.id(), std::move(impl)), node.id(), + /*stack_delta=*/0)); return context.ReplaceSubplan(node, std::move(path)); } @@ -918,7 +1286,7 @@ google::api::expr::runtime::FlatExprBuilder* GetFlatExprBuilder( absl::Status SelectOptimizationAstUpdater::UpdateAst(PlannerContext& context, Ast& ast) const { - RewriterImpl rewriter(ast, context); + RewriterImpl rewriter(ast, context, options_); AstRewrite(ast.mutable_root_expr(), rewriter); return rewriter.GetProgressStatus(); } @@ -927,6 +1295,9 @@ google::api::expr::runtime::ProgramOptimizerFactory CreateSelectOptimizationProgramOptimizer( const SelectOptimizationOptions& options) { return [=](PlannerContext& context, const Ast& ast) { + if (options.enable_field_descriptor_caching) { + context.set_resolve_types(); + } return std::make_unique(options); }; } @@ -940,15 +1311,19 @@ absl::Status EnableSelectOptimization( } flat_expr_builder->AddAstTransform( - std::make_unique()); + std::make_unique(options)); // Add overloads for select optimization signature. // These are never bound, only used to prevent the builder from failing on // the overloads check. CEL_RETURN_IF_ERROR(builder.function_registry().RegisterLazyFunction( FunctionDescriptor(kCelAttribute, false, {Kind::kAny, Kind::kList}))); + CEL_RETURN_IF_ERROR( + builder.function_registry().RegisterLazyFunction(FunctionDescriptor( + kCelAttribute, false, {Kind::kAny, Kind::kList, Kind::kAny}))); CEL_RETURN_IF_ERROR(builder.function_registry().RegisterLazyFunction( FunctionDescriptor(kCelHasField, false, {Kind::kAny, Kind::kList}))); + flat_expr_builder->enable_select_optimization(); // Add runtime implementation. flat_expr_builder->AddProgramOptimizer( CreateSelectOptimizationProgramOptimizer(options)); diff --git a/extensions/select_optimization.h b/extensions/select_optimization.h index 4de81b1b0..9c6bc51e9 100644 --- a/extensions/select_optimization.h +++ b/extensions/select_optimization.h @@ -35,10 +35,26 @@ struct SelectOptimizationOptions { // unimplemented for a given StructType. This option is exposed for testing or // to more closely match behavior of unoptimized expressions. bool force_fallback_implementation = false; + + // The minimum depth of a select chain to be considered for optimization. + // For example, if this is set to 2, then `a.b.c` will be optimized, but + // `a.b` will not be. + // + // This is applied in the pre-pass phase that rewrites the AST. If the + // optimization is already applied by a compiler / static optimizer, this + // option has no effect. + int min_field_depth_for_optimization = 0; + + // Whether to try to cache field descriptor lookups for the select + // optimization at plan time. + // + // This is temporary to make the change rollback safe for existing users. + bool enable_field_descriptor_caching = true; }; // Enable select optimization on the given RuntimeBuilder, replacing long -// select chains with a single operation. +// select chains with a single operation. Also enables planning ASTs tagged with +// the `select_optimization` runtime extension. // // This assumes that the type information at check time agrees with the // configured types at runtime. @@ -76,10 +92,15 @@ absl::Status EnableSelectOptimization( class SelectOptimizationAstUpdater : public google::api::expr::runtime::AstTransform { public: - SelectOptimizationAstUpdater() = default; + explicit SelectOptimizationAstUpdater( + const SelectOptimizationOptions& options = {}) + : options_(options) {} absl::Status UpdateAst(google::api::expr::runtime::PlannerContext& context, cel::Ast& ast) const override; + + private: + SelectOptimizationOptions options_; }; google::api::expr::runtime::ProgramOptimizerFactory diff --git a/extensions/select_optimization_test.cc b/extensions/select_optimization_test.cc index c07f4c6ad..70dc079d2 100644 --- a/extensions/select_optimization_test.cc +++ b/extensions/select_optimization_test.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -33,7 +34,6 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" -#include "absl/types/optional.h" #include "absl/types/span.h" #include "base/ast.h" #include "base/attribute.h" @@ -45,8 +45,11 @@ #include "common/decl_proto.h" #include "common/expr.h" #include "common/kind.h" -#include "common/memory.h" +#include "common/native_type.h" +#include "common/type.h" +#include "common/typeinfo.h" #include "common/value.h" +#include "common/values/custom_struct_value.h" #include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "compiler/optional.h" @@ -59,8 +62,6 @@ #include "eval/public/cel_type_registry.h" #include "eval/public/cel_value.h" #include "eval/public/structs/cel_proto_wrapper.h" -#include "eval/public/structs/legacy_type_adapter.h" -#include "eval/public/structs/legacy_type_info_apis.h" #include "extensions/protobuf/ast_converters.h" #include "internal/number.h" #include "internal/status_macros.h" @@ -72,12 +73,17 @@ #include "runtime/internal/issue_collector.h" #include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" #include "runtime/runtime_issue.h" #include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" #include "runtime/type_registry.h" +#include "testutil/test_external_extensions_descriptor_set.h" #include "cel/expr/conformance/proto2/test_all_types.pb.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/dynamic_message.h" #include "google/protobuf/extension_set.h" #include "google/protobuf/message.h" #include "google/protobuf/text_format.h" @@ -96,18 +102,10 @@ using ::google::api::expr::runtime::CelProtoWrapper; using ::google::api::expr::runtime::CelValue; using ::google::api::expr::runtime::FlatExprBuilder; using ::google::api::expr::runtime::FlatExpression; -using ::google::api::expr::runtime::LegacyTypeAccessApis; -using ::google::api::expr::runtime::LegacyTypeInfoApis; -using ::google::api::expr::runtime::LegacyTypeMutationApis; -using ::google::protobuf::Empty; -using ::testing::_; using ::testing::AllOf; -using ::testing::AnyOf; using ::testing::ElementsAre; using ::testing::Eq; using ::testing::HasSubstr; -using ::testing::NiceMock; -using ::testing::Return; using ::testing::SizeIs; using ::testing::Truly; @@ -232,74 +230,105 @@ absl::StatusOr> CompileForTestCase( return r.ReleaseAst(); } -class MockAccessApis : public LegacyTypeInfoApis, public LegacyTypeAccessApis { +class TestPartialQualifyStruct : public CustomStructValueInterface { public: - std::string DebugString( - const MessageWrapper& wrapped_message) const override { - return "MockAccessApis"; + explicit TestPartialQualifyStruct(Value leaf_value) + : leaf_value_(std::move(leaf_value)) {} + + std::string DebugString() const override { + return "TestPartialQualifyStruct"; + } + + absl::Status SerializeTo( + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::io::ZeroCopyOutputStream* output) const override { + return absl::UnimplementedError("SerializeTo"); } - absl::string_view GetTypename( - const MessageWrapper& wrapped_message) const override { - return "MockAccessApis"; + absl::Status ConvertToJsonObject( + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Message* json) const override { + return absl::UnimplementedError("ConvertToJsonObject"); } - const LegacyTypeAccessApis* GetAccessApis( - const MessageWrapper& wrapped_message) const override { - return this; + absl::string_view GetTypeName() const override { + return "cel.expr.conformance.proto2.NestedTestAllTypes"; } - const LegacyTypeMutationApis* GetMutationApis( - const MessageWrapper& wrapped_message) const override { - return nullptr; + bool IsZeroValue() const override { return false; } + + absl::Status GetFieldByName(absl::string_view name, + ProtoWrapperTypeOptions unboxing_options, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena, + Value* result) const override { + if (name == "child" || name == "payload" || name == "standalone_message") { + *result = CustomStructValue(this, arena); + return absl::OkStatus(); + } + if (name == "bb") { + *result = leaf_value_; + return absl::OkStatus(); + } + return common_internal::MakeNoSuchFieldError(name); } - absl::optional FindFieldByName( - absl::string_view field_name) const override { - return absl::nullopt; + absl::Status GetFieldByNumber(int64_t number, + ProtoWrapperTypeOptions unboxing_options, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena, + Value* result) const override { + return absl::UnimplementedError("GetFieldByNumber"); } - MOCK_METHOD(absl::StatusOr, GetField, - (absl::string_view field_name, - const CelValue::MessageWrapper& instance, - ProtoWrapperTypeOptions unboxing_option, - cel::MemoryManagerRef memory_manager), - (const, override)); - - MOCK_METHOD(absl::StatusOr, HasField, - (absl::string_view field_name, - const CelValue::MessageWrapper& value), - (const, override)); - - MOCK_METHOD(absl::StatusOr, - Qualify, - (absl::Span qualifiers, - const CelValue::MessageWrapper& instance, bool presence_test, - MemoryManagerRef memory_manager), - (const, override)); - - bool IsEqualTo( - const CelValue::MessageWrapper& instance, - const CelValue::MessageWrapper& other_instance) const override { + absl::StatusOr HasFieldByName(absl::string_view name) const override { + if (name == "child" || name == "payload" || name == "standalone_message" || + name == "bb") { + return true; + } return false; } - std::vector ListFields( - const CelValue::MessageWrapper& instance) const override { - return {}; + absl::StatusOr HasFieldByNumber(int64_t number) const override { + return absl::UnimplementedError("HasFieldByNumber"); } -}; -std::pair MakeMockLegacyMessage( - google::protobuf::Arena* arena) { - auto* mock_access_apis = - google::protobuf::Arena::Create>(arena); - auto* message = google::protobuf::Arena::Create(arena); + absl::Status ForEachField(ForEachFieldCallback callback, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena) const override { + return absl::OkStatus(); + } - CelValue::MessageWrapper::Builder wrapper(message); - return {mock_access_apis, - CelValue::CreateMessageWrapper(wrapper.Build(mock_access_apis))}; -} + absl::Status Qualify(absl::Span qualifiers, + bool presence_test, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena, Value* result, + int* count) const override { + if (qualifiers.size() >= 3) { + *count = 3; + *result = CustomStructValue(this, arena); + return absl::OkStatus(); + } + return absl::UnimplementedError("Qualify unsupported"); + } + + CustomStructValue Clone(google::protobuf::Arena* arena) const override { + return CustomStructValue(this, arena); + } + + NativeTypeId GetNativeTypeId() const override { + return cel::TypeId(); + } + + private: + Value leaf_value_; +}; absl::Status TestBindLegacyValue(absl::string_view variable, CelValue legacy_value, google::protobuf::Arena* arena, @@ -506,6 +535,47 @@ TEST_F(SelectOptimizationTest, AstTransformComplexSelect) { ElementsAre(SelectFieldEntry(1, "child"), SelectFieldEntry(1, "child"))); } +TEST_F(SelectOptimizationTest, AstTransformMinFieldDepth) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr ast, + CompileForTestCase( + "((false)? a.child.child : b.child).child.payload.single_int64")); + + SelectOptimizationOptions options; + options.min_field_depth_for_optimization = 2; + SelectOptimizationAstUpdater updater(options); + EXPECT_THAT(updater.UpdateAst(context_, *ast), IsOk()); + + const CallExpr& attr_call = ast->root_expr().call_expr(); + EXPECT_EQ(attr_call.function(), "cel.@attribute"); + + ASSERT_THAT(attr_call.args(), SizeIs(2)); + + EXPECT_THAT( + attr_call.args()[1].list_expr().elements(), + ElementsAre(SelectFieldEntry(1, "child"), SelectFieldEntry(2, "payload"), + SelectFieldEntry(2, "single_int64"))); + + const Expr& operand = attr_call.args()[0]; + + EXPECT_EQ(operand.call_expr().function(), cel::builtin::kTernary); + ASSERT_THAT(operand.call_expr().args(), SizeIs(3)); + + const Expr& true_branch = operand.call_expr().args()[1]; + + EXPECT_EQ(true_branch.call_expr().function(), "cel.@attribute"); + ASSERT_THAT(true_branch.call_expr().args(), SizeIs(2)); + + EXPECT_THAT( + true_branch.call_expr().args()[1].list_expr().elements(), + ElementsAre(SelectFieldEntry(1, "child"), SelectFieldEntry(1, "child"))); + + const Expr& false_branch = operand.call_expr().args()[2]; + EXPECT_TRUE(false_branch.has_select_expr()); + EXPECT_EQ(false_branch.select_expr().field(), "child"); + EXPECT_EQ(false_branch.select_expr().operand().ident_expr().name(), "b"); +} + TEST_F(SelectOptimizationTest, AstTransformMapIndexTraversal) { // nested_test_all_types.payload.map_string_message['$not_a_field'].bb ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, @@ -693,6 +763,58 @@ TEST_F(SelectOptimizationTest, AstTransformAnyDotTraversal) { EXPECT_EQ(operand.ident_expr().name(), "nested_test_all_types"); } +TEST_F(SelectOptimizationTest, AstTransformStructDotTraversal) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr ast, + CompileForTestCase( + "nested_test_all_types.payload.single_struct.key.subkey")); + + SelectOptimizationAstUpdater updater; + EXPECT_THAT(updater.UpdateAst(context_, *ast), IsOk()); + + // `.key` on `single_struct` (MapType) is included in the attribute call, + // while `.subkey` on `dyn` (`google.protobuf.Value`) remains an outer select. + EXPECT_EQ(ast->root_expr().select_expr().field(), "subkey"); + const CallExpr& attr_call = + ast->root_expr().select_expr().operand().call_expr(); + EXPECT_EQ(attr_call.function(), "cel.@attribute"); + + ASSERT_THAT(attr_call.args(), SizeIs(2)); + + EXPECT_THAT( + attr_call.args()[1].list_expr().elements(), + ElementsAre(SelectFieldEntry(2, "payload"), + SelectFieldEntry(103, "single_struct"), + SelectQualifier(AttributeQualifier::OfString("key")))); + + const Expr& operand = attr_call.args()[0]; + EXPECT_EQ(operand.ident_expr().name(), "nested_test_all_types"); +} + +TEST_F(SelectOptimizationTest, AstTransformValueDotTraversal) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr ast, + CompileForTestCase("nested_test_all_types.payload.single_value.key")); + + SelectOptimizationAstUpdater updater; + EXPECT_THAT(updater.UpdateAst(context_, *ast), IsOk()); + + // `.key` on `single_value` (`dyn`) is not merged into the attribute call. + EXPECT_EQ(ast->root_expr().select_expr().field(), "key"); + const CallExpr& attr_call = + ast->root_expr().select_expr().operand().call_expr(); + EXPECT_EQ(attr_call.function(), "cel.@attribute"); + + ASSERT_THAT(attr_call.args(), SizeIs(2)); + + EXPECT_THAT(attr_call.args()[1].list_expr().elements(), + ElementsAre(SelectFieldEntry(2, "payload"), + SelectFieldEntry(104, "single_value"))); + + const Expr& operand = attr_call.args()[0]; + EXPECT_EQ(operand.ident_expr().name(), "nested_test_all_types"); +} + TEST_F(SelectOptimizationTest, AstTransformRepeated) { // nested_test_all_types.payload.repeated_nested_message[1].bb ASSERT_OK_AND_ASSIGN( @@ -762,8 +884,11 @@ TEST_F(SelectOptimizationTest, ProgramOptimizerUnoptimizedAst) { google::protobuf::LinkMessageReflection(); FlatExprBuilder builder(env_, runtime_options_); + SelectOptimizationOptions select_options; + select_options.enable_field_descriptor_caching = true; - builder.AddProgramOptimizer(CreateSelectOptimizationProgramOptimizer()); + builder.AddProgramOptimizer( + CreateSelectOptimizationProgramOptimizer(select_options)); // nested_test_all_types.child.payload.standalone_message.bb ASSERT_OK_AND_ASSIGN( @@ -796,6 +921,46 @@ TEST_F(SelectOptimizationTest, ProgramOptimizerUnoptimizedAst) { EXPECT_EQ(result.GetInt().NativeValue(), 42); } +TEST_F(SelectOptimizationTest, FieldDescriptorCachingDisabled) { + google::protobuf::LinkMessageReflection(); + + FlatExprBuilder builder(env_, runtime_options_); + SelectOptimizationOptions select_options; + select_options.enable_field_descriptor_caching = false; + + builder.AddAstTransform( + std::make_unique(select_options)); + builder.AddProgramOptimizer( + CreateSelectOptimizationProgramOptimizer(select_options)); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr ast, + CompileForTestCase( + "nested_test_all_types.child.payload.standalone_message.bb")); + + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + + NestedTestAllTypes* var = google::protobuf::Arena::Create(&arena_); + var->mutable_child()->mutable_payload()->mutable_standalone_message()->set_bb( + 42); + + cel::Activation act; + act.InsertOrAssignValue("nested_test_all_types", + cel::ParsedMessageValue(var, &arena_)); + + auto state = plan.MakeEvaluatorState(env_->descriptor_pool.get(), + env_->MutableMessageFactory(), &arena_); + ASSERT_OK_AND_ASSIGN( + Value result, + plan.EvaluateWithCallback( + act, /*embedder_context=*/nullptr, + google::api::expr::runtime::EvaluationListener(), state)); + + ASSERT_TRUE(result->Is()) << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 42); +} + TEST_F(SelectOptimizationTest, MissingAttributeIndependentOfUnknown) { google::protobuf::LinkMessageReflection(); @@ -804,9 +969,13 @@ TEST_F(SelectOptimizationTest, MissingAttributeIndependentOfUnknown) { options.enable_missing_attribute_errors = true; FlatExprBuilder builder(env_, options); + SelectOptimizationOptions select_options; + select_options.enable_field_descriptor_caching = true; - builder.AddAstTransform(std::make_unique()); - builder.AddProgramOptimizer(CreateSelectOptimizationProgramOptimizer()); + builder.AddAstTransform( + std::make_unique(select_options)); + builder.AddProgramOptimizer( + CreateSelectOptimizationProgramOptimizer(select_options)); ASSERT_OK_AND_ASSIGN( std::unique_ptr ast, @@ -820,12 +989,13 @@ TEST_F(SelectOptimizationTest, MissingAttributeIndependentOfUnknown) { // activation only uses a ptr to the underlying message, persist them. NestedTestAllTypes var; - act.SetMissingPatterns( - {AttributePattern("nested_test_all_types", - { - AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("payload"), - })}); + ASSERT_THAT(act.SetMissingPatterns({AttributePattern( + "nested_test_all_types", + { + AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("payload"), + })}), + IsOk()); ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( R"pb( @@ -856,9 +1026,13 @@ TEST_F(SelectOptimizationTest, NullUnboxingOptionHonored) { options.enable_empty_wrapper_null_unboxing = true; FlatExprBuilder builder(env_, options); + SelectOptimizationOptions select_options; + select_options.enable_field_descriptor_caching = true; - builder.AddAstTransform(std::make_unique()); - builder.AddProgramOptimizer(CreateSelectOptimizationProgramOptimizer()); + builder.AddAstTransform( + std::make_unique(select_options)); + builder.AddProgramOptimizer( + CreateSelectOptimizationProgramOptimizer(select_options)); // nested_test_all_types.payload.single_int64_wrapper ASSERT_OK_AND_ASSIGN( @@ -869,16 +1043,64 @@ TEST_F(SelectOptimizationTest, NullUnboxingOptionHonored) { builder.CreateExpressionImpl(std::move(ast), nullptr)); cel::Activation act; - // activation only uses a ptr to the underlying message, persist them. - NestedTestAllTypes var; + NestedTestAllTypes generated_var; ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( R"pb( payload {} )pb", - &var)); - ASSERT_THAT(TestBindLegacyMessage("nested_test_all_types", var, &arena_, act), - IsOk()); + &generated_var)); + google::protobuf::Message* var = + runtime_internal::MakeTestingEnvDynamicProto(generated_var, &arena_); + ASSERT_THAT( + TestBindLegacyMessage("nested_test_all_types", *var, &arena_, act), + IsOk()); + + auto state = plan.MakeEvaluatorState(env_->descriptor_pool.get(), + env_->MutableMessageFactory(), &arena_); + ASSERT_OK_AND_ASSIGN( + Value result, + plan.EvaluateWithCallback( + act, /*embedder_context=*/nullptr, + google::api::expr::runtime::EvaluationListener(), state)); + + ASSERT_TRUE(result->Is()) << result->DebugString(); +} + +TEST_F(SelectOptimizationTest, NullUnboxingOptionHonoredModernParsedMessage) { + google::protobuf::LinkMessageReflection(); + + RuntimeOptions options = runtime_options_; + options.enable_empty_wrapper_null_unboxing = true; + + FlatExprBuilder builder(env_, options); + SelectOptimizationOptions select_options; + select_options.enable_field_descriptor_caching = true; + + builder.AddAstTransform( + std::make_unique(select_options)); + builder.AddProgramOptimizer( + CreateSelectOptimizationProgramOptimizer(select_options)); + + // nested_test_all_types.payload.single_int64_wrapper + ASSERT_OK_AND_ASSIGN( + std::unique_ptr ast, + CompileForTestCase("nested_test_all_types.payload.single_int64_wrapper")); + + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + + cel::Activation act; + NestedTestAllTypes generated_var; + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( + R"pb( + payload {} + )pb", + &generated_var)); + google::protobuf::Message* var = + runtime_internal::MakeTestingEnvDynamicProto(generated_var, &arena_); + act.InsertOrAssignValue("nested_test_all_types", + cel::ParsedMessageValue(var, &arena_)); auto state = plan.MakeEvaluatorState(env_->descriptor_pool.get(), env_->MutableMessageFactory(), &arena_); @@ -905,10 +1127,19 @@ struct ProgramOptimizerTestCase { class SelectOptimizationProgramOptimizerTest : public SelectOptimizationTest, - public testing::WithParamInterface {}; + public testing::WithParamInterface< + std::tuple> { + protected: + const ProgramOptimizerTestCase& test_case() const { + return std::get<0>(GetParam()); + } + + bool enable_field_descriptor_caching() const { + return std::get<1>(GetParam()); + } +}; TEST_P(SelectOptimizationProgramOptimizerTest, Default) { - const ProgramOptimizerTestCase& test_case = GetParam(); google::protobuf::LinkMessageReflection(); RuntimeOptions options = runtime_options_; @@ -916,42 +1147,185 @@ TEST_P(SelectOptimizationProgramOptimizerTest, Default) { options.enable_missing_attribute_errors = true; FlatExprBuilder builder(env_, options); + SelectOptimizationOptions select_options; + select_options.enable_field_descriptor_caching = + enable_field_descriptor_caching(); - builder.AddAstTransform(std::make_unique()); - builder.AddProgramOptimizer(CreateSelectOptimizationProgramOptimizer()); + builder.AddAstTransform( + std::make_unique(select_options)); + builder.AddProgramOptimizer( + CreateSelectOptimizationProgramOptimizer(select_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, - CompileForTestCase(test_case.expr)); + CompileForTestCase(test_case().expr)); ASSERT_OK_AND_ASSIGN(FlatExpression plan, builder.CreateExpressionImpl(std::move(ast), nullptr)); cel::Activation act; - // activation only uses a ptr to the underlying message, persist them. - std::vector> vars; - for (const auto& entry : test_case.vars) { - vars.push_back(std::make_unique()); + for (const auto& entry : test_case().vars) { + NestedTestAllTypes generated_var; ASSERT_TRUE( - google::protobuf::TextFormat::ParseFromString(entry.second, vars.back().get())); - ASSERT_THAT(TestBindLegacyMessage(entry.first, *vars.back(), &arena_, act), - IsOk()); + google::protobuf::TextFormat::ParseFromString(entry.second, &generated_var)); + google::protobuf::Message* var = + runtime_internal::MakeTestingEnvDynamicProto(generated_var, &arena_); + ASSERT_THAT(TestBindLegacyMessage(entry.first, *var, &arena_, act), IsOk()); } - if (test_case.setup_activation != nullptr) { - ASSERT_THAT(test_case.setup_activation(&arena_, act), IsOk()); + if (test_case().setup_activation != nullptr) { + ASSERT_THAT(test_case().setup_activation(&arena_, act), IsOk()); } - auto state = plan.MakeEvaluatorState(env_->descriptor_pool.get(), - env_->MutableMessageFactory(), &arena_); + google::api::expr::runtime::FlatExpressionEvaluatorState state = + plan.MakeEvaluatorState(env_->descriptor_pool.get(), + env_->MutableMessageFactory(), &arena_); absl::StatusOr result = plan.EvaluateWithCallback( act, /*embedder_context=*/nullptr, google::api::expr::runtime::EvaluationListener(), state); - ASSERT_NO_FATAL_FAILURE(test_case.expectations(result)); + ASSERT_NO_FATAL_FAILURE(test_case().expectations(result)); +} + +TEST_P(SelectOptimizationProgramOptimizerTest, ModernParsedMessage) { + google::protobuf::LinkMessageReflection(); + + RuntimeOptions options = runtime_options_; + options.unknown_processing = UnknownProcessingOptions::kAttributeAndFunction; + options.enable_missing_attribute_errors = true; + + FlatExprBuilder builder(env_, options); + SelectOptimizationOptions select_options; + select_options.enable_field_descriptor_caching = + enable_field_descriptor_caching(); + + builder.AddAstTransform( + std::make_unique(select_options)); + builder.AddProgramOptimizer( + CreateSelectOptimizationProgramOptimizer(select_options)); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + CompileForTestCase(test_case().expr)); + + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + + cel::Activation act; + for (const auto& entry : test_case().vars) { + NestedTestAllTypes generated_var; + ASSERT_TRUE( + google::protobuf::TextFormat::ParseFromString(entry.second, &generated_var)); + google::protobuf::Message* var = + runtime_internal::MakeTestingEnvDynamicProto(generated_var, &arena_); + act.InsertOrAssignValue(entry.first, cel::ParsedMessageValue(var, &arena_)); + } + + if (test_case().setup_activation != nullptr) { + ASSERT_THAT(test_case().setup_activation(&arena_, act), IsOk()); + } + + google::api::expr::runtime::FlatExpressionEvaluatorState state = + plan.MakeEvaluatorState(env_->descriptor_pool.get(), + env_->MutableMessageFactory(), &arena_); + absl::StatusOr result = plan.EvaluateWithCallback( + act, /*embedder_context=*/nullptr, + google::api::expr::runtime::EvaluationListener(), state); + + ASSERT_NO_FATAL_FAILURE(test_case().expectations(result)); +} + +TEST_P(SelectOptimizationProgramOptimizerTest, UnsafeParsedMessage) { + google::protobuf::LinkMessageReflection(); + + RuntimeOptions options = runtime_options_; + options.unknown_processing = UnknownProcessingOptions::kAttributeAndFunction; + options.enable_missing_attribute_errors = true; + + FlatExprBuilder builder(env_, options); + SelectOptimizationOptions select_options; + select_options.enable_field_descriptor_caching = + enable_field_descriptor_caching(); + + builder.AddAstTransform( + std::make_unique(select_options)); + builder.AddProgramOptimizer( + CreateSelectOptimizationProgramOptimizer(select_options)); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + CompileForTestCase(test_case().expr)); + + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + + cel::Activation act; + for (const auto& entry : test_case().vars) { + NestedTestAllTypes generated_var; + ASSERT_TRUE( + google::protobuf::TextFormat::ParseFromString(entry.second, &generated_var)); + google::protobuf::Message* var = + runtime_internal::MakeTestingEnvDynamicProto(generated_var, &arena_); + act.InsertOrAssignValue(entry.first, cel::UnsafeParsedMessageValue(var)); + } + + if (test_case().setup_activation != nullptr) { + ASSERT_THAT(test_case().setup_activation(&arena_, act), IsOk()); + } + + google::api::expr::runtime::FlatExpressionEvaluatorState state = + plan.MakeEvaluatorState(env_->descriptor_pool.get(), + env_->MutableMessageFactory(), &arena_); + absl::StatusOr result = plan.EvaluateWithCallback( + act, /*embedder_context=*/nullptr, + google::api::expr::runtime::EvaluationListener(), state); + + ASSERT_NO_FATAL_FAILURE(test_case().expectations(result)); +} + +TEST_P(SelectOptimizationProgramOptimizerTest, DifferentDescriptorPool) { + google::protobuf::LinkMessageReflection(); + + RuntimeOptions options = runtime_options_; + options.unknown_processing = UnknownProcessingOptions::kAttributeAndFunction; + options.enable_missing_attribute_errors = true; + + FlatExprBuilder builder(env_, options); + SelectOptimizationOptions select_options; + select_options.enable_field_descriptor_caching = + enable_field_descriptor_caching(); + + builder.AddAstTransform( + std::make_unique(select_options)); + builder.AddProgramOptimizer( + CreateSelectOptimizationProgramOptimizer(select_options)); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + CompileForTestCase(test_case().expr)); + + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + + cel::Activation act; + for (const auto& entry : test_case().vars) { + NestedTestAllTypes* var = + google::protobuf::Arena::Create(&arena_); + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(entry.second, var)); + act.InsertOrAssignValue(entry.first, cel::ParsedMessageValue(var, &arena_)); + } + + if (test_case().setup_activation != nullptr) { + ASSERT_THAT(test_case().setup_activation(&arena_, act), IsOk()); + } + + google::api::expr::runtime::FlatExpressionEvaluatorState state = + plan.MakeEvaluatorState(env_->descriptor_pool.get(), + env_->MutableMessageFactory(), &arena_); + absl::StatusOr result = plan.EvaluateWithCallback( + act, /*embedder_context=*/nullptr, + google::api::expr::runtime::EvaluationListener(), state); + + ASSERT_NO_FATAL_FAILURE(test_case().expectations(result)); } TEST_P(SelectOptimizationProgramOptimizerTest, ForceFallbackImpl) { - const ProgramOptimizerTestCase& test_case = GetParam(); google::protobuf::LinkMessageReflection(); RuntimeOptions options = runtime_options_; @@ -961,13 +1335,16 @@ TEST_P(SelectOptimizationProgramOptimizerTest, ForceFallbackImpl) { FlatExprBuilder builder(env_, options); SelectOptimizationOptions select_options; select_options.force_fallback_implementation = true; + select_options.enable_field_descriptor_caching = + enable_field_descriptor_caching(); - builder.AddAstTransform(std::make_unique()); + builder.AddAstTransform( + std::make_unique(select_options)); builder.AddProgramOptimizer( CreateSelectOptimizationProgramOptimizer(select_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, - CompileForTestCase(test_case.expr)); + CompileForTestCase(test_case().expr)); ASSERT_OK_AND_ASSIGN(FlatExpression plan, builder.CreateExpressionImpl(std::move(ast), nullptr)); @@ -975,7 +1352,7 @@ TEST_P(SelectOptimizationProgramOptimizerTest, ForceFallbackImpl) { cel::Activation act; // activation only uses a ptr to the underlying message, persist them. std::vector> vars; - for (const auto& entry : test_case.vars) { + for (const auto& entry : test_case().vars) { vars.push_back(std::make_unique()); ASSERT_TRUE( google::protobuf::TextFormat::ParseFromString(entry.second, vars.back().get())); @@ -983,764 +1360,1852 @@ TEST_P(SelectOptimizationProgramOptimizerTest, ForceFallbackImpl) { IsOk()); } - if (test_case.setup_activation != nullptr) { - ASSERT_THAT(test_case.setup_activation(&arena_, act), IsOk()); + if (test_case().setup_activation != nullptr) { + ASSERT_THAT(test_case().setup_activation(&arena_, act), IsOk()); } - auto state = plan.MakeEvaluatorState(env_->descriptor_pool.get(), - env_->MutableMessageFactory(), &arena_); + google::api::expr::runtime::FlatExpressionEvaluatorState state = + plan.MakeEvaluatorState(env_->descriptor_pool.get(), + env_->MutableMessageFactory(), &arena_); absl::StatusOr result = plan.EvaluateWithCallback( act, /*embedder_context=*/nullptr, google::api::expr::runtime::EvaluationListener(), state); - ASSERT_NO_FATAL_FAILURE(test_case.expectations(result)); + ASSERT_NO_FATAL_FAILURE(test_case().expectations(result)); } INSTANTIATE_TEST_SUITE_P( TestCases, SelectOptimizationProgramOptimizerTest, - testing::ValuesIn({ - { - "chained_select_success", - "nested_test_all_types.child.payload.standalone_message.bb", - {{"nested_test_all_types", - R"pb( - child { payload { standalone_message { bb: 42 } } } - )pb"}}, - ActivationSetupFn(), - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), 42); - }, - }, - { - "chained_select_defaults_success", - "nested_test_all_types.child.payload.standalone_message.bb", - {{"nested_test_all_types", R"pb()pb"}}, - ActivationSetupFn(), - - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), 0); - }, - }, - { - "chained_select_partial_success", - "nested_test_all_types.child.payload.standalone_message.bb", - {}, - [](google::protobuf::Arena* arena, Activation& act) { - auto mock_pair = MakeMockLegacyMessage(arena); - MockAccessApis* mock = mock_pair.first; - CelValue mocked_value = mock_pair.second; - ON_CALL(*mock, Qualify(SizeIs(4), _, /*presence_test=*/false, _)) - .WillByDefault( - Return(LegacyTypeAccessApis::LegacyQualifyResult{ - mocked_value, 3})); - ON_CALL(*mock, GetField("bb", _, _, _)) - .WillByDefault(Return(CelValue::CreateInt64(42))); - - // Support the forced-fallback case. - ON_CALL(*mock, GetField(AnyOf(Eq("child"), Eq("payload"), - Eq("standalone_message")), - _, _, _)) - .WillByDefault(Return(mocked_value)); - - return TestBindLegacyValue("nested_test_all_types", mocked_value, - arena, act); - }, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), 42); - }, - }, - { - "chained_select_presence_partial_present", - "has(nested_test_all_types.child.payload.standalone_message.bb)", - {}, - [](google::protobuf::Arena* arena, Activation& act) { - auto mock_pair = MakeMockLegacyMessage(arena); - MockAccessApis* mock = mock_pair.first; - CelValue mocked_value = mock_pair.second; - ON_CALL(*mock, Qualify(SizeIs(4), _, /*presence_test=*/true, _)) - .WillByDefault( - Return(LegacyTypeAccessApis::LegacyQualifyResult{ - mocked_value, 3})); - ON_CALL(*mock, HasField("bb", _)).WillByDefault(Return(true)); - ON_CALL(*mock, GetField("bb", _, _, _)) - .WillByDefault(Return(CelValue::CreateInt64(42))); - - // Support the forced-fallback case. - ON_CALL(*mock, GetField(AnyOf(Eq("child"), Eq("payload"), - Eq("standalone_message")), - _, _, _)) - .WillByDefault(Return(mocked_value)); - ON_CALL(*mock, HasField(AnyOf(Eq("child"), Eq("payload"), - Eq("standalone_message")), - _)) - .WillByDefault(Return(true)); - - return TestBindLegacyValue("nested_test_all_types", mocked_value, - arena, act); - }, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_TRUE(result.GetBool().NativeValue()); - }, - }, - { - "chained_select_not_bound", - "nested_test_all_types.child.payload.standalone_message.bb", - {}, // not set - ActivationSetupFn(), - - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_THAT(result.GetError().NativeValue(), - StatusIs(absl::StatusCode::kUnknown, - HasSubstr("nested_test_all_types"))); - }, - }, - { - // Some clients will use maps to represent a protobuf message at - // runtime. This is not yet supported. - "chained_select_map_as_root_unsupported", - "nested_test_all_types.child.payload.standalone_message.bb", - {}, // not set - [](google::protobuf::Arena* arena, Activation& act) -> absl::Status { - auto builder = cel::NewMapValueBuilder(arena); - CEL_RETURN_IF_ERROR( - builder->Put(cel::StringValue("child"), cel::NullValue())); - - auto value = std::move(*builder).Build(); - - act.InsertOrAssignValue("nested_test_all_types", - std::move(value)); - - return absl::OkStatus(); - }, - - [](const absl::StatusOr& got) { - EXPECT_THAT(got.status(), - StatusIs(absl::StatusCode::kInvalidArgument)); - }, - }, - { - // Some clients will use maps to represent a protobuf at runtime, - // this is not yet supported. - "chained_select_noncontainer_as_root_unsupported", - "nested_test_all_types.child.payload.standalone_message.bb", - {}, // not set - [](google::protobuf::Arena* arena, Activation& act) { - act.InsertOrAssignValue("nested_test_all_types", - cel::DurationValue(absl::Seconds(1))); - return absl::OkStatus(); - }, - - [](const absl::StatusOr& got) { - EXPECT_THAT(got.status(), - StatusIs(absl::StatusCode::kInvalidArgument)); - }, - }, - { - "complex_select_success", - "((false)? a.child.child : b.child).child.payload.single_int64", - {{"a", ""}, - {"b", - R"pb( - child { child { payload { single_int64: -42 } } } - )pb"}}, - ActivationSetupFn(), - - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), -42); - }, - }, - { - "chained_select_presence_present", - "has(nested_test_all_types.child.payload.standalone_message.bb)", - {{"nested_test_all_types", - R"pb( - child { payload { standalone_message { bb: 2 } } } - )pb"}}, - ActivationSetupFn(), - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_TRUE(result.GetBool().NativeValue()); - }, - }, - { - "chained_select_presence_not_present", - "has(nested_test_all_types.child.payload.standalone_message.bb)", - {{"nested_test_all_types", - R"pb( - child { payload { standalone_message {} } } - )pb"}}, - ActivationSetupFn(), - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_FALSE(result.GetBool().NativeValue()); - }, - }, - { - "select_with_map_supported", - "nested_test_all_types.payload.map_string_message['$not_a_field']." - "bb", - {{"nested_test_all_types", - R"pb( - payload { - map_string_message { - key: "$not_a_field", - value { bb: 5 } - } - } - )pb"}}, - ActivationSetupFn(), - - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), 5); - }, - }, - { - "select_with_map_no_such_key", - "nested_test_all_types.payload.map_string_message['$not_a_field']." - "bb", - {{"nested_test_all_types", - R"pb( - payload { - map_string_message { - key: "a_different_field", - value { bb: 5 } - } - } - )pb"}}, - ActivationSetupFn(), - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_THAT(result.GetError().NativeValue(), - StatusIs(absl::StatusCode::kNotFound, - AllOf(HasSubstr("Key not found"), - HasSubstr("$not_a_field")))); - }, - }, - { - "select_with_repeated_supported", - "nested_test_all_types.payload.repeated_nested_message[1].bb", - {{"nested_test_all_types", - R"pb( - payload { - repeated_nested_message {} - repeated_nested_message { bb: 7 } - } - )pb"}}, - ActivationSetupFn(), - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), 7); - }, - }, - { - "select_with_repeated_index_out_of_bounds", - "nested_test_all_types.payload.repeated_nested_message[1].bb", - {{"nested_test_all_types", - R"pb( - payload { repeated_nested_message {} } - )pb"}}, - ActivationSetupFn(), - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_THAT(result.GetError().NativeValue(), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("index out of bounds"))); - }, - }, - { - "unknown_field", - "((false)? a.child.child : b.child).child.payload.single_int64", - {{"a", ""}, - {"b", - R"pb( - child { child { payload { single_int64: -42 } } } - )pb"}}, - [](google::protobuf::Arena*, Activation& act) { - act.SetUnknownPatterns({AttributePattern( - "b", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("child")})}); - return absl::OkStatus(); - }, - - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_THAT( - result.GetUnknown().attribute_set(), - ElementsAre(Eq(Attribute( - "b", { - AttributeQualifier::OfString("child"), - AttributeQualifier::OfString("child"), - AttributeQualifier::OfString("payload"), - AttributeQualifier::OfString("single_int64"), - })))); - }, - }, - { - "unknown_field_partial", - "((false)? a.child.child : b.child).child.payload.single_int64", - {{"a", ""}, - {"b", - R"pb( - child { child { payload { single_int64: -42 } } } - )pb"}}, - [](google::protobuf::Arena*, Activation& act) { - act.SetUnknownPatterns({AttributePattern( - "b", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("child")})}); - return absl::OkStatus(); - }, - - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), -42); - }, - }, - { - "unknown_ident", - "((false)? a.child.child : b.child).child.payload.single_int64", - {{"a", ""}, - {"b", - R"pb( - child { child { payload { single_int64: -42 } } } - )pb"}}, - [](google::protobuf::Arena*, Activation& act) { - act.SetUnknownPatterns({ - AttributePattern("b", {}), - }); - return absl::OkStatus(); - }, - - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_THAT(result.GetUnknown().attribute_set(), - ElementsAre(Truly([](const Attribute& attr) { - return attr.variable_name() == "b"; - }))); - }, - }, - { - "unknown_pruned", - "((false)? a.child.child : b.child).child.payload.single_int64", - {{"a", ""}, - {"b", - R"pb( - child { child { payload { single_int64: -42 } } } - )pb"}}, - [](google::protobuf::Arena*, Activation& act) { - act.SetUnknownPatterns({ - AttributePattern("a", {}), - }); - return absl::OkStatus(); - }, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), -42); - }, - }, - { - "missing_field", - "custom_predicate(nested_test_all_types.child.payload.standalone_" - "message)", - {{"nested_test_all_types", - R"pb( - child { payload { standalone_message { bb: 20 } } } - )pb"}}, - [](google::protobuf::Arena*, Activation& act) { - act.SetMissingPatterns({AttributePattern( - "nested_test_all_types", - { - AttributeQualifierPattern::OfString("child"), - })}); - return absl::OkStatus(); - }, - - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_THAT(result.GetError().NativeValue().message(), - HasSubstr("nested_test_all_types.child.payload." - "standalone_message")); - }, - }, - { - "missing_field_partial", - "custom_predicate(nested_test_all_types.child.payload.standalone_" - "message)", - {{"nested_test_all_types", - R"pb( - child { payload { standalone_message { bb: 20 } } } - )pb"}}, - [](google::protobuf::Arena*, Activation& act) { - act.SetMissingPatterns({AttributePattern( - "b", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("child")})}); - return absl::OkStatus(); - }, - - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_TRUE(result.GetBool().NativeValue()); - }, - }, - { - "select_wrapper_int_leaf", - "nested_test_all_types.payload.single_int64_wrapper", - {{"nested_test_all_types", - R"pb( - payload { single_int64_wrapper { value: 10 } } - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), 10); - }, - }, - { - "select_repeated_leaf", - "nested_test_all_types.payload.repeated_int64", - {{"nested_test_all_types", - R"pb( - payload { repeated_int64: 10 } - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - }, - }, - { - "select_map_leaf", - "nested_test_all_types.payload.map_string_int64", - {{"nested_test_all_types", - R"pb( - payload { map_string_int64 { key: "key", value: 12 } } - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - }, - }, - { - "select_with_map_dot", - "nested_test_all_types.payload.map_string_message.field_like_key." - "bb", - {{"nested_test_all_types", - R"pb( - payload { - map_string_message { - key: "field_like_key", - value { bb: 42 } - } - } - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), 42); - }, - }, - { - "select_with_map_bool", - "nested_test_all_types.payload.map_bool_message[false].bb", - {{"nested_test_all_types", - R"pb( - payload { - map_bool_message { - key: false, - value { bb: 42 } - } - } - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), 42); - }, - }, - { - "select_with_map_int", - "nested_test_all_types.payload.map_int64_message[-1].bb", - {{"nested_test_all_types", - R"pb( - payload { - map_int64_message { - key: -1, - value { bb: 42 } - } - } - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), 42); - }, - }, - { - "select_with_map_uint", - "nested_test_all_types.payload.map_uint64_message[1u].bb", - {{"nested_test_all_types", - R"pb( - payload { - map_uint64_message { - key: 1, - value { bb: 42 } - } - } - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), 42); - }, - }, - { - "select_with_repeated", - "nested_test_all_types.payload.repeated_nested_message[1].bb", - {{"nested_test_all_types", - R"pb( - payload { - repeated_nested_message {} - repeated_nested_message { bb: 42 } - } - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), 42); - }, - }, - { - "select_with_any", - "nested_test_all_types.payload.single_any.single_int64", - {{"nested_test_all_types", - R"pb( - payload { - single_any { - [type.googleapis.com/cel.expr.conformance.proto2 - .TestAllTypes] { single_int64: 42 } - } - } - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_EQ(result.GetInt().NativeValue(), 42); - }, - }, - { - "has_repeated_leaf_true", - "has(nested_test_all_types.payload.repeated_int64)", - {{"nested_test_all_types", - R"pb( - payload { repeated_int64: 42 } - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_TRUE(result.GetBool().NativeValue()); - }, - }, - { - "has_repeated_leaf_false", - "has(nested_test_all_types.payload.repeated_int64)", - {{"nested_test_all_types", - R"pb( - payload {} - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_FALSE(result.GetBool().NativeValue()); - }, - }, - { - "has_map_leaf_true", - "has(nested_test_all_types.payload.map_string_int64)", - {{"nested_test_all_types", - R"pb( - payload { map_string_int64 { key: "string" value: 12 } } - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_TRUE(result.GetBool().NativeValue()); - }, - }, - { - "has_map_leaf_false", - "has(nested_test_all_types.payload.map_string_int64)", - {{"nested_test_all_types", - R"pb( - payload {} - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_FALSE(result.GetBool().NativeValue()); - }, - }, - { - "has_map_field_like_key", - "has(nested_test_all_types.payload.map_string_int64.field_like_" - "key)", - {{"nested_test_all_types", - R"pb( - payload { map_string_int64 { key: "field_like_key" value: 12 } } - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_TRUE(result.GetBool().NativeValue()); - }, - }, - { - "has_map_field_like_key_false", - "has(nested_test_all_types.payload.map_string_int64.field_like_" - "key)", - {{"nested_test_all_types", - R"pb( - payload { map_string_int64 { key: "wrong_key" value: 12 } } - )pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_FALSE(result.GetBool().NativeValue()); - }, - }, - { - "select_wrong_runtime_type", - "test_all_types.single_int64", - {{}}, - [](google::protobuf::Arena* arena, Activation& activation) { - activation.InsertOrAssignValue("test_all_types", - cel::IntValue(42)); - return absl::OkStatus(); - }, - [](const absl::StatusOr& got) { - EXPECT_THAT(got, StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("Expected struct type"))); - }, - }, - { - "select_with_struct", - "nested_test_all_types.payload.single_struct['key']['subkey']", - {{"nested_test_all_types", - R"pb(payload { - single_struct { - fields { - key: "key" - value { - struct_value { - fields { - key: "subkey" + testing::Combine( + testing::ValuesIn( + { + { + "chained_select_success", + "nested_test_all_types.child.payload.standalone_message.bb", + {{"nested_test_all_types", + R"pb( + child { payload { standalone_message { bb: 42 } } } + )pb"}}, + ActivationSetupFn(), + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 42); + }, + }, + { + "chained_select_defaults_success", + "nested_test_all_types.child.payload.standalone_message.bb", + {{"nested_test_all_types", R"pb()pb"}}, + ActivationSetupFn(), + + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 0); + }, + }, + { + "chained_select_partial_success", + "nested_test_all_types.child.payload.standalone_message.bb", + {}, + [](google::protobuf::Arena* arena, Activation& act) { + auto* custom_struct = + google::protobuf::Arena::Create( + arena, IntValue(42)); + act.InsertOrAssignValue( + "nested_test_all_types", + CustomStructValue(custom_struct, arena)); + return absl::OkStatus(); + }, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 42); + }, + }, + { + "chained_select_presence_partial_present", + "has(nested_test_all_types.child.payload.standalone_" + "message." + "bb)", + {}, + [](google::protobuf::Arena* arena, Activation& act) { + auto* custom_struct = + google::protobuf::Arena::Create( + arena, IntValue(42)); + act.InsertOrAssignValue( + "nested_test_all_types", + CustomStructValue(custom_struct, arena)); + return absl::OkStatus(); + }, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "chained_select_not_bound", + "nested_test_all_types.child.payload.standalone_message.bb", + {}, // not set + ActivationSetupFn(), + + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kUnknown, + HasSubstr("nested_test_all_types"))); + }, + }, + { + // Some clients will use maps to represent a protobuf + // message at runtime. This is not yet supported. + "chained_select_map_as_root_unsupported", + "nested_test_all_types.child.payload.standalone_message.bb", + {}, // not set + [](google::protobuf::Arena* arena, Activation& act) -> absl::Status { + auto builder = cel::NewMapValueBuilder(arena); + CEL_RETURN_IF_ERROR( + builder->Put(cel::StringValue::WrapUnsafe("child"), + cel::NullValue())); + + auto value = std::move(*builder).Build(); + + act.InsertOrAssignValue("nested_test_all_types", + std::move(value)); + + return absl::OkStatus(); + }, + + [](const absl::StatusOr& got) { + EXPECT_THAT(got.status(), + StatusIs(absl::StatusCode::kInvalidArgument)); + }, + }, + { + // Some clients will use maps to represent a protobuf at + // runtime, this is not yet supported. + "chained_select_noncontainer_as_root_unsupported", + "nested_test_all_types.child.payload.standalone_message.bb", + {}, // not set + [](google::protobuf::Arena* arena, Activation& act) { + act.InsertOrAssignValue( + "nested_test_all_types", + cel::DurationValue(absl::Seconds(1))); + return absl::OkStatus(); + }, + + [](const absl::StatusOr& got) { + EXPECT_THAT(got.status(), + StatusIs(absl::StatusCode::kInvalidArgument)); + }, + }, + { + "complex_select_success", + "((false)? a.child.child : " + "b.child).child.payload.single_int64", + {{"a", ""}, + {"b", + R"pb( + child { child { payload { single_int64: -42 } } } + )pb"}}, + ActivationSetupFn(), + + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), -42); + }, + }, + { + "chained_select_presence_present", + "has(nested_test_all_types.child.payload.standalone_" + "message." + "bb)", + {{"nested_test_all_types", + R"pb( + child { payload { standalone_message { bb: 2 } } } + )pb"}}, + ActivationSetupFn(), + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "chained_select_presence_not_present", + "has(nested_test_all_types.child.payload.standalone_" + "message." + "bb)", + {{"nested_test_all_types", + R"pb( + child { payload { standalone_message {} } } + )pb"}}, + ActivationSetupFn(), + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_FALSE(result.GetBool().NativeValue()); + }, + }, + { + "select_with_map_supported", + "nested_test_all_types.payload.map_string_message['$not_a_" + "field']." + "bb", + {{"nested_test_all_types", + R"pb( + payload { + map_string_message { + key: "$not_a_field", + value { bb: 5 } + } + } + )pb"}}, + ActivationSetupFn(), + + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 5); + }, + }, + { + "select_with_map_no_such_key", + "nested_test_all_types.payload.map_string_message['$not_a_" + "field']." + "bb", + {{"nested_test_all_types", + R"pb( + payload { + map_string_message { + key: "a_different_field", + value { bb: 5 } + } + } + )pb"}}, + ActivationSetupFn(), + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kNotFound, + AllOf(HasSubstr("Key not found"), + HasSubstr("$not_a_field")))); + }, + }, + { + "select_with_repeated_supported", + "nested_test_all_types.payload.repeated_nested_message[1]." + "bb", + {{"nested_test_all_types", + R"pb( + payload { + repeated_nested_message {} + repeated_nested_message { bb: 7 } + } + )pb"}}, + ActivationSetupFn(), + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 7); + }, + }, + { + "select_with_repeated_index_out_of_bounds", + "nested_test_all_types.payload.repeated_nested_message[1]." + "bb", + {{"nested_test_all_types", + R"pb( + payload { repeated_nested_message {} } + )pb"}}, + ActivationSetupFn(), + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("index out of bounds"))); + }, + }, + { + // -2^32, which would read element 0 if truncated to an int. + "select_with_repeated_index_beyond_int32", + "nested_test_all_types.payload.repeated_nested_message" + "[-4294967296].bb", + {{"nested_test_all_types", + R"pb( + payload { repeated_nested_message {} } + )pb"}}, + ActivationSetupFn(), + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kInvalidArgument)); + }, + }, + { + "unknown_field", + "((false)? a.child.child : " + "b.child).child.payload.single_int64", + {{"a", ""}, + {"b", + R"pb( + child { child { payload { single_int64: -42 } } } + )pb"}}, + [](google::protobuf::Arena*, Activation& act) { + return act.SetUnknownPatterns({AttributePattern( + "b", + {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("child")})}); + }, + + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT( + result.GetUnknown().ToAttributeSet(), + ElementsAre(Eq(Attribute( + "b", { + AttributeQualifier::OfString("child"), + AttributeQualifier::OfString("child"), + })))); + }, + }, + { + "unknown_field_partial", + "((false)? a.child.child : " + "b.child).child.payload.single_int64", + {{"a", ""}, + {"b", + R"pb( + child { child { payload { single_int64: -42 } } } + )pb"}}, + [](google::protobuf::Arena*, Activation& act) { + return act.SetUnknownPatterns({AttributePattern( + "b", + {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("child")})}); + }, + + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), -42); + }, + }, + { + "unknown_ident", + "((false)? a.child.child : " + "b.child).child.payload.single_int64", + {{"a", ""}, + {"b", + R"pb( + child { child { payload { single_int64: -42 } } } + )pb"}}, + [](google::protobuf::Arena*, Activation& act) { + return act.SetUnknownPatterns({ + AttributePattern("b", {}), + }); + }, + + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + ElementsAre(Truly([](const Attribute& attr) { + return attr.variable_name() == "b"; + }))); + }, + }, + { + "unknown_pruned", + "((false)? a.child.child : " + "b.child).child.payload.single_int64", + {{"a", ""}, + {"b", + R"pb( + child { child { payload { single_int64: -42 } } } + )pb"}}, + [](google::protobuf::Arena*, Activation& act) { + return act.SetUnknownPatterns({ + AttributePattern("a", {}), + }); + }, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), -42); + }, + }, + { + "missing_field", + "custom_predicate(nested_test_all_types.child.payload." + "standalone_" + "message)", + {{"nested_test_all_types", + R"pb( + child { payload { standalone_message { bb: 20 } } } + )pb"}}, + [](google::protobuf::Arena*, Activation& act) { + return act.SetMissingPatterns({AttributePattern( + "nested_test_all_types", + { + AttributeQualifierPattern::OfString("child"), + })}); + }, + + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue().message(), + HasSubstr("nested_test_all_types.child")); + }, + }, + { + "missing_field_partial", + "custom_predicate(nested_test_all_types.child.payload." + "standalone_" + "message)", + {{"nested_test_all_types", + R"pb( + child { payload { standalone_message { bb: 20 } } } + )pb"}}, + [](google::protobuf::Arena*, Activation& act) { + return act.SetMissingPatterns({AttributePattern( + "b", + {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("child")})}); + }, + + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "select_wrapper_int_leaf", + "nested_test_all_types.payload.single_int64_wrapper", + {{"nested_test_all_types", + R"pb( + payload { single_int64_wrapper { value: 10 } } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 10); + }, + }, + { + "select_repeated_leaf", + "nested_test_all_types.payload.repeated_int64", + {{"nested_test_all_types", + R"pb( + payload { repeated_int64: 10 } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + }, + }, + { + "select_map_leaf", + "nested_test_all_types.payload.map_string_int64", + {{"nested_test_all_types", + R"pb( + payload { map_string_int64 { key: "key", value: 12 } } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + }, + }, + { + "select_with_map_dot", + "nested_test_all_types.payload.map_string_message.field_" + "like_" + "key." + "bb", + {{"nested_test_all_types", + R"pb( + payload { + map_string_message { + key: "field_like_key", + value { bb: 42 } + } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 42); + }, + }, + { + "select_with_map_bool", + "nested_test_all_types.payload.map_bool_message[false].bb", + {{"nested_test_all_types", + R"pb( + payload { + map_bool_message { + key: false, + value { bb: 42 } + } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 42); + }, + }, + { + "select_with_map_int", + "nested_test_all_types.payload.map_int64_message[-1].bb", + {{"nested_test_all_types", + R"pb( + payload { + map_int64_message { + key: -1, + value { bb: 42 } + } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 42); + }, + }, + { + "select_with_map_uint", + "nested_test_all_types.payload.map_uint64_message[1u].bb", + {{"nested_test_all_types", + R"pb( + payload { + map_uint64_message { + key: 1, + value { bb: 42 } + } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 42); + }, + }, + { + "select_with_repeated", + "nested_test_all_types.payload.repeated_nested_message[1]." + "bb", + {{"nested_test_all_types", + R"pb( + payload { + repeated_nested_message {} + repeated_nested_message { bb: 42 } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 42); + }, + }, + { + "select_with_any", + "nested_test_all_types.payload.single_any.single_int64", + {{"nested_test_all_types", + R"pb( + payload { + single_any { + [type.googleapis.com/cel.expr.conformance.proto2 + .TestAllTypes] { single_int64: 42 } + } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 42); + }, + }, + { + "has_repeated_leaf_true", + "has(nested_test_all_types.payload.repeated_int64)", + {{"nested_test_all_types", + R"pb( + payload { repeated_int64: 42 } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "has_repeated_leaf_false", + "has(nested_test_all_types.payload.repeated_int64)", + {{"nested_test_all_types", + R"pb( + payload {} + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_FALSE(result.GetBool().NativeValue()); + }, + }, + { + "has_map_leaf_true", + "has(nested_test_all_types.payload.map_string_int64)", + {{"nested_test_all_types", + R"pb( + payload { map_string_int64 { key: "string" value: 12 } } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "has_map_leaf_false", + "has(nested_test_all_types.payload.map_string_int64)", + {{"nested_test_all_types", + R"pb( + payload {} + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_FALSE(result.GetBool().NativeValue()); + }, + }, + { + "has_map_field_like_key", + "has(nested_test_all_types.payload.map_string_int64.field_" + "like_" + "key)", + {{"nested_test_all_types", + R"pb( + payload { + map_string_int64 { key: "field_like_key" value: 12 } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "has_map_field_like_key_false", + "has(nested_test_all_types.payload.map_string_int64.field_" + "like_" + "key)", + {{"nested_test_all_types", + R"pb( + payload { + map_string_int64 { key: "wrong_key" value: 12 } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_FALSE(result.GetBool().NativeValue()); + }, + }, + { + "has_simple_true", + "has(nested_test_all_types.child)", + {{"nested_test_all_types", + R"pb( + child {} + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "has_simple_false", + "has(nested_test_all_types.child)", + {{"nested_test_all_types", R"pb()pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_FALSE(result.GetBool().NativeValue()); + }, + }, + { + "has_through_map_true", + "has(nested_test_all_types.payload.map_string_message['key'" + "]." + "bb)", + {{"nested_test_all_types", + R"pb( + payload { + map_string_message { + key: "key" + value { bb: 5 } + } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "has_through_map_false", + "has(nested_test_all_types.payload.map_string_message['key'" + "]." + "bb)", + {{"nested_test_all_types", + R"pb( + payload { + map_string_message { + key: "key" + value {} + } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_FALSE(result.GetBool().NativeValue()); + }, + }, + { + "has_through_map_no_such_key", + "has(nested_test_all_types.payload.map_string_message['" + "missing_" + "key'].bb)", + {{"nested_test_all_types", + R"pb( + payload { + map_string_message { + key: "other_key" + value { bb: 5 } + } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kNotFound, + AllOf(HasSubstr("Key not found"), + HasSubstr("missing_key")))); + }, + }, + { + "has_through_map_int32_overflow", + "has(nested_test_all_types.payload.map_int32_message[" + "2147483648].bb)", + {{"nested_test_all_types", + R"pb( + payload { + map_int32_message { + key: 1 + value { bb: 42 } + } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT( + result.GetError().NativeValue(), + testing::AnyOf( + StatusIs(absl::StatusCode::kOutOfRange, + HasSubstr("integer overflow")), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid map key type")))); + }, + }, + { + "has_through_repeated_true", + "has(nested_test_all_types.payload.repeated_nested_message[" + "1]." + "bb)", + {{"nested_test_all_types", + R"pb( + payload { + repeated_nested_message {} + repeated_nested_message { bb: 7 } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "has_through_repeated_false", + "has(nested_test_all_types.payload.repeated_nested_message[" + "1]." + "bb)", + {{"nested_test_all_types", + R"pb( + payload { + repeated_nested_message {} + repeated_nested_message {} + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_FALSE(result.GetBool().NativeValue()); + }, + }, + { + "has_through_repeated_index_out_of_bounds", + "has(nested_test_all_types.payload.repeated_nested_message[" + "1]." + "bb)", + {{"nested_test_all_types", + R"pb( + payload { repeated_nested_message { bb: 7 } } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("index out of bounds"))); + }, + }, + { + "has_through_repeated_negative_index", + "has(nested_test_all_types.payload.repeated_nested_message[" + "-1]." + "bb)", + {{"nested_test_all_types", + R"pb( + payload { repeated_nested_message { bb: 7 } } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kInvalidArgument)); + }, + }, + { + "select_wrong_runtime_type", + "test_all_types.single_int64", + {{}}, + [](google::protobuf::Arena* arena, Activation& activation) { + activation.InsertOrAssignValue("test_all_types", + cel::IntValue(42)); + return absl::OkStatus(); + }, + [](const absl::StatusOr& got) { + EXPECT_THAT(got, + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Expected struct type"))); + }, + }, + { + "select_with_struct", + "nested_test_all_types.payload.single_struct['key']['" + "subkey']", + {{"nested_test_all_types", + R"pb(payload { + single_struct { + fields { + key: "key" + value { + struct_value { + fields { + key: "subkey" + value { bool_value: true } + } + } + } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "select_with_list_value", + "nested_test_all_types.payload.list_value[0]['subkey']", + {{"nested_test_all_types", + R"pb(payload { + list_value { + values { + struct_value { + fields { + key: "subkey" + value { bool_value: true } + } + } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "select_with_value", + "nested_test_all_types.payload.single_value['key']['subkey'" + "]", + {{"nested_test_all_types", + R"pb(payload { + single_value { + struct_value { + fields { + key: "key" + value { + struct_value { + fields { + key: "subkey" + value { bool_value: true } + } + } + } + } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "select_with_map_int32", + "nested_test_all_types.payload.map_int32_message[-1].bb", + {{"nested_test_all_types", + R"pb( + payload { + map_int32_message { + key: -1, + value { bb: 42 } + } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 42); + }, + }, + { + "select_with_map_int32_overflow", + "nested_test_all_types.payload.map_int32_message[" + "2147483648]." + "bb", + {{"nested_test_all_types", + R"pb( + payload { + map_int32_message { + key: 1, + value { bb: 42 } + } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT( + result.GetError().NativeValue(), + testing::AnyOf( + StatusIs(absl::StatusCode::kOutOfRange, + HasSubstr("integer overflow")), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid map key type")))); + }, + }, + { + "select_with_map_int32_underflow", + "nested_test_all_types.payload.map_int32_message[-" + "2147483649]." + "bb", + {{"nested_test_all_types", + R"pb( + payload { + map_int32_message { + key: -1, + value { bb: 42 } + } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT( + result.GetError().NativeValue(), + testing::AnyOf( + StatusIs(absl::StatusCode::kOutOfRange, + HasSubstr("integer overflow")), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid map key type")))); + }, + }, + { + "select_with_map_uint32", + "nested_test_all_types.payload.map_uint32_message[1u].bb", + {{"nested_test_all_types", + R"pb( + payload { + map_uint32_message { + key: 1, + value { bb: 42 } + } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 42); + }, + }, + { + "select_with_map_uint32_overflow", + "nested_test_all_types.payload.map_uint32_message[" + "4294967296u]." + "bb", + {{"nested_test_all_types", + R"pb( + payload { + map_uint32_message { + key: 1, + value { bb: 42 } + } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT( + result.GetError().NativeValue(), + testing::AnyOf( + StatusIs(absl::StatusCode::kOutOfRange, + HasSubstr("unsigned integer over")), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid map key type")))); + }, + }, + { + "select_map_element_leaf", + "nested_test_all_types.payload.map_string_int64['key']", + {{"nested_test_all_types", + R"pb( + payload { map_string_int64 { key: "key", value: 12 } } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 12); + }, + }, + { + "select_map_dot_leaf", + "nested_test_all_types.payload.map_string_int64.field_like_" + "key", + {{"nested_test_all_types", + R"pb( + payload { + map_string_int64 { key: "field_like_key", value: 12 } + } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 12); + }, + }, + { + "select_repeated_element_leaf", + "nested_test_all_types.payload.repeated_int64[1]", + {{"nested_test_all_types", + R"pb( + payload { repeated_int64: 10 repeated_int64: 42 } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 42); + }, + }, + { + "select_with_repeated_negative_index", + "nested_test_all_types.payload.repeated_nested_message[-1]." + "bb", + {{"nested_test_all_types", + R"pb( + payload { repeated_nested_message { bb: 42 } } + )pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kInvalidArgument)); + }, + }, + { + "select_with_map_struct", + "nested_test_all_types.payload.map_string_struct['key']['" + "subkey']", + {{"nested_test_all_types", + R"pb(payload { + map_string_struct { + key: "key" + value { + fields { + key: "subkey" + value { bool_value: true } + } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "select_with_map_list_value", + "nested_test_all_types.payload.map_string_list_value['key']" + "[0][" + "'subkey']", + {{"nested_test_all_types", + R"pb(payload { + map_string_list_value { + key: "key" + value { + values { + struct_value { + fields { + key: "subkey" + value { bool_value: true } + } + } + } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "select_with_map_value", + "nested_test_all_types.payload.map_string_value['key']['" + "subkey'" + "]", + {{"nested_test_all_types", + R"pb(payload { + map_string_value { + key: "key" + value { + struct_value { + fields { + key: "subkey" + value { bool_value: true } + } + } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "select_with_repeated_struct", + "nested_test_all_types.payload.repeated_struct[0]['subkey'" + "]", + {{"nested_test_all_types", + R"pb(payload { + repeated_struct { + fields { + key: "subkey" + value { bool_value: true } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "select_with_repeated_list_value", + "nested_test_all_types.payload.repeated_list_value[0][0]['" + "subkey']", + {{"nested_test_all_types", + R"pb(payload { + repeated_list_value { + values { + struct_value { + fields { + key: "subkey" + value { bool_value: true } + } + } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "select_with_repeated_value", + "nested_test_all_types.payload.repeated_value[0]['subkey']", + {{"nested_test_all_types", + R"pb(payload { + repeated_value { + struct_value { + fields { + key: "subkey" + value { bool_value: true } + } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "select_with_bad_any_fallback", + "nested_test_all_types.payload.single_any['subkey']", + {{"nested_test_all_types", + R"pb(payload { + single_any { + type_url: "type.googleapis.com/UnknownType" + value: "invalid" + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + }, + }, + { + "select_struct_leaf", + "nested_test_all_types.payload.single_struct", + {{"nested_test_all_types", + R"pb(payload { + single_struct { + fields { + key: "key" + value { bool_value: true } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetMap().Size(), + absl_testing::IsOkAndHolds(1)); + }, + }, + { + "select_struct_leaf_unset", + "nested_test_all_types.payload.single_struct", + {{"nested_test_all_types", R"pb(payload {})pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetMap().Size(), + absl_testing::IsOkAndHolds(0)); + }, + }, + { + "select_value_leaf", + "nested_test_all_types.payload.single_value", + {{"nested_test_all_types", + R"pb(payload { single_value { bool_value: true } })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "select_value_leaf_unset", + "nested_test_all_types.payload.single_value", + {{"nested_test_all_types", R"pb(payload {})pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + EXPECT_TRUE(result->Is()) + << result->DebugString(); + }, + }, + { + "select_list_value_leaf", + "nested_test_all_types.payload.list_value", + {{"nested_test_all_types", + R"pb(payload { + list_value { values { bool_value: true } } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetList().Size(), + absl_testing::IsOkAndHolds(1)); + }, + }, + { + "select_list_value_leaf_unset", + "nested_test_all_types.payload.list_value", + {{"nested_test_all_types", R"pb(payload {})pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetList().Size(), + absl_testing::IsOkAndHolds(0)); + }, + }, + { + "has_struct_leaf_true", + "has(nested_test_all_types.payload.single_struct)", + {{"nested_test_all_types", + R"pb(payload { single_struct {} })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "has_struct_leaf_false", + "has(nested_test_all_types.payload.single_struct)", + {{"nested_test_all_types", R"pb(payload {})pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_FALSE(result.GetBool().NativeValue()); + }, + }, + { + "has_value_leaf_true", + "has(nested_test_all_types.payload.single_value)", + {{"nested_test_all_types", R"pb(payload { + single_value { + null_value: NULL_VALUE + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "has_value_leaf_false", + "has(nested_test_all_types.payload.single_value)", + {{"nested_test_all_types", R"pb(payload {})pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_FALSE(result.GetBool().NativeValue()); + }, + }, + { + "has_list_value_leaf_true", + "has(nested_test_all_types.payload.list_value)", + {{"nested_test_all_types", + R"pb(payload { list_value {} })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "has_list_value_leaf_false", + "has(nested_test_all_types.payload.list_value)", + {{"nested_test_all_types", R"pb(payload {})pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_FALSE(result.GetBool().NativeValue()); + }, + }, + { + "select_map_struct_leaf", + "nested_test_all_types.payload.map_string_struct['key']", + {{"nested_test_all_types", + R"pb(payload { + map_string_struct { + key: "key" + value { + fields { + key: "subkey" + value { bool_value: true } + } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetMap().Size(), + absl_testing::IsOkAndHolds(1)); + }, + }, + { + "select_map_list_value_leaf", + "nested_test_all_types.payload.map_string_list_value['key'" + "]", + {{"nested_test_all_types", + R"pb(payload { + map_string_list_value { + key: "key" + value { values { bool_value: true } } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetList().Size(), + absl_testing::IsOkAndHolds(1)); + }, + }, + { + "select_map_value_leaf", + "nested_test_all_types.payload.map_string_value['key']", + {{"nested_test_all_types", + R"pb(payload { + map_string_value { + key: "key" value { bool_value: true } } - } - } - } - } - })pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_TRUE(result.GetBool().NativeValue()); - }, - }, - { - "select_with_list_value", - "nested_test_all_types.payload.list_value[0]['subkey']", - {{"nested_test_all_types", - R"pb(payload { - list_value { - values { - struct_value { - fields { - key: "subkey" - value { bool_value: true } - } - } - } - } - })pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_TRUE(result.GetBool().NativeValue()); - }, - }, - { - "select_with_value", - "nested_test_all_types.payload.single_value['key']['subkey']", - {{"nested_test_all_types", - R"pb(payload { - single_value { - struct_value { - fields { - key: "key" - value { - struct_value { + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "select_repeated_struct_leaf", + "nested_test_all_types.payload.repeated_struct[0]", + {{"nested_test_all_types", + R"pb(payload { + repeated_struct { fields { key: "subkey" value { bool_value: true } } } - } - } - } - } - })pb"}}, - nullptr, - [](const absl::StatusOr& got) { - ASSERT_OK_AND_ASSIGN(Value result, got); - ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_TRUE(result.GetBool().NativeValue()); - }, - }, - }), - - [](const testing::TestParamInfo& info) { - return info.param.case_name; + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetMap().Size(), + absl_testing::IsOkAndHolds(1)); + }, + }, + { + "select_repeated_list_value_leaf", + "nested_test_all_types.payload.repeated_list_value[0]", + {{"nested_test_all_types", + R"pb(payload { + repeated_list_value { values { bool_value: true } } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_THAT(result.GetList().Size(), + absl_testing::IsOkAndHolds(1)); + }, + }, + { + "select_repeated_value_leaf", + "nested_test_all_types.payload.repeated_value[0]", + {{"nested_test_all_types", R"pb(payload { + repeated_value { + bool_value: true + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "select_with_struct_dot", + "nested_test_all_types.payload.single_struct.key.subkey", + {{"nested_test_all_types", + R"pb(payload { + single_struct { + fields { + key: "key" + value { + struct_value { + fields { + key: "subkey" + value { bool_value: true } + } + } + } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "has_with_struct_dot_true", + "has(nested_test_all_types.payload.single_struct.key)", + {{"nested_test_all_types", + R"pb(payload { + single_struct { + fields { + key: "key" + value { bool_value: false } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "has_with_struct_dot_false", + "has(nested_test_all_types.payload.single_struct.key)", + {{"nested_test_all_types", + R"pb(payload { single_struct {} })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_FALSE(result.GetBool().NativeValue()); + }, + }, + { + "has_with_struct_nested_dot_true", + "has(nested_test_all_types.payload.single_struct.key." + "subkey)", + {{"nested_test_all_types", + R"pb(payload { + single_struct { + fields { + key: "key" + value { + struct_value { + fields { + key: "subkey" + value { bool_value: false } + } + } + } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "has_with_struct_nested_dot_false", + "has(nested_test_all_types.payload.single_struct.key." + "subkey)", + {{"nested_test_all_types", + R"pb(payload { + single_struct { + fields { + key: "key" + value { struct_value {} } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_FALSE(result.GetBool().NativeValue()); + }, + }, + { + "select_with_value_dot", + "nested_test_all_types.payload.single_value.key.subkey", + {{"nested_test_all_types", + R"pb(payload { + single_value { + struct_value { + fields { + key: "key" + value { + struct_value { + fields { + key: "subkey" + value { bool_value: true } + } + } + } + } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "has_with_value_dot_true", + "has(nested_test_all_types.payload.single_value.key)", + {{"nested_test_all_types", + R"pb(payload { + single_value { + struct_value { + fields { + key: "key" + value { bool_value: false } + } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + { + "has_with_value_dot_false", + "has(nested_test_all_types.payload.single_value.key)", + {{"nested_test_all_types", + R"pb(payload { single_value { struct_value {} } })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_FALSE(result.GetBool().NativeValue()); + }, + }, + { + "select_with_list_value_dot", + "nested_test_all_types.payload.list_value[0].subkey", + {{"nested_test_all_types", + R"pb(payload { + list_value { + values { + struct_value { + fields { + key: "subkey" + value { bool_value: true } + } + } + } + } + })pb"}}, + nullptr, + [](const absl::StatusOr& got) { + ASSERT_OK_AND_ASSIGN(Value result, got); + ASSERT_TRUE(result->Is()) + << result->DebugString(); + EXPECT_TRUE(result.GetBool().NativeValue()); + }, + }, + }), + testing::Bool()), + [](const testing::TestParamInfo< + SelectOptimizationProgramOptimizerTest::ParamType>& info) { + return absl::StrCat(std::get<0>(info.param).case_name, "_", + std::get<1>(info.param) ? "cached" : "uncached"); }); // Tests covering unexpected / malformed ASTs. @@ -1765,6 +3230,86 @@ class SelectOptimizationUnexpectedAstTest : public SelectOptimizationTest { return element; } + Expr NextInt(int64_t value) { + Expr result = NextExpr(); + result.mutable_const_expr().set_int_value(value); + return result; + } + + Expr NextString(absl::string_view value) { + Expr result = NextExpr(); + result.mutable_const_expr().set_string_value(value); + return result; + } + + template + Expr NextList(Exprs... elements) { + Expr result = NextExpr(); + (result.mutable_list_expr().add_elements().set_expr(std::move(elements)), + ...); + return result; + } + + Expr NextUint(uint64_t value) { + Expr result = NextExpr(); + result.mutable_const_expr().set_uint64_value(value); + return result; + } + + Expr NextBool(bool value) { + Expr result = NextExpr(); + result.mutable_const_expr().set_bool_value(value); + return result; + } + + // Plans `fn(ident, args...)` with only the select program optimizer. + template + absl::Status PlanSelectCall(absl::string_view fn, Exprs... args) { + auto ast = std::make_unique(NextExpr(), SourceInfo()); + auto& call = ast->mutable_root_expr().mutable_call_expr(); + call.set_function(fn); + call.add_args().mutable_ident_expr().set_name("ident"); + (call.mutable_args().push_back(std::move(args)), ...); + FlatExprBuilder builder(env_, runtime_options_); + builder.AddProgramOptimizer(CreateSelectOptimizationProgramOptimizer()); + return builder.CreateExpressionImpl(std::move(ast), nullptr).status(); + } + + // Helper for testing malformed / type mismatched optimized select chains. + absl::StatusOr EvalTypedSelectCall(absl::string_view fn, + Expr select_list, + const NestedTestAllTypes& msg) { + auto ast = std::make_unique(NextExpr(), SourceInfo()); + CallExpr& call = ast->mutable_root_expr().mutable_call_expr(); + call.set_function(fn); + Expr ident = NextExpr(); + int64_t ident_id = ident.id(); + ident.mutable_ident_expr().set_name("ident"); + call.mutable_args().push_back(std::move(ident)); + call.mutable_args().push_back(std::move(select_list)); + ast->mutable_type_map()[ident_id] = TypeSpec( + MessageTypeSpec("cel.expr.conformance.proto2.NestedTestAllTypes")); + + SelectOptimizationOptions select_options; + select_options.enable_field_descriptor_caching = true; + FlatExprBuilder builder(env_, runtime_options_); + builder.AddProgramOptimizer( + CreateSelectOptimizationProgramOptimizer(select_options)); + CEL_ASSIGN_OR_RETURN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + + cel::Activation act; + google::protobuf::Message* var = + runtime_internal::MakeTestingEnvDynamicProto(msg, &arena_); + act.InsertOrAssignValue("ident", cel::ParsedMessageValue(var, &arena_)); + google::api::expr::runtime::FlatExpressionEvaluatorState state = + plan.MakeEvaluatorState(env_->descriptor_pool.get(), + env_->MutableMessageFactory(), &arena_); + return plan.EvaluateWithCallback( + act, /*embedder_context=*/nullptr, + google::api::expr::runtime::EvaluationListener(), state); + } + protected: int64_t next_id_; }; @@ -1919,37 +3464,316 @@ TEST_F(SelectOptimizationUnexpectedAstTest, StatusIs(absl::StatusCode::kInvalidArgument)); } -TEST_F(SelectOptimizationUnexpectedAstTest, OptionalNotYetSupported) { - std::unique_ptr ast = std::make_unique(NextExpr(), SourceInfo()); +TEST_F(SelectOptimizationUnexpectedAstTest, TypedTupleAttributeAccepted) { + using FieldDescriptor = google::protobuf::FieldDescriptor; + // [field_num, field_name, type_code] hops, a map leaf + // [field_num, field_name, -1, [key_type, value_type]], and a 3rd argument + // that only binds the result type. + EXPECT_THAT( + PlanSelectCall( + kCelAttribute, + NextList(NextList(NextInt(1), NextString("field"), + NextInt(FieldDescriptor::TYPE_MESSAGE)), + NextList(NextInt(2), NextString("map_field"), NextInt(-1), + NextList(NextInt(FieldDescriptor::TYPE_STRING), + NextInt(FieldDescriptor::TYPE_INT32)))), + NextInt(0)), + IsOk()); +} - ast->mutable_root_expr().mutable_call_expr().set_function(kCelAttribute); - auto& call_args = ast->mutable_root_expr().mutable_call_expr().mutable_args(); - call_args.emplace_back(NextExpr()).mutable_ident_expr().set_name("ident"); +TEST_F(SelectOptimizationUnexpectedAstTest, TooManyArguments) { + EXPECT_THAT( + PlanSelectCall(kCelHasField, + NextList(NextList(NextInt(1), NextString("field"))), + NextInt(0)), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid cel.attribute call"))); + EXPECT_THAT( + PlanSelectCall(kCelAttribute, + NextList(NextList(NextInt(1), NextString("field"))), + NextInt(0), NextInt(0)), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid cel.attribute call"))); +} - auto& list_expr = call_args.emplace_back(NextExpr()).mutable_list_expr(); - auto& fields = list_expr.mutable_elements() - .emplace_back(NextListExprElement()) - .mutable_expr() - .mutable_list_expr() - .mutable_elements(); +TEST_F(SelectOptimizationUnexpectedAstTest, + MalformedSelectPathTooManyElements) { + EXPECT_THAT( + PlanSelectCall(kCelAttribute, + NextList(NextList(NextInt(1), NextString("field"), + NextInt(5), NextInt(0), NextInt(0)))), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid cel.attribute select list"))); +} - fields.emplace_back(NextListExprElement()) - .mutable_expr() - .mutable_const_expr() - .set_int64_value(1); - fields.emplace_back(NextListExprElement()) - .mutable_expr() - .mutable_const_expr() - .set_string_value("field"); +TEST_F(SelectOptimizationUnexpectedAstTest, CachedFieldDescriptorMapKeyErrors) { + NestedTestAllTypes msg; + + // map (field 63) with int key + { + ASSERT_OK_AND_ASSIGN( + Value result, + EvalTypedSelectCall( + kCelAttribute, + NextList(NextList(NextInt(2), NextString("payload")), + NextList(NextInt(63), NextString("map_bool_bool")), + NextInt(1)), + msg)); + ASSERT_TRUE(result->Is()) << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid map key type"))); + } - call_args.emplace_back(NextExpr()).mutable_const_expr().set_int64_value(0); + // map (field 77) with string key + { + ASSERT_OK_AND_ASSIGN( + Value result, + EvalTypedSelectCall( + kCelAttribute, + NextList(NextList(NextInt(2), NextString("payload")), + NextList(NextInt(77), NextString("map_int32_int32")), + NextString("bad_key")), + msg)); + ASSERT_TRUE(result->Is()) << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid map key type"))); + } - FlatExprBuilder builder(env_, runtime_options_); + // map (field 89) with bool key + { + ASSERT_OK_AND_ASSIGN( + Value result, + EvalTypedSelectCall( + kCelAttribute, + NextList(NextList(NextInt(2), NextString("payload")), + NextList(NextInt(89), NextString("map_int64_int64")), + NextBool(true)), + msg)); + ASSERT_TRUE(result->Is()) << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid map key type"))); + } - builder.AddProgramOptimizer(CreateSelectOptimizationProgramOptimizer()); + // map (field 201) with int key + { + ASSERT_OK_AND_ASSIGN( + Value result, + EvalTypedSelectCall( + kCelAttribute, + NextList(NextList(NextInt(2), NextString("payload")), + NextList(NextInt(201), NextString("map_uint32_uint32")), + NextInt(1)), + msg)); + ASSERT_TRUE(result->Is()) << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid map key type"))); + } - EXPECT_THAT(builder.CreateExpressionImpl(std::move(ast), nullptr), - StatusIs(absl::StatusCode::kUnimplemented)); + // map (field 213) with int key + { + ASSERT_OK_AND_ASSIGN( + Value result, + EvalTypedSelectCall( + kCelAttribute, + NextList(NextList(NextInt(2), NextString("payload")), + NextList(NextInt(213), NextString("map_uint64_uint64")), + NextInt(1)), + msg)); + ASSERT_TRUE(result->Is()) << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid map key type"))); + } + + // map (field 61) with int key + { + ASSERT_OK_AND_ASSIGN( + Value result, + EvalTypedSelectCall( + kCelAttribute, + NextList(NextList(NextInt(2), NextString("payload")), + NextList(NextInt(61), NextString("map_string_string")), + NextInt(1)), + msg)); + ASSERT_TRUE(result->Is()) << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid map key type"))); + } +} + +TEST_F(SelectOptimizationUnexpectedAstTest, + CachedFieldDescriptorMapPresenceErrors) { + NestedTestAllTypes msg; + + // has(ident.payload.map_int64_int64[1]) -> non-string qualifier for has() + { + ASSERT_OK_AND_ASSIGN( + Value result, + EvalTypedSelectCall( + kCelHasField, + NextList(NextList(NextInt(2), NextString("payload")), + NextList(NextInt(89), NextString("map_int64_int64")), + NextInt(1)), + msg)); + ASSERT_TRUE(result->Is()) << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kUnknown, + HasSubstr("No matching overloads found : has"))); + } + + // has(ident.payload.map_int64_int64.key) -> string qualifier on int64 map + { + ASSERT_OK_AND_ASSIGN( + Value result, + EvalTypedSelectCall( + kCelHasField, + NextList(NextList(NextInt(2), NextString("payload")), + NextList(NextInt(89), NextString("map_int64_int64")), + NextString("key")), + msg)); + ASSERT_TRUE(result->Is()) << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid map key type"))); + } +} + +TEST_F(SelectOptimizationUnexpectedAstTest, CachedFieldDescriptorListErrors) { + NestedTestAllTypes msg; + + // has(ident.payload.repeated_int32[0]) -> presence test on list element + { + ASSERT_OK_AND_ASSIGN( + Value result, + EvalTypedSelectCall( + kCelHasField, + NextList(NextList(NextInt(2), NextString("payload")), + NextList(NextInt(31), NextString("repeated_int32")), + NextInt(0)), + msg)); + ASSERT_TRUE(result->Is()) << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kUnknown, + HasSubstr("No matching overloads found : has"))); + } + + // ident.payload.repeated_int32['bad_index'] -> non-int64 list index + { + ASSERT_OK_AND_ASSIGN( + Value result, + EvalTypedSelectCall( + kCelAttribute, + NextList(NextList(NextInt(2), NextString("payload")), + NextList(NextInt(31), NextString("repeated_int32")), + NextString("bad_index")), + msg)); + ASSERT_TRUE(result->Is()) << result->DebugString(); + EXPECT_THAT(result.GetError().NativeValue(), + StatusIs(absl::StatusCode::kUnknown, + HasSubstr("No matching overloads found : _[_]"))); + } +} + +// Tests for overlay pool with cross-pool extensions. +// We specifically recommend against doing this, but some clients do it. +// Behavior is not totally consistent with normal select implementation so +// this test is just confirming we don't crash or return garbage. +TEST_F(SelectOptimizationTest, + CachedFieldDescriptorCrossPoolExtensionFallback) { + auto overlay_pool = std::make_shared( + google::protobuf::DescriptorPool::generated_pool()); + ASSERT_NE( + overlay_pool->BuildFile(test::GetTestExternalExtensionsFileDescriptor()), + nullptr); + + CompilerOptions compiler_options; + compiler_options.parser_options.enable_quoted_identifiers = true; + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler_builder, + cel::NewCompilerBuilder(overlay_pool, compiler_options)); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + const google::protobuf::Descriptor* desc = overlay_pool->FindMessageTypeByName( + "cel.expr.conformance.proto2.NestedTestAllTypes"); + ASSERT_NE(desc, nullptr); + ASSERT_THAT(compiler_builder->GetCheckerBuilder().AddVariable( + MakeVariableDecl("ident", MessageType(desc))), + IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + compiler_builder->Build()); + ASSERT_OK_AND_ASSIGN( + cel::ValidationResult validation, + compiler->Compile( + "ident.payload.`cel.cpp.testutil.test_external_extensions_message`." + "int64_field")); + ASSERT_TRUE(validation.IsValid()) << validation.FormatError(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, validation.ReleaseAst()); + + ASSERT_OK_AND_ASSIGN( + cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder(overlay_pool, runtime_options_)); + SelectOptimizationOptions select_options; + select_options.enable_field_descriptor_caching = true; + ASSERT_THAT(EnableSelectOptimization(runtime_builder, select_options), + IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + const google::protobuf::FieldDescriptor* payload_field = + desc->FindFieldByName("payload"); + ASSERT_NE(payload_field, nullptr); + const google::protobuf::FieldDescriptor* ext_field = overlay_pool->FindExtensionByName( + "cel.cpp.testutil.test_external_extensions_message"); + ASSERT_NE(ext_field, nullptr); + const google::protobuf::FieldDescriptor* int64_field = + ext_field->message_type()->FindFieldByName("int64_field"); + ASSERT_NE(int64_field, nullptr); + + google::protobuf::DynamicMessageFactory overlay_factory(overlay_pool.get()); + google::protobuf::Message* msg = overlay_factory.GetPrototype(desc)->New(&arena_); + google::protobuf::Message* payload = msg->GetReflection()->MutableMessage( + msg, payload_field, &overlay_factory); + google::protobuf::Message* ext_msg = payload->GetReflection()->MutableMessage( + payload, ext_field, &overlay_factory); + ext_msg->GetReflection()->SetInt64(ext_msg, int64_field, 99); + + // This works for the SelectOptimization, but not normal field access. Here, + // we are able to resolve the extension using the message reflection extension + // table (by number) without having to look up against the descriptor pool + // directly. + cel::Activation act; + act.InsertOrAssignValue("ident", cel::ParsedMessageValue(msg, &arena_)); + cel::EvaluateOptions eval_options; + eval_options.message_factory = &overlay_factory; + ASSERT_OK_AND_ASSIGN(Value result, + program->Evaluate(&arena_, act, eval_options)); + ASSERT_TRUE(result->Is()) << result->DebugString(); + EXPECT_EQ(result.GetInt().NativeValue(), 99); + + // Also test when the extension message is unset, where Reflection::GetMessage + // has to construct the default message instance. + google::protobuf::Message* unset_msg = overlay_factory.GetPrototype(desc)->New(&arena_); + act.InsertOrAssignValue("ident", cel::ParsedMessageValue(unset_msg, &arena_)); + ASSERT_OK_AND_ASSIGN(Value unset_result, + program->Evaluate(&arena_, act, eval_options)); + ASSERT_TRUE(unset_result->Is()) << unset_result->DebugString(); + EXPECT_EQ(unset_result.GetInt().NativeValue(), 0); + + // Also test when `payload` comes from a different MessageFactory (the + // generated factory) while `runtime` uses its default DynamicMessageFactory. + NestedTestAllTypes* empty_generated = + NestedTestAllTypes::default_instance().New(&arena_); + act.InsertOrAssignValue("ident", + cel::ParsedMessageValue(empty_generated, &arena_)); + ASSERT_OK_AND_ASSIGN(Value underlay_result, program->Evaluate(&arena_, act)); + EXPECT_TRUE(underlay_result->Is()) + << underlay_result->DebugString(); } } // namespace diff --git a/extensions/set_membership_optimization.cc b/extensions/set_membership_optimization.cc new file mode 100644 index 000000000..dbb36176d --- /dev/null +++ b/extensions/set_membership_optimization.cc @@ -0,0 +1,678 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "extensions/set_membership_optimization.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "absl/hash/hash.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "base/builtins.h" +#include "common/ast.h" +#include "common/constant.h" +#include "common/expr.h" +#include "common/internal/attribute_trail.h" +#include "common/native_type.h" +#include "common/standard_definitions.h" +#include "common/type.h" +#include "common/value.h" +#include "eval/compiler/flat_expr_builder.h" +#include "eval/compiler/flat_expr_builder_extensions.h" +#include "eval/eval/bitset_steps.h" +#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_base.h" +#include "internal/casts.h" +#include "internal/status_macros.h" +#include "runtime/internal/errors.h" +#include "runtime/internal/runtime_friend_access.h" +#include "runtime/internal/runtime_impl.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" + +namespace cel::extensions { +namespace { + +using ::google::api::expr::runtime::ExecutionFrame; +using ::google::api::expr::runtime::ExecutionFrameBase; +using ::google::api::expr::runtime::ExecutionPath; +using ::google::api::expr::runtime::ExecutionPathView; +using ::google::api::expr::runtime::ExpressionStep; +using ::google::api::expr::runtime::ExpressionStepBase; +using ::google::api::expr::runtime::ExtractBit; +using ::google::api::expr::runtime::FlatExprBuilder; +using ::google::api::expr::runtime::GetIfConstant; +using ::google::api::expr::runtime::PlannerContext; +using ::google::api::expr::runtime::ProgramOptimizer; +using ::google::api::expr::runtime::ProgramOptimizerFactory; + +// Repeated here to support @in without adding dependency on the sets +// extension. +constexpr absl::string_view kSetsContains = "sets.contains"; +constexpr absl::string_view kSetsIntersects = "sets.intersects"; +constexpr absl::string_view kSetsEquivalent = "sets.equivalent"; + +constexpr absl::string_view kSetsContainsOverloadId = "list_sets_contains_list"; +constexpr absl::string_view kSetsIntersectsOverloadId = + "list_sets_intersects_list"; +constexpr absl::string_view kSetsEquivalentOverloadId = + "list_sets_equivalent_list"; + +struct StringSetHash { + using is_transparent = void; + size_t operator()(absl::string_view v) const { return absl::HashOf(v); } + size_t operator()(const StringValue& v) const { return absl::HashOf(v); } +}; + +struct StringSetEq { + using is_transparent = void; + bool operator()(absl::string_view lhs, absl::string_view rhs) const { + return lhs == rhs; + } + bool operator()(absl::string_view lhs, const StringValue& rhs) const { + return lhs == rhs; + } + bool operator()(const StringValue& lhs, absl::string_view rhs) const { + return lhs == rhs; + } + bool operator()(const StringValue& lhs, const StringValue& rhs) const { + return lhs == rhs; + } +}; + +using StringSet = absl::flat_hash_set; + +enum class StringSetListOp { + kIntersects, + kContains, + kEquivalent, +}; + +absl::string_view StringSetListOpName(StringSetListOp op) { + switch (op) { + case StringSetListOp::kIntersects: + return kSetsIntersects; + case StringSetListOp::kContains: + return kSetsContains; + case StringSetListOp::kEquivalent: + return kSetsEquivalent; + } + return ""; +} + +class StringSetInStep final : public ExpressionStepBase { + public: + StringSetInStep(StringSet set, int64_t expr_id) + : ExpressionStepBase(expr_id), set_(std::move(set)) {} + + void Evaluate(ExecutionFrame* frame) const override { + if (!frame->value_stack().HasEnough(1)) { + frame->Abort(absl::InternalError("StringSetInStep: stack underflow")); + return; + } + const Value& arg = frame->value_stack().Peek(); + if (arg.IsError()) { + return; + } + if (frame->unknown_processing_enabled()) { + if (auto unknown = + cel::common_internal::PartiallyIdentifyAndMergeUnknowns( + frame->value_stack().GetSpan(1), + frame->value_stack().GetAttributeSpan(1), + frame->unknown_tree()); + unknown.has_value()) { + frame->value_stack().PopAndPush(1, *unknown); + return; + } + } + bool found = arg.IsString() && set_.contains(arg.GetString()); + frame->value_stack().PopAndPush(BoolValue(found)); + } + + private: + StringSet set_; +}; + +class StringSetListStep final : public ExpressionStepBase { + public: + StringSetListStep(StringSetListOp op, StringSet set, int64_t expr_id) + : ExpressionStepBase(expr_id), op_(op), set_(std::move(set)) {} + + void Evaluate(ExecutionFrame* frame) const override { + if (!frame->value_stack().HasEnough(1)) { + frame->Abort(absl::InternalError("StringSetListStep: stack underflow")); + return; + } + const Value& arg = frame->value_stack().Peek(); + if (arg.IsError()) { + return; + } + if (frame->unknown_processing_enabled()) { + if (auto unknown = + cel::common_internal::PartiallyIdentifyAndMergeUnknowns( + frame->value_stack().GetSpan(1), + frame->value_stack().GetAttributeSpan(1), + frame->unknown_tree()); + unknown.has_value()) { + frame->value_stack().PopAndPush(1, *unknown); + return; + } + } + + if (!arg.IsList()) { + frame->value_stack().PopAndPush(cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError( + StringSetListOpName(op_)), + frame->arena())); + return; + } + + const ListValue& list = arg.GetList(); + absl::StatusOr result = EvaluateList(*frame, list); + if (!result.ok()) { + frame->Abort(std::move(result).status()); + return; + } + frame->value_stack().PopAndPush(BoolValue(*result)); + } + + private: + absl::StatusOr EvaluateList(ExecutionFrameBase& frame, + const ListValue& list) const { + switch (op_) { + case StringSetListOp::kIntersects: { + bool intersects = false; + CEL_RETURN_IF_ERROR(list.ForEach( + [&](size_t /*index*/, + const Value& element) -> absl::StatusOr { + if (element.IsString() && set_.contains(element.GetString())) { + intersects = true; + return false; + } + return true; + }, + frame.descriptor_pool(), frame.message_factory(), frame.arena())); + return intersects; + } + case StringSetListOp::kContains: { + CEL_ASSIGN_OR_RETURN(size_t list_size, list.Size()); + if (list_size < set_.size()) { + return false; + } + absl::flat_hash_set matched; + CEL_RETURN_IF_ERROR(list.ForEach( + [&](size_t index, const Value& element) -> absl::StatusOr { + if (element.IsString()) { + auto it = set_.find(element.GetString()); + if (it != set_.end()) { + matched.insert(*it); + if (matched.size() == set_.size()) { + return false; + } + } + } + size_t remaining = index < list_size ? list_size - 1 - index : 0; + if (remaining < set_.size() - matched.size()) { + return false; + } + return true; + }, + frame.descriptor_pool(), frame.message_factory(), frame.arena())); + return matched.size() == set_.size(); + } + case StringSetListOp::kEquivalent: { + CEL_ASSIGN_OR_RETURN(size_t list_size, list.Size()); + if (list_size < set_.size()) { + return false; + } + bool all_in_set = true; + absl::flat_hash_set matched; + CEL_RETURN_IF_ERROR(list.ForEach( + [&](size_t /*index*/, + const Value& element) -> absl::StatusOr { + if (!element.IsString()) { + all_in_set = false; + return false; + } + auto it = set_.find(element.GetString()); + if (it == set_.end()) { + all_in_set = false; + return false; + } + matched.insert(*it); + return true; + }, + frame.descriptor_pool(), frame.message_factory(), frame.arena())); + return all_in_set && matched.size() == set_.size(); + } + } + return false; + } + + StringSetListOp op_; + StringSet set_; +}; + +std::optional TryExtractBitmaskFromListExpr( + const ListExpr& list_expr) { + uint64_t bitmask = 0; + + for (const ListExprElement& element : list_expr.elements()) { + if (element.optional() || !element.has_expr() || + !element.expr().has_const_expr()) { + return std::nullopt; + } + const Constant& constant = element.expr().const_expr(); + ConstantKindCase kind = constant.kind_case(); + if (kind != ConstantKindCase::kInt && kind != ConstantKindCase::kUint) { + return std::nullopt; + } + + if (kind == ConstantKindCase::kInt) { + int64_t v = constant.int_value(); + if (v < 0 || v >= 64) { + return std::nullopt; + } + bitmask |= (uint64_t{1} << v); + } else { + uint64_t v = constant.uint_value(); + if (v >= 64) { + return std::nullopt; + } + bitmask |= (uint64_t{1} << v); + } + } + + return bitmask; +} + +std::optional TryExtractBitmaskFromFoldedList( + PlannerContext& context, const ListValue& list_value) { + uint64_t bitmask = 0; + bool valid = true; + + absl::Status status = list_value.ForEach( + [&](size_t /*index*/, const Value& element) -> absl::StatusOr { + uint64_t bit = ExtractBit(element); + if (bit == 0) { + valid = false; + return false; + } + bitmask |= bit; + return true; + }, + context.descriptor_pool(), context.MutableMessageFactory(), + context.MutableArena()); + + if (!status.ok() || !valid) { + return std::nullopt; + } + return bitmask; +} + +std::optional TryExtractBitmask(PlannerContext& context, + const Expr& list_arg) { + ExecutionPathView subplan = context.GetSubplan(list_arg); + if (subplan.empty()) { + return std::nullopt; + } + if (subplan.size() == 1) { + Value constant_value; + if (GetIfConstant(subplan[0], constant_value) && constant_value.IsList()) { + return TryExtractBitmaskFromFoldedList(context, constant_value.GetList()); + } + // Otherwise, fall through. This may happen for empty lists. + } + + if (list_arg.has_list_expr()) { + return TryExtractBitmaskFromListExpr(list_arg.list_expr()); + } + + return std::nullopt; +} + +bool IsStringListType(const PlannerContext& context, int64_t expr_id) { + std::optional type = context.GetResolvedType(expr_id); + if (!type.has_value() || !type->IsList()) { + return false; + } + return type->GetList().GetElement().IsString(); +} + +std::optional TryExtractStringSetFromListExpr( + const ListExpr& list_expr, size_t min_set_size) { + if (list_expr.elements().empty() || + list_expr.elements().size() < min_set_size) { + return std::nullopt; + } + StringSet set; + set.reserve(list_expr.elements().size()); + for (const ListExprElement& element : list_expr.elements()) { + if (element.optional() || !element.has_expr() || + !element.expr().has_const_expr()) { + return std::nullopt; + } + const Constant& constant = element.expr().const_expr(); + if (constant.kind_case() != ConstantKindCase::kString) { + return std::nullopt; + } + set.insert(constant.string_value()); + } + return set; +} + +std::optional TryExtractStringSetFromFoldedList( + PlannerContext& context, const ListValue& list_value, size_t min_set_size) { + absl::StatusOr size = list_value.Size(); + if (!size.ok() || *size == 0 || *size < min_set_size) { + return std::nullopt; + } + StringSet set; + set.reserve(*size); + bool valid = true; + + absl::Status status = list_value.ForEach( + [&](size_t /*index*/, const Value& element) -> absl::StatusOr { + if (!element.IsString()) { + valid = false; + return false; + } + set.insert(element.GetString().ToString()); + return true; + }, + context.descriptor_pool(), context.MutableMessageFactory(), + context.MutableArena()); + + if (!status.ok() || !valid) { + return std::nullopt; + } + return set; +} + +std::optional TryExtractStringSet(PlannerContext& context, + const Expr& list_arg, + size_t min_set_size) { + if (!IsStringListType(context, list_arg.id())) { + return std::nullopt; + } + ExecutionPathView subplan = context.GetSubplan(list_arg); + if (subplan.empty()) { + return std::nullopt; + } + if (subplan.size() == 1) { + Value constant_value; + if (!GetIfConstant(subplan[0], constant_value) || + !constant_value.IsList()) { + return std::nullopt; + } + return TryExtractStringSetFromFoldedList(context, constant_value.GetList(), + min_set_size); + } + + if (list_arg.has_list_expr()) { + return TryExtractStringSetFromListExpr(list_arg.list_expr(), min_set_size); + } + + return std::nullopt; +} + +bool IsInFunction(absl::string_view fn) { + return fn == cel::builtin::kIn || fn == cel::builtin::kInDeprecated || + fn == cel::builtin::kInFunction; +} + +bool IsTargetFunction(absl::string_view fn) { + return IsInFunction(fn) || fn == kSetsContains || fn == kSetsIntersects || + fn == kSetsEquivalent; +} + +bool HasExpectedOverloadOrUnchecked(const Ast& ast, int64_t node_id, + absl::string_view expected_overload_id) { + const Reference* reference = ast.GetReference(node_id); + if (reference == nullptr || reference->overload_id().empty()) { + return true; + } + return reference->overload_id().size() == 1 && + reference->overload_id()[0] == expected_overload_id; +} + +class SetMembershipOptimizer : public ProgramOptimizer { + public: + explicit SetMembershipOptimizer( + const Ast& ast, const SetMembershipOptimizationOptions& options) + : ast_(ast), options_(options) {} + + absl::Status OnPreVisit(PlannerContext& context, const Expr& node) override { + return absl::OkStatus(); + } + + absl::Status OnPostVisit(PlannerContext& context, const Expr& node) override { + if (!node.has_call_expr()) { + return absl::OkStatus(); + } + + const CallExpr& call = node.call_expr(); + if (call.has_target() || call.args().size() != 2) { + return absl::OkStatus(); + } + + absl::string_view fn = call.function(); + if (!IsTargetFunction(fn)) { + return absl::OkStatus(); + } + + const Expr& arg0 = call.args()[0]; + const Expr& arg1 = call.args()[1]; + if (!context.NodeHasInspectablePlan(arg0) || + !context.NodeHasInspectablePlan(arg1)) { + return absl::OkStatus(); + } + + if (IsInFunction(fn)) { + if (!HasExpectedOverloadOrUnchecked(ast_, node.id(), + StandardOverloadIds::kInList)) { + return absl::OkStatus(); + } + std::optional bitmask = TryExtractBitmask(context, arg1); + if (bitmask.has_value()) { + return ReplaceWithOptimizedStep( + context, node, arg0, + ExpressionStep::MakeBitIsSetStep(*bitmask, node.id())); + } + std::optional str_set = + TryExtractStringSet(context, arg1, options_.min_set_size); + if (str_set.has_value()) { + return ReplaceWithOptimizedStep( + context, node, arg0, + ExpressionStep::MakeGenericStep(std::make_unique( + std::move(*str_set), node.id()), + node.id(), /*stack_delta=*/0)); + } + return absl::OkStatus(); + } + + if (fn == kSetsContains) { + if (!HasExpectedOverloadOrUnchecked(ast_, node.id(), + kSetsContainsOverloadId)) { + return absl::OkStatus(); + } + std::optional bitmask = TryExtractBitmask(context, arg1); + if (bitmask.has_value()) { + return ReplaceWithOptimizedStep( + context, node, arg0, + ExpressionStep::MakeBitAndEqualStep(*bitmask, node.id())); + } + std::optional str_set = + TryExtractStringSet(context, arg1, options_.min_set_size); + if (str_set.has_value()) { + return ReplaceWithOptimizedStep( + context, node, arg0, + ExpressionStep::MakeGenericStep( + std::make_unique( + StringSetListOp::kContains, std::move(*str_set), node.id()), + node.id(), /*stack_delta=*/0)); + } + return absl::OkStatus(); + } + + if (fn == kSetsIntersects) { + if (!HasExpectedOverloadOrUnchecked(ast_, node.id(), + kSetsIntersectsOverloadId)) { + return absl::OkStatus(); + } + std::optional bitmask = TryExtractBitmask(context, arg1); + if (bitmask.has_value()) { + return ReplaceWithOptimizedStep( + context, node, arg0, + ExpressionStep::MakeBitAndIsNonZeroStep(*bitmask, node.id())); + } + bitmask = TryExtractBitmask(context, arg0); + if (bitmask.has_value()) { + return ReplaceWithOptimizedStep( + context, node, arg1, + ExpressionStep::MakeBitAndIsNonZeroStep(*bitmask, node.id())); + } + std::optional str_set = + TryExtractStringSet(context, arg1, options_.min_set_size); + if (str_set.has_value()) { + return ReplaceWithOptimizedStep( + context, node, arg0, + ExpressionStep::MakeGenericStep(std::make_unique( + StringSetListOp::kIntersects, + std::move(*str_set), node.id()), + node.id(), /*stack_delta=*/0)); + } + str_set = TryExtractStringSet(context, arg0, options_.min_set_size); + if (str_set.has_value()) { + return ReplaceWithOptimizedStep( + context, node, arg1, + ExpressionStep::MakeGenericStep(std::make_unique( + StringSetListOp::kIntersects, + std::move(*str_set), node.id()), + node.id(), /*stack_delta=*/0)); + } + return absl::OkStatus(); + } + + if (fn == kSetsEquivalent) { + if (!HasExpectedOverloadOrUnchecked(ast_, node.id(), + kSetsEquivalentOverloadId)) { + return absl::OkStatus(); + } + std::optional bitmask = TryExtractBitmask(context, arg1); + if (bitmask.has_value()) { + return ReplaceWithOptimizedStep( + context, node, arg0, + ExpressionStep::MakeBitEqualStep(*bitmask, node.id())); + } + bitmask = TryExtractBitmask(context, arg0); + if (bitmask.has_value()) { + return ReplaceWithOptimizedStep( + context, node, arg1, + ExpressionStep::MakeBitEqualStep(*bitmask, node.id())); + } + std::optional str_set = + TryExtractStringSet(context, arg1, options_.min_set_size); + if (str_set.has_value()) { + return ReplaceWithOptimizedStep( + context, node, arg0, + ExpressionStep::MakeGenericStep(std::make_unique( + StringSetListOp::kEquivalent, + std::move(*str_set), node.id()), + node.id(), /*stack_delta=*/0)); + } + str_set = TryExtractStringSet(context, arg0, options_.min_set_size); + if (str_set.has_value()) { + return ReplaceWithOptimizedStep( + context, node, arg1, + ExpressionStep::MakeGenericStep(std::make_unique( + StringSetListOp::kEquivalent, + std::move(*str_set), node.id()), + node.id(), /*stack_delta=*/0)); + } + return absl::OkStatus(); + } + + // We try to fail early to minimize overhead. This is effectively + // unreachable. + return absl::OkStatus(); + } + + private: + static absl::Status ReplaceWithOptimizedStep(PlannerContext& context, + const Expr& call_node, + const Expr& var_arg, + ExpressionStep step) { + CEL_ASSIGN_OR_RETURN(ExecutionPath path, context.ExtractSubplan(var_arg)); + path.push_back(std::move(step)); + return context.ReplaceSubplan(call_node, std::move(path)); + } + + // TODO(b/572083203): expose this in the planner context instead of capturing + // a reference. + const Ast& ast_; + SetMembershipOptimizationOptions options_; +}; + +FlatExprBuilder* GetFlatExprBuilder(RuntimeBuilder& builder) { + Runtime& runtime = + runtime_internal::RuntimeFriendAccess::GetMutableRuntime(builder); + if (runtime_internal::RuntimeFriendAccess::RuntimeTypeId(runtime) == + NativeTypeId::For()) { + runtime_internal::RuntimeImpl& runtime_impl = + cel::internal::down_cast(runtime); + return &runtime_impl.expr_builder(); + } + return nullptr; +} + +} // namespace + +ProgramOptimizerFactory CreateSetMembershipOptimizationProgramOptimizer( + const SetMembershipOptimizationOptions& options) { + return [options](PlannerContext& context, const Ast& ast) + -> absl::StatusOr> { + if (!context.options().enable_heterogeneous_equality) { + return nullptr; + } + context.set_resolve_types(); + return std::make_unique(ast, options); + }; +} + +absl::Status EnableSetMembershipOptimization( + cel::RuntimeBuilder& builder, + const SetMembershipOptimizationOptions& options) { + FlatExprBuilder* flat_expr_builder = GetFlatExprBuilder(builder); + ABSL_DCHECK(flat_expr_builder != nullptr); + if (flat_expr_builder == nullptr) { + // This should not be possible at present, but we do check the downcast + // in case of new implementations in the future. + return absl::InvalidArgumentError( + "SetMembershipOptimization requires default runtime implementation"); + } + flat_expr_builder->AddProgramOptimizer( + CreateSetMembershipOptimizationProgramOptimizer(options)); + return absl::OkStatus(); +} + +} // namespace cel::extensions diff --git a/extensions/set_membership_optimization.h b/extensions/set_membership_optimization.h new file mode 100644 index 000000000..acf4db5a0 --- /dev/null +++ b/extensions/set_membership_optimization.h @@ -0,0 +1,70 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_EXTENSIONS_SET_MEMBERSHIP_OPTIMIZATION_H_ +#define THIRD_PARTY_CEL_CPP_EXTENSIONS_SET_MEMBERSHIP_OPTIMIZATION_H_ + +#include + +#include "absl/status/status.h" +#include "eval/compiler/flat_expr_builder_extensions.h" +#include "runtime/runtime_builder.h" + +namespace cel::extensions { + +// Options for set membership optimization. +struct SetMembershipOptimizationOptions { + // Minimum size of a non-bitset constant list (e.g. list) to optimize + // using a hash set. Optimistically assumes the input list elements are + // unique. + size_t min_set_size = 4; +}; + +// Enable set membership optimization on the given RuntimeBuilder. +// +// This should only be called *once* on a given runtime builder. +// +// This optimization assumes that `enable_heterogeneous_equality` is enabled. +// +// When enabled, the runtime will attempt to optimize branches that appear +// to implement a set membership test against a constant list. For small +// integer lists, this is implemented via a precomputed bitset. list is +// emphasized to cover the common case of testing protobuf enums against a +// constant list. For constant list of size >= `options.min_set_size`, +// this is implemented via a precomputed hash set. +// +// Examples: +// foo in [EnumA, EnumB, EnumC] +// sets.contains(foos, [EnumA, EnumB, EnumC]) // foos has all +// sets.intersects(foos, [EnumA, EnumB, EnumC]) // foos has any +// sets.equivalent(foos, [EnumA, EnumB, EnumC]) +// +// Should be called after `EnableConstantFolding` if enabled. +absl::Status EnableSetMembershipOptimization( + cel::RuntimeBuilder& builder, + const SetMembershipOptimizationOptions& options = {}); + +// =============================================================== +// Implementation details -- CEL users should not depend on these. +// Exposed here for enabling on Legacy APIs. They expose internal details +// which are not guaranteed to be stable. +// =============================================================== + +google::api::expr::runtime::ProgramOptimizerFactory +CreateSetMembershipOptimizationProgramOptimizer( + const SetMembershipOptimizationOptions& options = {}); + +} // namespace cel::extensions + +#endif // THIRD_PARTY_CEL_CPP_EXTENSIONS_SET_MEMBERSHIP_OPTIMIZATION_H_ diff --git a/extensions/set_membership_optimization_test.cc b/extensions/set_membership_optimization_test.cc new file mode 100644 index 000000000..aff9f6dc5 --- /dev/null +++ b/extensions/set_membership_optimization_test.cc @@ -0,0 +1,810 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "extensions/set_membership_optimization.h" + +#include +#include +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "base/attribute.h" +#include "checker/type_checker_builder.h" +#include "checker/validation_result.h" +#include "common/ast.h" +#include "common/decl.h" +#include "common/standard_definitions.h" +#include "common/type.h" +#include "common/value.h" +#include "common/value_kind.h" +#include "common/value_testing.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" +#include "eval/compiler/constant_folding.h" +#include "eval/compiler/flat_expr_builder.h" +#include "eval/compiler/qualified_reference_resolver.h" +#include "eval/eval/evaluator_core.h" +#include "extensions/bindings_ext.h" +#include "extensions/sets_functions.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "runtime/activation.h" +#include "runtime/constant_folding.h" +#include "runtime/function_adapter.h" +#include "runtime/internal/runtime_env.h" +#include "runtime/internal/runtime_env_testing.h" +#include "runtime/reference_resolver.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_functions.h" +#include "runtime/standard_runtime_builder_factory.h" +#include "cel/expr/conformance/proto2/test_all_types.pb.h" +#include "google/protobuf/arena.h" + +namespace cel::extensions { +namespace { + +using ::absl_testing::IsOk; +using ::cel::runtime_internal::NewTestingRuntimeEnv; +using ::cel::test::BoolValueIs; +using ::cel::test::ValueKindIs; +using ::google::api::expr::runtime::ExpressionStepKind; +using ::google::api::expr::runtime::FlatExprBuilder; +using ::google::api::expr::runtime::FlatExpression; +using ::google::api::expr::runtime::NewReferenceResolverExtension; +using ::google::api::expr::runtime::ReferenceResolverOption; +using ::testing::ElementsAre; +using ::testing::Not; +using ::testing::SizeIs; + +MATCHER_P(StepKindIs, expected_kind, "") { return arg.kind() == expected_kind; } + +struct BitsetPlanTestCase { + absl::string_view expr; + ExpressionStepKind expected_kind; +}; + +absl::StatusOr> CreateTestCompiler() { + CEL_ASSIGN_OR_RETURN( + std::unique_ptr builder, + NewCompilerBuilder(cel::internal::GetTestingDescriptorPool())); + CEL_RETURN_IF_ERROR(builder->AddLibrary(StandardCompilerLibrary())); + CEL_RETURN_IF_ERROR(builder->AddLibrary(SetsCompilerLibrary())); + CEL_RETURN_IF_ERROR(builder->AddLibrary(BindingsCompilerLibrary())); + + TypeCheckerBuilder& checker = builder->GetCheckerBuilder(); + checker.set_container("cel.expr.conformance.proto2"); + CEL_RETURN_IF_ERROR(checker.AddContextDeclaration( + "cel.expr.conformance.proto2.TestAllTypes")); + CEL_RETURN_IF_ERROR(checker.AddVariable(MakeVariableDecl("x", IntType()))); + CEL_RETURN_IF_ERROR(checker.AddVariable(MakeVariableDecl("u", UintType()))); + CEL_RETURN_IF_ERROR(checker.AddVariable(MakeVariableDecl("d", DynType()))); + CEL_RETURN_IF_ERROR(checker.AddVariable(MakeVariableDecl("s", StringType()))); + CEL_RETURN_IF_ERROR(checker.AddVariable( + MakeVariableDecl("int_list", ListType(checker.arena(), IntType())))); + CEL_RETURN_IF_ERROR(checker.AddVariable( + MakeVariableDecl("uint_list", ListType(checker.arena(), UintType())))); + CEL_RETURN_IF_ERROR(checker.AddVariable( + MakeVariableDecl("dyn_list", ListType(checker.arena(), DynType())))); + CEL_RETURN_IF_ERROR(checker.AddVariable( + MakeVariableDecl("str_list", ListType(checker.arena(), StringType())))); + + return builder->Build(); +} + +absl::StatusOr> CompileTestExpr(absl::string_view expr) { + static const Compiler* const kCompiler = []() { + absl::StatusOr> compiler = CreateTestCompiler(); + ABSL_CHECK_OK(compiler.status()); + return compiler->release(); + }(); + + CEL_ASSIGN_OR_RETURN(ValidationResult result, kCompiler->Compile(expr)); + if (!result.IsValid()) { + return absl::InvalidArgumentError(result.FormatError()); + } + return result.ReleaseAst(); +} + +absl::StatusOr> BuildTestRuntime( + bool enable_constant_folding, UnknownProcessingOptions unknown_processing = + UnknownProcessingOptions::kDisabled) { + RuntimeOptions options; + options.container = "cel.expr.conformance.proto2"; + options.enable_heterogeneous_equality = true; + options.unknown_processing = unknown_processing; + + CEL_ASSIGN_OR_RETURN(RuntimeBuilder builder, + CreateStandardRuntimeBuilder( + cel::internal::GetTestingDescriptorPool(), options)); + CEL_RETURN_IF_ERROR( + RegisterSetsFunctions(builder.function_registry(), options)); + CEL_RETURN_IF_ERROR( + EnableReferenceResolver(builder, ReferenceResolverEnabled::kAlways)); + if (enable_constant_folding) { + CEL_RETURN_IF_ERROR(EnableConstantFolding(builder)); + } + CEL_RETURN_IF_ERROR(EnableSetMembershipOptimization(builder)); + return std::move(builder).Build(); +} + +Value MakeIntList(google::protobuf::Arena* arena, const std::vector& values) { + ListValueBuilderPtr builder = NewListValueBuilder(arena); + for (int64_t v : values) { + ABSL_CHECK_OK(builder->Add(IntValue(v))); + } + return std::move(*builder).Build(); +} + +Value MakeUintList(google::protobuf::Arena* arena, const std::vector& values) { + ListValueBuilderPtr builder = NewListValueBuilder(arena); + for (uint64_t v : values) { + ABSL_CHECK_OK(builder->Add(UintValue(v))); + } + return std::move(*builder).Build(); +} + +Value MakeValueList(google::protobuf::Arena* arena, const std::vector& values) { + ListValueBuilderPtr builder = NewListValueBuilder(arena); + for (const Value& v : values) { + ABSL_CHECK_OK(builder->Add(v)); + } + return std::move(*builder).Build(); +} + +TEST(SetMembershipOptimizationPlanTest, EmitsCompactBitsetPlan) { + RuntimeOptions options; + options.container = "cel.expr.conformance.proto2"; + options.enable_heterogeneous_equality = true; + std::shared_ptr env = + NewTestingRuntimeEnv(); + ASSERT_THAT(RegisterSetsFunctions(env->function_registry, options), IsOk()); + FlatExprBuilder builder(std::move(env), options); + builder.set_container(options.container); + builder.AddAstTransform( + NewReferenceResolverExtension(ReferenceResolverOption::kAlways)); + builder.AddProgramOptimizer( + CreateSetMembershipOptimizationProgramOptimizer()); + + const BitsetPlanTestCase test_cases[] = { + {"x in [1, 2, 3, 4]", ExpressionStepKind::kBitIsSet}, + {"d in []", ExpressionStepKind::kBitIsSet}, + {"sets.contains(int_list, [1, 2, 3, 4])", + ExpressionStepKind::kBitAndEqual}, + {"sets.contains(str_list, [])", ExpressionStepKind::kBitAndEqual}, + {"sets.intersects(int_list, [1, 2, 3, 4])", + ExpressionStepKind::kBitAndIsNonZero}, + {"sets.intersects([1, 2, 3, 4], int_list)", + ExpressionStepKind::kBitAndIsNonZero}, + {"sets.intersects(str_list, [])", ExpressionStepKind::kBitAndIsNonZero}, + {"sets.equivalent(int_list, [1, 2, 3, 4])", + ExpressionStepKind::kBitEqual}, + {"sets.equivalent([1, 2, 3, 4], int_list)", + ExpressionStepKind::kBitEqual}, + {"sets.equivalent(str_list, [])", ExpressionStepKind::kBitEqual}, + }; + for (const BitsetPlanTestCase& tc : test_cases) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, CompileTestExpr(tc.expr)); + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + // Optimized plan consists of just 2 steps: reading the variable and + // evaluating the bitset step. + EXPECT_THAT(plan.path(), + ElementsAre(StepKindIs(ExpressionStepKind::kIdentifier), + StepKindIs(tc.expected_kind))) + << tc.expr; + } +} + +TEST(SetMembershipOptimizationPlanTest, + EmitsCompactBitsetPlanWithConstantFoldedList) { + RuntimeOptions options; + options.container = "cel.expr.conformance.proto2"; + options.enable_heterogeneous_equality = true; + std::shared_ptr env = + NewTestingRuntimeEnv(); + ASSERT_THAT(RegisterStandardFunctions(env->function_registry, options), + IsOk()); + ASSERT_THAT(RegisterSetsFunctions(env->function_registry, options), IsOk()); + FlatExprBuilder builder(std::move(env), options); + builder.set_container(options.container); + builder.AddAstTransform( + NewReferenceResolverExtension(ReferenceResolverOption::kAlways)); + builder.AddProgramOptimizer( + cel::runtime_internal::CreateConstantFoldingOptimizer()); + builder.AddProgramOptimizer( + CreateSetMembershipOptimizationProgramOptimizer()); + + const BitsetPlanTestCase test_cases[] = { + {"x in ([1, 2] + [3, 4])", ExpressionStepKind::kBitIsSet}, + {"sets.contains(int_list, [1, 2] + [3, 4])", + ExpressionStepKind::kBitAndEqual}, + {"sets.intersects(int_list, [1, 2] + [3, 4])", + ExpressionStepKind::kBitAndIsNonZero}, + {"sets.equivalent([1, 2] + [3, 4], int_list)", + ExpressionStepKind::kBitEqual}, + }; + for (const BitsetPlanTestCase& tc : test_cases) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, CompileTestExpr(tc.expr)); + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + // Constant folding folds `[1, 2] + [3, 4]` into a single constant ListValue + // step, which SetMembershipOptimization then replaces with a 2-step plan. + EXPECT_THAT(plan.path(), + ElementsAre(StepKindIs(ExpressionStepKind::kIdentifier), + StepKindIs(tc.expected_kind))) + << tc.expr; + } +} + +TEST(SetMembershipOptimizationPlanTest, EmitsCompactStringSetPlan) { + RuntimeOptions options; + options.container = "cel.expr.conformance.proto2"; + options.enable_heterogeneous_equality = true; + std::shared_ptr env = + NewTestingRuntimeEnv(); + ASSERT_THAT(RegisterStandardFunctions(env->function_registry, options), + IsOk()); + ASSERT_THAT(RegisterSetsFunctions(env->function_registry, options), IsOk()); + FlatExprBuilder builder(std::move(env), options); + builder.set_container(options.container); + builder.AddAstTransform( + NewReferenceResolverExtension(ReferenceResolverOption::kAlways)); + builder.AddProgramOptimizer( + CreateSetMembershipOptimizationProgramOptimizer()); + + for (absl::string_view expr_str : { + "s in ['a', 'b', 'c', 'd']", + "d in ['a', 'b', 'c', 'd']", + "sets.contains(str_list, ['a', 'b', 'c', 'd'])", + "sets.intersects(str_list, ['a', 'b', 'c', 'd'])", + "sets.intersects(['a', 'b', 'c', 'd'], str_list)", + "sets.equivalent(str_list, ['a', 'b', 'c', 'd'])", + "sets.equivalent(['a', 'b', 'c', 'd'], str_list)", + }) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, CompileTestExpr(expr_str)); + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + // Optimized plan consists of just 2 steps: reading the variable and + // evaluating the generic string set step. + EXPECT_THAT(plan.path(), + ElementsAre(StepKindIs(ExpressionStepKind::kIdentifier), + StepKindIs(ExpressionStepKind::kGenericLogic))) + << expr_str; + } + + // Constant string lists smaller than default min_set_size (4) are not + // optimized. + for (absl::string_view expr_str : { + "s in ['a', 'b', 'c']", + "sets.contains(str_list, ['a', 'b', 'c'])", + "sets.intersects(str_list, ['a', 'b', 'c'])", + "sets.equivalent(str_list, ['a', 'b', 'c'])", + }) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, CompileTestExpr(expr_str)); + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + EXPECT_THAT(plan.path(), Not(SizeIs(2))) << expr_str; + } + + // Constant list containing strings is not optimized as a string set. + for ( + absl::string_view expr_str : { + "s in [dyn('a'), dyn('b'), dyn('c'), dyn('d')]", + "sets.contains(str_list, [dyn('a'), dyn('b'), dyn('c'), dyn('d')])", + "sets.intersects(str_list, [dyn('a'), dyn('b'), dyn('c'), dyn('d')])", + "sets.equivalent(str_list, [dyn('a'), dyn('b'), dyn('c'), dyn('d')])", + }) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, CompileTestExpr(expr_str)); + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + EXPECT_THAT(plan.path(), Not(SizeIs(2))) << expr_str; + } +} + +TEST(SetMembershipOptimizationPlanTest, + RespectsCustomMinSetSizeForConstantStringSets) { + RuntimeOptions options; + options.container = "cel.expr.conformance.proto2"; + options.enable_heterogeneous_equality = true; + std::shared_ptr env = + NewTestingRuntimeEnv(); + ASSERT_THAT(RegisterSetsFunctions(env->function_registry, options), IsOk()); + FlatExprBuilder builder(std::move(env), options); + builder.set_container(options.container); + builder.AddAstTransform( + NewReferenceResolverExtension(ReferenceResolverOption::kAlways)); + builder.AddProgramOptimizer(CreateSetMembershipOptimizationProgramOptimizer( + SetMembershipOptimizationOptions{.min_set_size = 2})); + + for (absl::string_view expr_str : { + "s in ['a', 'b']", + "sets.contains(str_list, ['a', 'b'])", + "sets.intersects(str_list, ['a', 'b'])", + "sets.equivalent(str_list, ['a', 'b'])", + }) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, CompileTestExpr(expr_str)); + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + EXPECT_THAT(plan.path(), + ElementsAre(StepKindIs(ExpressionStepKind::kIdentifier), + StepKindIs(ExpressionStepKind::kGenericLogic))) + << expr_str; + } + + // Size 1 is still below min_set_size = 2. + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + CompileTestExpr("s in ['a']")); + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + EXPECT_THAT(plan.path(), Not(SizeIs(2))); + } +} + +TEST(SetMembershipOptimizationPlanTest, + EmitsCompactStringSetPlanWithConstantFoldedList) { + RuntimeOptions options; + options.container = "cel.expr.conformance.proto2"; + options.enable_heterogeneous_equality = true; + std::shared_ptr env = + NewTestingRuntimeEnv(); + ASSERT_THAT(RegisterStandardFunctions(env->function_registry, options), + IsOk()); + ASSERT_THAT(RegisterSetsFunctions(env->function_registry, options), IsOk()); + FlatExprBuilder builder(std::move(env), options); + builder.set_container(options.container); + builder.AddAstTransform( + NewReferenceResolverExtension(ReferenceResolverOption::kAlways)); + builder.AddProgramOptimizer( + cel::runtime_internal::CreateConstantFoldingOptimizer()); + builder.AddProgramOptimizer( + CreateSetMembershipOptimizationProgramOptimizer()); + + for (absl::string_view expr_str : { + "s in (['a', 'b'] + ['c', 'd'])", + "sets.contains(str_list, ['a', 'b'] + ['c', 'd'])", + "sets.intersects(str_list, ['a', 'b'] + ['c', 'd'])", + "sets.equivalent(['a', 'b'] + ['c', 'd'], str_list)", + }) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, CompileTestExpr(expr_str)); + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + EXPECT_THAT(plan.path(), + ElementsAre(StepKindIs(ExpressionStepKind::kIdentifier), + StepKindIs(ExpressionStepKind::kGenericLogic))) + << expr_str; + } +} + +TEST(SetMembershipOptimizationPlanTest, + EmitsCompactPlanWithInlinedBoundConstantSet) { + RuntimeOptions options; + options.container = "cel.expr.conformance.proto2"; + options.enable_heterogeneous_equality = true; + options.enable_subexpression_inlining = true; + std::shared_ptr env = + NewTestingRuntimeEnv(); + ASSERT_THAT(RegisterStandardFunctions(env->function_registry, options), + IsOk()); + ASSERT_THAT(RegisterSetsFunctions(env->function_registry, options), IsOk()); + FlatExprBuilder builder(std::move(env), options); + builder.set_container(options.container); + builder.AddAstTransform( + NewReferenceResolverExtension(ReferenceResolverOption::kAlways)); + builder.AddProgramOptimizer( + cel::runtime_internal::CreateConstantFoldingOptimizer()); + builder.AddProgramOptimizer( + CreateSetMembershipOptimizationProgramOptimizer()); + + for (absl::string_view expr_str : { + "cel.bind(set_var, [1, 2, 3, 4], x in set_var)", + "cel.bind(set_var, ['a', 'b', 'c', 'd'], s in set_var)", + }) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, CompileTestExpr(expr_str)); + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + ASSERT_FALSE(plan.subexpressions().empty()); + // Main expression has 3 steps: Identifier(x/s), Bitset/StringSet step, and + // ClearSlot. + EXPECT_THAT(plan.subexpressions()[0], SizeIs(3)) << expr_str; + } +} + +TEST(SetMembershipOptimizationPlanTest, EmitsCompactBitsetPlanForUncheckedAst) { + RuntimeOptions options; + options.container = "cel.expr.conformance.proto2"; + options.enable_heterogeneous_equality = true; + std::shared_ptr env = + NewTestingRuntimeEnv(); + ASSERT_THAT(RegisterSetsFunctions(env->function_registry, options), IsOk()); + FlatExprBuilder builder(std::move(env), options); + builder.set_container(options.container); + builder.AddAstTransform( + NewReferenceResolverExtension(ReferenceResolverOption::kAlways)); + builder.AddProgramOptimizer( + CreateSetMembershipOptimizationProgramOptimizer()); + + const BitsetPlanTestCase test_cases[] = { + {"x in [1, 2, 3, 4]", ExpressionStepKind::kBitIsSet}, + {"sets.contains(int_list, [1, 2, 3, 4])", + ExpressionStepKind::kBitAndEqual}, + {"sets.intersects(int_list, [1, 2, 3, 4])", + ExpressionStepKind::kBitAndIsNonZero}, + {"sets.equivalent(int_list, [1, 2, 3, 4])", + ExpressionStepKind::kBitEqual}, + }; + for (const BitsetPlanTestCase& tc : test_cases) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, CompileTestExpr(tc.expr)); + ast->mutable_reference_map().clear(); + ast->mutable_type_map().clear(); + ast->set_is_checked(false); + ASSERT_OK_AND_ASSIGN(FlatExpression plan, + builder.CreateExpressionImpl(std::move(ast), nullptr)); + EXPECT_THAT(plan.path(), + ElementsAre(StepKindIs(ExpressionStepKind::kIdentifier), + StepKindIs(tc.expected_kind))) + << tc.expr; + } +} + +TEST(SetMembershipOptimizationPlanTest, DoesNotOptimizeCustomOverload) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr compiler_builder, + NewCompilerBuilder(cel::internal::GetTestingDescriptorPool())); + ASSERT_THAT(compiler_builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(compiler_builder->AddLibrary(SetsCompilerLibrary()), IsOk()); + + TypeCheckerBuilder& checker = compiler_builder->GetCheckerBuilder(); + ASSERT_THAT(checker.AddVariable(MakeVariableDecl("x", IntType())), IsOk()); + ASSERT_THAT(checker.AddVariable(MakeVariableDecl("s", StringType())), IsOk()); + ListType int_list_type(checker.arena(), IntType()); + + ASSERT_OK_AND_ASSIGN( + FunctionDecl custom_in, + MakeFunctionDecl(std::string(StandardFunctions::kIn), + MakeOverloadDecl("string_in_int_list", BoolType(), + StringType(), int_list_type))); + ASSERT_THAT(checker.MergeFunction(custom_in), IsOk()); + + ASSERT_OK_AND_ASSIGN( + FunctionDecl custom_contains, + MakeFunctionDecl("sets.contains", + MakeOverloadDecl("int_sets_contains_list", BoolType(), + IntType(), int_list_type))); + ASSERT_THAT(checker.MergeFunction(custom_contains), IsOk()); + + ASSERT_OK_AND_ASSIGN( + FunctionDecl custom_intersects, + MakeFunctionDecl("sets.intersects", + MakeOverloadDecl("int_sets_intersects_list", BoolType(), + IntType(), int_list_type))); + ASSERT_THAT(checker.MergeFunction(custom_intersects), IsOk()); + + ASSERT_OK_AND_ASSIGN( + FunctionDecl custom_equivalent, + MakeFunctionDecl("sets.equivalent", + MakeOverloadDecl("int_sets_equivalent_list", BoolType(), + IntType(), int_list_type))); + ASSERT_THAT(checker.MergeFunction(custom_equivalent), IsOk()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + compiler_builder->Build()); + + RuntimeOptions options; + options.enable_heterogeneous_equality = true; + std::shared_ptr env = + NewTestingRuntimeEnv(); + ASSERT_THAT(RegisterSetsFunctions(env->function_registry, options), IsOk()); + for (absl::string_view fn_name : + {"sets.contains", "sets.intersects", "sets.equivalent"}) { + ASSERT_THAT(env->function_registry.Register( + BinaryFunctionAdapter:: + CreateDescriptor(fn_name, /*receiver_style=*/false), + BinaryFunctionAdapter:: + WrapFunction([](int64_t x, const ListValue&) { + return x == 42; + })), + IsOk()); + } + FlatExprBuilder plan_builder(std::move(env), options); + plan_builder.AddAstTransform( + NewReferenceResolverExtension(ReferenceResolverOption::kAlways)); + plan_builder.AddProgramOptimizer( + CreateSetMembershipOptimizationProgramOptimizer()); + + for (absl::string_view expr_str : { + "s in [1, 2, 3, 4]", + "sets.contains(x, [1, 2, 3, 4])", + "sets.intersects(x, [1, 2, 3, 4])", + "sets.equivalent(x, [1, 2, 3, 4])", + }) { + ASSERT_OK_AND_ASSIGN(ValidationResult validation, + compiler->Compile(expr_str)); + ASSERT_TRUE(validation.IsValid()) << validation.FormatError(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, validation.ReleaseAst()); + ASSERT_OK_AND_ASSIGN(FlatExpression plan, plan_builder.CreateExpressionImpl( + std::move(ast), nullptr)); + EXPECT_THAT(plan.path(), Not(SizeIs(2))) << expr_str; + } + + ASSERT_OK_AND_ASSIGN(RuntimeBuilder runtime_builder, + CreateStandardRuntimeBuilder( + cel::internal::GetTestingDescriptorPool(), options)); + ASSERT_THAT( + RegisterSetsFunctions(runtime_builder.function_registry(), options), + IsOk()); + for (absl::string_view fn_name : + {"sets.contains", "sets.intersects", "sets.equivalent"}) { + ASSERT_THAT(runtime_builder.function_registry().Register( + BinaryFunctionAdapter:: + CreateDescriptor(fn_name, /*receiver_style=*/false), + BinaryFunctionAdapter:: + WrapFunction([](int64_t x, const ListValue&) { + return x == 42; + })), + IsOk()); + } + ASSERT_THAT(EnableReferenceResolver(runtime_builder, + ReferenceResolverEnabled::kAlways), + IsOk()); + ASSERT_THAT(EnableSetMembershipOptimization(runtime_builder), IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + + google::protobuf::Arena arena; + Activation activation; + activation.InsertOrAssignValue("x", IntValue(42)); + + for (absl::string_view expr_str : { + "sets.contains(x, [1, 2, 3, 4])", + "sets.intersects(x, [1, 2, 3, 4])", + "sets.equivalent(x, [1, 2, 3, 4])", + }) { + ASSERT_OK_AND_ASSIGN(ValidationResult validation, + compiler->Compile(expr_str)); + ASSERT_TRUE(validation.IsValid()) << validation.FormatError(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, validation.ReleaseAst()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); + EXPECT_THAT(result, BoolValueIs(true)) << expr_str; + } +} + +struct EvalTestCase { + std::string expr; + bool expected; +}; + +class SetMembershipOptimizationEvalTest + : public ::testing::TestWithParam {}; + +TEST_P(SetMembershipOptimizationEvalTest, EvaluatesCorrectly) { + const bool enable_constant_folding = GetParam(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + BuildTestRuntime(enable_constant_folding)); + + google::protobuf::Arena arena; + Activation activation; + activation.InsertOrAssignValue("x", IntValue(2)); + activation.InsertOrAssignValue("u", UintValue(63)); + activation.InsertOrAssignValue("d", DoubleValue(2.0)); + activation.InsertOrAssignValue("s", StringValue::From("b", &arena)); + activation.InsertOrAssignValue("int_list", MakeIntList(&arena, {0, 2, 63})); + activation.InsertOrAssignValue("uint_list", MakeUintList(&arena, {0, 2})); + activation.InsertOrAssignValue( + "dyn_list", MakeValueList(&arena, {IntValue(0), DoubleValue(2.0)})); + activation.InsertOrAssignValue( + "str_list", MakeValueList(&arena, {StringValue::From("a", &arena), + StringValue::From("b", &arena), + StringValue::From("c", &arena), + StringValue::From("d", &arena)})); + + const std::vector test_cases = { + // @in with int, uint, dyn(double), and protobuf enums + {"x in [0, 2, 63]", true}, + {"x in [0, 1, 63]", false}, + {"u in [0u, 2u, 63u]", true}, + {"u in [0u, 2u]", false}, + {"d in [0, 2, 63]", true}, + {"d in [0, 1, 63]", false}, + {"dyn(2.5) in [0, 2, 63]", false}, + {"-1 in [0, 2, 63]", false}, + {"64 in [0, 2, 63]", false}, + {"x in [TestAllTypes.NestedEnum.FOO, TestAllTypes.NestedEnum.BAR, " + "TestAllTypes.NestedEnum.BAZ]", + true}, + {"x in [TestAllTypes.NestedEnum.FOO, TestAllTypes.NestedEnum.BAR]", + false}, + + // sets.contains + {"sets.contains(int_list, [0, 2])", true}, + {"sets.contains(int_list, [0, 2, 63])", true}, + {"sets.contains(int_list, [0, 1, 2])", false}, + {"sets.contains(uint_list, [0u, 2u])", true}, + {"sets.contains(uint_list, [0u, 1u])", false}, + {"sets.contains(dyn_list, [0, 2])", true}, + {"sets.contains(int_list, [TestAllTypes.NestedEnum.FOO, " + "TestAllTypes.NestedEnum.BAZ])", + true}, + // Non-optimizable 1st arg constant list still evaluates via standard fn + {"sets.contains([0, 2, 63], int_list)", true}, + {"sets.contains([0, 2], int_list)", false}, + + // sets.intersects (both argument orders) + {"sets.intersects(int_list, [1, 2])", true}, + {"sets.intersects([1, 2], int_list)", true}, + {"sets.intersects(int_list, [1, 3])", false}, + {"sets.intersects([1, 3], int_list)", false}, + {"sets.intersects(uint_list, [2u, 5u])", true}, + {"sets.intersects([2u, 5u], uint_list)", true}, + {"sets.intersects(dyn_list, [2, 5])", true}, + + // sets.equivalent (both argument orders) + {"sets.equivalent(int_list, [0, 2, 63])", true}, + {"sets.equivalent([0, 2, 63], int_list)", true}, + {"sets.equivalent(int_list, [0, 2, 2, 63])", true}, + {"sets.equivalent(int_list, [0, 2])", false}, + {"sets.equivalent([0, 2], int_list)", false}, + {"sets.equivalent(int_list, [0, 1, 2, 63])", false}, + {"sets.equivalent(dyn_list, [0, 2])", true}, + {"sets.equivalent([0, 2], dyn_list)", true}, + {"sets.equivalent([0, 2, 64], [0, 2])", false}, + {"sets.equivalent([0, 2, -1], [0, 2])", false}, + {"sets.equivalent([dyn(0), dyn(2), dyn(1.5)], [0, 2])", false}, + + // Empty constant lists + {"x in []", false}, + {"u in []", false}, + {"d in []", false}, + {"'hello' in []", false}, + {"sets.contains(int_list, [])", true}, + {"sets.contains(str_list, [])", true}, + {"sets.intersects(int_list, [])", false}, + {"sets.intersects([], int_list)", false}, + {"sets.intersects(str_list, [])", false}, + {"sets.equivalent(int_list, [])", false}, + {"sets.equivalent([], int_list)", false}, + {"sets.equivalent(str_list, [])", false}, + {"sets.equivalent(int_list.filter(i, i < 0), [])", true}, + {"sets.equivalent(str_list.filter(s, s == 'none'), [])", true}, + + // Constant string sets (size >= 4) + {"s in ['a', 'b', 'c', 'd']", true}, + {"s in ['a', 'c', 'd', 'e']", false}, + {"d in ['a', 'b', 'c', 'd']", false}, + {"sets.contains(str_list, ['a', 'b', 'c', 'd'])", true}, + {"sets.contains(str_list, ['a', 'b', 'b', 'c'])", true}, + {"sets.contains(str_list, ['a', 'b', 'c', 'e'])", false}, + {"sets.contains(['a', 'b', 'c'], ['a', 'b', 'c', 'd'])", false}, + {"sets.contains([dyn('a'), dyn('b'), dyn('c'), dyn('d'), dyn(1)], " + "['a', 'b', 'c', 'd'])", + true}, + {"sets.intersects(str_list, ['b', 'e', 'f', 'g'])", true}, + {"sets.intersects(['b', 'e', 'f', 'g'], str_list)", true}, + {"sets.intersects(str_list, ['e', 'f', 'g', 'h'])", false}, + {"sets.intersects(['e', 'f', 'g', 'h'], str_list)", false}, + {"sets.intersects([dyn(1), dyn('d')], ['a', 'b', 'c', 'd'])", true}, + {"sets.equivalent(str_list, ['d', 'c', 'b', 'a'])", true}, + {"sets.equivalent(['d', 'c', 'b', 'a'], str_list)", true}, + {"sets.equivalent(str_list, ['a', 'b', 'b', 'c', 'd'])", true}, + {"sets.equivalent(['a', 'a', 'b', 'c', 'd'], " + "['a', 'b', 'c', 'd'])", + true}, + {"sets.equivalent(str_list, ['a', 'b', 'c', 'e'])", false}, + {"sets.equivalent(['a', 'b', 'c', 'e'], str_list)", false}, + {"sets.equivalent(['a', 'b', 'c'], ['a', 'b', 'c', 'd'])", false}, + {"sets.equivalent([dyn('a'), dyn('b'), dyn('c'), dyn('d'), dyn(1)], " + "['a', 'b', 'c', 'd'])", + false}, + + // Fallback when constant list has out-of-range values, mixed types, or + // small string lists (< 4 elements) + {"x in [-1, 2, 100]", true}, + {"100 in [-1, 2, 100]", true}, + {"-1 in [-1, 2, 100]", true}, + {"x in [1, 64]", false}, + {"d in [dyn(1), dyn(2u), dyn(3.5)]", true}, + {"'a' in ['a', 'b', 'c']", true}, + {"sets.contains(str_list, ['a', 'b'])", true}, + {"sets.intersects(str_list, ['b', 'e'])", true}, + {"sets.equivalent(['a', 'b'], ['b', 'a'])", true}, + + // Heterogeneous non-numeric dynamic elements with constant int sets + {"dyn('1') in [0, 1, 2]", false}, + {"sets.contains([dyn('1'), dyn(1)], [1])", true}, + {"sets.intersects([1], [dyn('1'), dyn(2)])", false}, + {"sets.equivalent([1], [dyn('1'), dyn(1)])", false}, + + // Constant-folded list expression when constant folding is enabled + {"x in ([0, 1] + [2, 3])", true}, + {"sets.contains(int_list, [0] + [2])", true}, + {"s in (['a', 'b'] + ['c', 'd'])", true}, + {"sets.contains(str_list, ['a', 'b'] + ['c', 'd'])", true}, + {"sets.intersects(str_list, ['e', 'f'] + ['g', 'd'])", true}, + {"sets.equivalent(str_list, ['d', 'c'] + ['b', 'a'])", true}, + }; + + for (const EvalTestCase& tc : test_cases) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, CompileTestExpr(tc.expr)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); + EXPECT_THAT(result, BoolValueIs(tc.expected)) << "Expression: " << tc.expr; + } +} + +INSTANTIATE_TEST_SUITE_P(SetMembershipOptimizationEvalTest, + SetMembershipOptimizationEvalTest, testing::Bool()); + +TEST(SetMembershipOptimizationUnknownAndErrorTest, + PropagatesUnknownsAndErrors) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr runtime, + BuildTestRuntime(/*enable_constant_folding=*/true, + UnknownProcessingOptions::kAttributeAndFunction)); + + google::protobuf::Arena arena; + Activation activation; + ASSERT_THAT( + activation.SetUnknownPatterns( + {AttributePattern("x", {}), AttributePattern("s", {}), + AttributePattern("int_list", {}), AttributePattern("str_list", {})}), + IsOk()); + + for (absl::string_view expr_str : { + "x in [1, 2, 3]", + "sets.contains(int_list, [1, 2])", + "sets.intersects(int_list, [1, 2])", + "sets.equivalent(int_list, [1, 2])", + "s in ['a', 'b', 'c', 'd']", + "sets.contains(str_list, ['a', 'b', 'c', 'd'])", + "sets.intersects(str_list, ['a', 'b', 'c', 'd'])", + "sets.equivalent(str_list, ['a', 'b', 'c', 'd'])", + }) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, CompileTestExpr(expr_str)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); + EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)) << expr_str; + } + + // Runtime error in operand propagates as ErrorValue. + for (absl::string_view expr_str : { + "(1 / 0) in [1, 2, 3]", + "dyn(1 / 0) in ['a', 'b', 'c', 'd']", + "sets.contains(dyn(1 / 0), ['a', 'b', 'c', 'd'])", + "sets.intersects(dyn(1 / 0), ['a', 'b', 'c', 'd'])", + "sets.equivalent(dyn(1 / 0), ['a', 'b', 'c', 'd'])", + }) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, CompileTestExpr(expr_str)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); + EXPECT_THAT(result, ValueKindIs(ValueKind::kError)) << expr_str; + } +} + +} // namespace +} // namespace cel::extensions diff --git a/extensions/sets_functions_benchmark_test.cc b/extensions/sets_functions_benchmark_test.cc index 0b51f1464..acea32381 100644 --- a/extensions/sets_functions_benchmark_test.cc +++ b/extensions/sets_functions_benchmark_test.cc @@ -17,52 +17,65 @@ #include #include -#include "cel/expr/syntax.pb.h" #include "absl/base/nullability.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/str_replace.h" +#include "checker/type_checker_builder.h" +#include "checker/validation_result.h" +#include "common/ast.h" +#include "common/decl.h" +#include "common/legacy_value.h" +#include "common/minimal_descriptor_pool.h" +#include "common/type.h" #include "common/value.h" -#include "eval/internal/interop.h" -#include "eval/public/activation.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_options.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_list_impl.h" +#include "extensions/set_membership_optimization.h" #include "extensions/sets_functions.h" #include "internal/benchmark.h" #include "internal/status_macros.h" #include "internal/testing.h" -#include "parser/parser.h" +#include "runtime/activation.h" +#include "runtime/constant_folding.h" +#include "runtime/reference_resolver.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" #include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" #include "google/protobuf/arena.h" namespace cel::extensions { namespace { -using ::cel::Value; -using ::cel::expr::ParsedExpr; -using ::google::api::expr::parser::Parse; -using ::google::api::expr::runtime::Activation; +using ::absl_testing::IsOk; +using ::google::api::expr::runtime::CelList; using ::google::api::expr::runtime::CelValue; using ::google::api::expr::runtime::ContainerBackedListImpl; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::InterpreterOptions; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; -enum class ListImpl : int { kLegacy = 0, kWrappedModern = 1, kRhsConstant = 2 }; +enum class ListImpl : int { + kLegacy = 0, + kModern = 1, + kRhsConstant = 2, + kRhsConstantOptimized = 3, +}; int ToNumber(ListImpl impl) { return static_cast(impl); } ListImpl FromNumber(int number) { switch (number) { case 0: return ListImpl::kLegacy; case 1: - return ListImpl::kWrappedModern; + return ListImpl::kModern; case 2: return ListImpl::kRhsConstant; + case 3: + return ListImpl::kRhsConstantOptimized; default: return ListImpl::kLegacy; } @@ -73,16 +86,19 @@ struct TestCase { std::string expr; ListImpl list_impl; int size; - CelValue result; + bool result; std::string MakeLabel(int len) const { std::string list_impl; switch (this->list_impl) { + case ListImpl::kRhsConstantOptimized: + list_impl = "rhs_constant_opt"; + break; case ListImpl::kRhsConstant: list_impl = "rhs_constant"; break; - case ListImpl::kWrappedModern: - list_impl = "wrapped_modern"; + case ListImpl::kModern: + list_impl = "modern"; break; case ListImpl::kLegacy: list_impl = "legacy"; @@ -93,43 +109,9 @@ struct TestCase { } }; -class ListStorage { - public: - virtual ~ListStorage() = default; -}; - -class LegacyListStorage : public ListStorage { - public: - LegacyListStorage(ContainerBackedListImpl x, ContainerBackedListImpl y) - : x_(std::move(x)), y_(std::move(y)) {} - - CelValue x() { return CelValue::CreateList(&x_); } - CelValue y() { return CelValue::CreateList(&y_); } - - private: - ContainerBackedListImpl x_; - ContainerBackedListImpl y_; -}; - -class ModernListStorage : public ListStorage { - public: - ModernListStorage(Value x, Value y) : x_(std::move(x)), y_(std::move(y)) {} - - CelValue x() { - return interop_internal::ModernValueToLegacyValueOrDie(&arena_, x_); - } - CelValue y() { - return interop_internal::ModernValueToLegacyValueOrDie(&arena_, y_); - } - - private: - google::protobuf::Arena arena_; - Value x_; - Value y_; -}; - -absl::StatusOr> RegisterLegacyLists( - bool overlap, int len, Activation& activation) { +absl::Status RegisterLegacyLists(bool overlap, int len, + google::protobuf::Arena* absl_nonnull arena, + Activation& activation) { std::vector x; std::vector y; x.reserve(len + 1); @@ -144,30 +126,33 @@ absl::StatusOr> RegisterLegacyLists( y.push_back(CelValue::CreateInt64(2)); } - auto result = std::make_unique( - ContainerBackedListImpl(std::move(x)), - ContainerBackedListImpl(std::move(y))); + const CelList* x_list = + google::protobuf::Arena::Create(arena, std::move(x)); + const CelList* y_list = + google::protobuf::Arena::Create(arena, std::move(y)); - activation.InsertValue("x", result->x()); - activation.InsertValue("y", result->y()); - return result; + CEL_ASSIGN_OR_RETURN(Value x_val, + cel::ModernValue(arena, CelValue::CreateList(x_list))); + CEL_ASSIGN_OR_RETURN(Value y_val, + cel::ModernValue(arena, CelValue::CreateList(y_list))); + + activation.InsertOrAssignValue("x", std::move(x_val)); + activation.InsertOrAssignValue("y", std::move(y_val)); + return absl::OkStatus(); } // Constant list literal that has the same elements as the bound test cases. std::string ConstantList(bool overlap, int len) { - std::string list_body; - for (int i = 0; i < len; i++) { - } return absl::StrCat("[", overlap ? "1, " : "", absl::StrJoin(std::vector(len, "2"), ", "), "]"); } -absl::StatusOr> RegisterModernLists( - bool overlap, int len, google::protobuf::Arena* absl_nonnull arena, - Activation& activation) { - auto x_builder = cel::NewListValueBuilder(arena); - auto y_builder = cel::NewListValueBuilder(arena); +absl::Status RegisterModernLists(bool overlap, int len, + google::protobuf::Arena* absl_nonnull arena, + Activation& activation) { + ListValueBuilderPtr x_builder = cel::NewListValueBuilder(arena); + ListValueBuilderPtr y_builder = cel::NewListValueBuilder(arena); x_builder->Reserve(len + 1); y_builder->Reserve(len + 1); @@ -182,81 +167,151 @@ absl::StatusOr> RegisterModernLists( CEL_RETURN_IF_ERROR(y_builder->Add(cel::IntValue(2))); } - auto x = std::move(*x_builder).Build(); - auto y = std::move(*y_builder).Build(); - auto result = std::make_unique(std::move(x), std::move(y)); - activation.InsertValue("x", result->x()); - activation.InsertValue("y", result->y()); + activation.InsertOrAssignValue("x", std::move(*x_builder).Build()); + activation.InsertOrAssignValue("y", std::move(*y_builder).Build()); - return result; + return absl::OkStatus(); } -absl::StatusOr> RegisterLists( - bool overlap, int len, bool use_modern, google::protobuf::Arena* absl_nonnull arena, - Activation& activation) { +absl::Status RegisterLists(bool overlap, int len, bool use_modern, + google::protobuf::Arena* absl_nonnull arena, + Activation& activation) { if (use_modern) { return RegisterModernLists(overlap, len, arena, activation); } else { - return RegisterLegacyLists(overlap, len, activation); + return RegisterLegacyLists(overlap, len, arena, activation); } } +absl::StatusOr> CreateBenchmarkCompiler() { + CEL_ASSIGN_OR_RETURN(std::unique_ptr builder, + NewCompilerBuilder(cel::GetMinimalDescriptorPool())); + CEL_RETURN_IF_ERROR(builder->AddLibrary(StandardCompilerLibrary())); + CEL_RETURN_IF_ERROR(builder->AddLibrary(SetsCompilerLibrary())); + + TypeCheckerBuilder& checker = builder->GetCheckerBuilder(); + CEL_RETURN_IF_ERROR(checker.AddVariable( + MakeVariableDecl("x", ListType(checker.arena(), IntType())))); + CEL_RETURN_IF_ERROR(checker.AddVariable( + MakeVariableDecl("y", ListType(checker.arena(), IntType())))); + CEL_RETURN_IF_ERROR(checker.AddVariable( + MakeVariableDecl("str_x", ListType(checker.arena(), StringType())))); + CEL_RETURN_IF_ERROR(checker.AddVariable( + MakeVariableDecl("str_y", ListType(checker.arena(), StringType())))); + CEL_RETURN_IF_ERROR( + checker.AddVariable(MakeVariableDecl("str_elem", StringType()))); + + return builder->Build(); +} + +absl::StatusOr> CreateBenchmarkRuntime( + bool enable_set_opt, google::protobuf::Arena* absl_nonnull arena) { + RuntimeOptions options; + CEL_ASSIGN_OR_RETURN( + RuntimeBuilder builder, + CreateStandardRuntimeBuilder(cel::GetMinimalDescriptorPool(), options)); + CEL_RETURN_IF_ERROR( + RegisterSetsFunctions(builder.function_registry(), options)); + CEL_RETURN_IF_ERROR( + EnableReferenceResolver(builder, ReferenceResolverEnabled::kAlways)); + CEL_RETURN_IF_ERROR(EnableConstantFolding(builder, arena)); + if (enable_set_opt) { + CEL_RETURN_IF_ERROR(EnableSetMembershipOptimization(builder)); + } + return std::move(builder).Build(); +} + void RunBenchmark(const TestCase& test_case, benchmark::State& state) { - bool lists_overlap = test_case.result.BoolOrDie(); + bool lists_overlap = test_case.result; std::string expr = test_case.expr; - if (test_case.list_impl == ListImpl::kRhsConstant) { + if (test_case.list_impl == ListImpl::kRhsConstant || + test_case.list_impl == ListImpl::kRhsConstantOptimized) { expr = absl::StrReplaceAll( expr, {{"y", ConstantList(lists_overlap, test_case.size)}}); } - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(expr)); - google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + CreateBenchmarkCompiler()); + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(expr)); + ASSERT_TRUE(validation_result.IsValid()) << validation_result.FormatError(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + validation_result.ReleaseAst()); - InterpreterOptions options; - options.constant_folding = true; - options.constant_arena = &arena; - options.enable_qualified_identifier_rewrites = true; - auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); - ASSERT_OK(RegisterSetsFunctions(builder->GetRegistry()->InternalGetRegistry(), - cel::RuntimeOptions{})); + google::protobuf::Arena arena; ASSERT_OK_AND_ASSIGN( - auto cel_expr, builder->CreateExpression(&(parsed_expr.expr()), nullptr)); + std::unique_ptr runtime, + CreateBenchmarkRuntime( + test_case.list_impl == ListImpl::kRhsConstantOptimized, &arena)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); Activation activation; - ASSERT_OK_AND_ASSIGN( - auto storage, - RegisterLists(test_case.result.BoolOrDie(), test_case.size, - test_case.list_impl == ListImpl::kWrappedModern, &arena, - activation)); + ASSERT_THAT(RegisterLists(test_case.result, test_case.size, + test_case.list_impl != ListImpl::kLegacy, &arena, + activation), + IsOk()); state.SetLabel(test_case.MakeLabel(test_case.size)); for (auto _ : state) { - ASSERT_OK_AND_ASSIGN(CelValue result, - cel_expr->Evaluate(activation, &arena)); + ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); ASSERT_TRUE(result.IsBool()); - ASSERT_EQ(result.BoolOrDie(), test_case.result.BoolOrDie()) + ASSERT_EQ(result.GetBool().NativeValue(), test_case.result) << test_case.test_name; } } -void BM_SetsIntersectsTrue(benchmark::State& state) { +void BM_SetsContainsTrue(benchmark::State& state) { ListImpl impl = FromNumber(state.range(0)); int size = state.range(1); - RunBenchmark({"sets.intersects_true", "sets.intersects(x, y)", impl, size, - CelValue::CreateBool(true)}, + RunBenchmark({"sets.contains_true", "sets.contains(x, y)", impl, size, true}, state); } +void BM_SetsContainsFalse(benchmark::State& state) { + ListImpl impl = FromNumber(state.range(0)); + int size = state.range(1); + + RunBenchmark( + {"sets.contains_false", "sets.contains(x, y)", impl, size, false}, state); +} + +void BM_SetsContainsComprehensionTrue(benchmark::State& state) { + ListImpl impl = FromNumber(state.range(0)); + int size = state.range(1); + + RunBenchmark( + {"comprehension_contains_true", "x.all(i, i in y)", impl, size, true}, + state); +} + +void BM_SetsContainsComprehensionFalse(benchmark::State& state) { + ListImpl impl = FromNumber(state.range(0)); + int size = state.range(1); + + RunBenchmark( + {"comprehension_contains_false", "x.all(i, i in y)", impl, size, false}, + state); +} + +void BM_SetsIntersectsTrue(benchmark::State& state) { + ListImpl impl = FromNumber(state.range(0)); + int size = state.range(1); + + RunBenchmark( + {"sets.intersects_true", "sets.intersects(x, y)", impl, size, true}, + state); +} + void BM_SetsIntersectsFalse(benchmark::State& state) { ListImpl impl = FromNumber(state.range(0)); int size = state.range(1); - RunBenchmark({"sets.intersects_false", "sets.intersects(x, y)", impl, size, - CelValue::CreateBool(false)}, - state); + RunBenchmark( + {"sets.intersects_false", "sets.intersects(x, y)", impl, size, false}, + state); } void BM_SetsIntersectsComprehensionTrue(benchmark::State& state) { @@ -264,7 +319,7 @@ void BM_SetsIntersectsComprehensionTrue(benchmark::State& state) { int size = state.range(1); RunBenchmark({"comprehension_intersects_true", "x.exists(i, i in y)", impl, - size, CelValue::CreateBool(true)}, + size, true}, state); } @@ -273,7 +328,7 @@ void BM_SetsIntersectsComprehensionFalse(benchmark::State& state) { int size = state.range(1); RunBenchmark({"comprehension_intersects_false", "x.exists(i, i in y)", impl, - size, CelValue::CreateBool(false)}, + size, false}, state); } @@ -281,50 +336,54 @@ void BM_SetsEquivalentTrue(benchmark::State& state) { ListImpl impl = FromNumber(state.range(0)); int size = state.range(1); - RunBenchmark({"sets.equivalent_true", "sets.equivalent(x, y)", impl, size, - CelValue::CreateBool(true)}, - state); + RunBenchmark( + {"sets.equivalent_true", "sets.equivalent(x, y)", impl, size, true}, + state); } void BM_SetsEquivalentFalse(benchmark::State& state) { ListImpl impl = FromNumber(state.range(0)); int size = state.range(1); - RunBenchmark({"sets.equivalent_false", "sets.equivalent(x, y)", impl, size, - CelValue::CreateBool(false)}, - state); + RunBenchmark( + {"sets.equivalent_false", "sets.equivalent(x, y)", impl, size, false}, + state); } void BM_SetsEquivalentComprehensionTrue(benchmark::State& state) { ListImpl impl = FromNumber(state.range(0)); int size = state.range(1); - RunBenchmark( - {"comprehension_equivalent_true", "x.all(i, i in y) && y.all(j, j in x)", - impl, size, CelValue::CreateBool(true)}, - state); + RunBenchmark({"comprehension_equivalent_true", + "x.all(i, i in y) && y.all(j, j in x)", impl, size, true}, + state); } void BM_SetsEquivalentComprehensionFalse(benchmark::State& state) { ListImpl impl = FromNumber(state.range(0)); int size = state.range(1); - RunBenchmark( - {"comprehension_equivalent_false", "x.all(i, i in y) && y.all(j, j in x)", - impl, size, CelValue::CreateBool(false)}, - state); + RunBenchmark({"comprehension_equivalent_false", + "x.all(i, i in y) && y.all(j, j in x)", impl, size, false}, + state); } template void BenchArgs(Benchmark* bench) { for (ListImpl impl : - {ListImpl::kLegacy, ListImpl::kWrappedModern, ListImpl::kRhsConstant}) { + {ListImpl::kLegacy, ListImpl::kModern, ListImpl::kRhsConstant, + ListImpl::kRhsConstantOptimized}) { for (int size : {1, 8, 32, 64, 256}) { bench->ArgPair(ToNumber(impl), size); } } } +BENCHMARK(BM_SetsContainsComprehensionTrue)->Apply(BenchArgs); +BENCHMARK(BM_SetsContainsComprehensionFalse)->Apply(BenchArgs); +BENCHMARK(BM_SetsContainsTrue)->Apply(BenchArgs); +BENCHMARK(BM_SetsContainsFalse)->Apply(BenchArgs); + BENCHMARK(BM_SetsIntersectsComprehensionTrue)->Apply(BenchArgs); BENCHMARK(BM_SetsIntersectsComprehensionFalse)->Apply(BenchArgs); BENCHMARK(BM_SetsIntersectsTrue)->Apply(BenchArgs); @@ -335,5 +394,204 @@ BENCHMARK(BM_SetsEquivalentComprehensionFalse)->Apply(BenchArgs); BENCHMARK(BM_SetsEquivalentTrue)->Apply(BenchArgs); BENCHMARK(BM_SetsEquivalentFalse)->Apply(BenchArgs); +enum class StringSetBenchOp { + kIn, + kIntersects, + kContains, + kEquivalent, +}; + +std::string ConstantStringList(int len) { + std::vector elems; + elems.reserve(len); + for (int i = 0; i < len; ++i) { + elems.push_back(absl::StrCat("'str_", i, "'")); + } + return absl::StrCat("[", absl::StrJoin(elems, ", "), "]"); +} + +absl::Status RegisterStringBenchValues(StringSetBenchOp op, bool expected, + int len, + google::protobuf::Arena* absl_nonnull arena, + Activation& activation) { + ListValueBuilderPtr y_builder = cel::NewListValueBuilder(arena); + y_builder->Reserve(len); + for (int i = 0; i < len; ++i) { + CEL_RETURN_IF_ERROR( + y_builder->Add(cel::StringValue::From(absl::StrCat("str_", i), arena))); + } + activation.InsertOrAssignValue("str_y", std::move(*y_builder).Build()); + + if (op == StringSetBenchOp::kIn) { + // True: best case / short-circuit (matches first element). + // False: worst case (scans all elements). + std::string elem = expected ? "str_0" : "str_missing"; + activation.InsertOrAssignValue("str_elem", + cel::StringValue::From(elem, arena)); + return absl::OkStatus(); + } + + ListValueBuilderPtr x_builder = cel::NewListValueBuilder(arena); + x_builder->Reserve(len); + if (op == StringSetBenchOp::kIntersects) { + // True: best case / short-circuit (matches on the first element). + // False: worst case (disjoint lists, scans all elements). + std::string first = expected ? "str_0" : "other_0"; + CEL_RETURN_IF_ERROR(x_builder->Add(cel::StringValue::From(first, arena))); + for (int i = 1; i < len; ++i) { + CEL_RETURN_IF_ERROR(x_builder->Add( + cel::StringValue::From(absl::StrCat("other_", i), arena))); + } + } else { + // For contains and equivalent: + // True: worst case (str_x has all elements in reverse order, exercising the + // full loop and matched set allocation). + // False: best case / short-circuit (disjoint elements starting with + // "other_0", short-circuiting on the first element for equivalent and + // avoiding matched set allocation for contains). + if (expected) { + for (int i = len - 1; i >= 0; --i) { + CEL_RETURN_IF_ERROR(x_builder->Add( + cel::StringValue::From(absl::StrCat("str_", i), arena))); + } + } else { + for (int i = 0; i < len; ++i) { + CEL_RETURN_IF_ERROR(x_builder->Add( + cel::StringValue::From(absl::StrCat("other_", i), arena))); + } + } + } + activation.InsertOrAssignValue("str_x", std::move(*x_builder).Build()); + return absl::OkStatus(); +} + +void RunStringSetBenchmark(StringSetBenchOp op, const TestCase& test_case, + benchmark::State& state) { + std::string expr = test_case.expr; + if (test_case.list_impl == ListImpl::kRhsConstant || + test_case.list_impl == ListImpl::kRhsConstantOptimized) { + expr = absl::StrReplaceAll(expr, + {{"str_y", ConstantStringList(test_case.size)}}); + } + + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + CreateBenchmarkCompiler()); + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(expr)); + ASSERT_TRUE(validation_result.IsValid()) << validation_result.FormatError(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + validation_result.ReleaseAst()); + + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr runtime, + CreateBenchmarkRuntime( + test_case.list_impl == ListImpl::kRhsConstantOptimized, &arena)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + Activation activation; + ASSERT_THAT(RegisterStringBenchValues(op, test_case.result, test_case.size, + &arena, activation), + IsOk()); + + state.SetLabel(test_case.MakeLabel(test_case.size)); + for (auto _ : state) { + ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); + ASSERT_TRUE(result.IsBool()); + ASSERT_EQ(result.GetBool().NativeValue(), test_case.result) + << test_case.test_name; + } +} + +void BM_StringSetsInTrue(benchmark::State& state) { + ListImpl impl = FromNumber(state.range(0)); + int size = state.range(1); + RunStringSetBenchmark(StringSetBenchOp::kIn, + {"str_in_true", "str_elem in str_y", impl, size, true}, + state); +} + +void BM_StringSetsInFalse(benchmark::State& state) { + ListImpl impl = FromNumber(state.range(0)); + int size = state.range(1); + RunStringSetBenchmark( + StringSetBenchOp::kIn, + {"str_in_false", "str_elem in str_y", impl, size, false}, state); +} + +void BM_StringSetsIntersectsTrue(benchmark::State& state) { + ListImpl impl = FromNumber(state.range(0)); + int size = state.range(1); + RunStringSetBenchmark(StringSetBenchOp::kIntersects, + {"str_sets.intersects_true", + "sets.intersects(str_x, str_y)", impl, size, true}, + state); +} + +void BM_StringSetsIntersectsFalse(benchmark::State& state) { + ListImpl impl = FromNumber(state.range(0)); + int size = state.range(1); + RunStringSetBenchmark(StringSetBenchOp::kIntersects, + {"str_sets.intersects_false", + "sets.intersects(str_x, str_y)", impl, size, false}, + state); +} + +void BM_StringSetsContainsTrue(benchmark::State& state) { + ListImpl impl = FromNumber(state.range(0)); + int size = state.range(1); + RunStringSetBenchmark(StringSetBenchOp::kContains, + {"str_sets.contains_true", + "sets.contains(str_x, str_y)", impl, size, true}, + state); +} + +void BM_StringSetsContainsFalse(benchmark::State& state) { + ListImpl impl = FromNumber(state.range(0)); + int size = state.range(1); + RunStringSetBenchmark(StringSetBenchOp::kContains, + {"str_sets.contains_false", + "sets.contains(str_x, str_y)", impl, size, false}, + state); +} + +void BM_StringSetsEquivalentTrue(benchmark::State& state) { + ListImpl impl = FromNumber(state.range(0)); + int size = state.range(1); + RunStringSetBenchmark(StringSetBenchOp::kEquivalent, + {"str_sets.equivalent_true", + "sets.equivalent(str_x, str_y)", impl, size, true}, + state); +} + +void BM_StringSetsEquivalentFalse(benchmark::State& state) { + ListImpl impl = FromNumber(state.range(0)); + int size = state.range(1); + RunStringSetBenchmark(StringSetBenchOp::kEquivalent, + {"str_sets.equivalent_false", + "sets.equivalent(str_x, str_y)", impl, size, false}, + state); +} + +template +void StringBenchArgs(Benchmark* bench) { + for (ListImpl impl : + {ListImpl::kRhsConstant, ListImpl::kRhsConstantOptimized}) { + for (int size : {1, 4, 8, 16, 32, 64, 256}) { + bench->ArgPair(ToNumber(impl), size); + } + } +} + +BENCHMARK(BM_StringSetsInTrue)->Apply(StringBenchArgs); +BENCHMARK(BM_StringSetsInFalse)->Apply(StringBenchArgs); +BENCHMARK(BM_StringSetsIntersectsTrue)->Apply(StringBenchArgs); +BENCHMARK(BM_StringSetsIntersectsFalse)->Apply(StringBenchArgs); +BENCHMARK(BM_StringSetsContainsTrue)->Apply(StringBenchArgs); +BENCHMARK(BM_StringSetsContainsFalse)->Apply(StringBenchArgs); +BENCHMARK(BM_StringSetsEquivalentTrue)->Apply(StringBenchArgs); +BENCHMARK(BM_StringSetsEquivalentFalse)->Apply(StringBenchArgs); + } // namespace } // namespace cel::extensions diff --git a/extensions/sets_functions_test.cc b/extensions/sets_functions_test.cc index dc6768f34..e2f191a9f 100644 --- a/extensions/sets_functions_test.cc +++ b/extensions/sets_functions_test.cc @@ -16,6 +16,7 @@ #include #include +#include #include "cel/expr/syntax.pb.h" #include "absl/status/status_matchers.h" @@ -23,6 +24,8 @@ #include "checker/validation_result.h" #include "common/ast_proto.h" #include "common/minimal_descriptor_pool.h" +#include "common/value.h" +#include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "eval/public/activation.h" #include "eval/public/builtin_func_registrar.h" @@ -31,8 +34,13 @@ #include "eval/public/cel_function_adapter.h" #include "eval/public/cel_options.h" #include "eval/public/cel_value.h" +#include "extensions/set_membership_optimization.h" #include "internal/testing.h" +#include "runtime/activation.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" #include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" #include "google/protobuf/arena.h" namespace cel::extensions { @@ -93,6 +101,83 @@ TEST_P(CelSetsFunctionsTest, EndToEnd) { EXPECT_TRUE(out.BoolOrDie()) << test_info.expr << " -> " << out.DebugString(); } +TEST_P(CelSetsFunctionsTest, RuntimeEndToEnd) { + const TestInfo& test_info = GetParam(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler_builder, + NewCompilerBuilder(cel::GetMinimalDescriptorPool())); + + ASSERT_THAT(compiler_builder->AddLibrary(StandardCheckerLibrary()), IsOk()); + ASSERT_THAT(compiler_builder->AddLibrary(SetsCompilerLibrary()), IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult compiled, + compiler->Compile(test_info.expr)); + + ASSERT_TRUE(compiled.IsValid()) << compiled.FormatError(); + + RuntimeOptions options; + options.enable_heterogeneous_equality = true; + options.enable_empty_wrapper_null_unboxing = true; + options.enable_qualified_type_identifiers = true; + ASSERT_OK_AND_ASSIGN( + RuntimeBuilder runtime_builder, + CreateStandardRuntimeBuilder(cel::GetMinimalDescriptorPool(), options)); + ASSERT_THAT( + RegisterSetsFunctions(runtime_builder.function_registry(), options), + IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(*compiled.ReleaseAst())); + Arena arena; + cel::Activation activation; + ASSERT_OK_AND_ASSIGN(Value out, program->Evaluate(&arena, activation)); + ASSERT_TRUE(out.IsBool()) << test_info.expr << " -> " << out.DebugString(); + EXPECT_TRUE(out.GetBool().NativeValue()) + << test_info.expr << " -> " << out.DebugString(); +} + +TEST_P(CelSetsFunctionsTest, RuntimeEndToEndWithSetMembershipOptimization) { + const TestInfo& test_info = GetParam(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler_builder, + NewCompilerBuilder(cel::GetMinimalDescriptorPool())); + + ASSERT_THAT(compiler_builder->AddLibrary(StandardCheckerLibrary()), IsOk()); + ASSERT_THAT(compiler_builder->AddLibrary(SetsCompilerLibrary()), IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult compiled, + compiler->Compile(test_info.expr)); + + ASSERT_TRUE(compiled.IsValid()) << compiled.FormatError(); + + RuntimeOptions options; + options.enable_heterogeneous_equality = true; + options.enable_empty_wrapper_null_unboxing = true; + options.enable_qualified_type_identifiers = true; + ASSERT_OK_AND_ASSIGN( + RuntimeBuilder runtime_builder, + CreateStandardRuntimeBuilder(cel::GetMinimalDescriptorPool(), options)); + ASSERT_THAT( + RegisterSetsFunctions(runtime_builder.function_registry(), options), + IsOk()); + ASSERT_THAT(EnableSetMembershipOptimization(runtime_builder), IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(*compiled.ReleaseAst())); + Arena arena; + cel::Activation activation; + ASSERT_OK_AND_ASSIGN(Value out, program->Evaluate(&arena, activation)); + ASSERT_TRUE(out.IsBool()) << test_info.expr << " -> " << out.DebugString(); + EXPECT_TRUE(out.GetBool().NativeValue()) + << test_info.expr << " -> " << out.DebugString(); +} + INSTANTIATE_TEST_SUITE_P( CelSetsFunctionsTest, CelSetsFunctionsTest, testing::ValuesIn({ @@ -130,6 +215,7 @@ INSTANTIATE_TEST_SUITE_P( {"sets.equivalent([1], [1u, 1.0])"}, {"sets.equivalent([1], [1u, 1.0])"}, {"sets.equivalent([1, 2, 3], [3u, 2.0, 1])"}, + {"!sets.equivalent([-1], [])"}, {"!sets.equivalent([2, 1], [1])"}, {"!sets.equivalent([1], [1, 2])"}, {"!sets.equivalent([1, 2], [2u, 2, 2.0])"}, diff --git a/extensions/strings.cc b/extensions/strings.cc index 54fda20d6..831600353 100644 --- a/extensions/strings.cc +++ b/extensions/strings.cc @@ -34,6 +34,7 @@ #include "eval/public/cel_options.h" #include "extensions/formatting.h" #include "internal/status_macros.h" +#include "runtime/function.h" #include "runtime/function_adapter.h" #include "runtime/function_registry.h" #include "runtime/runtime_options.h" @@ -107,8 +108,9 @@ absl::StatusOr Replace1( return string.Replace(old_sub, new_sub, -1, arena); } -Value CharAt(const StringValue& string, int64_t pos) { - return string.CharAt(pos); +Value CharAt(const StringValue& string, int64_t pos, + const Function::InvokeContext& context) { + return string.CharAt(pos, context.arena()); } int64_t IndexOf2(const StringValue& haystack, const StringValue& needle) { @@ -116,10 +118,11 @@ int64_t IndexOf2(const StringValue& haystack, const StringValue& needle) { } Value IndexOf3(const StringValue& haystack, const StringValue& needle, - int64_t pos) { + int64_t pos, const Function::InvokeContext& context) { if (pos > haystack.Size()) { - return ErrorValue{ - absl::InvalidArgumentError(absl::StrCat("index out of range: ", pos))}; + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrCat("index out of range: ", pos)), + context.arena()); } return IntValue(haystack.IndexOf(needle, pos).value_or(-1)); } @@ -129,20 +132,23 @@ int64_t LastIndexOf2(const StringValue& haystack, const StringValue& needle) { } Value LastIndexOf3(const StringValue& haystack, const StringValue& needle, - int64_t pos) { + int64_t pos, const Function::InvokeContext& context) { if (pos < 0 || pos > haystack.Size()) { - return ErrorValue{ - absl::InvalidArgumentError(absl::StrCat("index out of range: ", pos))}; + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrCat("index out of range: ", pos)), + context.arena()); } return IntValue(haystack.LastIndexOf(needle, pos).value_or(-1)); } -Value Substring2(const StringValue& string, int64_t start) { - return string.Substring(start); +Value Substring2(const StringValue& string, int64_t start, + const Function::InvokeContext& context) { + return string.Substring(start, context.arena()); } -Value Substring3(const StringValue& string, int64_t start, int64_t end) { - return string.Substring(start, end); +Value Substring3(const StringValue& string, int64_t start, int64_t end, + const Function::InvokeContext& context) { + return string.Substring(start, end, context.arena()); } StringValue Trim(const StringValue& string) { return string.Trim(); } diff --git a/internal/BUILD b/internal/BUILD index 3891c635d..b6cabfb75 100644 --- a/internal/BUILD +++ b/internal/BUILD @@ -42,6 +42,27 @@ cc_test( ], ) +cc_library( + name = "arena_tree", + srcs = ["arena_tree.cc"], + hdrs = ["arena_tree.h"], + deps = [ + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/log:absl_check", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "arena_tree_test", + srcs = ["arena_tree_test.cc"], + deps = [ + ":arena_tree", + ":testing", + "@com_google_protobuf//:protobuf", + ], +) + cc_library( name = "new", srcs = ["new.cc"], @@ -86,12 +107,25 @@ cc_library( ], ) +cc_library( + name = "runfiles", + srcs = ["runfiles.cc"], + hdrs = ["runfiles.h"], + deps = [ + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@rules_cc//cc/runfiles", + ], +) + cc_library( name = "status_builder", hdrs = ["status_builder.h"], deps = [ "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_builder", ], ) @@ -145,9 +179,9 @@ cc_library( name = "status_macros", hdrs = ["status_macros.h"], deps = [ - ":status_builder", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_macros", ], ) @@ -294,6 +328,7 @@ cc_library( "testing.h", ], deps = [ + ":status_builder", ":status_macros", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings", @@ -311,6 +346,7 @@ cc_library( "testing.h", ], deps = [ + ":status_builder", ":status_macros", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings", @@ -691,7 +727,6 @@ cc_test( ":testing_descriptor_pool", ":testing_message_factory", ":well_known_types", - "//common:memory", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:die_if_null", @@ -701,7 +736,6 @@ cc_test( "@com_google_absl//absl/strings:cord", "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/time", - "@com_google_absl//absl/types:variant", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", "@com_google_protobuf//:any_cc_proto", "@com_google_protobuf//:duration_cc_proto", @@ -777,7 +811,6 @@ cc_library( ":number", ":status_macros", ":well_known_types", - "//common:memory", "//extensions/protobuf/internal:map_reflection", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", @@ -806,12 +839,12 @@ cc_test( ":testing_descriptor_pool", ":testing_message_factory", ":well_known_types", - "//common:allocator", "//common:memory", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:die_if_null", + "@com_google_absl//absl/memory", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:cord", diff --git a/internal/arena_tree.cc b/internal/arena_tree.cc new file mode 100644 index 000000000..8db8fde6c --- /dev/null +++ b/internal/arena_tree.cc @@ -0,0 +1,345 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "internal/arena_tree.h" + +namespace cel::internal { + +namespace { + +void ArenaTreeRotateLeft(ArenaTreeNodeBase** head, ArenaTreeNodeBase* elem) { + ArenaTreeNodeBase* tmp = ArenaTreeNodeGetRight(elem); + if (ArenaTreeNodeSetRight(elem, ArenaTreeNodeGetLeft(tmp)) != nullptr) { + ArenaTreeNodeSetParent(ArenaTreeNodeGetLeft(tmp), elem); + } + ArenaTreeNodeBase* parent = ArenaTreeNodeGetParent(elem); + ArenaTreeNodeSetParent(tmp, parent); + if (parent != nullptr) { + if (elem == ArenaTreeNodeGetLeft(parent)) { + ArenaTreeNodeSetLeft(parent, tmp); + } else { + ArenaTreeNodeSetRight(parent, tmp); + } + } else { + *head = tmp; + } + ArenaTreeNodeSetLeft(tmp, elem); + ArenaTreeNodeSetParent(elem, tmp); +} + +void ArenaTreeRotateRight(ArenaTreeNodeBase** head, ArenaTreeNodeBase* elem) { + ArenaTreeNodeBase* tmp = ArenaTreeNodeGetLeft(elem); + if (ArenaTreeNodeSetLeft(elem, ArenaTreeNodeGetRight(tmp)) != nullptr) { + ArenaTreeNodeSetParent(ArenaTreeNodeGetRight(tmp), elem); + } + ArenaTreeNodeBase* parent = ArenaTreeNodeGetParent(elem); + ArenaTreeNodeSetParent(tmp, parent); + if (parent != nullptr) { + if (elem == ArenaTreeNodeGetLeft(parent)) { + ArenaTreeNodeSetLeft(parent, tmp); + } else { + ArenaTreeNodeSetRight(parent, tmp); + } + } else { + *head = tmp; + } + ArenaTreeNodeSetRight(tmp, elem); + ArenaTreeNodeSetParent(elem, tmp); +} + +void ArenaTreeRemoveColor(ArenaTreeNodeBase** head, ArenaTreeNodeBase* parent, + ArenaTreeNodeBase* elem) { + ArenaTreeNodeBase* tmp; + while ((elem == nullptr || + ArenaTreeNodeGetColor(elem) == ArenaTreeNodeColor::kBlack) && + elem != *head) { + if (ArenaTreeNodeGetLeft(parent) == elem) { + tmp = ArenaTreeNodeGetRight(parent); + if (ArenaTreeNodeGetColor(tmp) == ArenaTreeNodeColor::kRed) { + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kRed); + ArenaTreeRotateLeft(head, parent); + tmp = ArenaTreeNodeGetRight(parent); + } + if ((ArenaTreeNodeGetLeft(tmp) == nullptr || + ArenaTreeNodeGetColor(ArenaTreeNodeGetLeft(tmp)) == + ArenaTreeNodeColor::kBlack) && + (ArenaTreeNodeGetRight(tmp) == nullptr || + ArenaTreeNodeGetColor(ArenaTreeNodeGetRight(tmp)) == + ArenaTreeNodeColor::kBlack)) { + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kRed); + elem = parent; + parent = ArenaTreeNodeGetParent(elem); + } else { + if (ArenaTreeNodeGetRight(tmp) == nullptr || + ArenaTreeNodeGetColor(ArenaTreeNodeGetRight(tmp)) == + ArenaTreeNodeColor::kBlack) { + ArenaTreeNodeBase* left; + if ((left = ArenaTreeNodeGetLeft(tmp)) != nullptr) { + ArenaTreeNodeSetColor(left, ArenaTreeNodeColor::kBlack); + } + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kRed); + ArenaTreeRotateRight(head, tmp); + tmp = ArenaTreeNodeGetRight(parent); + } + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeGetColor(parent)); + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kBlack); + if (ArenaTreeNodeGetRight(tmp) != nullptr) { + ArenaTreeNodeSetColor(ArenaTreeNodeGetRight(tmp), + ArenaTreeNodeColor::kBlack); + } + ArenaTreeRotateLeft(head, parent); + elem = *head; + break; + } + } else { + tmp = ArenaTreeNodeGetLeft(parent); + if (ArenaTreeNodeGetColor(tmp) == ArenaTreeNodeColor::kRed) { + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kRed); + ArenaTreeRotateRight(head, parent); + tmp = ArenaTreeNodeGetLeft(parent); + } + if ((ArenaTreeNodeGetLeft(tmp) == nullptr || + ArenaTreeNodeGetColor(ArenaTreeNodeGetLeft(tmp)) == + ArenaTreeNodeColor::kBlack) && + (ArenaTreeNodeGetRight(tmp) == nullptr || + ArenaTreeNodeGetColor(ArenaTreeNodeGetRight(tmp)) == + ArenaTreeNodeColor::kBlack)) { + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kRed); + elem = parent; + parent = ArenaTreeNodeGetParent(elem); + } else { + if (ArenaTreeNodeGetLeft(tmp) == nullptr || + ArenaTreeNodeGetColor(ArenaTreeNodeGetLeft(tmp)) == + ArenaTreeNodeColor::kBlack) { + ArenaTreeNodeBase* right; + if ((right = ArenaTreeNodeGetRight(tmp)) != nullptr) { + ArenaTreeNodeSetColor(right, ArenaTreeNodeColor::kBlack); + } + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kRed); + ArenaTreeRotateLeft(head, tmp); + tmp = ArenaTreeNodeGetLeft(parent); + } + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeGetColor(parent)); + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kBlack); + if (ArenaTreeNodeGetLeft(tmp) != nullptr) { + ArenaTreeNodeSetColor(ArenaTreeNodeGetLeft(tmp), + ArenaTreeNodeColor::kBlack); + } + ArenaTreeRotateRight(head, parent); + elem = *head; + break; + } + } + } + if (elem != nullptr) { + ArenaTreeNodeSetColor(elem, ArenaTreeNodeColor::kBlack); + } +} + +} // namespace + +const ArenaTreeNodeBase* ArenaTreeNext(const ArenaTreeNodeBase* node) { + if (node != nullptr) { + const ArenaTreeNodeBase* right = ArenaTreeNodeGetRight(node); + if (right != nullptr) { + node = right; + const ArenaTreeNodeBase* left; + while ((left = ArenaTreeNodeGetLeft(node)) != nullptr) { + node = left; + } + } else { + const ArenaTreeNodeBase* parent = ArenaTreeNodeGetParent(node); + if (parent != nullptr && node == ArenaTreeNodeGetLeft(parent)) { + node = parent; + } else { + while (parent != nullptr && ArenaTreeNodeGetRight(parent) == node) { + node = ArenaTreeNodeGetParent(node); + parent = ArenaTreeNodeGetParent(node); + } + node = parent; + } + } + } + return node; +} + +const ArenaTreeNodeBase* ArenaTreePrev(const ArenaTreeNodeBase* node) { + if (node != nullptr) { + const ArenaTreeNodeBase* left = ArenaTreeNodeGetLeft(node); + if (left != nullptr) { + node = left; + const ArenaTreeNodeBase* right; + while ((right = ArenaTreeNodeGetRight(node)) != nullptr) { + node = right; + } + } else { + const ArenaTreeNodeBase* parent = ArenaTreeNodeGetParent(node); + if (parent != nullptr && node == ArenaTreeNodeGetRight(parent)) { + node = parent; + } else { + while (parent != nullptr && ArenaTreeNodeGetLeft(parent) == node) { + node = ArenaTreeNodeGetParent(node); + parent = ArenaTreeNodeGetParent(node); + } + node = parent; + } + } + } + return node; +} + +const ArenaTreeNodeBase* ArenaTreeMin(const ArenaTreeNodeBase* node) { + const ArenaTreeNodeBase* tmp = node; + const ArenaTreeNodeBase* parent = nullptr; + while (tmp != nullptr) { + parent = tmp; + tmp = ArenaTreeNodeGetLeft(tmp); + } + return parent; +} + +const ArenaTreeNodeBase* ArenaTreeMax(const ArenaTreeNodeBase* node) { + const ArenaTreeNodeBase* tmp = node; + const ArenaTreeNodeBase* parent = nullptr; + while (tmp != nullptr) { + parent = tmp; + tmp = ArenaTreeNodeGetRight(tmp); + } + return parent; +} + +void ArenaTreeRemove(ArenaTreeNodeBase** head, ArenaTreeNodeBase* elem) { + ArenaTreeNodeBase* child; + ArenaTreeNodeBase* parent; + ArenaTreeNodeBase* const old = elem; + ArenaTreeNodeColor color; + if (ArenaTreeNodeGetLeft(elem) == nullptr) { + child = ArenaTreeNodeGetRight(elem); + } else if (ArenaTreeNodeGetRight(elem) == nullptr) { + child = ArenaTreeNodeGetLeft(elem); + } else { + ArenaTreeNodeBase* left; + elem = ArenaTreeNodeGetRight(elem); + while ((left = ArenaTreeNodeGetLeft(elem)) != nullptr) { + elem = left; + } + child = ArenaTreeNodeGetRight(elem); + parent = ArenaTreeNodeGetParent(elem); + color = ArenaTreeNodeGetColor(elem); + if (child != nullptr) { + ArenaTreeNodeSetParent(child, parent); + } + if (parent != nullptr) { + if (ArenaTreeNodeGetLeft(parent) == elem) { + ArenaTreeNodeSetLeft(parent, child); + } else { + ArenaTreeNodeSetRight(parent, child); + } + } else { + *head = child; + } + if (ArenaTreeNodeGetParent(elem) == old) { + parent = elem; + } + ArenaTreeNodeCopy(elem, old); + ArenaTreeNodeBase* old_parent = ArenaTreeNodeGetParent(old); + if (old_parent != nullptr) { + if (ArenaTreeNodeGetLeft(old_parent) == old) { + ArenaTreeNodeSetLeft(old_parent, elem); + } else { + ArenaTreeNodeSetRight(old_parent, elem); + } + } else { + *head = elem; + } + ArenaTreeNodeSetParent(ArenaTreeNodeGetLeft(old), elem); + if (ArenaTreeNodeGetRight(old) != nullptr) { + ArenaTreeNodeSetParent(ArenaTreeNodeGetRight(old), elem); + } + goto color; + } + parent = ArenaTreeNodeGetParent(elem); + color = ArenaTreeNodeGetColor(elem); + if (child != nullptr) { + ArenaTreeNodeSetParent(child, parent); + } + if (parent != nullptr) { + if (ArenaTreeNodeGetLeft(parent) == elem) { + ArenaTreeNodeSetLeft(parent, child); + } else { + ArenaTreeNodeSetRight(parent, child); + } + } else { + *head = child; + } +color: + if (color == ArenaTreeNodeColor::kBlack) { + ArenaTreeRemoveColor(head, parent, child); + } + ArenaTreeNodeClear(old); +} + +void ArenaTreeInsertColor(ArenaTreeNodeBase** head, ArenaTreeNodeBase* elem) { + ArenaTreeNodeBase* parent; + ArenaTreeNodeBase* grandparent; + ArenaTreeNodeBase* tmp; + while ((parent = ArenaTreeNodeGetParent(elem)) != nullptr && + ArenaTreeNodeGetColor(parent) == ArenaTreeNodeColor::kRed) { + grandparent = ArenaTreeNodeGetParent(parent); + if (parent == ArenaTreeNodeGetLeft(grandparent)) { + tmp = ArenaTreeNodeGetRight(grandparent); + if (tmp != nullptr && + ArenaTreeNodeGetColor(tmp) == ArenaTreeNodeColor::kRed) { + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(grandparent, ArenaTreeNodeColor::kRed); + elem = grandparent; + continue; + } + if (ArenaTreeNodeGetRight(parent) == elem) { + ArenaTreeRotateLeft(head, parent); + tmp = parent; + parent = elem; + elem = tmp; + } + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(grandparent, ArenaTreeNodeColor::kRed); + ArenaTreeRotateRight(head, grandparent); + } else { + tmp = ArenaTreeNodeGetLeft(grandparent); + if (tmp != nullptr && + ArenaTreeNodeGetColor(tmp) == ArenaTreeNodeColor::kRed) { + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(grandparent, ArenaTreeNodeColor::kRed); + elem = grandparent; + continue; + } + if (ArenaTreeNodeGetLeft(parent) == elem) { + ArenaTreeRotateRight(head, parent); + tmp = parent; + parent = elem; + elem = tmp; + } + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(grandparent, ArenaTreeNodeColor::kRed); + ArenaTreeRotateLeft(head, grandparent); + } + } + ArenaTreeNodeSetColor(*head, ArenaTreeNodeColor::kBlack); +} + +} // namespace cel::internal diff --git a/internal/arena_tree.h b/internal/arena_tree.h new file mode 100644 index 000000000..05f0e74bd --- /dev/null +++ b/internal/arena_tree.h @@ -0,0 +1,390 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ArenaTree is a low level implementation of an RBTree tailored for use with +// google::protobuf::Arena. It avoids needing to deal with using a custom Allocator with +// std::set, std::map, absl::btree_set, or absl::btree_map, where the options +// are manually construct on the arena to avoid registering the destructor +// (technically undefined behavior) or registering the destructor and running it +// even though it is a no-op at runtime. Using absl::btree_set or +// absl::btree_map also does not preserve pointer stability and they have a +// higher overhead for small containers. +// +// Note that you should not use this unless you are really sure you need to. In +// most cases you should prefer std::set, std::map, absl::btree_set, or +// absl::btree_map. + +#ifndef THIRD_PARTY_CEL_CPP_INTERNAL_ARENA_TREE_H_ +#define THIRD_PARTY_CEL_CPP_INTERNAL_ARENA_TREE_H_ + +#include +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "google/protobuf/arena.h" + +namespace cel::internal { + +enum class ArenaTreeNodeColor : uintptr_t { + kBlack = 0, + kRed = 1, +}; + +class ArenaTreeNodeBase; + +[[nodiscard]] +const ArenaTreeNodeBase* ArenaTreeNext(const ArenaTreeNodeBase* node); + +[[nodiscard]] +const ArenaTreeNodeBase* ArenaTreePrev(const ArenaTreeNodeBase* node); + +[[nodiscard]] +const ArenaTreeNodeBase* ArenaTreeMin(const ArenaTreeNodeBase* node); + +[[nodiscard]] +const ArenaTreeNodeBase* ArenaTreeMax(const ArenaTreeNodeBase* node); + +template +using IsDerivedFromArenaTreeNodeBase = std::conjunction< + std::is_base_of, + std::negation>>>; + +template +constexpr bool kIsDerivedFromArenaTreeNodeBase = + IsDerivedFromArenaTreeNodeBase::value; + +template +using EnableIfDerivedFromArenaTreeNodeBase = + std::enable_if_t, U>; + +template +[[nodiscard]] +inline EnableIfDerivedFromArenaTreeNodeBase ArenaTreeNext(T* node) { + return static_cast(const_cast( + (ArenaTreeNext)(static_cast(node)))); +} + +template +[[nodiscard]] +inline EnableIfDerivedFromArenaTreeNodeBase ArenaTreePrev(T* node) { + return static_cast(const_cast( + (ArenaTreePrev)(static_cast(node)))); +} + +template +[[nodiscard]] +inline EnableIfDerivedFromArenaTreeNodeBase ArenaTreeMin(T* node) { + return static_cast(const_cast( + (ArenaTreeMin)(static_cast(node)))); +} + +template +[[nodiscard]] +inline EnableIfDerivedFromArenaTreeNodeBase ArenaTreeMax(T* node) { + return static_cast(const_cast( + (ArenaTreeMax)(static_cast(node)))); +} + +[[nodiscard]] +ArenaTreeNodeBase* ArenaTreeNodeGetParent(const ArenaTreeNodeBase* node); + +void ArenaTreeNodeSetParent(ArenaTreeNodeBase* node, ArenaTreeNodeBase* parent); + +[[nodiscard]] +ArenaTreeNodeColor ArenaTreeNodeGetColor(const ArenaTreeNodeBase* node); + +void ArenaTreeNodeSetColor(ArenaTreeNodeBase* node, ArenaTreeNodeColor color); + +void ArenaTreeNodeSet(ArenaTreeNodeBase* node, ArenaTreeNodeBase* parent); + +void ArenaTreeNodeCopy(ArenaTreeNodeBase* dst, const ArenaTreeNodeBase* src); + +void ArenaTreeNodeClear(ArenaTreeNodeBase* node); + +[[nodiscard]] +ArenaTreeNodeBase* ArenaTreeNodeGetLeft(const ArenaTreeNodeBase* node); + +ArenaTreeNodeBase* ArenaTreeNodeSetLeft(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* left); + +[[nodiscard]] +ArenaTreeNodeBase* ArenaTreeNodeGetRight(const ArenaTreeNodeBase* node); + +ArenaTreeNodeBase* ArenaTreeNodeSetRight(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* right); + +class ArenaTreeNodeBase { + private: + uintptr_t parent_and_color_ = 0; + ArenaTreeNodeBase* left_ = nullptr; + ArenaTreeNodeBase* right_ = nullptr; + + friend ArenaTreeNodeBase* ArenaTreeNodeGetParent( + const ArenaTreeNodeBase* node); + friend void ArenaTreeNodeSetParent(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* parent); + friend ArenaTreeNodeColor ArenaTreeNodeGetColor( + const ArenaTreeNodeBase* node); + friend void ArenaTreeNodeSetColor(ArenaTreeNodeBase* node, + ArenaTreeNodeColor color); + friend void ArenaTreeNodeSet(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* parent); + friend void ArenaTreeNodeClear(ArenaTreeNodeBase* node); + friend void ArenaTreeNodeCopy(ArenaTreeNodeBase* dst, + const ArenaTreeNodeBase* src); + friend ArenaTreeNodeBase* ArenaTreeNodeGetLeft(const ArenaTreeNodeBase* node); + friend ArenaTreeNodeBase* ArenaTreeNodeSetLeft(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* left); + friend ArenaTreeNodeBase* ArenaTreeNodeGetRight( + const ArenaTreeNodeBase* node); + friend ArenaTreeNodeBase* ArenaTreeNodeSetRight(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* right); +}; + +[[nodiscard]] +inline ArenaTreeNodeBase* ArenaTreeNodeGetParent( + const ArenaTreeNodeBase* node) { + return reinterpret_cast(node->parent_and_color_ & + ~uintptr_t{1}); +} + +inline void ArenaTreeNodeSetParent(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* parent) { + node->parent_and_color_ = + static_cast(ArenaTreeNodeGetColor(node)) | + reinterpret_cast(parent); +} + +[[nodiscard]] +inline ArenaTreeNodeColor ArenaTreeNodeGetColor(const ArenaTreeNodeBase* node) { + return static_cast(node->parent_and_color_ & + uintptr_t{1}); +} + +inline void ArenaTreeNodeSetColor(ArenaTreeNodeBase* node, + ArenaTreeNodeColor color) { + node->parent_and_color_ = + reinterpret_cast(ArenaTreeNodeGetParent(node)) | + static_cast(color); +} + +inline void ArenaTreeNodeSet(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* parent) { + node->parent_and_color_ = reinterpret_cast(parent) | + static_cast(ArenaTreeNodeColor::kRed); + node->left_ = node->right_ = nullptr; +} + +inline void ArenaTreeNodeClear(ArenaTreeNodeBase* node) { + node->parent_and_color_ = 0; + node->left_ = node->right_ = nullptr; +} + +inline void ArenaTreeNodeCopy(ArenaTreeNodeBase* dst, + const ArenaTreeNodeBase* src) { + dst->parent_and_color_ = src->parent_and_color_; + dst->left_ = src->left_; + dst->right_ = src->right_; +} + +[[nodiscard]] +inline ArenaTreeNodeBase* ArenaTreeNodeGetLeft(const ArenaTreeNodeBase* node) { + return node->left_; +} + +inline ArenaTreeNodeBase* ArenaTreeNodeSetLeft(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* left) { + return node->left_ = left; +} + +[[nodiscard]] +inline ArenaTreeNodeBase* ArenaTreeNodeGetRight(const ArenaTreeNodeBase* node) { + return node->right_; +} + +inline ArenaTreeNodeBase* ArenaTreeNodeSetRight(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* right) { + return node->right_ = right; +} + +void ArenaTreeRemove(ArenaTreeNodeBase** head, ArenaTreeNodeBase* elem); + +template +[[nodiscard]] +inline EnableIfDerivedFromArenaTreeNodeBase ArenaTreeRemove(T** head, + T* elem) { + ArenaTreeNodeBase* head_base = *head; + (ArenaTreeRemove)(&head_base, static_cast(elem)); + *head = static_cast(head_base); +} + +void ArenaTreeInsertColor(ArenaTreeNodeBase** head, ArenaTreeNodeBase* elem); + +template +class ArenaTreeNode; + +template +[[nodiscard]] +const T& ArenaTreeNodeGetValue(const ArenaTreeNode* node); + +template +class ArenaTreeNode : public ArenaTreeNodeBase { + public: + template + explicit ArenaTreeNode(Args&&... args) + : ArenaTreeNodeBase(), value_(std::forward(args)...) {} + + private: + template + friend const U& ArenaTreeNodeGetValue(const ArenaTreeNode* node); + + T value_; +}; + +template +[[nodiscard]] inline const T& ArenaTreeNodeGetValue( + const ArenaTreeNode* node) { + return node->value_; +} + +template +class ArenaTreeNodeCrtp : public ArenaTreeNodeBase { + public: + using ArenaTreeNodeBase::ArenaTreeNodeBase; +}; + +template +[[nodiscard]] inline const T& ArenaTreeNodeGetValue( + const ArenaTreeNodeCrtp* node) { + return *static_cast(node); +} + +template +[[nodiscard]] +T* ArenaTreeInsert(T** head, T* elem, const Compare& compare) { + T* tmp = *head; + T* parent = nullptr; + int diff = 0; + while (tmp != nullptr) { + parent = tmp; + diff = std::invoke(compare, (ArenaTreeNodeGetValue)(elem), + (ArenaTreeNodeGetValue)(parent)); + if (diff < 0) { + tmp = static_cast((ArenaTreeNodeGetLeft)(tmp)); + } else if (diff > 0) { + tmp = static_cast((ArenaTreeNodeGetRight)(tmp)); + } else { + return tmp; + } + } + (ArenaTreeNodeSet)(elem, parent); + if (parent != nullptr) { + if (diff < 0) { + (ArenaTreeNodeSetLeft)(parent, elem); + } else { + (ArenaTreeNodeSetRight)(parent, elem); + } + } else { + *head = elem; + } + ArenaTreeNodeBase* head_base = *head; + (ArenaTreeInsertColor)(&head_base, elem); + *head = static_cast(head_base); + return elem; +} + +template +struct ArenaTreeNodeConstructor { + template + void operator()(Args&&... args) const { + ABSL_DCHECK(*out == nullptr); + *out = google::protobuf::Arena::Create(arena, std::forward(args)...); + } + + google::protobuf::Arena* const arena; + T** out; +}; + +template +[[nodiscard]] +std::pair ArenaTreeLazyEmplace(google::protobuf::Arena* arena, T** head, + const K& key, const Compare& compare, + Emplacer&& emplacer) { + T* tmp = *head; + T* parent = nullptr; + int diff = 0; + while (tmp != nullptr) { + parent = tmp; + diff = std::invoke(compare, key, (ArenaTreeNodeGetValue)(parent)); + if (diff < 0) { + tmp = static_cast((ArenaTreeNodeGetLeft)(tmp)); + } else if (diff > 0) { + tmp = static_cast((ArenaTreeNodeGetRight)(tmp)); + } else { + return {tmp, false}; + } + } + T* elem = nullptr; + ArenaTreeNodeConstructor constructor{ + .arena = arena, + .out = &elem, + }; + std::invoke(std::forward(emplacer), + static_cast&>(constructor)); + ABSL_DCHECK(elem != nullptr); + (ArenaTreeNodeSet)(elem, parent); + if (parent != nullptr) { + if (diff < 0) { + (ArenaTreeNodeSetLeft)(parent, elem); + } else { + (ArenaTreeNodeSetRight)(parent, elem); + } + } else { + *head = elem; + } + ArenaTreeNodeBase* head_base = *head; + (ArenaTreeInsertColor)(&head_base, elem); + *head = static_cast(head_base); + return {elem, true}; +} + +template +[[nodiscard]] +const T* ArenaTreeFind(const T* head, const K& key, const Compare& compare) { + const T* tmp = head; + while (tmp != nullptr) { + int diff = std::invoke(compare, key, (ArenaTreeNodeGetValue)(tmp)); + if (diff < 0) { + tmp = static_cast((ArenaTreeNodeGetLeft)(tmp)); + } else if (diff > 0) { + tmp = static_cast((ArenaTreeNodeGetRight)(tmp)); + } else { + return tmp; + } + } + return nullptr; +} + +template +[[nodiscard]] +T* ArenaTreeFind(T* head, const K& key, const Compare& compare) { + return const_cast( + (ArenaTreeFind)(static_cast(head), key, compare)); +} + +} // namespace cel::internal + +#endif // THIRD_PARTY_CEL_CPP_INTERNAL_ARENA_TREE_H_ diff --git a/internal/arena_tree_test.cc b/internal/arena_tree_test.cc new file mode 100644 index 000000000..9f6dfdbf7 --- /dev/null +++ b/internal/arena_tree_test.cc @@ -0,0 +1,627 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "internal/arena_tree.h" + +#include +#include +#include +#include +#include + +#include "internal/testing.h" +#include "google/protobuf/arena.h" + +namespace cel::internal { +namespace { + +using ::testing::ElementsAre; +using ::testing::IsEmpty; +using ::testing::IsNull; + +using TestNode = ArenaTreeNode; + +struct TestNodeCompare { + int operator()(int lhs, int rhs) const { + if (lhs < rhs) { + return -1; + } + if (lhs > rhs) { + return 1; + } + return 0; + } +}; + +struct TestNodeCrtp : ArenaTreeNodeCrtp { + explicit TestNodeCrtp(int value) : ArenaTreeNodeCrtp(), value(value) {} + + int value; +}; + +struct TestNodeCrtpCompare { + int operator()(int lhs, int rhs) const { + if (lhs < rhs) { + return -1; + } + if (lhs > rhs) { + return 1; + } + return 0; + } + + int operator()(int lhs, const TestNodeCrtp& rhs) const { + return (*this)(lhs, rhs.value); + } + + int operator()(const TestNodeCrtp& lhs, int rhs) const { + return (*this)(lhs.value, rhs); + } + + int operator()(const TestNodeCrtp& lhs, const TestNodeCrtp& rhs) const { + return (*this)(lhs.value, rhs.value); + } +}; + +template +int VerifySubtree(const T* node, const T* expected_parent, const T* lower_bound, + const T* upper_bound, const Compare& compare) { + if (node == nullptr) { + return 1; + } + EXPECT_EQ(ArenaTreeNodeGetParent(node), expected_parent); + if (lower_bound != nullptr) { + EXPECT_LT(compare(ArenaTreeNodeGetValue(lower_bound), + ArenaTreeNodeGetValue(node)), + 0); + } + if (upper_bound != nullptr) { + EXPECT_LT(compare(ArenaTreeNodeGetValue(node), + ArenaTreeNodeGetValue(upper_bound)), + 0); + } + const auto* left = static_cast(ArenaTreeNodeGetLeft(node)); + const auto* right = static_cast(ArenaTreeNodeGetRight(node)); + if (ArenaTreeNodeGetColor(node) == ArenaTreeNodeColor::kRed) { + if (left != nullptr) { + EXPECT_EQ(ArenaTreeNodeGetColor(left), ArenaTreeNodeColor::kBlack); + } + if (right != nullptr) { + EXPECT_EQ(ArenaTreeNodeGetColor(right), ArenaTreeNodeColor::kBlack); + } + } + int left_black_height = VerifySubtree(left, node, lower_bound, node, compare); + int right_black_height = + VerifySubtree(right, node, node, upper_bound, compare); + EXPECT_EQ(left_black_height, right_black_height); + return left_black_height + + (ArenaTreeNodeGetColor(node) == ArenaTreeNodeColor::kBlack ? 1 : 0); +} + +template +std::vector VerifyTree(T* head, const Compare& compare) { + if (head == nullptr) { + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + return {}; + } + EXPECT_EQ(ArenaTreeNodeGetColor(head), ArenaTreeNodeColor::kBlack); + VerifySubtree(static_cast(head), static_cast(nullptr), + static_cast(nullptr), static_cast(nullptr), + compare); + + std::vector forward; + for (const T* curr = ArenaTreeMin(static_cast(head)); + curr != nullptr; curr = static_cast(ArenaTreeNext(curr))) { + forward.push_back(curr); + EXPECT_EQ(ArenaTreeFind(static_cast(head), + ArenaTreeNodeGetValue(curr), compare), + curr); + EXPECT_EQ(ArenaTreeFind(head, ArenaTreeNodeGetValue(curr), compare), curr); + } + EXPECT_FALSE(forward.empty()); + EXPECT_EQ(forward.front(), ArenaTreeMin(head)); + EXPECT_EQ(forward.back(), ArenaTreeMax(head)); + + std::vector backward; + for (T* curr = ArenaTreeMax(head); curr != nullptr; + curr = ArenaTreePrev(curr)) { + backward.push_back(curr); + } + std::reverse(backward.begin(), backward.end()); + EXPECT_EQ(forward.size(), backward.size()); + for (size_t i = 0; i < forward.size(); ++i) { + EXPECT_EQ(forward[i], backward[i]); + } + + std::vector values; + values.reserve(forward.size()); + for (const T* node : forward) { + if constexpr (std::is_same_v) { + values.push_back(ArenaTreeNodeGetValue(node)); + } else { + values.push_back(ArenaTreeNodeGetValue(node).value); + } + } + return values; +} + +TEST(ArenaTree, Empty) { + TestNode* head = nullptr; + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); +} + +TEST(ArenaTree, Single) { + TestNode* head = nullptr; + auto node1 = std::make_unique(1); + EXPECT_EQ(ArenaTreeInsert(&head, node1.get(), TestNodeCompare{}), + node1.get()); + EXPECT_EQ(ArenaTreePrev(head), nullptr); + EXPECT_EQ(ArenaTreeNext(head), nullptr); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node1.get()); + EXPECT_EQ(ArenaTreeFind(head, 1, TestNodeCompare{}), node1.get()); + ArenaTreeRemove(&head, node1.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); + EXPECT_THAT(ArenaTreeFind(head, 1, TestNodeCompare{}), IsNull()); +} + +TEST(ArenaTree, Couple) { + TestNode* head = nullptr; + auto node1 = std::make_unique(1); + auto node2 = std::make_unique(2); + + EXPECT_EQ(ArenaTreeInsert(&head, node1.get(), TestNodeCompare{}), + node1.get()); + EXPECT_EQ(ArenaTreeInsert(&head, node2.get(), TestNodeCompare{}), + node2.get()); + EXPECT_EQ(ArenaTreePrev(node1.get()), nullptr); + EXPECT_EQ(ArenaTreePrev(node2.get()), node1.get()); + EXPECT_EQ(ArenaTreeNext(node1.get()), node2.get()); + EXPECT_EQ(ArenaTreeNext(node2.get()), nullptr); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node2.get()); + + ArenaTreeRemove(&head, node1.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_EQ(ArenaTreeMin(head), node2.get()); + EXPECT_EQ(ArenaTreeMax(head), node2.get()); + + ArenaTreeRemove(&head, node2.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); + + EXPECT_EQ(ArenaTreeInsert(&head, node2.get(), TestNodeCompare{}), + node2.get()); + EXPECT_EQ(ArenaTreeInsert(&head, node1.get(), TestNodeCompare{}), + node1.get()); + EXPECT_EQ(ArenaTreePrev(node1.get()), nullptr); + EXPECT_EQ(ArenaTreePrev(node2.get()), node1.get()); + EXPECT_EQ(ArenaTreeNext(node1.get()), node2.get()); + EXPECT_EQ(ArenaTreeNext(node2.get()), nullptr); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node2.get()); + + ArenaTreeRemove(&head, node2.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node1.get()); + + ArenaTreeRemove(&head, node1.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); +} + +TEST(ArenaTree, CrtpEmpty) { + TestNodeCrtp* head = nullptr; + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); +} + +TEST(ArenaTree, CrtpSingle) { + TestNodeCrtp* head = nullptr; + auto node1 = std::make_unique(1); + EXPECT_EQ(ArenaTreeInsert(&head, node1.get(), TestNodeCrtpCompare{}), + node1.get()); + EXPECT_EQ(ArenaTreePrev(head), nullptr); + EXPECT_EQ(ArenaTreeNext(head), nullptr); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node1.get()); + EXPECT_EQ(ArenaTreeFind(head, 1, TestNodeCrtpCompare{}), node1.get()); + ArenaTreeRemove(&head, node1.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); + EXPECT_THAT(ArenaTreeFind(head, 1, TestNodeCrtpCompare{}), IsNull()); +} + +TEST(ArenaTree, CrtpCouple) { + TestNodeCrtp* head = nullptr; + auto node1 = std::make_unique(1); + auto node2 = std::make_unique(2); + + EXPECT_EQ(ArenaTreeInsert(&head, node1.get(), TestNodeCrtpCompare{}), + node1.get()); + EXPECT_EQ(ArenaTreeInsert(&head, node2.get(), TestNodeCrtpCompare{}), + node2.get()); + EXPECT_EQ(ArenaTreePrev(node1.get()), nullptr); + EXPECT_EQ(ArenaTreePrev(node2.get()), node1.get()); + EXPECT_EQ(ArenaTreeNext(node1.get()), node2.get()); + EXPECT_EQ(ArenaTreeNext(node2.get()), nullptr); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node2.get()); + + ArenaTreeRemove(&head, node1.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_EQ(ArenaTreeMin(head), node2.get()); + EXPECT_EQ(ArenaTreeMax(head), node2.get()); + + ArenaTreeRemove(&head, node2.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); + + EXPECT_EQ(ArenaTreeInsert(&head, node2.get(), TestNodeCrtpCompare{}), + node2.get()); + EXPECT_EQ(ArenaTreeInsert(&head, node1.get(), TestNodeCrtpCompare{}), + node1.get()); + EXPECT_EQ(ArenaTreePrev(node1.get()), nullptr); + EXPECT_EQ(ArenaTreePrev(node2.get()), node1.get()); + EXPECT_EQ(ArenaTreeNext(node1.get()), node2.get()); + EXPECT_EQ(ArenaTreeNext(node2.get()), nullptr); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node2.get()); + + ArenaTreeRemove(&head, node2.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node1.get()); + + ArenaTreeRemove(&head, node1.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); +} + +TEST(ArenaTree, InsertDuplicateReturnsExistingNode) { + TestNode* head = nullptr; + auto node2 = std::make_unique(2); + auto node1 = std::make_unique(1); + auto node3 = std::make_unique(3); + auto dup2 = std::make_unique(2); + auto dup1 = std::make_unique(1); + auto dup3 = std::make_unique(3); + + EXPECT_EQ(ArenaTreeInsert(&head, node2.get(), TestNodeCompare{}), + node2.get()); + EXPECT_EQ(ArenaTreeInsert(&head, node1.get(), TestNodeCompare{}), + node1.get()); + EXPECT_EQ(ArenaTreeInsert(&head, node3.get(), TestNodeCompare{}), + node3.get()); + + EXPECT_EQ(ArenaTreeInsert(&head, dup2.get(), TestNodeCompare{}), node2.get()); + EXPECT_EQ(ArenaTreeInsert(&head, dup1.get(), TestNodeCompare{}), node1.get()); + EXPECT_EQ(ArenaTreeInsert(&head, dup3.get(), TestNodeCompare{}), node3.get()); + EXPECT_THAT(VerifyTree(head, TestNodeCompare{}), ElementsAre(1, 2, 3)); +} + +TEST(ArenaTree, LazyEmplace) { + google::protobuf::Arena arena; + TestNode* head = nullptr; + + int emplace_calls = 0; + auto emplace_val = [&](int key) { + return ArenaTreeLazyEmplace(&arena, &head, key, TestNodeCompare{}, + [&](const auto& ctor) { + ++emplace_calls; + ctor(key); + }); + }; + + auto [node2, inserted2] = emplace_val(2); + EXPECT_TRUE(inserted2); + EXPECT_EQ(ArenaTreeNodeGetValue(node2), 2); + EXPECT_EQ(emplace_calls, 1); + + auto [node1, inserted1] = emplace_val(1); + EXPECT_TRUE(inserted1); + EXPECT_EQ(ArenaTreeNodeGetValue(node1), 1); + EXPECT_EQ(emplace_calls, 2); + + auto [node3, inserted3] = emplace_val(3); + EXPECT_TRUE(inserted3); + EXPECT_EQ(ArenaTreeNodeGetValue(node3), 3); + EXPECT_EQ(emplace_calls, 3); + + // Duplicate keys should not invoke the emplacer. + auto [dup2, dup_inserted2] = emplace_val(2); + EXPECT_FALSE(dup_inserted2); + EXPECT_EQ(dup2, node2); + EXPECT_EQ(emplace_calls, 3); + + auto [dup1, dup_inserted1] = emplace_val(1); + EXPECT_FALSE(dup_inserted1); + EXPECT_EQ(dup1, node1); + EXPECT_EQ(emplace_calls, 3); + + auto [dup3, dup_inserted3] = emplace_val(3); + EXPECT_FALSE(dup_inserted3); + EXPECT_EQ(dup3, node3); + EXPECT_EQ(emplace_calls, 3); + + EXPECT_THAT(VerifyTree(head, TestNodeCompare{}), ElementsAre(1, 2, 3)); +} + +TEST(ArenaTree, CrtpLazyEmplace) { + google::protobuf::Arena arena; + TestNodeCrtp* head = nullptr; + + int emplace_calls = 0; + auto emplace_val = [&](int key) { + return ArenaTreeLazyEmplace(&arena, &head, key, TestNodeCrtpCompare{}, + [&](const auto& ctor) { + ++emplace_calls; + ctor(key); + }); + }; + + auto [node2, inserted2] = emplace_val(2); + EXPECT_TRUE(inserted2); + EXPECT_EQ(ArenaTreeNodeGetValue(node2).value, 2); + EXPECT_EQ(emplace_calls, 1); + + auto [node1, inserted1] = emplace_val(1); + EXPECT_TRUE(inserted1); + EXPECT_EQ(ArenaTreeNodeGetValue(node1).value, 1); + EXPECT_EQ(emplace_calls, 2); + + auto [node3, inserted3] = emplace_val(3); + EXPECT_TRUE(inserted3); + EXPECT_EQ(ArenaTreeNodeGetValue(node3).value, 3); + EXPECT_EQ(emplace_calls, 3); + + auto [dup2, dup_inserted2] = emplace_val(2); + EXPECT_FALSE(dup_inserted2); + EXPECT_EQ(dup2, node2); + EXPECT_EQ(emplace_calls, 3); + + EXPECT_THAT(VerifyTree(head, TestNodeCrtpCompare{}), ElementsAre(1, 2, 3)); +} + +TEST(ArenaTree, FindLeftRightAndMissing) { + google::protobuf::Arena arena; + TestNode* head = nullptr; + for (int key : {8, 4, 12, 2, 6, 10, 14}) { + auto [node, inserted] = + ArenaTreeLazyEmplace(&arena, &head, key, TestNodeCompare{}, + [key](const auto& ctor) { ctor(key); }); + EXPECT_TRUE(inserted); + } + + const TestNode* const_head = head; + for (int key : {2, 4, 6, 8, 10, 12, 14}) { + TestNode* found = ArenaTreeFind(head, key, TestNodeCompare{}); + ASSERT_NE(found, nullptr); + EXPECT_EQ(ArenaTreeNodeGetValue(found), key); + EXPECT_EQ(ArenaTreeFind(const_head, key, TestNodeCompare{}), found); + } + for (int missing : {1, 3, 5, 7, 9, 11, 13, 15}) { + EXPECT_THAT(ArenaTreeFind(head, missing, TestNodeCompare{}), IsNull()); + EXPECT_THAT(ArenaTreeFind(const_head, missing, TestNodeCompare{}), + IsNull()); + } +} + +TEST(ArenaTree, InsertAndRemoveAscendingAndDescending) { + constexpr int kCount = 64; + std::vector> nodes; + nodes.reserve(kCount); + for (int i = 0; i < kCount; ++i) { + nodes.push_back(std::make_unique(i)); + } + + // Insert ascending, remove ascending. + TestNode* head = nullptr; + for (int i = 0; i < kCount; ++i) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[i].get(), TestNodeCompare{}), + nodes[i].get()); + VerifyTree(head, TestNodeCompare{}); + } + for (int i = 0; i < kCount; ++i) { + ArenaTreeRemove(&head, nodes[i].get()); + VerifyTree(head, TestNodeCompare{}); + } + EXPECT_THAT(head, IsNull()); + + // Insert descending, remove descending. + for (int i = kCount - 1; i >= 0; --i) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[i].get(), TestNodeCompare{}), + nodes[i].get()); + VerifyTree(head, TestNodeCompare{}); + } + for (int i = kCount - 1; i >= 0; --i) { + ArenaTreeRemove(&head, nodes[i].get()); + VerifyTree(head, TestNodeCompare{}); + } + EXPECT_THAT(head, IsNull()); + + // Insert ascending, remove descending. + for (int i = 0; i < kCount; ++i) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[i].get(), TestNodeCompare{}), + nodes[i].get()); + } + VerifyTree(head, TestNodeCompare{}); + for (int i = kCount - 1; i >= 0; --i) { + ArenaTreeRemove(&head, nodes[i].get()); + VerifyTree(head, TestNodeCompare{}); + } + EXPECT_THAT(head, IsNull()); + + // Insert descending, remove ascending. + for (int i = kCount - 1; i >= 0; --i) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[i].get(), TestNodeCompare{}), + nodes[i].get()); + } + VerifyTree(head, TestNodeCompare{}); + for (int i = 0; i < kCount; ++i) { + ArenaTreeRemove(&head, nodes[i].get()); + VerifyTree(head, TestNodeCompare{}); + } + EXPECT_THAT(head, IsNull()); +} + +TEST(ArenaTree, InsertZigZagAndRemoveInternalNodes) { + constexpr int kCount = 63; + std::vector> nodes; + nodes.reserve(kCount); + for (int i = 0; i < kCount; ++i) { + nodes.push_back(std::make_unique(i)); + } + + // Insert in a zig-zag pattern (alternating low and high values) to exercise + // Left-Right and Right-Left triangle rotations in ArenaTreeInsertColor. + TestNode* head = nullptr; + int lo = 0; + int hi = kCount - 1; + while (lo <= hi) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[lo].get(), TestNodeCompare{}), + nodes[lo].get()); + VerifyTree(head, TestNodeCompare{}); + if (lo < hi) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[hi].get(), TestNodeCompare{}), + nodes[hi].get()); + VerifyTree(head, TestNodeCompare{}); + } + ++lo; + --hi; + } + + // Repeatedly remove left child of root, right child of root, and root itself + // to cover 2-child removals where old_parent is null, old is a left child, + // and old is a right child. + int step = 0; + while (head != nullptr) { + TestNode* target = head; + if (step % 3 == 0 && ArenaTreeNodeGetLeft(head) != nullptr) { + target = static_cast(ArenaTreeNodeGetLeft(head)); + } else if (step % 3 == 1 && ArenaTreeNodeGetRight(head) != nullptr) { + target = static_cast(ArenaTreeNodeGetRight(head)); + } + ArenaTreeRemove(&head, target); + VerifyTree(head, TestNodeCompare{}); + ++step; + } +} + +TEST(ArenaTree, RemoveSingleLeftAndSingleRightChildCases) { + // Specifically exercise ArenaTreeRemove when the removed node has only a left + // child or only a right child (as root, left child of parent, and right child + // of parent). + std::vector> nodes; + for (int i = 0; i < 10; ++i) { + nodes.push_back(std::make_unique(i)); + } + + // Case 1: Node with only a left child as left child of parent and right child + // of parent. + TestNode* head = nullptr; + for (int idx : {5, 2, 8, 1, 7}) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[idx].get(), TestNodeCompare{}), + nodes[idx].get()); + } + // Node 2 has only left child 1; Node 8 has only left child 7. + ArenaTreeRemove(&head, nodes[2].get()); + VerifyTree(head, TestNodeCompare{}); + ArenaTreeRemove(&head, nodes[8].get()); + VerifyTree(head, TestNodeCompare{}); + while (head != nullptr) { + ArenaTreeRemove(&head, head); + } + + // Case 2: Node with only a right child as left child of parent and right + // child of parent. + for (int idx : {5, 2, 8, 3, 9}) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[idx].get(), TestNodeCompare{}), + nodes[idx].get()); + } + // Node 2 has only right child 3; Node 8 has only right child 9. + ArenaTreeRemove(&head, nodes[2].get()); + VerifyTree(head, TestNodeCompare{}); + ArenaTreeRemove(&head, nodes[8].get()); + VerifyTree(head, TestNodeCompare{}); + while (head != nullptr) { + ArenaTreeRemove(&head, head); + } +} + +TEST(ArenaTree, AllPermutationsInsertAndRemove) { + constexpr int kPermSize = 6; + std::vector> nodes; + nodes.reserve(kPermSize); + std::vector insert_order(kPermSize); + for (int i = 0; i < kPermSize; ++i) { + nodes.push_back(std::make_unique(i)); + insert_order[i] = i; + } + + do { + TestNode* head = nullptr; + for (int idx : insert_order) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[idx].get(), TestNodeCompare{}), + nodes[idx].get()); + } + EXPECT_THAT(VerifyTree(head, TestNodeCompare{}), + ElementsAre(0, 1, 2, 3, 4, 5)); + + // Remove in insertion order. + for (int idx : insert_order) { + ArenaTreeRemove(&head, nodes[idx].get()); + } + EXPECT_THAT(VerifyTree(head, TestNodeCompare{}), IsEmpty()); + + // Re-insert and remove in reverse insertion order. + for (int idx : insert_order) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[idx].get(), TestNodeCompare{}), + nodes[idx].get()); + } + for (int i = kPermSize - 1; i >= 0; --i) { + ArenaTreeRemove(&head, nodes[insert_order[i]].get()); + } + EXPECT_THAT(VerifyTree(head, TestNodeCompare{}), IsEmpty()); + } while (std::next_permutation(insert_order.begin(), insert_order.end())); +} + +} // namespace +} // namespace cel::internal diff --git a/internal/empty_descriptors.cc b/internal/empty_descriptors.cc index 05e3843a5..43fe90006 100644 --- a/internal/empty_descriptors.cc +++ b/internal/empty_descriptors.cc @@ -15,6 +15,7 @@ #include "internal/empty_descriptors.h" #include +#include #include "google/protobuf/descriptor.pb.h" #include "absl/base/attributes.h" @@ -39,7 +40,7 @@ const google::protobuf::DescriptorPool* absl_nonnull GetEmptyDescriptorPool() { static const google::protobuf::DescriptorPool* absl_nonnull const pool = []() { google::protobuf::FileDescriptorSet file_desc_set; ABSL_CHECK(file_desc_set.ParseFromArray( // Crash OK - kEmptyDescriptorSet, ABSL_ARRAYSIZE(kEmptyDescriptorSet))); + kEmptyDescriptorSet, std::size(kEmptyDescriptorSet))); auto* pool = new google::protobuf::DescriptorPool(); for (const auto& file_desc : file_desc_set.file()) { ABSL_CHECK(pool->BuildFile(file_desc) != nullptr); // Crash OK diff --git a/internal/json.cc b/internal/json.cc index 630ceb267..cdd4c1a5d 100644 --- a/internal/json.cc +++ b/internal/json.cc @@ -1417,7 +1417,7 @@ class JsonMapIterator final { } private: - absl::variant variant_; + std::variant variant_; }; class JsonAccessor { diff --git a/internal/json.h b/internal/json.h index d32c42741..e35909d0e 100644 --- a/internal/json.h +++ b/internal/json.h @@ -26,7 +26,7 @@ namespace cel::internal { // Converts the given message to its `google.protobuf.Value` equivalent -// representation. This is similar to `proto2::json::MessageToJsonString()`, +// representation. This is similar to `google::protobuf::json::MessageToJsonString()`, // except that this results in structured serialization. absl::Status MessageToJson( const google::protobuf::Message& message, @@ -45,7 +45,7 @@ absl::Status MessageToJson( google::protobuf::Message* absl_nonnull result); // Converts the given message field to its `google.protobuf.Value` equivalent -// representation. This is similar to `proto2::json::MessageToJsonString()`, +// representation. This is similar to `google::protobuf::json::MessageToJsonString()`, // except that this results in structured serialization. absl::Status MessageFieldToJson( const google::protobuf::Message& message, diff --git a/internal/message_equality.cc b/internal/message_equality.cc index 945cca8df..bfa12883c 100644 --- a/internal/message_equality.cc +++ b/internal/message_equality.cc @@ -33,7 +33,6 @@ #include "absl/strings/string_view.h" #include "absl/time/time.h" #include "absl/types/variant.h" -#include "common/memory.h" #include "extensions/protobuf/internal/map_reflection.h" #include "internal/json.h" #include "internal/number.h" @@ -86,10 +85,10 @@ class EquatableMessage final }; using EquatableValue = - absl::variant; + std::variant; struct NullValueEqualer { bool operator()(std::nullptr_t, std::nullptr_t) const { return true; } @@ -380,11 +379,11 @@ absl::StatusOr AsEquatableValue( case Descriptor::WELLKNOWNTYPE_DURATION: CEL_RETURN_IF_ERROR( reflection.duration_reflection.Initialize(descriptor)); - return reflection.duration_reflection.ToAbslDuration(message); + return reflection.duration_reflection.UnsafeToAbslDuration(message); case Descriptor::WELLKNOWNTYPE_TIMESTAMP: CEL_RETURN_IF_ERROR( reflection.timestamp_reflection.Initialize(descriptor)); - return reflection.timestamp_reflection.ToAbslTime(message); + return reflection.timestamp_reflection.UnsafeToAbslTime(message); case Descriptor::WELLKNOWNTYPE_ANY: return EquatableAny(message); default: @@ -462,7 +461,7 @@ absl::StatusOr MapValueAsEquatableValue( MessageFactory* absl_nonnull factory, EquatableValueReflection& reflection, const google::protobuf::MapValueConstRef& value, const FieldDescriptor* absl_nonnull field, std::string& scratch, - Unique& unpacked) { + google::protobuf::Arena::UniquePtr& unpacked) { if (IsAnyField(field)) { CEL_ASSIGN_OR_RETURN(unpacked, well_known_types::UnpackAnyIfResolveable( arena, reflection.any_reflection, @@ -516,7 +515,8 @@ absl::StatusOr RepeatedFieldAsEquatableValue( google::protobuf::Arena* absl_nonnull arena, const DescriptorPool* absl_nonnull pool, MessageFactory* absl_nonnull factory, EquatableValueReflection& reflection, const Message& message, const FieldDescriptor* absl_nonnull field, - int index, std::string& scratch, Unique& unpacked) { + int index, std::string& scratch, + google::protobuf::Arena::UniquePtr& unpacked) { if (IsAnyField(field)) { const auto& field_value = message.GetReflection()->GetRepeatedMessage(message, field, index); @@ -834,8 +834,8 @@ class MessageEqualsState final { auto rhs_well_known_type = rhs_descriptor->well_known_type(); const Message* absl_nonnull lhs_ptr = &lhs; const Message* absl_nonnull rhs_ptr = &rhs; - Unique lhs_unpacked; - Unique rhs_unpacked; + google::protobuf::Arena::UniquePtr lhs_unpacked; + google::protobuf::Arena::UniquePtr rhs_unpacked; // Deal with any first. We could in theory check if we should bother // unpacking, but that is more complicated. We can always implement it // later. @@ -845,7 +845,7 @@ class MessageEqualsState final { well_known_types::UnpackAnyIfResolveable( &arena_, lhs_reflection_.any_reflection, lhs, pool_, factory_)); if (lhs_unpacked) { - lhs_ptr = cel::to_address(lhs_unpacked); + lhs_ptr = lhs_unpacked.get(); lhs_descriptor = lhs_ptr->GetDescriptor(); lhs_well_known_type = lhs_descriptor->well_known_type(); } @@ -856,7 +856,7 @@ class MessageEqualsState final { well_known_types::UnpackAnyIfResolveable( &arena_, rhs_reflection_.any_reflection, rhs, pool_, factory_)); if (rhs_unpacked) { - rhs_ptr = cel::to_address(rhs_unpacked); + rhs_ptr = rhs_unpacked.get(); rhs_descriptor = rhs_ptr->GetDescriptor(); rhs_well_known_type = rhs_descriptor->well_known_type(); } @@ -906,9 +906,9 @@ class MessageEqualsState final { } auto lhs_begin = ConstMapBegin(*lhs_reflection, lhs, *lhs_field); const auto lhs_end = ConstMapEnd(*lhs_reflection, lhs, *lhs_field); - Unique lhs_unpacked; + google::protobuf::Arena::UniquePtr lhs_unpacked; EquatableValue lhs_value; - Unique rhs_unpacked; + google::protobuf::Arena::UniquePtr rhs_unpacked; EquatableValue rhs_value; google::protobuf::MapKey rhs_map_key; google::protobuf::MapValueConstRef rhs_map_value; @@ -960,9 +960,9 @@ class MessageEqualsState final { if (size != rhs_reflection->FieldSize(rhs, rhs_field)) { return false; } - Unique lhs_unpacked; + google::protobuf::Arena::UniquePtr lhs_unpacked; EquatableValue lhs_value; - Unique rhs_unpacked; + google::protobuf::Arena::UniquePtr rhs_unpacked; EquatableValue rhs_value; for (int i = 0; i < size; ++i) { CEL_ASSIGN_OR_RETURN(lhs_value, @@ -1007,8 +1007,8 @@ class MessageEqualsState final { } const Message* absl_nonnull lhs_ptr = &lhs; const Message* absl_nonnull rhs_ptr = &rhs; - Unique lhs_unpacked; - Unique rhs_unpacked; + google::protobuf::Arena::UniquePtr lhs_unpacked; + google::protobuf::Arena::UniquePtr rhs_unpacked; if (lhs_field != nullptr && IsAnyField(lhs_field)) { CEL_ASSIGN_OR_RETURN(lhs_unpacked, well_known_types::UnpackAnyIfResolveable( @@ -1016,7 +1016,7 @@ class MessageEqualsState final { lhs.GetReflection()->GetMessage(lhs, lhs_field), pool_, factory_)); if (lhs_unpacked) { - lhs_ptr = cel::to_address(lhs_unpacked); + lhs_ptr = lhs_unpacked.get(); lhs_field = nullptr; } } else if (lhs_field == nullptr && IsAny(lhs)) { @@ -1025,7 +1025,7 @@ class MessageEqualsState final { well_known_types::UnpackAnyIfResolveable( &arena_, lhs_reflection_.any_reflection, lhs, pool_, factory_)); if (lhs_unpacked) { - lhs_ptr = cel::to_address(lhs_unpacked); + lhs_ptr = lhs_unpacked.get(); } } if (rhs_field != nullptr && IsAnyField(rhs_field)) { @@ -1035,7 +1035,7 @@ class MessageEqualsState final { rhs.GetReflection()->GetMessage(rhs, rhs_field), pool_, factory_)); if (rhs_unpacked) { - rhs_ptr = cel::to_address(rhs_unpacked); + rhs_ptr = rhs_unpacked.get(); rhs_field = nullptr; } } else if (rhs_field == nullptr && IsAny(rhs)) { @@ -1044,7 +1044,7 @@ class MessageEqualsState final { well_known_types::UnpackAnyIfResolveable( &arena_, rhs_reflection_.any_reflection, rhs, pool_, factory_)); if (rhs_unpacked) { - rhs_ptr = cel::to_address(rhs_unpacked); + rhs_ptr = rhs_unpacked.get(); } } EquatableValue lhs_value; @@ -1093,7 +1093,7 @@ class MessageEqualsState final { return false; } const Message* absl_nullable rhs_packed = nullptr; - Unique rhs_unpacked; + google::protobuf::Arena::UniquePtr rhs_unpacked; if (rhs_field != nullptr && IsAnyField(rhs_field)) { rhs_packed = &rhs.GetReflection()->GetMessage(rhs, rhs_field); } else if (rhs_field == nullptr && IsAny(rhs)) { @@ -1124,8 +1124,8 @@ class MessageEqualsState final { const Message* absl_nonnull rhs_message = rhs_field != nullptr ? &rhs.GetReflection()->GetMessage(rhs, rhs_field) - : rhs_unpacked != nullptr ? cel::to_address(rhs_unpacked) - : &rhs; + : rhs_unpacked.get() != nullptr ? rhs_unpacked.get() + : &rhs; const auto* rhs_descriptor = rhs_message->GetDescriptor(); const auto rhs_well_known_type = rhs_descriptor->well_known_type(); switch (rhs_well_known_type) { @@ -1174,7 +1174,7 @@ class MessageEqualsState final { return false; } const Message* absl_nullable lhs_packed = nullptr; - Unique lhs_unpacked; + google::protobuf::Arena::UniquePtr lhs_unpacked; if (lhs_field != nullptr && IsAnyField(lhs_field)) { lhs_packed = &lhs.GetReflection()->GetMessage(lhs, lhs_field); } else if (lhs_field == nullptr && IsAny(lhs)) { @@ -1205,8 +1205,8 @@ class MessageEqualsState final { const Message* absl_nonnull lhs_message = lhs_field != nullptr ? &lhs.GetReflection()->GetMessage(lhs, lhs_field) - : lhs_unpacked != nullptr ? cel::to_address(lhs_unpacked) - : &lhs; + : lhs_unpacked.get() != nullptr ? lhs_unpacked.get() + : &lhs; const auto* lhs_descriptor = lhs_message->GetDescriptor(); const auto lhs_well_known_type = lhs_descriptor->well_known_type(); switch (lhs_well_known_type) { @@ -1262,7 +1262,7 @@ class MessageEqualsState final { return false; } const Message* absl_nullable rhs_packed = nullptr; - Unique rhs_unpacked; + google::protobuf::Arena::UniquePtr rhs_unpacked; if (rhs_field != nullptr && IsAnyField(rhs_field)) { rhs_packed = &rhs.GetReflection()->GetMessage(rhs, rhs_field); } else if (rhs_field == nullptr && IsAny(rhs)) { @@ -1293,8 +1293,8 @@ class MessageEqualsState final { const Message* absl_nonnull rhs_message = rhs_field != nullptr ? &rhs.GetReflection()->GetMessage(rhs, rhs_field) - : rhs_unpacked != nullptr ? cel::to_address(rhs_unpacked) - : &rhs; + : rhs_unpacked.get() != nullptr ? rhs_unpacked.get() + : &rhs; const auto* rhs_descriptor = rhs_message->GetDescriptor(); const auto rhs_well_known_type = rhs_descriptor->well_known_type(); switch (rhs_well_known_type) { @@ -1342,7 +1342,7 @@ class MessageEqualsState final { return false; } const Message* absl_nullable lhs_packed = nullptr; - Unique lhs_unpacked; + google::protobuf::Arena::UniquePtr lhs_unpacked; if (lhs_field != nullptr && IsAnyField(lhs_field)) { lhs_packed = &lhs.GetReflection()->GetMessage(lhs, lhs_field); } else if (lhs_field == nullptr && IsAny(lhs)) { @@ -1373,8 +1373,8 @@ class MessageEqualsState final { const Message* absl_nonnull lhs_message = lhs_field != nullptr ? &lhs.GetReflection()->GetMessage(lhs, lhs_field) - : lhs_unpacked != nullptr ? cel::to_address(lhs_unpacked) - : &lhs; + : lhs_unpacked.get() != nullptr ? lhs_unpacked.get() + : &lhs; const auto* lhs_descriptor = lhs_message->GetDescriptor(); const auto lhs_well_known_type = lhs_descriptor->well_known_type(); switch (lhs_well_known_type) { diff --git a/internal/message_equality_test.cc b/internal/message_equality_test.cc index 318138d9b..c08116181 100644 --- a/internal/message_equality_test.cc +++ b/internal/message_equality_test.cc @@ -14,6 +14,7 @@ #include "internal/message_equality.h" +#include #include #include #include @@ -27,12 +28,12 @@ #include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/log/die_if_null.h" +#include "absl/memory/memory.h" #include "absl/status/status_matchers.h" #include "absl/strings/cord.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/allocator.h" #include "common/memory.h" #include "internal/message_type_name.h" #include "internal/parse_text_proto.h" @@ -394,18 +395,18 @@ void PackMessageTo(const google::protobuf::Message& message, google::protobuf::M reflection.SetValue(instance, value); } -absl::optional, +absl::optional, const google::protobuf::FieldDescriptor* absl_nonnull>> PackTestAllTypesProto3Field(const google::protobuf::Message& message, const google::protobuf::FieldDescriptor* absl_nonnull field) { if (field->is_map()) { - return absl::nullopt; + return std::nullopt; } if (field->is_repeated() && field->type() == google::protobuf::FieldDescriptor::TYPE_MESSAGE) { const auto* descriptor = message.GetDescriptor(); const auto* any_field = descriptor->FindFieldByName("repeated_any"); - auto packed = WrapShared(message.New(), NewDeleteAllocator<>{}); + auto packed = absl::WrapUnique(message.New()); const int size = message.GetReflection()->FieldSize(message, field); for (int i = 0; i < size; ++i) { PackMessageTo( @@ -413,19 +414,19 @@ PackTestAllTypesProto3Field(const google::protobuf::Message& message, packed->GetReflection()->AddMessage(cel::to_address(packed), any_field)); } - return std::pair{packed, any_field}; + return std::pair{std::move(packed), any_field}; } if (!field->is_repeated() && field->type() == google::protobuf::FieldDescriptor::TYPE_MESSAGE) { const auto* descriptor = message.GetDescriptor(); const auto* any_field = descriptor->FindFieldByName("single_any"); - auto packed = WrapShared(message.New(), NewDeleteAllocator<>{}); + auto packed = absl::WrapUnique(message.New()); PackMessageTo(message.GetReflection()->GetMessage(message, field), packed->GetReflection()->MutableMessage( cel::to_address(packed), any_field)); - return std::pair{packed, any_field}; + return std::pair{std::move(packed), any_field}; } - return absl::nullopt; + return std::nullopt; } TEST_P(UnaryMessageFieldEqualsTest, Equals) { @@ -499,10 +500,10 @@ TEST_P(UnaryMessageFieldEqualsTest, Equals) { << rhs_field->name(); } // Test `google.protobuf.Any`. - absl::optional, + absl::optional, const google::protobuf::FieldDescriptor* absl_nonnull>> lhs_any = PackTestAllTypesProto3Field(*lhs_message, lhs_field); - absl::optional, + absl::optional, const google::protobuf::FieldDescriptor* absl_nonnull>> rhs_any = PackTestAllTypesProto3Field(*rhs_message, rhs_field); if (lhs_any) { diff --git a/internal/minimal_descriptors.cc b/internal/minimal_descriptors.cc index f0b96e838..b8d56923d 100644 --- a/internal/minimal_descriptors.cc +++ b/internal/minimal_descriptors.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include @@ -43,7 +44,7 @@ const google::protobuf::FileDescriptorSet* GetMinimumFileDescriptorSet() { static google::protobuf::FileDescriptorSet* const file_desc_set = []() { google::protobuf::FileDescriptorSet* file_desc_set = new google::protobuf::FileDescriptorSet(); ABSL_CHECK(file_desc_set->ParseFromArray( // Crash OK - kMinimalDescriptorSet, ABSL_ARRAYSIZE(kMinimalDescriptorSet))); + kMinimalDescriptorSet, std::size(kMinimalDescriptorSet))); return file_desc_set; }(); return file_desc_set; diff --git a/internal/overflow.cc b/internal/overflow.cc index 8cc209384..79d1f24ad 100644 --- a/internal/overflow.cc +++ b/internal/overflow.cc @@ -32,7 +32,7 @@ constexpr int64_t kInt32Min = std::numeric_limits::lowest(); constexpr int64_t kInt64Max = std::numeric_limits::max(); constexpr int64_t kInt64Min = std::numeric_limits::lowest(); constexpr uint64_t kUint32Max = std::numeric_limits::max(); -ABSL_ATTRIBUTE_UNUSED constexpr uint64_t kUint64Max = +[[maybe_unused]] constexpr uint64_t kUint64Max = std::numeric_limits::max(); constexpr uint64_t kUintToIntMax = static_cast(kInt64Max); constexpr double kDoubleToIntMax = static_cast(kInt64Max); diff --git a/internal/proto_matchers.h b/internal/proto_matchers.h index 76d844036..02250634b 100644 --- a/internal/proto_matchers.h +++ b/internal/proto_matchers.h @@ -21,7 +21,6 @@ #include "absl/log/absl_check.h" #include "absl/memory/memory.h" -#include "internal/casts.h" #include "internal/testing.h" #include "google/protobuf/message.h" #include "google/protobuf/message_lite.h" @@ -43,13 +42,13 @@ class TextProtoMatcher { bool MatchAndExplain(const google::protobuf::MessageLite& p, ::testing::MatchResultListener* listener) const { - return MatchAndExplain(cel::internal::down_cast(p), + return MatchAndExplain(google::protobuf::DownCastMessage(p), listener); } bool MatchAndExplain(const google::protobuf::MessageLite* p, ::testing::MatchResultListener* listener) const { - return MatchAndExplain(cel::internal::down_cast(p), + return MatchAndExplain(google::protobuf::DownCastMessage(p), listener); } @@ -58,7 +57,7 @@ class TextProtoMatcher { auto message = absl::WrapUnique(p.New()); ABSL_CHECK(google::protobuf::TextFormat::ParseFromString(expected_, message.get())); return google::protobuf::util::MessageDifferencer::Equals( - *message, cel::internal::down_cast(p)); + *message, google::protobuf::DownCastMessage(p)); } bool MatchAndExplain(const google::protobuf::Message* p, @@ -66,7 +65,7 @@ class TextProtoMatcher { auto message = absl::WrapUnique(p->New()); ABSL_CHECK(google::protobuf::TextFormat::ParseFromString(expected_, message.get())); return google::protobuf::util::MessageDifferencer::Equals( - *message, cel::internal::down_cast(*p)); + *message, google::protobuf::DownCastMessage(*p)); } inline void DescribeTo(::std::ostream* os) const { *os << expected_; } @@ -93,13 +92,13 @@ class ProtoMatcher { bool MatchAndExplain(const google::protobuf::MessageLite& p, ::testing::MatchResultListener* listener) const { - return MatchAndExplain(cel::internal::down_cast(p), + return MatchAndExplain(google::protobuf::DownCastMessage(p), listener); } bool MatchAndExplain(const google::protobuf::MessageLite* p, ::testing::MatchResultListener* listener) const { - return MatchAndExplain(cel::internal::down_cast(p), + return MatchAndExplain(google::protobuf::DownCastMessage(p), listener); } diff --git a/internal/runfiles.cc b/internal/runfiles.cc new file mode 100644 index 000000000..bffbfa9d1 --- /dev/null +++ b/internal/runfiles.cc @@ -0,0 +1,53 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "internal/runfiles.h" + +#include +#include +#include + +#include "rules_cc/cc/runfiles/runfiles.h" +#include "absl/log/absl_check.h" + +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace cel::internal { + +std::string ResolveRunfilesPath(absl::string_view path) { + using ::rules_cc::cc::runfiles::Runfiles; + static Runfiles* runfiles = []() { + std::string error; + auto runfiles = + Runfiles::CreateForTest(BAZEL_CURRENT_REPOSITORY, &error); + ABSL_QCHECK(runfiles != nullptr) + << absl::StrCat("failed to init runfiles", error); + return runfiles; + }(); + return runfiles->Rlocation(std::string(path)); +} + +absl::Status GetFileContents(absl::string_view path, std::string* out) { + std::ifstream file{std::string(path)}; + if (!file.is_open()) { + return absl::NotFoundError(absl::StrCat("Failed to open file: ", path)); + } + out->append((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + return absl::OkStatus(); +} + +} // namespace cel::internal diff --git a/internal/runfiles.h b/internal/runfiles.h new file mode 100644 index 000000000..11fdcf337 --- /dev/null +++ b/internal/runfiles.h @@ -0,0 +1,36 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Utilities for working with bazel runfiles. + +#ifndef THIRD_PARTY_CEL_CPP_INTERNAL_RUNFILES_H_ +#define THIRD_PARTY_CEL_CPP_INTERNAL_RUNFILES_H_ + +#include + +#include "absl/status/status.h" +#include "absl/strings/string_view.h" + +namespace cel::internal { + +// Resolves a path relative to the runfiles directory. +// Intended for resolving test cases from cel-spec and cel-policy. +std::string ResolveRunfilesPath(absl::string_view path); + +// Read contents of a file at a resolved path to a string. +absl::Status GetFileContents(absl::string_view path, std::string* out); + +} // namespace cel::internal + +#endif // THIRD_PARTY_CEL_CPP_INTERNAL_RUNFILES_H_ diff --git a/internal/status_builder.h b/internal/status_builder.h index 9caa6c462..80cdd7712 100644 --- a/internal/status_builder.h +++ b/internal/status_builder.h @@ -15,99 +15,14 @@ #ifndef THIRD_PARTY_CEL_CPP_INTERNAL_STATUS_BUILDER_H_ #define THIRD_PARTY_CEL_CPP_INTERNAL_STATUS_BUILDER_H_ -#include -#include - -#include "absl/base/attributes.h" -#include "absl/status/status.h" +#include "absl/base/macros.h" +#include "absl/status/status_builder.h" namespace cel::internal { -class StatusBuilder; - -template -inline constexpr bool StatusBuilderResultMatches = - std::is_same_v>, Expected>; - -template -using StatusBuilderPurePolicy = std::enable_if_t< - StatusBuilderResultMatches, - std::invoke_result_t>; - -template -using StatusBuilderSideEffect = - std::enable_if_t, - std::invoke_result_t>; - -template -using StatusBuilderConversion = std::enable_if_t< - !StatusBuilderResultMatches && - !StatusBuilderResultMatches, - std::invoke_result_t>; - -class StatusBuilder final { - public: - StatusBuilder() = default; - - explicit StatusBuilder(const absl::Status& status) : status_(status) {} - - StatusBuilder(const StatusBuilder&) = default; - - StatusBuilder(StatusBuilder&&) = default; - - ~StatusBuilder() = default; - - StatusBuilder& operator=(const StatusBuilder&) = default; - - StatusBuilder& operator=(StatusBuilder&&) = default; - - bool ok() const { return status_.ok(); } - - absl::StatusCode code() const { return status_.code(); } - - operator absl::Status() const& { return status_; } // NOLINT - - operator absl::Status() && { return std::move(status_); } // NOLINT - - template - auto With( - Adaptor&& adaptor) & -> StatusBuilderPurePolicy { - return std::forward(adaptor)(*this); - } - template - ABSL_MUST_USE_RESULT auto With( - Adaptor&& - adaptor) && -> StatusBuilderPurePolicy { - return std::forward(adaptor)(std::move(*this)); - } - - template - auto With( - Adaptor&& adaptor) & -> StatusBuilderSideEffect { - return std::forward(adaptor)(*this); - } - template - ABSL_MUST_USE_RESULT auto With( - Adaptor&& - adaptor) && -> StatusBuilderSideEffect { - return std::forward(adaptor)(std::move(*this)); - } - - template - auto With( - Adaptor&& adaptor) & -> StatusBuilderConversion { - return std::forward(adaptor)(*this); - } - template - ABSL_MUST_USE_RESULT auto With( - Adaptor&& - adaptor) && -> StatusBuilderConversion { - return std::forward(adaptor)(std::move(*this)); - } - - private: - absl::Status status_; -}; +using StatusBuilder + [[deprecated("Use absl::StatusBuilder instead")]] ABSL_REFACTOR_INLINE = + ::absl::StatusBuilder; } // namespace cel::internal diff --git a/internal/status_macros.h b/internal/status_macros.h index a4b662df6..8ab9a4914 100644 --- a/internal/status_macros.h +++ b/internal/status_macros.h @@ -15,138 +15,12 @@ #ifndef THIRD_PARTY_CEL_CPP_INTERNAL_STATUS_MACROS_H_ #define THIRD_PARTY_CEL_CPP_INTERNAL_STATUS_MACROS_H_ -#include +#include "absl/status/status_macros.h" -#include "absl/base/optimization.h" -#include "absl/status/status.h" -#include "internal/status_builder.h" +// Deprecated: use ABSL_RETURN_IF_ERROR directly. +#define CEL_RETURN_IF_ERROR(expr) ABSL_RETURN_IF_ERROR(expr) -#define CEL_RETURN_IF_ERROR(expr) \ - CEL_INTERNAL_STATUS_MACROS_IMPL_ELSE_BLOCKER_ \ - if (::cel::internal::StatusAdaptor cel_internal_status_macro = {(expr)}) { \ - } else /* NOLINT */ \ - return cel_internal_status_macro.Consume() - -// The GNU compiler historically emitted warnings for obscure usages of -// `if (foo) if (bar) {} else`. This suppresses that. - -// clang-format off -#define CEL_INTERNAL_STATUS_MACROS_IMPL_ELSE_BLOCKER_ \ - switch (0) case 0: default: /* NOLINT */ -// clang-format on - -#define CEL_ASSIGN_OR_RETURN(...) \ - CEL_INTERNAL_STATUS_MACROS_GET_VARIADIC_( \ - (__VA_ARGS__, CEL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_3_, \ - CEL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_2_)) \ - (__VA_ARGS__) - -// The following are macro magic to select either the 2 arg variant or 3 arg -// variant of CEL_ASSIGN_OR_RETURN. - -#define CEL_INTERNAL_STATUS_MACROS_GET_VARIADIC_HELPER_(_1, _2, _3, NAME, ...) \ - NAME -#define CEL_INTERNAL_STATUS_MACROS_GET_VARIADIC_(args) \ - CEL_INTERNAL_STATUS_MACROS_GET_VARIADIC_HELPER_ args - -#define CEL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_2_(lhs, rexpr) \ - CEL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_( \ - CEL_INTERNAL_STATUS_MACROS_CONCAT(_status_or_value, __LINE__), lhs, \ - rexpr, \ - return absl::Status(std::move(CEL_INTERNAL_STATUS_MACROS_CONCAT( \ - _status_or_value, __LINE__)) \ - .status())) - -#define CEL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_3_(lhs, rexpr, \ - error_expression) \ - CEL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_( \ - CEL_INTERNAL_STATUS_MACROS_CONCAT(_status_or_value, __LINE__), lhs, \ - rexpr, \ - ::cel::internal::StatusBuilder _( \ - std::move( \ - CEL_INTERNAL_STATUS_MACROS_CONCAT(_status_or_value, __LINE__)) \ - .status()); \ - (void)_; /* error_expression is allowed to not use this variable */ \ - return (error_expression)) - -// Common implementation of CEL_ASSIGN_OR_RETURN. Both the 2 arg variant and 3 -// arg variant are implemented by this macro. - -#define CEL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_(statusor, lhs, rexpr, \ - error_expression) \ - auto statusor = (rexpr); \ - if (ABSL_PREDICT_FALSE(!statusor.ok())) { \ - error_expression; \ - } \ - CEL_INTERNAL_STATUS_MACROS_UNPARENTHESIZE_IF_PARENTHESIZED(lhs) = \ - std::move(statusor).value() - -#define CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_INNER(...) \ - CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_INNER_HELPER((__VA_ARGS__, 0, 1)) - -// MSVC historically expands variadic macros incorrectly, so another level of -// indirection is required. -#define CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_INNER_HELPER(args) \ - CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_INNER_I args -#define CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_INNER_I(e0, e1, is_empty, ...) \ - is_empty - -#define CEL_INTERNAL_STATUS_MACROS_IS_EMPTY(...) \ - CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_I(__VA_ARGS__) -#define CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_I(...) \ - CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_INNER(_, ##__VA_ARGS__) - -#define CEL_INTERNAL_STATUS_MACROS_IF_1(_Then, _Else) _Then -#define CEL_INTERNAL_STATUS_MACROS_IF_0(_Then, _Else) _Else -#define CEL_INTERNAL_STATUS_MACROS_IF(_Cond, _Then, _Else) \ - CEL_INTERNAL_STATUS_MACROS_CONCAT(CEL_INTERNAL_STATUS_MACROS_IF_, _Cond) \ - (_Then, _Else) - -#define CEL_INTERNAL_STATUS_MACROS_EAT(...) -#define CEL_INTERNAL_STATUS_MACROS_REM(...) __VA_ARGS__ -#define CEL_INTERNAL_STATUS_MACROS_EMPTY() - -// Expands to 1 if the input is surrounded by parenthesis, 0 otherwise. -#define CEL_INTERNAL_STATUS_MACROS_IS_PARENTHESIZED(...) \ - CEL_INTERNAL_STATUS_MACROS_IS_EMPTY( \ - CEL_INTERNAL_STATUS_MACROS_EAT __VA_ARGS__) - -// If the input is surrounded by parenthesis, remove them. Otherwise expand it -// unchanged. -#define CEL_INTERNAL_STATUS_MACROS_UNPARENTHESIZE_IF_PARENTHESIZED(...) \ - CEL_INTERNAL_STATUS_MACROS_IF( \ - CEL_INTERNAL_STATUS_MACROS_IS_PARENTHESIZED(__VA_ARGS__), \ - CEL_INTERNAL_STATUS_MACROS_REM, CEL_INTERNAL_STATUS_MACROS_EMPTY()) \ - __VA_ARGS__ - -#define CEL_INTERNAL_STATUS_MACROS_CONCAT_HELPER(x, y) x##y -#define CEL_INTERNAL_STATUS_MACROS_CONCAT(x, y) \ - CEL_INTERNAL_STATUS_MACROS_CONCAT_HELPER(x, y) - -namespace cel::internal { - -class StatusAdaptor final { - public: - StatusAdaptor() = default; - - StatusAdaptor(const StatusAdaptor&) = delete; - - StatusAdaptor(StatusAdaptor&&) = delete; - - StatusAdaptor(const absl::Status& status) : builder_(status) {} // NOLINT - - StatusAdaptor& operator=(const StatusAdaptor&) = delete; - - StatusAdaptor& operator=(StatusAdaptor&&) = delete; - - StatusBuilder&& Consume() { return std::move(builder_); } - - explicit operator bool() const { return ABSL_PREDICT_TRUE(builder_.ok()); } - - private: - StatusBuilder builder_; -}; - -} // namespace cel::internal +// Deprecated: use ABSL_ASSIGN_OR_RETURN directly. +#define CEL_ASSIGN_OR_RETURN(...) ABSL_ASSIGN_OR_RETURN(__VA_ARGS__) #endif // THIRD_PARTY_CEL_CPP_INTERNAL_STATUS_MACROS_H_ diff --git a/internal/strings.cc b/internal/strings.cc index a272aaa46..144ae91de 100644 --- a/internal/strings.cc +++ b/internal/strings.cc @@ -14,13 +14,13 @@ #include "internal/strings.h" +#include #include #include "absl/base/attributes.h" #include "absl/status/status.h" #include "absl/strings/ascii.h" #include "absl/strings/cord.h" -#include "absl/strings/escaping.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" @@ -441,62 +441,100 @@ bool UnescapeInternal(absl::string_view source, absl::string_view closing_str, return true; } -std::string EscapeInternal(absl::string_view src, bool escape_all_bytes, - char escape_quote_char) { - std::string dest; - // Worst case size is every byte has to be hex escaped, so 4 char for every - // byte. - dest.reserve(src.size() * 4); - bool last_hex_escape = false; // true if last output char was \xNN. - const char* p = src.data(); - const char* end = p + src.size(); - for (; p < end; ++p) { - unsigned char c = static_cast(*p); - bool is_hex_escape = false; - switch (c) { - case '\n': - dest.append("\\n"); - break; - case '\r': - dest.append("\\r"); - break; - case '\t': - dest.append("\\t"); +[[nodiscard]] +bool EscapeChar(std::string* dest, unsigned char c, bool escape_all_bytes, + char escape_quote_char, bool last_hex_escape) { + bool is_hex_escape = false; + switch (c) { + case '\n': + dest->append("\\n"); + break; + case '\r': + dest->append("\\r"); + break; + case '\t': + dest->append("\\t"); + break; + case '\\': + dest->append("\\\\"); + break; + case '\'': + ABSL_FALLTHROUGH_INTENDED; + case '\"': + ABSL_FALLTHROUGH_INTENDED; + case '`': + // Escape only quote chars that match escape_quote_char. + if (escape_quote_char == 0 || c == escape_quote_char) { + dest->push_back('\\'); + } + dest->push_back(c); + break; + default: + // Note that if we emit \xNN and the src character after that is a hex + // digit then that digit must be escaped too to prevent it being + // interpreted as part of the character code by C. + if ((!escape_all_bytes || c < 0x80) && + (!absl::ascii_isprint(c) || + (last_hex_escape && absl::ascii_isxdigit(c)))) { + dest->append("\\x"); + dest->push_back(kHexTable[c / 16]); + dest->push_back(kHexTable[c % 16]); + is_hex_escape = true; + } else { + dest->push_back(c); break; + } + } + return is_hex_escape; +} + +template +void EscapeStringInternal(std::string* dest, Iter src_begin, const Iter src_end, + bool escape_all_bytes, char escape_quote_char) { + bool last_hex_escape = false; // true if last output char was \xNN. + for (; src_begin != src_end; ++src_begin) { + last_hex_escape = + EscapeChar(dest, static_cast(*src_begin), + escape_all_bytes, escape_quote_char, last_hex_escape); + } +} + +void EscapeByte(std::string* dest, unsigned char b, bool escape_all_bytes, + char escape_quote_char) { + if (escape_all_bytes || !absl::ascii_isprint(b)) { + dest->append("\\x"); + dest->push_back(kHexTable[b / 16]); + dest->push_back(kHexTable[b % 16]); + } else { + switch (b) { + // Note that we only handle printable escape characters here. All + // unprintable (\n, \r, \t, etc.) are hex escaped above. case '\\': - dest.append("\\\\"); + dest->append("\\\\"); break; case '\'': - ABSL_FALLTHROUGH_INTENDED; - case '\"': - ABSL_FALLTHROUGH_INTENDED; + case '"': case '`': // Escape only quote chars that match escape_quote_char. - if (escape_quote_char == 0 || c == escape_quote_char) { - dest.push_back('\\'); + if (escape_quote_char == 0 || b == escape_quote_char) { + dest->push_back('\\'); } - dest.push_back(c); + dest->push_back(static_cast(b)); break; default: - // Note that if we emit \xNN and the src character after that is a hex - // digit then that digit must be escaped too to prevent it being - // interpreted as part of the character code by C. - if ((!escape_all_bytes || c < 0x80) && - (!absl::ascii_isprint(c) || - (last_hex_escape && absl::ascii_isxdigit(c)))) { - dest.append("\\x"); - dest.push_back(kHexTable[c / 16]); - dest.push_back(kHexTable[c % 16]); - is_hex_escape = true; - } else { - dest.push_back(c); - break; - } + dest->push_back(static_cast(b)); + break; } - last_hex_escape = is_hex_escape; } - dest.shrink_to_fit(); - return dest; +} + +template +void EscapeBytesInternal(std::string* dest, Iter src_begin, const Iter src_end, + bool escape_all_bytes, char escape_quote_char) { + for (; src_begin != src_end; ++src_begin) { + EscapeByte(dest, static_cast(*src_begin), escape_all_bytes, + escape_quote_char); + } } bool MayBeTripleQuotedString(absl::string_view str) { @@ -550,42 +588,25 @@ absl::StatusOr UnescapeBytes(absl::string_view str) { } std::string EscapeString(absl::string_view str) { - return EscapeInternal(str, true, '\0'); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4); + EscapeStringInternal(&dest, str.begin(), str.end(), true, '\0'); + dest.shrink_to_fit(); + return dest; } std::string EscapeBytes(absl::string_view str, bool escape_all_bytes, char escape_quote_char) { - std::string escaped_bytes; - const char* p = str.data(); - const char* end = p + str.size(); - for (; p < end; ++p) { - unsigned char c = *p; - if (escape_all_bytes || !absl::ascii_isprint(c)) { - escaped_bytes += "\\x"; - escaped_bytes += absl::BytesToHexString(absl::string_view(p, 1)); - } else { - switch (c) { - // Note that we only handle printable escape characters here. All - // unprintable (\n, \r, \t, etc.) are hex escaped above. - case '\\': - escaped_bytes += "\\\\"; - break; - case '\'': - case '"': - case '`': - // Escape only quote chars that match escape_quote_char. - if (escape_quote_char == 0 || c == escape_quote_char) { - escaped_bytes += '\\'; - } - escaped_bytes += c; - break; - default: - escaped_bytes += c; - break; - } - } - } - return escaped_bytes; + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4); + EscapeBytesInternal(&dest, str.begin(), str.end(), escape_all_bytes, + escape_quote_char); + dest.shrink_to_fit(); + return dest; } absl::StatusOr ParseStringLiteral(absl::string_view str) { @@ -649,38 +670,143 @@ absl::StatusOr ParseBytesLiteral(absl::string_view str) { } std::string FormatStringLiteral(absl::string_view str) { - absl::string_view quote = - (str.find('"') != str.npos && str.find('\'') == str.npos) ? "'" : "\""; - return absl::StrCat(quote, EscapeInternal(str, true, quote[0]), quote); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 2); + FormatStringLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; } std::string FormatStringLiteral(const absl::Cord& str) { - if (auto flat = str.TryFlat(); flat) { - return FormatStringLiteral(*flat); - } - return FormatStringLiteral(static_cast(str)); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 2); + FormatStringLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; +} + +void FormatStringLiteralTo(absl::string_view str, std::string* dest) { + const char quote = + absl::StrContains(str, '"') && !absl::StrContains(str, '\'') ? '\'' : '"'; + dest->push_back(quote); + EscapeStringInternal(dest, str.begin(), str.end(), true, quote); + dest->push_back(quote); +} + +void FormatStringLiteralTo(const absl::Cord& str, std::string* dest) { + const char quote = (str.Contains("\"") && !str.Contains("'")) ? '\'' : '"'; + dest->push_back(quote); + EscapeStringInternal(dest, str.char_begin(), str.char_end(), true, quote); + dest->push_back(quote); } std::string FormatSingleQuotedStringLiteral(absl::string_view str) { - return absl::StrCat("'", EscapeInternal(str, true, '\''), "'"); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 2); + FormatSingleQuotedStringLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; +} + +void FormatSingleQuotedStringLiteralTo(absl::string_view str, + std::string* dest) { + dest->push_back('\''); + EscapeStringInternal(dest, str.begin(), str.end(), true, '\''); + dest->push_back('\''); } std::string FormatDoubleQuotedStringLiteral(absl::string_view str) { - return absl::StrCat("\"", EscapeInternal(str, true, '"'), "\""); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 2); + FormatDoubleQuotedStringLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; +} + +void FormatDoubleQuotedStringLiteralTo(absl::string_view str, + std::string* dest) { + dest->push_back('"'); + EscapeStringInternal(dest, str.begin(), str.end(), true, '"'); + dest->push_back('"'); } std::string FormatBytesLiteral(absl::string_view str) { - absl::string_view quote = - (str.find('"') != str.npos && str.find('\'') == str.npos) ? "'" : "\""; - return absl::StrCat("b", quote, EscapeBytes(str, false, quote[0]), quote); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 3); + FormatBytesLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; +} + +std::string FormatBytesLiteral(const absl::Cord& str) { + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 3); + FormatBytesLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; +} + +void FormatBytesLiteralTo(absl::string_view str, std::string* dest) { + const char quote = + absl::StrContains(str, '"') && !absl::StrContains(str, '\'') ? '\'' : '"'; + dest->push_back('b'); + dest->push_back(quote); + EscapeBytesInternal(dest, str.begin(), str.end(), false, quote); + dest->push_back(quote); +} + +void FormatBytesLiteralTo(const absl::Cord& str, std::string* dest) { + const char quote = (str.Contains("\"") && !str.Contains("'")) ? '\'' : '"'; + dest->push_back('b'); + dest->push_back(quote); + EscapeBytesInternal(dest, str.char_begin(), str.char_end(), false, quote); + dest->push_back(quote); } std::string FormatSingleQuotedBytesLiteral(absl::string_view str) { - return absl::StrCat("b'", EscapeBytes(str, false, '\''), "'"); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 3); + FormatSingleQuotedBytesLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; +} + +void FormatSingleQuotedBytesLiteralTo(absl::string_view str, + std::string* dest) { + dest->append("b'"); + EscapeBytesInternal(dest, str.begin(), str.end(), false, '\''); + dest->push_back('\''); } std::string FormatDoubleQuotedBytesLiteral(absl::string_view str) { - return absl::StrCat("b\"", EscapeBytes(str, false, '"'), "\""); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 3); + FormatDoubleQuotedBytesLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; +} + +void FormatDoubleQuotedBytesLiteralTo(absl::string_view str, + std::string* dest) { + dest->append("b\""); + EscapeBytesInternal(dest, str.begin(), str.end(), false, '"'); + dest->push_back('"'); } absl::StatusOr ParseIdentifier(absl::string_view str) { diff --git a/internal/strings.h b/internal/strings.h index ae82a14fd..97a972835 100644 --- a/internal/strings.h +++ b/internal/strings.h @@ -61,26 +61,37 @@ absl::StatusOr ParseBytesLiteral(absl::string_view str); // May choose to quote with ' or " to produce nicer output. std::string FormatStringLiteral(absl::string_view str); std::string FormatStringLiteral(const absl::Cord& str); +void FormatStringLiteralTo(absl::string_view str, std::string* dest); +void FormatStringLiteralTo(const absl::Cord& str, std::string* dest); // Return a quoted and escaped CEL string literal for . // Always uses single quotes. std::string FormatSingleQuotedStringLiteral(absl::string_view str); +void FormatSingleQuotedStringLiteralTo(absl::string_view str, + std::string* dest); // Return a quoted and escaped CEL string literal for . // Always uses double quotes. std::string FormatDoubleQuotedStringLiteral(absl::string_view str); +void FormatDoubleQuotedStringLiteralTo(absl::string_view str, + std::string* dest); // Return a quoted and escaped CEL bytes literal for . // Prefixes with b and may choose to quote with ' or " to produce nicer output. std::string FormatBytesLiteral(absl::string_view str); +std::string FormatBytesLiteral(const absl::Cord& str); +void FormatBytesLiteralTo(absl::string_view str, std::string* dest); +void FormatBytesLiteralTo(const absl::Cord& str, std::string* dest); // Return a quoted and escaped CEL bytes literal for . // Prefixes with b and always uses single quotes. std::string FormatSingleQuotedBytesLiteral(absl::string_view str); +void FormatSingleQuotedBytesLiteralTo(absl::string_view str, std::string* dest); // Return a quoted and escaped CEL bytes literal for . // Prefixes with b and always uses double quotes. std::string FormatDoubleQuotedBytesLiteral(absl::string_view str); +void FormatDoubleQuotedBytesLiteralTo(absl::string_view str, std::string* dest); // Parse a CEL identifier. absl::StatusOr ParseIdentifier(absl::string_view str); diff --git a/internal/strings_test.cc b/internal/strings_test.cc index fcdb6d4ec..3d950d260 100644 --- a/internal/strings_test.cc +++ b/internal/strings_test.cc @@ -86,7 +86,7 @@ void TestBytesEscaping(const std::string& unquoted, const std::string& quoted) { } // takes a byte literal of the form b'...', b'''...''' -void TestBytesLiteral(const std::string& quoted) { +void TestBytesLiteralString(const std::string& quoted) { // Parse the literal. ASSERT_OK_AND_ASSIGN(auto unquoted, ParseBytesLiteral(quoted)); @@ -103,6 +103,30 @@ void TestBytesLiteral(const std::string& quoted) { TestBytesEscaping(unquoted, quoted); } +// takes a byte literal of the form b'...', b'''...''' +void TestBytesLiteralCord(const std::string& quoted) { + // Parse the literal. + ASSERT_OK_AND_ASSIGN(auto unquoted, ParseBytesLiteral(quoted)); + + // Take the parsed literal and turn it back to a literal. + std::string requoted = FormatBytesLiteral(absl::Cord(unquoted)); + // Parse it again. + ASSERT_OK_AND_ASSIGN(auto unquoted2, ParseBytesLiteral(requoted)); + // Test the parsed literal forms for equality, not the unparsed forms. + // This is because the unparsed forms can have different representations for + // the same data, i.e., \000 and \x00. + EXPECT_EQ(unquoted, unquoted2) + << "unquoted : " << unquoted << "\nunquoted2: " << unquoted2; + + TestBytesEscaping(unquoted, quoted); +} + +// takes a byte literal of the form b'...', b'''...''' +void TestBytesLiteral(const std::string& quoted) { + TestBytesLiteralString(quoted); + TestBytesLiteralCord(quoted); +} + // takes a raw byte literal of the form rb'...', br'...', rb'''...''' // or br'''...'''. is the expected parsed form of . void TestQuotedRawBytesLiteral(const std::string& unquoted, diff --git a/internal/testing.h b/internal/testing.h index e1b9f7498..a909a8c6d 100644 --- a/internal/testing.h +++ b/internal/testing.h @@ -18,14 +18,15 @@ #include "gmock/gmock.h" // IWYU pragma: export #include "gtest/gtest.h" // IWYU pragma: export #include "absl/status/status_matchers.h" +#include "internal/status_builder.h" #include "internal/status_macros.h" // IWYU pragma: keep #ifndef ASSERT_OK -#define ASSERT_OK(expr) ASSERT_THAT(expr, ::absl_testing::IsOk()) +#define ASSERT_OK(expr) ABSL_ASSERT_OK(expr) #endif #ifndef EXPECT_OK -#define EXPECT_OK(expr) EXPECT_THAT(expr, ::absl_testing::IsOk()) +#define EXPECT_OK(expr) ABSL_EXPECT_OK(expr) #endif #ifndef ASSERT_OK_AND_ASSIGN diff --git a/internal/time.cc b/internal/time.cc index 45945613d..c42212b68 100644 --- a/internal/time.cc +++ b/internal/time.cc @@ -42,7 +42,7 @@ absl::Duration MaxDuration() { // intent is to widen the CEL spec to support the larger range and match // google.protobuf.Duration from protocol buffer messages, which this // implementation currently supports. - // TODO(google/cel-spec/issues/214): revisit + // TODO(cel-expr/cel-spec/issues/214): revisit return absl::Seconds(google::protobuf::util::TimeUtil::kDurationMaxSeconds) + absl::Nanoseconds(google::protobuf::util::TimeUtil::kDurationMaxNanoseconds); } @@ -52,7 +52,7 @@ absl::Duration MinDuration() { // intent is to widen the CEL spec to support the larger range and match // google.protobuf.Duration from protocol buffer messages, which this // implementation currently supports. - // TODO(google/cel-spec/issues/214): revisit + // TODO(cel-expr/cel-spec/issues/214): revisit return absl::Seconds(google::protobuf::util::TimeUtil::kDurationMinSeconds) + absl::Nanoseconds(google::protobuf::util::TimeUtil::kDurationMinNanoseconds); } diff --git a/internal/well_known_types.cc b/internal/well_known_types.cc index dee029534..f48d77f7f 100644 --- a/internal/well_known_types.cc +++ b/internal/well_known_types.cc @@ -1489,7 +1489,7 @@ google::protobuf::Message* absl_nonnull ValueReflection::MutableStructValue( return message->GetReflection()->MutableMessage(message, struct_value_field_); } -Unique ValueReflection::ReleaseListValue( +google::protobuf::Arena::UniquePtr ValueReflection::ReleaseListValue( google::protobuf::Message* absl_nonnull message) const { ABSL_DCHECK(IsInitialized()); ABSL_DCHECK_EQ(message->GetDescriptor(), descriptor_); @@ -1497,12 +1497,12 @@ Unique ValueReflection::ReleaseListValue( if (!reflection->HasField(*message, list_value_field_)) { reflection->MutableMessage(message, list_value_field_); } - return WrapUnique( - reflection->UnsafeArenaReleaseMessage(message, list_value_field_), - message->GetArena()); + return google::protobuf::Arena::UnsafeWrapUniquePtr( + message->GetArena(), + reflection->UnsafeArenaReleaseMessage(message, list_value_field_)); } -Unique ValueReflection::ReleaseStructValue( +google::protobuf::Arena::UniquePtr ValueReflection::ReleaseStructValue( google::protobuf::Message* absl_nonnull message) const { ABSL_DCHECK(IsInitialized()); ABSL_DCHECK_EQ(message->GetDescriptor(), descriptor_); @@ -1510,9 +1510,9 @@ Unique ValueReflection::ReleaseStructValue( if (!reflection->HasField(*message, struct_value_field_)) { reflection->MutableMessage(message, struct_value_field_); } - return WrapUnique( - reflection->UnsafeArenaReleaseMessage(message, struct_value_field_), - message->GetArena()); + return google::protobuf::Arena::UnsafeWrapUniquePtr( + message->GetArena(), + reflection->UnsafeArenaReleaseMessage(message, struct_value_field_)); } absl::StatusOr GetValueReflection( @@ -1899,9 +1899,9 @@ namespace { // it as `ListValue`. If adapted is empty, we return as a reference. If adapted // is present, message must be a reference to the value held in adapted and it // will be returned by value. -absl::StatusOr AdaptListValue(google::protobuf::Arena* absl_nullable arena, - const google::protobuf::Message& message, - Unique adapted) { +absl::StatusOr AdaptListValue( + google::protobuf::Arena* absl_nullable arena, const google::protobuf::Message& message, + google::protobuf::Arena::UniquePtr adapted) { ABSL_DCHECK(!adapted || &message == cel::to_address(adapted)); const auto* descriptor = message.GetDescriptor(); if (ABSL_PREDICT_FALSE(descriptor == nullptr)) { @@ -1922,9 +1922,9 @@ absl::StatusOr AdaptListValue(google::protobuf::Arena* absl_nullable // as `Struct`. If adapted is empty, we return as a reference. If adapted is // present, message must be a reference to the value held in adapted and it will // be returned by value. -absl::StatusOr AdaptStruct(google::protobuf::Arena* absl_nullable arena, - const google::protobuf::Message& message, - Unique adapted) { +absl::StatusOr AdaptStruct( + google::protobuf::Arena* absl_nullable arena, const google::protobuf::Message& message, + google::protobuf::Arena::UniquePtr adapted) { ABSL_DCHECK(!adapted || &message == cel::to_address(adapted)); const auto* descriptor = message.GetDescriptor(); if (ABSL_PREDICT_FALSE(descriptor == nullptr)) { @@ -1942,14 +1942,14 @@ absl::StatusOr AdaptStruct(google::protobuf::Arena* absl_nullable arena, // AdaptAny recursively unpacks a protocol buffer message which is an instance // of `google.protobuf.Any`. -absl::StatusOr> AdaptAny( +absl::StatusOr> AdaptAny( google::protobuf::Arena* absl_nullable arena, AnyReflection& reflection, const google::protobuf::Message& message, const Descriptor* absl_nonnull descriptor, const DescriptorPool* absl_nonnull pool, google::protobuf::MessageFactory* absl_nonnull factory, bool error_if_unresolveable) { ABSL_DCHECK_EQ(descriptor->well_known_type(), Descriptor::WELLKNOWNTYPE_ANY); const google::protobuf::Message* absl_nonnull to_unwrap = &message; - Unique unwrapped; + google::protobuf::Arena::UniquePtr unwrapped; std::string type_url_scratch; std::string value_scratch; do { @@ -1979,7 +1979,14 @@ absl::StatusOr> AdaptAny( "unable to build prototype for type name: ", type_url_view)); } BytesValue value = reflection.GetValue(*to_unwrap, value_scratch); - Unique unpacked = WrapUnique(prototype->New(arena), arena); + google::protobuf::Arena::UniquePtr unpacked = + google::protobuf::Arena::UnsafeWrapUniquePtr(arena, prototype->New(arena)); + // TODO(b/557267722): Extensions that are not included in the same + // descriptor pool as the resolved descriptor will be treated as unknown + // fields. Extending messages like this should be exceedingly rare and is + // hard to support safely. + // + // See CodedInputStream::SetExtensionRegistry. const bool ok = absl::visit(absl::Overload( [&](absl::string_view string) -> bool { return unpacked->ParseFromString(string); @@ -2008,7 +2015,7 @@ absl::StatusOr> AdaptAny( } // namespace -absl::StatusOr> UnpackAnyFrom( +absl::StatusOr> UnpackAnyFrom( google::protobuf::Arena* absl_nullable arena, AnyReflection& reflection, const google::protobuf::Message& message, const google::protobuf::DescriptorPool* absl_nonnull pool, @@ -2019,11 +2026,12 @@ absl::StatusOr> UnpackAnyFrom( factory, /*error_if_unresolveable=*/true); } -absl::StatusOr> UnpackAnyIfResolveable( - google::protobuf::Arena* absl_nullable arena, AnyReflection& reflection, - const google::protobuf::Message& message, - const google::protobuf::DescriptorPool* absl_nonnull pool, - google::protobuf::MessageFactory* absl_nonnull factory) { +absl::StatusOr> +UnpackAnyIfResolveable(google::protobuf::Arena* absl_nullable arena, + AnyReflection& reflection, + const google::protobuf::Message& message, + const google::protobuf::DescriptorPool* absl_nonnull pool, + google::protobuf::MessageFactory* absl_nonnull factory) { ABSL_DCHECK_EQ(message.GetDescriptor()->well_known_type(), Descriptor::WELLKNOWNTYPE_ANY); return AdaptAny(arena, reflection, message, message.GetDescriptor(), pool, @@ -2041,7 +2049,7 @@ absl::StatusOr AdaptFromMessage( message.GetTypeName())); } const google::protobuf::Message* absl_nonnull to_adapt; - Unique adapted; + google::protobuf::Arena::UniquePtr adapted; Descriptor::WellKnownType well_known_type = descriptor->well_known_type(); if (well_known_type == Descriptor::WELLKNOWNTYPE_ANY) { AnyReflection reflection; @@ -2114,13 +2122,15 @@ absl::StatusOr AdaptFromMessage( case Descriptor::WELLKNOWNTYPE_ANY: // This is unreachable, as AdaptAny() above recursively unpacks. ABSL_UNREACHABLE(); + // Don't check that time values are in range on field access. Assume + // error will propagate if they are used in any arithmetic. case Descriptor::WELLKNOWNTYPE_DURATION: { CEL_ASSIGN_OR_RETURN(auto reflection, GetDurationReflection(descriptor)); - return reflection.ToAbslDuration(*to_adapt); + return reflection.UnsafeToAbslDuration(*to_adapt); } case Descriptor::WELLKNOWNTYPE_TIMESTAMP: { CEL_ASSIGN_OR_RETURN(auto reflection, GetTimestampReflection(descriptor)); - return reflection.ToAbslTime(*to_adapt); + return reflection.UnsafeToAbslTime(*to_adapt); } case Descriptor::WELLKNOWNTYPE_VALUE: { CEL_ASSIGN_OR_RETURN(auto reflection, GetValueReflection(descriptor)); @@ -2174,7 +2184,7 @@ absl::StatusOr AdaptFromMessage( if (adapted) { return adapted; } - return absl::monostate{}; + return std::monostate{}; } } diff --git a/internal/well_known_types.h b/internal/well_known_types.h index f63e5e76b..3e09b286b 100644 --- a/internal/well_known_types.h +++ b/internal/well_known_types.h @@ -50,7 +50,6 @@ #include "absl/time/time.h" #include "absl/types/variant.h" #include "common/any.h" -#include "common/memory.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/map_field.h" @@ -1007,10 +1006,10 @@ class ValueReflection final { google::protobuf::Message* absl_nonnull MutableStructValue( google::protobuf::Message* absl_nonnull message) const; - Unique ReleaseListValue( + google::protobuf::Arena::UniquePtr ReleaseListValue( google::protobuf::Message* absl_nonnull message) const; - Unique ReleaseStructValue( + google::protobuf::Arena::UniquePtr ReleaseStructValue( google::protobuf::Message* absl_nonnull message) const; private: @@ -1278,18 +1277,18 @@ absl::StatusOr GetFieldMaskReflection( const google::protobuf::Descriptor* absl_nonnull descriptor ABSL_ATTRIBUTE_LIFETIME_BOUND); -using ListValuePtr = Unique; +using ListValuePtr = google::protobuf::Arena::UniquePtr; using ListValueConstRef = std::reference_wrapper; -using StructPtr = Unique; +using StructPtr = google::protobuf::Arena::UniquePtr; using StructConstRef = std::reference_wrapper; // Variant holding `std::reference_wrapper` or `Unique`, either of which is an -// instance of `google.protobuf.ListValue` which is either a generated message -// or dynamic message. +// google::protobuf::Message>` or `google::protobuf::Arena::UniquePtr`, either of +// which is an instance of `google.protobuf.ListValue` which is either a +// generated message or dynamic message. class ListValue final : public absl::variant { using absl::variant::variant; }; @@ -1316,9 +1315,9 @@ inline absl::variant&& AsVariant( } // Variant holding `std::reference_wrapper` or `Unique`, either of which is an -// instance of `google.protobuf.Struct` which is either a generated message or -// dynamic message. +// google::protobuf::Message>` or `google::protobuf::Arena::UniquePtr`, either of +// which is an instance of `google.protobuf.Struct` which is either a generated +// message or dynamic message. class Struct final : public absl::variant { public: using absl::variant::variant; @@ -1342,13 +1341,14 @@ inline absl::variant&& AsVariant(Struct&& value) { } // Variant capable of representing any unwrapped well known type or message. -using Value = absl::variant>; +using Value = + absl::variant>; // Unpacks the given instance of `google.protobuf.Any`. -absl::StatusOr> UnpackAnyFrom( +absl::StatusOr> UnpackAnyFrom( google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND, AnyReflection& reflection, const google::protobuf::Message& message, const google::protobuf::DescriptorPool* absl_nonnull pool @@ -1356,7 +1356,8 @@ absl::StatusOr> UnpackAnyFrom( google::protobuf::MessageFactory* absl_nonnull factory ABSL_ATTRIBUTE_LIFETIME_BOUND); // Unpacks the given instance of `google.protobuf.Any` if it is resolvable. -absl::StatusOr> UnpackAnyIfResolveable( +absl::StatusOr> +UnpackAnyIfResolveable( google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND, AnyReflection& reflection, const google::protobuf::Message& message, const google::protobuf::DescriptorPool* absl_nonnull pool diff --git a/internal/well_known_types_test.cc b/internal/well_known_types_test.cc index 0d2c9fe33..aae66c51c 100644 --- a/internal/well_known_types_test.cc +++ b/internal/well_known_types_test.cc @@ -34,8 +34,6 @@ #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" -#include "absl/types/variant.h" -#include "common/memory.h" #include "internal/message_type_name.h" #include "internal/minimal_descriptor_pool.h" #include "internal/parse_text_proto.h" @@ -341,6 +339,44 @@ TEST_F(ReflectionTest, Duration_Dynamic) { StatusIs(absl::StatusCode::kInvalidArgument)); } +TEST_F(ReflectionTest, Duration_ToAbslDuration) { + auto* value = MakeDynamic(); + ASSERT_OK_AND_ASSIGN( + auto reflection, + GetDurationReflection(ABSL_DIE_IF_NULL(value->GetDescriptor()))); + + reflection.SetSeconds(value, 1); + reflection.SetNanos(value, 1); + EXPECT_THAT(reflection.ToAbslDuration(*value), + IsOkAndHolds(absl::Seconds(1) + absl::Nanoseconds(1))); + EXPECT_EQ(reflection.UnsafeToAbslDuration(*value), + absl::Seconds(1) + absl::Nanoseconds(1)); + + reflection.SetSeconds(value, 0x7fffffffffffffff); + reflection.SetNanos(value, 1); + EXPECT_THAT(reflection.ToAbslDuration(*value), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("invalid duration seconds: "))); + EXPECT_EQ(reflection.UnsafeToAbslDuration(*value), + absl::Seconds(0x7fffffffffffffff) + absl::Nanoseconds(1)); + + reflection.SetSeconds(value, 1); + reflection.SetNanos(value, 0x7fffffff); + EXPECT_THAT(reflection.ToAbslDuration(*value), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("invalid duration nanoseconds: "))); + EXPECT_EQ(reflection.UnsafeToAbslDuration(*value), + absl::Seconds(1) + absl::Nanoseconds(0x7fffffff)); + + reflection.SetSeconds(value, -1); + reflection.SetNanos(value, 1); + EXPECT_THAT(reflection.ToAbslDuration(*value), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("duration sign mismatch: "))); + EXPECT_EQ(reflection.UnsafeToAbslDuration(*value), + absl::Seconds(-1) + absl::Nanoseconds(1)); +} + TEST_F(ReflectionTest, Timestamp_Generated) { auto* value = MakeGenerated(); EXPECT_EQ(TimestampReflection::GetSeconds(*value), 0); @@ -389,6 +425,39 @@ TEST_F(ReflectionTest, Timestamp_Dynamic) { StatusIs(absl::StatusCode::kInvalidArgument)); } +TEST_F(ReflectionTest, Timestamp_ToAbslTime) { + auto* value = MakeDynamic(); + ASSERT_OK_AND_ASSIGN( + auto reflection, + GetTimestampReflection(ABSL_DIE_IF_NULL(value->GetDescriptor()))); + + reflection.SetSeconds(value, 1); + reflection.SetNanos(value, 1); + EXPECT_THAT(reflection.ToAbslTime(*value), + IsOkAndHolds(absl::UnixEpoch() + absl::Seconds(1) + + absl::Nanoseconds(1))); + EXPECT_EQ(reflection.UnsafeToAbslTime(*value), + absl::UnixEpoch() + absl::Seconds(1) + absl::Nanoseconds(1)); + + reflection.SetSeconds(value, 0x7fffffffffffffff); + reflection.SetNanos(value, 1); + EXPECT_THAT(reflection.ToAbslTime(*value), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("invalid timestamp seconds: "))); + EXPECT_EQ(reflection.UnsafeToAbslTime(*value), + absl::UnixEpoch() + absl::Seconds(0x7fffffffffffffff) + + absl::Nanoseconds(1)); + + reflection.SetSeconds(value, 1); + reflection.SetNanos(value, 0x7fffffff); + EXPECT_THAT(reflection.ToAbslTime(*value), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("invalid timestamp nanoseconds: "))); + EXPECT_EQ( + reflection.UnsafeToAbslTime(*value), + absl::UnixEpoch() + absl::Seconds(1) + absl::Nanoseconds(0x7fffffff)); +} + TEST_F(ReflectionTest, Value_Generated) { auto* value = MakeGenerated(); EXPECT_EQ(ValueReflection::GetKindCase(*value), @@ -698,16 +767,16 @@ TEST_F(AdaptFromMessageTest, Duration_SecondsOutOfRange) { auto message = DynamicParseTextProto( R"pb(seconds: 0x7fffffffffffffff nanos: 1)pb"); EXPECT_THAT(AdaptFromMessage(*message), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("invalid duration seconds: "))); + IsOkAndHolds(VariantWith( + absl::Seconds(0x7fffffffffffffff) + absl::Nanoseconds(1)))); } TEST_F(AdaptFromMessageTest, Duration_NanosOutOfRange) { auto message = DynamicParseTextProto( R"pb(seconds: 1 nanos: 0x7fffffff)pb"); EXPECT_THAT(AdaptFromMessage(*message), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("invalid duration nanoseconds: "))); + IsOkAndHolds(VariantWith( + absl::Seconds(1) + absl::Nanoseconds(0x7fffffff)))); } TEST_F(AdaptFromMessageTest, Duration_SignMismatch) { @@ -715,8 +784,8 @@ TEST_F(AdaptFromMessageTest, Duration_SignMismatch) { DynamicParseTextProto(R"pb(seconds: -1 nanos: 1)pb"); EXPECT_THAT(AdaptFromMessage(*message), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("duration sign mismatch: "))); + IsOkAndHolds(VariantWith(absl::Seconds(-1) + + absl::Nanoseconds(1)))); } TEST_F(AdaptFromMessageTest, Timestamp) { @@ -733,16 +802,18 @@ TEST_F(AdaptFromMessageTest, Timestamp_SecondsOutOfRange) { auto message = DynamicParseTextProto( R"pb(seconds: 0x7fffffffffffffff nanos: 1)pb"); EXPECT_THAT(AdaptFromMessage(*message), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("invalid timestamp seconds: "))); + IsOkAndHolds(VariantWith( + absl::UnixEpoch() + absl::Seconds(0x7fffffffffffffff) + + absl::Nanoseconds(1)))); } TEST_F(AdaptFromMessageTest, Timestamp_NanosOutOfRange) { auto message = DynamicParseTextProto( R"pb(seconds: 1 nanos: 0x7fffffff)pb"); EXPECT_THAT(AdaptFromMessage(*message), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("invalid timestamp nanoseconds: "))); + IsOkAndHolds( + VariantWith(absl::UnixEpoch() + absl::Seconds(1) + + absl::Nanoseconds(0x7fffffff)))); } TEST_F(AdaptFromMessageTest, Value_NullValue) { @@ -806,7 +877,7 @@ TEST_F(AdaptFromMessageTest, Struct) { TEST_F(AdaptFromMessageTest, TestAllTypesProto3) { auto message = DynamicParseTextProto(R"pb()pb"); EXPECT_THAT(AdaptFromMessage(*message), - IsOkAndHolds(VariantWith(absl::monostate()))); + IsOkAndHolds(VariantWith(std::monostate()))); } TEST_F(AdaptFromMessageTest, Any_BoolValue) { @@ -954,8 +1025,10 @@ TEST_F(AdaptFromMessageTest, Any_Struct) { TEST_F(AdaptFromMessageTest, Any_TestAllTypesProto3) { auto message = DynamicParseTextProto( R"pb(type_url: "type.googleapis.com/cel.expr.conformance.proto3.TestAllTypes")pb"); - EXPECT_THAT(AdaptFromMessage(*message), - IsOkAndHolds(VariantWith>(NotNull()))); + EXPECT_THAT( + AdaptFromMessage(*message), + IsOkAndHolds( + VariantWith>(NotNull()))); } TEST_F(AdaptFromMessageTest, Any_BadTypeUrlDomain) { diff --git a/parser/BUILD b/parser/BUILD index 63813bb59..f29765835 100644 --- a/parser/BUILD +++ b/parser/BUILD @@ -27,47 +27,50 @@ cc_library( hdrs = [ "parser.h", ], - copts = [ - "-fexceptions", + deps = [ + ":macro", + ":macro_registry", + ":options", + ":parser_interface", + ":source_factory", + "//common:ast", + "//common:source", + "//common/ast:expr_proto", + "//common/ast:source_info_proto", + "//internal:status_macros", + "//parser/internal:antlr_parser", + "//parser/internal:pratt_parser", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", + ], +) + +cc_library( + name = "pratt_parser", + srcs = [ + "parser.cc", + ], + hdrs = [ + "parser.h", ], defines = [ - "ANTLR4CPP_STATIC", + "EXCLUDE_CEL_ANTLR_PARSER", ], deps = [ ":macro", - ":macro_expr_factory", ":macro_registry", ":options", ":parser_interface", ":source_factory", "//common:ast", - "//common:constant", - "//common:expr_factory", - "//common:operators", "//common:source", "//common/ast:expr_proto", "//common/ast:source_info_proto", - "//internal:lexis", "//internal:status_macros", - "//internal:strings", - "//internal:utf8", - "//parser/internal:cel_cc_parser", - "@antlr4-cpp-runtime", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/cleanup", - "@com_google_absl//absl/container:btree", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/functional:overload", - "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/status", + "//parser/internal:pratt_parser", "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", - "@com_google_absl//absl/types:variant", + "@com_google_absl//absl/strings:string_view", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", ], ) @@ -162,6 +165,9 @@ cc_library( hdrs = [ "source_factory.h", ], + deps = [ + "@com_google_absl//absl/container:flat_hash_map", + ], ) cc_library( @@ -184,9 +190,36 @@ cc_test( ":source_factory", "//common:constant", "//common:expr", + "//common:expr_printer", + "//common:source", + "//internal:testing", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:optional", + "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", + ], +) + +cc_test( + name = "pratt_parser_test", + srcs = ["parser_test.cc"], + defines = [ + "EXCLUDE_CEL_ANTLR_PARSER", + ], + deps = [ + ":macro", + ":options", + ":parser_interface", + ":pratt_parser", + ":source_factory", + "//common:constant", + "//common:expr", + "//common:expr_printer", "//common:source", "//internal:testing", - "//testutil:expr_printer", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", @@ -208,10 +241,10 @@ cc_test( ":source_factory", "//common:constant", "//common:expr", + "//common:expr_printer", "//common:source", "//internal:benchmark", "//internal:testing", - "//testutil:expr_printer", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", @@ -244,9 +277,11 @@ cc_library( ":options", "//common:ast", "//common:source", + "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/functional:any_invocable", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", ], ) diff --git a/parser/internal/BUILD b/parser/internal/BUILD index af815588e..d679e2853 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -13,12 +13,44 @@ # limitations under the License. load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_test.bzl", "cc_test") load("//bazel:antlr.bzl", "antlr_cc_library") package(default_visibility = ["//visibility:public"]) licenses(["notice"]) +cc_library( + name = "ast_factory_interface", + hdrs = ["ast_factory_interface.h"], + deps = [ + "@com_google_absl//absl/functional:function_ref", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/types:span", + ], +) + +cc_library( + name = "ast_factory", + srcs = ["ast_factory.cc"], + hdrs = ["ast_factory.h"], + deps = [ + ":ast_factory_interface", + "//common:expr", + "//common:expr_factory", + "//internal:status_macros", + "//parser:macro", + "//parser:macro_expr_factory", + "//parser:macro_registry", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/functional:function_ref", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + ], +) + cc_library( name = "options", hdrs = ["options.h"], @@ -29,3 +61,216 @@ antlr_cc_library( src = "Cel.g4", package = "cel_parser_internal", ) + +cc_library( + name = "antlr_parser", + srcs = ["antlr_parser.cc"], + hdrs = ["antlr_parser.h"], + copts = [ + "-fexceptions", + ], + defines = [ + "ANTLR4CPP_STATIC", + ], + features = ["-use_header_modules"], # Incompatible with -fexceptions + deps = [ + ":cel_cc_parser", + "//common:ast", + "//common:constant", + "//common:expr", + "//common:expr_factory", + "//common:operators", + "//common:source", + "//internal:lexis", + "//internal:status_macros", + "//internal:strings", + "//parser:macro", + "//parser:macro_expr_factory", + "//parser:macro_registry", + "//parser:options", + "//parser:parser_interface", + "//parser:source_factory", + "@antlr4-cpp-runtime", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/container:btree", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/functional:overload", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", + "@com_google_absl//absl/types:variant", + ], +) + +cc_test( + name = "antlr_parser_test", + srcs = ["antlr_parser_test.cc"], + deps = [ + ":antlr_parser", + "//common:ast", + "//common:navigable_ast", + "//common:source", + "//internal:status_macros", + "//internal:testing", + "//parser:options", + "//parser:parser_interface", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + ], +) + +cc_library( + name = "lexer", + srcs = ["lexer.cc"], + hdrs = ["lexer.h"], + deps = [ + "//common:source", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/functional:function_ref", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + ], +) + +cc_library( + name = "pratt_parser_worker", + srcs = ["pratt_parser_worker.cc"], + hdrs = ["pratt_parser_worker.h"], + deps = [ + ":ast_factory_interface", + ":lexer", + "//common:operators", + "//common:source", + "//internal:lexis", + "//internal:strings", + "//parser:options", + "//parser:parser_interface", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", + ], +) + +cc_library( + name = "pratt_parser", + srcs = ["pratt_parser.cc"], + hdrs = ["pratt_parser.h"], + deps = [ + ":ast_factory", + ":pratt_parser_worker", + "//common:ast", + "//common:expr", + "//common:source", + "//internal:status_macros", + "//parser:macro", + "//parser:macro_registry", + "//parser:options", + "//parser:parser_interface", + "//parser:source_factory", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:span", + ], +) + +cc_test( + name = "ast_factory_test", + srcs = ["ast_factory_test.cc"], + deps = [ + ":ast_factory", + ":ast_factory_interface", + "//common:constant", + "//common:expr", + "//internal:testing", + "//parser:macro", + "//parser:macro_expr_factory", + "//parser:macro_registry", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + ], +) + +cc_test( + name = "lexer_test", + srcs = ["lexer_test.cc"], + deps = [ + ":lexer", + "//common:source", + "//internal:testing", + ], +) + +cc_test( + name = "pratt_parser_test", + srcs = ["pratt_parser_test.cc"], + deps = [ + ":lexer", + ":pratt_parser", + ":pratt_parser_worker", + "//common:ast", + "//common:constant", + "//common:expr", + "//common:expr_printer", + "//common:source", + "//internal:status_macros", + "//internal:testing", + "//parser:macro", + "//parser:macro_expr_factory", + "//parser:options", + "//parser:parser_interface", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + ], +) + +cc_test( + name = "pratt_parser_benchmark", + srcs = ["pratt_parser_benchmark.cc"], + tags = ["benchmark"], + deps = [ + ":pratt_parser", + "//common:source", + "//internal:benchmark", + "//internal:testing", + "//parser", + "//parser:options", + "//parser:parser_interface", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings", + ], +) diff --git a/parser/internal/antlr_parser.cc b/parser/internal/antlr_parser.cc new file mode 100644 index 000000000..14f125c84 --- /dev/null +++ b/parser/internal/antlr_parser.cc @@ -0,0 +1,1869 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/antlr_parser.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/macros.h" +#include "absl/base/nullability.h" +#include "absl/base/optimization.h" +#include "absl/cleanup/cleanup.h" +#include "absl/container/btree_map.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/functional/overload.h" +#include "absl/log/absl_check.h" +#include "absl/log/check.h" +#include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" +#include "absl/types/variant.h" +#include "antlr4-runtime.h" +#include "common/ast.h" +#include "common/constant.h" +#include "common/expr.h" +#include "common/expr_factory.h" +#include "common/operators.h" +#include "common/source.h" +#include "internal/lexis.h" +#include "internal/status_macros.h" +#include "internal/strings.h" +#pragma push_macro("IN") +#undef IN +#include "parser/internal/CelBaseVisitor.h" +#include "parser/internal/CelLexer.h" +#include "parser/internal/CelParser.h" +#pragma pop_macro("IN") +#include "parser/macro.h" +#include "parser/macro_expr_factory.h" +#include "parser/macro_registry.h" +#include "parser/options.h" +#include "parser/parser_interface.h" +#include "parser/source_factory.h" + +namespace cel { + +namespace { + +constexpr const char kHiddenAccumulatorVariableName[] = "@result"; + +std::any ExprPtrToAny(std::unique_ptr&& expr) { + return std::make_any(expr.release()); +} + +std::any ExprToAny(Expr&& expr) { + return ExprPtrToAny(std::make_unique(std::move(expr))); +} + +std::unique_ptr ExprPtrFromAny(std::any&& any) { + return absl::WrapUnique(std::any_cast(std::move(any))); +} + +Expr ExprFromAny(std::any&& any) { + auto expr = ExprPtrFromAny(std::move(any)); + return std::move(*expr); +} + +struct ParserError { + std::string message; + SourceRange range; +}; + +std::string DisplayParserError(const cel::Source& source, + SourceLocation location, + absl::string_view message) { + return absl::StrCat(absl::StrFormat("ERROR: %s:%zu:%zu: %s", + source.description(), location.line, + // add one to the 0-based column + location.column + 1, message), + source.DisplayErrorLocation(location)); +} + +int32_t PositiveOrMax(int32_t value) { + return value >= 0 ? value : std::numeric_limits::max(); +} + +SourceRange SourceRangeFromToken(const antlr4::Token* token) { + SourceRange range; + if (token != nullptr) { + if (auto start = token->getStartIndex(); start != INVALID_INDEX) { + range.begin = static_cast(start); + } + if (auto end = token->getStopIndex(); end != INVALID_INDEX) { + range.end = static_cast(end + 1); + } + } + return range; +} + +SourceRange SourceRangeFromParserRuleContext( + const antlr4::ParserRuleContext* context) { + SourceRange range; + if (context != nullptr) { + if (auto start = context->getStart() != nullptr + ? context->getStart()->getStartIndex() + : INVALID_INDEX; + start != INVALID_INDEX) { + range.begin = static_cast(start); + } + if (auto end = context->getStop() != nullptr + ? context->getStop()->getStopIndex() + : INVALID_INDEX; + end != INVALID_INDEX) { + range.end = static_cast(end + 1); + } + } + return range; +} + +} // namespace + +class ParserMacroExprFactory final : public MacroExprFactory { + public: + explicit ParserMacroExprFactory(const cel::Source& source, + int expression_node_limit) + : source_(source), expression_node_limit_(expression_node_limit) {} + + void BeginMacro(SourceRange macro_position) { + macro_position_ = macro_position; + } + + void EndMacro() { macro_position_ = SourceRange{}; } + + Expr ReportError(absl::string_view message) override { + return ReportError(macro_position_, message); + } + + Expr ReportError(int64_t expr_id, absl::string_view message) { + return ReportError(GetSourceRange(expr_id), message); + } + + Expr ReportError(SourceRange range, absl::string_view message) { + ++error_count_; + if (errors_.size() <= 100) { + errors_.push_back(ParserError{std::string(message), range}); + } + return NewUnspecified(NextId(range)); + } + + Expr ReportErrorAt(const Expr& expr, absl::string_view message) override { + return ReportError(GetSourceRange(expr.id()), message); + } + + SourceRange GetSourceRange(int64_t id) const { + if (auto it = positions_.find(id); it != positions_.end()) { + return it->second; + } + return SourceRange{}; + } + + int64_t NextId(const SourceRange& range) { + auto id = expr_id_++; + if (id > expression_node_limit_ && !node_limit_exceeded_) { + node_limit_exceeded_ = true; + ReportError(range, "expression node limit exceeded"); + } + if (range.begin != -1 || range.end != -1) { + positions_.insert(std::pair{id, range}); + } + return id; + } + + bool is_node_limit_exceeded() const { return node_limit_exceeded_; } + + bool HasErrors() const { return error_count_ != 0; } + + std::vector CollectIssues() { + // Errors are collected as they are encountered, not by their location + // within the source. To have a more stable error message as implementation + // details change, we sort the collected errors by their source location + // first. + std::stable_sort( + errors_.begin(), errors_.end(), + [](const ParserError& lhs, const ParserError& rhs) -> bool { + auto lhs_begin = PositiveOrMax(lhs.range.begin); + auto lhs_end = PositiveOrMax(lhs.range.end); + auto rhs_begin = PositiveOrMax(rhs.range.begin); + auto rhs_end = PositiveOrMax(rhs.range.end); + return lhs_begin < rhs_begin || + (lhs_begin == rhs_begin && lhs_end < rhs_end); + }); + // Build the summary error message using the sorted errors. + bool errors_truncated = error_count_ > 100; + std::vector issues; + issues.reserve( + errors_.size() + + errors_truncated); // Reserve space for the transform and an + // additional element when truncation occurs. + std::transform( + errors_.begin(), errors_.end(), std::back_inserter(issues), + [this](const ParserError& error) { + auto location = + source_.GetLocation(error.range.begin).value_or(SourceLocation{}); + return cel::ParseIssue(location, error.message); + }); + if (errors_truncated) { + issues.push_back(cel::ParseIssue( + absl::StrCat(error_count_ - 100, " more errors were truncated."))); + } + return issues; + } + + void AddMacroCall(int64_t macro_id, absl::string_view function, + absl::optional target, std::vector arguments) { + macro_calls_.insert( + {macro_id, target.has_value() + ? NewMemberCall(0, function, std::move(*target), + std::move(arguments)) + : NewCall(0, function, std::move(arguments))}); + } + + Expr BuildMacroCallArg(const Expr& expr) { + if (auto it = macro_calls_.find(expr.id()); it != macro_calls_.end()) { + return NewUnspecified(expr.id()); + } + return absl::visit( + absl::Overload( + [this, &expr](const UnspecifiedExpr&) -> Expr { + return NewUnspecified(expr.id()); + }, + [this, &expr](const Constant& const_expr) -> Expr { + return NewConst(expr.id(), const_expr); + }, + [this, &expr](const IdentExpr& ident_expr) -> Expr { + return NewIdent(expr.id(), ident_expr.name()); + }, + [this, &expr](const SelectExpr& select_expr) -> Expr { + return select_expr.test_only() + ? NewPresenceTest( + expr.id(), + BuildMacroCallArg(select_expr.operand()), + select_expr.field()) + : NewSelect(expr.id(), + BuildMacroCallArg(select_expr.operand()), + select_expr.field()); + }, + [this, &expr](const CallExpr& call_expr) -> Expr { + std::vector macro_arguments; + macro_arguments.reserve(call_expr.args().size()); + for (const auto& argument : call_expr.args()) { + macro_arguments.push_back(BuildMacroCallArg(argument)); + } + absl::optional macro_target; + if (call_expr.has_target()) { + macro_target = BuildMacroCallArg(call_expr.target()); + } + return macro_target.has_value() + ? NewMemberCall(expr.id(), call_expr.function(), + std::move(*macro_target), + std::move(macro_arguments)) + : NewCall(expr.id(), call_expr.function(), + std::move(macro_arguments)); + }, + [this, &expr](const ListExpr& list_expr) -> Expr { + std::vector macro_elements; + macro_elements.reserve(list_expr.elements().size()); + for (const auto& element : list_expr.elements()) { + auto& cloned_element = macro_elements.emplace_back(); + if (element.has_expr()) { + cloned_element.set_expr(BuildMacroCallArg(element.expr())); + } + cloned_element.set_optional(element.optional()); + } + return NewList(expr.id(), std::move(macro_elements)); + }, + [this, &expr](const StructExpr& struct_expr) -> Expr { + std::vector macro_fields; + macro_fields.reserve(struct_expr.fields().size()); + for (const auto& field : struct_expr.fields()) { + auto& macro_field = macro_fields.emplace_back(); + macro_field.set_id(field.id()); + macro_field.set_name(field.name()); + macro_field.set_value(BuildMacroCallArg(field.value())); + macro_field.set_optional(field.optional()); + } + return NewStruct(expr.id(), struct_expr.name(), + std::move(macro_fields)); + }, + [this, &expr](const MapExpr& map_expr) -> Expr { + std::vector macro_entries; + macro_entries.reserve(map_expr.entries().size()); + for (const auto& entry : map_expr.entries()) { + auto& macro_entry = macro_entries.emplace_back(); + macro_entry.set_id(entry.id()); + macro_entry.set_key(BuildMacroCallArg(entry.key())); + macro_entry.set_value(BuildMacroCallArg(entry.value())); + macro_entry.set_optional(entry.optional()); + } + return NewMap(expr.id(), std::move(macro_entries)); + }, + [this, &expr](const ComprehensionExpr& comprehension_expr) -> Expr { + return NewComprehension( + expr.id(), comprehension_expr.iter_var(), + BuildMacroCallArg(comprehension_expr.iter_range()), + comprehension_expr.accu_var(), + BuildMacroCallArg(comprehension_expr.accu_init()), + BuildMacroCallArg(comprehension_expr.loop_condition()), + BuildMacroCallArg(comprehension_expr.loop_step()), + BuildMacroCallArg(comprehension_expr.result())); + }), + expr.kind()); + } + + using ExprFactory::NewBoolConst; + using ExprFactory::NewBytesConst; + using ExprFactory::NewCall; + using ExprFactory::NewComprehension; + using ExprFactory::NewConst; + using ExprFactory::NewDoubleConst; + using ExprFactory::NewIdent; + using ExprFactory::NewIntConst; + using ExprFactory::NewList; + using ExprFactory::NewListElement; + using ExprFactory::NewMap; + using ExprFactory::NewMapEntry; + using ExprFactory::NewMemberCall; + using ExprFactory::NewNullConst; + using ExprFactory::NewPresenceTest; + using ExprFactory::NewSelect; + using ExprFactory::NewStringConst; + using ExprFactory::NewStruct; + using ExprFactory::NewStructField; + using ExprFactory::NewUintConst; + using ExprFactory::NewUnspecified; + + const absl::btree_map& positions() const { + return positions_; + } + + const absl::flat_hash_map& macro_calls() const { + return macro_calls_; + } + + absl::flat_hash_map release_macro_calls() { + using std::swap; + absl::flat_hash_map result; + swap(result, macro_calls_); + return result; + } + + void EraseId(ExprId id) { + positions_.erase(id); + if (expr_id_ == id + 1) { + --expr_id_; + } + } + + protected: + int64_t NextId() override { return NextId(macro_position_); } + + int64_t CopyId(int64_t id) override { + if (id == 0) { + return 0; + } + return NextId(GetSourceRange(id)); + } + + private: + int64_t expr_id_ = 1; + absl::btree_map positions_; + absl::flat_hash_map macro_calls_; + std::vector errors_; + size_t error_count_ = 0; + const Source& source_; + int expression_node_limit_; + bool node_limit_exceeded_ = false; + SourceRange macro_position_; +}; + +} // namespace cel + +namespace cel::parser_internal { + +namespace { + +using ::antlr4::CharStream; +using ::antlr4::CommonTokenStream; +using ::antlr4::DefaultErrorStrategy; +using ::antlr4::ParseCancellationException; +using ::antlr4::Parser; +using ::antlr4::ParserRuleContext; +using ::antlr4::Token; +using ::antlr4::misc::IntervalSet; +using ::antlr4::tree::ErrorNode; +using ::antlr4::tree::ParseTreeListener; +using ::antlr4::tree::TerminalNode; +using ::cel_parser_internal::CelBaseVisitor; +using ::cel_parser_internal::CelLexer; +using ::cel_parser_internal::CelParser; +using ::google::api::expr::common::CelOperator; +using ::google::api::expr::common::ReverseLookupOperator; + +class CodePointStream final : public CharStream { + public: + CodePointStream(cel::SourceContentView buffer, absl::string_view source_name) + : buffer_(buffer), + source_name_(source_name), + size_(buffer_.size()), + index_(0) {} + + void consume() override { + if (ABSL_PREDICT_FALSE(index_ >= size_)) { + ABSL_ASSERT(LA(1) == IntStream::EOF); + throw antlr4::IllegalStateException("cannot consume EOF"); + } + index_++; + } + + size_t LA(ptrdiff_t i) override { + if (ABSL_PREDICT_FALSE(i == 0)) { + return 0; + } + auto p = static_cast(index_); + if (i < 0) { + i++; + if (p + i - 1 < 0) { + return IntStream::EOF; + } + } + if (p + i - 1 >= static_cast(size_)) { + return IntStream::EOF; + } + return buffer_.at(static_cast(p + i - 1)); + } + + ptrdiff_t mark() override { return -1; } + + void release(ptrdiff_t marker) override {} + + size_t index() override { return index_; } + + void seek(size_t index) override { index_ = std::min(index, size_); } + + size_t size() override { return size_; } + + std::string getSourceName() const override { + return source_name_.empty() ? IntStream::UNKNOWN_SOURCE_NAME + : std::string(source_name_); + } + + std::string getText(const antlr4::misc::Interval& interval) override { + if (ABSL_PREDICT_FALSE(interval.a < 0 || interval.b < 0)) { + return std::string(); + } + size_t start = static_cast(interval.a); + if (ABSL_PREDICT_FALSE(start >= size_)) { + return std::string(); + } + size_t stop = static_cast(interval.b); + if (ABSL_PREDICT_FALSE(stop >= size_)) { + stop = size_ - 1; + } + return buffer_.ToString(static_cast(start), + static_cast(stop) + 1); + } + + std::string toString() const override { return buffer_.ToString(); } + + private: + cel::SourceContentView const buffer_; + const absl::string_view source_name_; + const size_t size_; + size_t index_; +}; + +// Scoped helper for incrementing the parse recursion count. +// Increments on creation, decrements on destruction (stack unwind). +class ScopedIncrement final { + public: + explicit ScopedIncrement(int& recursion_depth) + : recursion_depth_(recursion_depth) { + ++recursion_depth_; + } + + ~ScopedIncrement() { --recursion_depth_; } + + private: + int& recursion_depth_; +}; + +// balancer performs tree balancing on operators whose arguments are of equal +// precedence. +// +// The purpose of the balancer is to ensure a compact serialization format for +// the logical &&, || operators which have a tendency to create long DAGs which +// are skewed in one direction. Since the operators are commutative re-ordering +// the terms *must not* affect the evaluation result. +// +// Based on code from //third_party/cel/go/parser/helper.go +class ExpressionBalancer final { + public: + ExpressionBalancer(cel::ParserMacroExprFactory& factory, std::string function, + Expr expr); + + // addTerm adds an operation identifier and term to the set of terms to be + // balanced. + void AddTerm(int64_t op, Expr term); + + // balance creates a balanced tree from the sub-terms and returns the final + // Expr value. + Expr Balance(bool enable_variadic = false); + + private: + // balancedTree recursively balances the terms provided to a commutative + // operator. + Expr BalancedTree(int lo, int hi); + + private: + cel::ParserMacroExprFactory& factory_; + std::string function_; + std::vector terms_; + std::vector ops_; +}; + +ExpressionBalancer::ExpressionBalancer(cel::ParserMacroExprFactory& factory, + std::string function, Expr expr) + : factory_(factory), function_(std::move(function)) { + terms_.push_back(std::move(expr)); +} + +void ExpressionBalancer::AddTerm(int64_t op, Expr term) { + terms_.push_back(std::move(term)); + ops_.push_back(op); +} + +Expr ExpressionBalancer::Balance(bool enable_variadic) { + if (terms_.size() == 1) { + return std::move(terms_[0]); + } + if (enable_variadic) { + return factory_.NewCall(ops_[0], function_, std::move(terms_)); + } + return BalancedTree(0, ops_.size() - 1); +} + +Expr ExpressionBalancer::BalancedTree(int lo, int hi) { + int mid = (lo + hi + 1) / 2; + + std::vector arguments; + arguments.reserve(2); + + if (mid == lo) { + arguments.push_back(std::move(terms_[mid])); + } else { + arguments.push_back(BalancedTree(lo, mid - 1)); + } + + if (mid == hi) { + arguments.push_back(std::move(terms_[mid + 1])); + } else { + arguments.push_back(BalancedTree(mid + 1, hi)); + } + return factory_.NewCall(ops_[mid], function_, std::move(arguments)); +} + +std::string FormatIssues(const cel::Source& source, + absl::Span issues) { + return absl::StrJoin( + issues, "\n", [&source](std::string* out, const cel::ParseIssue& issue) { + absl::StrAppend( + out, DisplayParserError(source, issue.location(), issue.message())); + }); +} + +class ParserVisitor final : public CelBaseVisitor, + public antlr4::BaseErrorListener { + public: + ParserVisitor(const cel::Source& source, int max_recursion_depth, + int max_expression_node_count, + const cel::MacroRegistry& macro_registry, + bool add_macro_calls = false, + bool enable_optional_syntax = false, + bool enable_quoted_identifiers = false, + bool enable_variadic_logical_operators = false, + bool fold_unary_operators = false) + : source_(source), + factory_(source_, max_expression_node_count), + macro_registry_(macro_registry), + recursion_depth_(0), + max_recursion_depth_(max_recursion_depth), + add_macro_calls_(add_macro_calls), + enable_optional_syntax_(enable_optional_syntax), + enable_quoted_identifiers_(enable_quoted_identifiers), + enable_variadic_logical_operators_(enable_variadic_logical_operators), + fold_unary_operators_(fold_unary_operators) {} + + ~ParserVisitor() override = default; + + std::any visit(antlr4::tree::ParseTree* tree) override; + + std::any visitStart(CelParser::StartContext* ctx) override; + std::any visitExpr(CelParser::ExprContext* ctx) override; + std::any visitConditionalOr(CelParser::ConditionalOrContext* ctx) override; + std::any visitConditionalAnd(CelParser::ConditionalAndContext* ctx) override; + std::any visitRelation(CelParser::RelationContext* ctx) override; + std::any visitCalc(CelParser::CalcContext* ctx) override; + std::any visitUnary(CelParser::UnaryContext* ctx); + std::any visitLogicalNot(CelParser::LogicalNotContext* ctx) override; + std::any visitNegate(CelParser::NegateContext* ctx) override; + std::any visitSelect(CelParser::SelectContext* ctx) override; + std::any visitMemberCall(CelParser::MemberCallContext* ctx) override; + std::any visitIndex(CelParser::IndexContext* ctx) override; + std::any visitCreateMessage(CelParser::CreateMessageContext* ctx) override; + std::any visitFieldInitializerList( + CelParser::FieldInitializerListContext* ctx) override; + std::vector visitFields( + CelParser::FieldInitializerListContext* ctx); + std::any visitGlobalCall(CelParser::GlobalCallContext* ctx) override; + std::any visitIdent(CelParser::IdentContext* ctx) override; + std::any visitNested(CelParser::NestedContext* ctx) override; + std::any visitCreateList(CelParser::CreateListContext* ctx) override; + std::vector visitList(CelParser::ListInitContext* ctx); + std::vector visitList(CelParser::ExprListContext* ctx); + std::any visitCreateMap(CelParser::CreateMapContext* ctx) override; + std::any visitConstantLiteral( + CelParser::ConstantLiteralContext* ctx) override; + std::any visitPrimaryExpr(CelParser::PrimaryExprContext* ctx) override; + std::any visitMemberExpr(CelParser::MemberExprContext* ctx) override; + + std::any visitMapInitializerList( + CelParser::MapInitializerListContext* ctx) override; + std::vector visitEntries( + CelParser::MapInitializerListContext* ctx); + std::any visitInt(CelParser::IntContext* ctx) override; + std::any visitUint(CelParser::UintContext* ctx) override; + std::any visitDouble(CelParser::DoubleContext* ctx) override; + std::any visitString(CelParser::StringContext* ctx) override; + std::any visitBytes(CelParser::BytesContext* ctx) override; + std::any visitBoolTrue(CelParser::BoolTrueContext* ctx) override; + std::any visitBoolFalse(CelParser::BoolFalseContext* ctx) override; + std::any visitNull(CelParser::NullContext* ctx) override; + // Note: this is destructive and intended to be called after the parse is + // finished. + cel::SourceInfo GetSourceInfo(); + EnrichedSourceInfo enriched_source_info() const; + void syntaxError(antlr4::Recognizer* recognizer, + antlr4::Token* offending_symbol, size_t line, size_t col, + const std::string& msg, std::exception_ptr e) override; + bool HasErrored() const; + + std::vector CollectIssues(); + + private: + template + Expr GlobalCallOrMacro(int64_t expr_id, absl::string_view function, + Args&&... args) { + std::vector arguments; + arguments.reserve(sizeof...(Args)); + (arguments.push_back(std::forward(args)), ...); + return GlobalCallOrMacroImpl(expr_id, function, std::move(arguments)); + } + + Expr GlobalCallOrMacroImpl(int64_t expr_id, absl::string_view function, + std::vector args); + Expr ReceiverCallOrMacroImpl(int64_t expr_id, absl::string_view function, + Expr target, std::vector args); + std::string ExtractQualifiedName(antlr4::ParserRuleContext* ctx, + const Expr& e); + + std::string NormalizeIdentifier(CelParser::EscapeIdentContext* ctx); + std::any VisitUnaryOps(const std::vector& ops, + CelParser::MemberContext* member, + absl::string_view op_name); + // Attempt to unnest parse context. + // + // Walk the parse tree to the first complex term to reduce recursive depth in + // the visit* calls. + antlr4::tree::ParseTree* UnnestContext(antlr4::tree::ParseTree* tree); + + private: + const cel::Source& source_; + cel::ParserMacroExprFactory factory_; + const cel::MacroRegistry& macro_registry_; + int recursion_depth_; + const int max_recursion_depth_; + const bool add_macro_calls_; + const bool enable_optional_syntax_; + const bool enable_quoted_identifiers_; + const bool enable_variadic_logical_operators_; + const bool fold_unary_operators_; +}; + +template >> +T* tree_as(antlr4::tree::ParseTree* tree) { + return dynamic_cast(tree); +} + +std::any ParserVisitor::visit(antlr4::tree::ParseTree* tree) { + ScopedIncrement inc(recursion_depth_); + if (recursion_depth_ > max_recursion_depth_) { + return ExprToAny(factory_.ReportError( + absl::StrFormat("Exceeded max recursion depth of %d when parsing.", + max_recursion_depth_))); + } + tree = UnnestContext(tree); + if (auto* ctx = tree_as(tree)) { + return visitStart(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitExpr(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitConditionalAnd(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitConditionalOr(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitRelation(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitCalc(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitLogicalNot(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitPrimaryExpr(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitMemberExpr(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitSelect(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitMemberCall(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitMapInitializerList(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitNegate(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitIndex(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitUnary(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitCreateList(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitCreateMessage(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitCreateMap(ctx); + } + + if (tree) { + return ExprToAny( + factory_.ReportError(SourceRangeFromParserRuleContext( + tree_as(tree)), + "unknown parsetree type")); + } + return ExprToAny(factory_.ReportError("<> parsetree")); +} + +std::any ParserVisitor::visitPrimaryExpr(CelParser::PrimaryExprContext* pctx) { + CelParser::PrimaryContext* primary = pctx->primary(); + if (auto* ctx = tree_as(primary)) { + return visitNested(ctx); + } else if (auto* ctx = tree_as(primary)) { + return visitIdent(ctx); + } else if (auto* ctx = tree_as(primary)) { + return visitGlobalCall(ctx); + } else if (auto* ctx = tree_as(primary)) { + return visitCreateList(ctx); + } else if (auto* ctx = tree_as(primary)) { + return visitCreateMap(ctx); + } else if (auto* ctx = tree_as(primary)) { + return visitCreateMessage(ctx); + } else if (auto* ctx = tree_as(primary)) { + return visitConstantLiteral(ctx); + } + if (factory_.HasErrors()) { + // ANTLR creates PrimaryContext rather than a derived class during certain + // error conditions. This is odd, but we ignore it as we already have errors + // that occurred. + return ExprToAny(factory_.NewUnspecified(factory_.NextId({}))); + } + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(pctx), + "invalid primary expression")); +} + +std::any ParserVisitor::visitMemberExpr(CelParser::MemberExprContext* mctx) { + CelParser::MemberContext* member = mctx->member(); + if (auto* ctx = tree_as(member)) { + return visitPrimaryExpr(ctx); + } else if (auto* ctx = tree_as(member)) { + return visitSelect(ctx); + } else if (auto* ctx = tree_as(member)) { + return visitMemberCall(ctx); + } else if (auto* ctx = tree_as(member)) { + return visitIndex(ctx); + } + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(mctx), + "unsupported simple expression")); +} + +std::any ParserVisitor::visitStart(CelParser::StartContext* ctx) { + return visit(ctx->expr()); +} + +antlr4::tree::ParseTree* ParserVisitor::UnnestContext( + antlr4::tree::ParseTree* tree) { + antlr4::tree::ParseTree* last = nullptr; + while (tree != last) { + last = tree; + + if (auto* ctx = tree_as(tree)) { + tree = ctx->expr(); + } + + if (auto* ctx = tree_as(tree)) { + if (ctx->op != nullptr) { + return ctx; + } + tree = ctx->e; + } + + if (auto* ctx = tree_as(tree)) { + if (!ctx->ops.empty()) { + return ctx; + } + tree = ctx->e; + } + + if (auto* ctx = tree_as(tree)) { + if (!ctx->ops.empty()) { + return ctx; + } + tree = ctx->e; + } + + if (auto* ctx = tree_as(tree)) { + if (ctx->calc() == nullptr) { + return ctx; + } + tree = ctx->calc(); + } + + if (auto* ctx = tree_as(tree)) { + if (ctx->unary() == nullptr) { + return ctx; + } + tree = ctx->unary(); + } + + if (auto* ctx = tree_as(tree)) { + tree = ctx->member(); + } + + if (auto* ctx = tree_as(tree)) { + if (auto* nested = tree_as(ctx->primary())) { + tree = nested->e; + } else { + return ctx; + } + } + } + + return tree; +} + +std::any ParserVisitor::visitExpr(CelParser::ExprContext* ctx) { + auto result = ExprFromAny(visit(ctx->e)); + if (!ctx->op) { + return ExprToAny(std::move(result)); + } + std::vector arguments; + arguments.reserve(3); + arguments.push_back(std::move(result)); + int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + arguments.push_back(ExprFromAny(visit(ctx->e1))); + arguments.push_back(ExprFromAny(visit(ctx->e2))); + + return ExprToAny( + factory_.NewCall(op_id, CelOperator::CONDITIONAL, std::move(arguments))); +} + +std::any ParserVisitor::visitConditionalOr( + CelParser::ConditionalOrContext* ctx) { + auto result = ExprFromAny(visit(ctx->e)); + if (ctx->ops.empty()) { + return ExprToAny(std::move(result)); + } + ExpressionBalancer b(factory_, CelOperator::LOGICAL_OR, std::move(result)); + for (size_t i = 0; i < ctx->ops.size(); ++i) { + auto op = ctx->ops[i]; + if (i >= ctx->e1.size()) { + return ExprToAny( + factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "unexpected character, wanted '||'")); + } + auto next = ExprFromAny(visit(ctx->e1[i])); + int64_t op_id = factory_.NextId(SourceRangeFromToken(op)); + b.AddTerm(op_id, std::move(next)); + } + return ExprToAny(b.Balance(enable_variadic_logical_operators_)); +} + +std::any ParserVisitor::visitConditionalAnd( + CelParser::ConditionalAndContext* ctx) { + auto result = ExprFromAny(visit(ctx->e)); + if (ctx->ops.empty()) { + return ExprToAny(std::move(result)); + } + ExpressionBalancer b(factory_, CelOperator::LOGICAL_AND, std::move(result)); + for (size_t i = 0; i < ctx->ops.size(); ++i) { + auto op = ctx->ops[i]; + if (i >= ctx->e1.size()) { + return ExprToAny( + factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "unexpected character, wanted '&&'")); + } + auto next = ExprFromAny(visit(ctx->e1[i])); + int64_t op_id = factory_.NextId(SourceRangeFromToken(op)); + b.AddTerm(op_id, std::move(next)); + } + return ExprToAny(b.Balance(enable_variadic_logical_operators_)); +} + +std::any ParserVisitor::visitRelation(CelParser::RelationContext* ctx) { + if (ctx->calc()) { + return visit(ctx->calc()); + } + std::string op_text; + if (ctx->op) { + op_text = ctx->op->getText(); + } + auto op = ReverseLookupOperator(op_text); + if (op) { + auto lhs = ExprFromAny(visit(ctx->relation(0))); + int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + auto rhs = ExprFromAny(visit(ctx->relation(1))); + return ExprToAny( + GlobalCallOrMacro(op_id, *op, std::move(lhs), std::move(rhs))); + } + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "operator not found")); +} + +std::any ParserVisitor::visitCalc(CelParser::CalcContext* ctx) { + if (ctx->unary()) { + return visit(ctx->unary()); + } + std::string op_text; + if (ctx->op) { + op_text = ctx->op->getText(); + } + auto op = ReverseLookupOperator(op_text); + if (op) { + auto lhs = ExprFromAny(visit(ctx->calc(0))); + int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + auto rhs = ExprFromAny(visit(ctx->calc(1))); + return ExprToAny( + GlobalCallOrMacro(op_id, *op, std::move(lhs), std::move(rhs))); + } + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "operator not found")); +} + +std::any ParserVisitor::visitUnary(CelParser::UnaryContext* ctx) { + return ExprToAny(factory_.NewStringConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), "<>")); +} + +std::any ParserVisitor::VisitUnaryOps(const std::vector& ops, + CelParser::MemberContext* member, + absl::string_view op_name) { + if (fold_unary_operators_) { + if (ops.size() % 2 == 0) { + return visit(member); + } + int64_t op_id = factory_.NextId(SourceRangeFromToken(ops[0])); + auto target = ExprFromAny(visit(member)); + return ExprToAny(GlobalCallOrMacro(op_id, op_name, std::move(target))); + } + + std::vector op_ids; + op_ids.reserve(ops.size()); + for (const auto* op : ops) { + op_ids.push_back(factory_.NextId(SourceRangeFromToken(op))); + } + + auto target = ExprFromAny(visit(member)); + for (int i = static_cast(op_ids.size()) - 1; i >= 0; --i) { + target = GlobalCallOrMacro(op_ids[i], op_name, std::move(target)); + } + return ExprToAny(std::move(target)); +} + +std::any ParserVisitor::visitLogicalNot(CelParser::LogicalNotContext* ctx) { + return VisitUnaryOps(ctx->ops, ctx->member(), CelOperator::LOGICAL_NOT); +} + +std::any ParserVisitor::visitNegate(CelParser::NegateContext* ctx) { + return VisitUnaryOps(ctx->ops, ctx->member(), CelOperator::NEGATE); +} + +std::string ParserVisitor::NormalizeIdentifier( + CelParser::EscapeIdentContext* ctx) { + if (auto* raw_id = tree_as(ctx); raw_id) { + return raw_id->id->getText(); + } + if (auto* escaped_id = tree_as(ctx); + escaped_id) { + if (!enable_quoted_identifiers_) { + factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "unsupported syntax '`'"); + } + auto escaped_id_text = escaped_id->id->getText(); + return escaped_id_text.substr(1, escaped_id_text.size() - 2); + } + + // Fallthrough might occur if the parser is in an error state. + return ""; +} + +std::any ParserVisitor::visitSelect(CelParser::SelectContext* ctx) { + auto operand = ExprFromAny(visit(ctx->member())); + // Handle the error case where no valid identifier is specified. + if (!ctx->id || !ctx->op) { + return ExprToAny(factory_.NewUnspecified( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); + } + auto id = NormalizeIdentifier(ctx->id); + if (ctx->opt != nullptr) { + if (!enable_optional_syntax_) { + return ExprToAny(factory_.ReportError( + SourceRangeFromParserRuleContext(ctx), "unsupported syntax '.?'")); + } + auto op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + std::vector arguments; + arguments.reserve(2); + arguments.push_back(std::move(operand)); + arguments.push_back(factory_.NewStringConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), std::move(id))); + return ExprToAny(factory_.NewCall(op_id, "_?._", std::move(arguments))); + } + return ExprToAny( + factory_.NewSelect(factory_.NextId(SourceRangeFromToken(ctx->op)), + std::move(operand), std::move(id))); +} + +std::any ParserVisitor::visitMemberCall(CelParser::MemberCallContext* ctx) { + auto operand = ExprFromAny(visit(ctx->member())); + // Handle the error case where no valid identifier is specified. + if (!ctx->id) { + return ExprToAny(factory_.NewUnspecified( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); + } + auto id = ctx->id->getText(); + int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->open)); + auto args = visitList(ctx->args); + return ExprToAny( + ReceiverCallOrMacroImpl(op_id, id, std::move(operand), std::move(args))); +} + +std::any ParserVisitor::visitIndex(CelParser::IndexContext* ctx) { + auto target = ExprFromAny(visit(ctx->member())); + int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + auto index = ExprFromAny(visit(ctx->index)); + if (!enable_optional_syntax_ && ctx->opt != nullptr) { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "unsupported syntax '.?'")); + } + return ExprToAny(GlobalCallOrMacro( + op_id, ctx->opt != nullptr ? "_[?_]" : CelOperator::INDEX, + std::move(target), std::move(index))); +} + +std::any ParserVisitor::visitCreateMessage( + CelParser::CreateMessageContext* ctx) { + std::vector parts; + parts.reserve(ctx->ids.size()); + for (const auto* id : ctx->ids) { + parts.push_back(id->getText()); + } + std::string name; + if (ctx->leadingDot) { + name.push_back('.'); + name.append(absl::StrJoin(parts, ".")); + } else { + name = absl::StrJoin(parts, "."); + } + int64_t obj_id = factory_.NextId(SourceRangeFromParserRuleContext(ctx)); + std::vector fields; + if (ctx->entries) { + fields = visitFields(ctx->entries); + } + return ExprToAny( + factory_.NewStruct(obj_id, std::move(name), std::move(fields))); +} + +std::any ParserVisitor::visitFieldInitializerList( + CelParser::FieldInitializerListContext* ctx) { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "<>")); +} + +std::vector ParserVisitor::visitFields( + CelParser::FieldInitializerListContext* ctx) { + std::vector res; + if (!ctx || ctx->fields.empty()) { + return res; + } + + res.reserve(ctx->fields.size()); + for (size_t i = 0; i < ctx->fields.size(); ++i) { + if (i >= ctx->cols.size() || i >= ctx->values.size()) { + // This is the result of a syntax error detected elsewhere. + return res; + } + auto* f = ctx->fields[i]; + if (!f->escapeIdent()) { + ABSL_DCHECK(HasErrored()); + // This is the result of a syntax error detected elsewhere. + return res; + } + + std::string id = NormalizeIdentifier(f->escapeIdent()); + + int64_t init_id = factory_.NextId(SourceRangeFromToken(ctx->cols[i])); + if (!enable_optional_syntax_ && f->opt) { + factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "unsupported syntax '?'"); + continue; + } + auto value = ExprFromAny(visit(ctx->values[i])); + res.push_back(factory_.NewStructField(init_id, std::move(id), + std::move(value), f->opt != nullptr)); + } + + return res; +} + +std::any ParserVisitor::visitIdent(CelParser::IdentContext* ctx) { + std::string ident_name; + if (ctx->leadingDot) { + ident_name = "."; + } + if (!ctx->id) { + return ExprToAny(factory_.NewUnspecified( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); + } + // check if ID is in reserved identifiers + if (cel::internal::LexisIsReserved(ctx->id->getText())) { + return ExprToAny(factory_.ReportError( + SourceRangeFromParserRuleContext(ctx), + absl::StrFormat("reserved identifier: %s", ctx->id->getText()))); + } + + ident_name += ctx->id->getText(); + + return ExprToAny(factory_.NewIdent( + factory_.NextId(SourceRangeFromToken(ctx->id)), std::move(ident_name))); +} + +std::any ParserVisitor::visitGlobalCall(CelParser::GlobalCallContext* ctx) { + std::string ident_name; + if (ctx->leadingDot) { + ident_name = "."; + } + if (!ctx->id || !ctx->op) { + return ExprToAny(factory_.NewUnspecified( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); + } + // check if ID is in reserved identifiers + if (cel::internal::LexisIsReserved(ctx->id->getText())) { + return ExprToAny(factory_.ReportError( + SourceRangeFromParserRuleContext(ctx), + absl::StrFormat("reserved identifier: %s", ctx->id->getText()))); + } + + ident_name += ctx->id->getText(); + + int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + auto args = visitList(ctx->args); + return ExprToAny( + GlobalCallOrMacroImpl(op_id, std::move(ident_name), std::move(args))); +} + +std::any ParserVisitor::visitNested(CelParser::NestedContext* ctx) { + return visit(ctx->e); +} + +std::any ParserVisitor::visitCreateList(CelParser::CreateListContext* ctx) { + int64_t list_id = factory_.NextId(SourceRangeFromParserRuleContext(ctx)); + auto elems = visitList(ctx->elems); + return ExprToAny(factory_.NewList(list_id, std::move(elems))); +} + +std::vector ParserVisitor::visitList( + CelParser::ListInitContext* ctx) { + std::vector rv; + if (!ctx) return rv; + rv.reserve(ctx->elems.size()); + for (size_t i = 0; i < ctx->elems.size(); ++i) { + auto* expr_ctx = ctx->elems[i]; + if (expr_ctx == nullptr) { + return rv; + } + if (!enable_optional_syntax_ && expr_ctx->opt != nullptr) { + factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "unsupported syntax '?'"); + // Still generate an ID to detect node limit exceeded. + factory_.NextId(SourceRangeFromParserRuleContext(ctx)); + rv.push_back(factory_.NewListElement(factory_.NewUnspecified(0), false)); + continue; + } + rv.push_back(factory_.NewListElement(ExprFromAny(visitExpr(expr_ctx->e)), + expr_ctx->opt != nullptr)); + } + return rv; +} + +std::vector ParserVisitor::visitList(CelParser::ExprListContext* ctx) { + std::vector rv; + if (!ctx) return rv; + std::transform(ctx->e.begin(), ctx->e.end(), std::back_inserter(rv), + [this](CelParser::ExprContext* expr_ctx) { + return ExprFromAny(visitExpr(expr_ctx)); + }); + return rv; +} + +std::any ParserVisitor::visitCreateMap(CelParser::CreateMapContext* ctx) { + int64_t struct_id = factory_.NextId(SourceRangeFromParserRuleContext(ctx)); + std::vector entries; + if (ctx->entries) { + entries = visitEntries(ctx->entries); + } + return ExprToAny(factory_.NewMap(struct_id, std::move(entries))); +} + +std::any ParserVisitor::visitConstantLiteral( + CelParser::ConstantLiteralContext* clctx) { + CelParser::LiteralContext* literal = clctx->literal(); + if (auto* ctx = tree_as(literal)) { + return visitInt(ctx); + } else if (auto* ctx = tree_as(literal)) { + return visitUint(ctx); + } else if (auto* ctx = tree_as(literal)) { + return visitDouble(ctx); + } else if (auto* ctx = tree_as(literal)) { + return visitString(ctx); + } else if (auto* ctx = tree_as(literal)) { + return visitBytes(ctx); + } else if (auto* ctx = tree_as(literal)) { + return visitBoolFalse(ctx); + } else if (auto* ctx = tree_as(literal)) { + return visitBoolTrue(ctx); + } else if (auto* ctx = tree_as(literal)) { + return visitNull(ctx); + } + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(clctx), + "invalid constant literal expression")); +} + +std::any ParserVisitor::visitMapInitializerList( + CelParser::MapInitializerListContext* ctx) { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "<>")); +} + +std::vector ParserVisitor::visitEntries( + CelParser::MapInitializerListContext* ctx) { + std::vector res; + if (!ctx || ctx->keys.empty()) { + return res; + } + + res.reserve(ctx->cols.size()); + for (size_t i = 0; i < ctx->cols.size(); ++i) { + auto id = factory_.NextId(SourceRangeFromToken(ctx->cols[i])); + if (!enable_optional_syntax_ && ctx->keys[i]->opt) { + factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "unsupported syntax '?'"); + // Still generate an ID to detect node limit exceeded. + factory_.NextId(SourceRangeFromParserRuleContext(ctx)); + factory_.NextId(SourceRangeFromParserRuleContext(ctx)); + res.push_back(factory_.NewMapEntry(0, factory_.NewUnspecified(0), + factory_.NewUnspecified(0), false)); + continue; + } + auto key = ExprFromAny(visit(ctx->keys[i]->e)); + auto value = ExprFromAny(visit(ctx->values[i])); + res.push_back(factory_.NewMapEntry(id, std::move(key), std::move(value), + ctx->keys[i]->opt != nullptr)); + } + return res; +} + +std::any ParserVisitor::visitInt(CelParser::IntContext* ctx) { + std::string value; + if (ctx->sign) { + value = ctx->sign->getText(); + } + value += ctx->tok->getText(); + int64_t int_value; + if (absl::StartsWith(ctx->tok->getText(), "0x")) { + if (absl::SimpleHexAtoi(value, &int_value)) { + return ExprToAny(factory_.NewIntConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), int_value)); + } else { + return ExprToAny(factory_.ReportError( + SourceRangeFromParserRuleContext(ctx), "invalid hex int literal")); + } + } + if (absl::SimpleAtoi(value, &int_value)) { + return ExprToAny(factory_.NewIntConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), int_value)); + } else { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "invalid int literal")); + } +} + +std::any ParserVisitor::visitUint(CelParser::UintContext* ctx) { + std::string value = ctx->tok->getText(); + // trim the 'u' designator included in the uint literal. + if (!value.empty()) { + value.resize(value.size() - 1); + } + uint64_t uint_value; + if (absl::StartsWith(ctx->tok->getText(), "0x")) { + if (absl::SimpleHexAtoi(value, &uint_value)) { + return ExprToAny(factory_.NewUintConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), uint_value)); + } else { + return ExprToAny(factory_.ReportError( + SourceRangeFromParserRuleContext(ctx), "invalid hex uint literal")); + } + } + if (absl::SimpleAtoi(value, &uint_value)) { + return ExprToAny(factory_.NewUintConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), uint_value)); + } else { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "invalid uint literal")); + } +} + +std::any ParserVisitor::visitDouble(CelParser::DoubleContext* ctx) { + std::string value; + if (ctx->sign) { + value = ctx->sign->getText(); + } + value += ctx->tok->getText(); + double double_value; + if (absl::SimpleAtod(value, &double_value)) { + return ExprToAny(factory_.NewDoubleConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), double_value)); + } else { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "invalid double literal")); + } +} + +std::any ParserVisitor::visitString(CelParser::StringContext* ctx) { + auto status_or_value = cel::internal::ParseStringLiteral(ctx->tok->getText()); + if (!status_or_value.ok()) { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + status_or_value.status().message())); + } + return ExprToAny(factory_.NewStringConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), + std::move(status_or_value).value())); +} + +std::any ParserVisitor::visitBytes(CelParser::BytesContext* ctx) { + auto status_or_value = cel::internal::ParseBytesLiteral(ctx->tok->getText()); + if (!status_or_value.ok()) { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + status_or_value.status().message())); + } + return ExprToAny(factory_.NewBytesConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), + std::move(status_or_value).value())); +} + +std::any ParserVisitor::visitBoolTrue(CelParser::BoolTrueContext* ctx) { + return ExprToAny(factory_.NewBoolConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), true)); +} + +std::any ParserVisitor::visitBoolFalse(CelParser::BoolFalseContext* ctx) { + return ExprToAny(factory_.NewBoolConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), false)); +} + +std::any ParserVisitor::visitNull(CelParser::NullContext* ctx) { + return ExprToAny(factory_.NewNullConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); +} + +cel::SourceInfo ParserVisitor::GetSourceInfo() { + cel::SourceInfo source_info; + source_info.set_location(std::string(source_.description())); + for (const auto& positions : factory_.positions()) { + source_info.mutable_positions().insert( + std::pair{positions.first, positions.second.begin}); + } + source_info.mutable_line_offsets().reserve(source_.line_offsets().size()); + for (const auto& line_offset : source_.line_offsets()) { + source_info.mutable_line_offsets().push_back(line_offset); + } + + source_info.mutable_macro_calls() = factory_.release_macro_calls(); + return source_info; +} + +EnrichedSourceInfo ParserVisitor::enriched_source_info() const { + absl::flat_hash_map> offsets; + offsets.reserve(factory_.positions().size()); + for (const auto& positions : factory_.positions()) { + offsets.insert( + std::pair{positions.first, + std::pair{positions.second.begin, positions.second.end - 1}}); + } + return EnrichedSourceInfo(std::move(offsets)); +} + +void ParserVisitor::syntaxError(antlr4::Recognizer* recognizer, + antlr4::Token* offending_symbol, size_t line, + size_t col, const std::string& msg, + std::exception_ptr e) { + cel::SourceRange range; + if (auto position = source_.GetPosition(cel::SourceLocation{ + static_cast(line), static_cast(col)}); + position) { + range.begin = *position; + } + factory_.ReportError(range, absl::StrCat("Syntax error: ", msg)); +} + +bool ParserVisitor::HasErrored() const { return factory_.HasErrors(); } + +std::vector ParserVisitor::CollectIssues() { + return factory_.CollectIssues(); +} + +Expr ParserVisitor::GlobalCallOrMacroImpl(int64_t expr_id, + absl::string_view function, + std::vector args) { + auto macro = macro_registry_.FindMacro(function, args.size(), false); + if (!macro) { + return factory_.NewCall(expr_id, function, std::move(args)); + } + if (factory_.is_node_limit_exceeded()) { + return factory_.ReportError( + factory_.GetSourceRange(expr_id), + "could not expand macro: expression node limit exceeded"); + } + std::vector macro_args; + if (add_macro_calls_) { + macro_args.reserve(args.size()); + for (const auto& arg : args) { + macro_args.push_back(factory_.BuildMacroCallArg(arg)); + } + } + factory_.BeginMacro(factory_.GetSourceRange(expr_id)); + auto expr = macro->Expand(factory_, std::nullopt, absl::MakeSpan(args)); + factory_.EndMacro(); + if (expr) { + if (add_macro_calls_) { + factory_.AddMacroCall(expr->id(), function, std::nullopt, + std::move(macro_args)); + } + // We did not end up using `expr_id`. Delete metadata. + factory_.EraseId(expr_id); + return std::move(*expr); + } + return factory_.NewCall(expr_id, function, std::move(args)); +} + +Expr ParserVisitor::ReceiverCallOrMacroImpl(int64_t expr_id, + absl::string_view function, + Expr target, + std::vector args) { + auto macro = macro_registry_.FindMacro(function, args.size(), true); + if (!macro) { + return factory_.NewMemberCall(expr_id, function, std::move(target), + std::move(args)); + } + if (factory_.is_node_limit_exceeded()) { + return factory_.ReportError( + factory_.GetSourceRange(expr_id), + "could not expand macro: expression node limit exceeded"); + } + + Expr macro_target; + std::vector macro_args; + if (add_macro_calls_) { + macro_args.reserve(args.size()); + macro_target = factory_.BuildMacroCallArg(target); + for (const auto& arg : args) { + macro_args.push_back(factory_.BuildMacroCallArg(arg)); + } + } + factory_.BeginMacro(factory_.GetSourceRange(expr_id)); + auto expr = macro->Expand(factory_, std::ref(target), absl::MakeSpan(args)); + factory_.EndMacro(); + if (expr) { + if (add_macro_calls_) { + factory_.AddMacroCall(expr->id(), function, std::move(macro_target), + std::move(macro_args)); + } + // We did not end up using `expr_id`. Delete metadata. + factory_.EraseId(expr_id); + return std::move(*expr); + } + + return factory_.NewMemberCall(expr_id, function, std::move(target), + std::move(args)); +} + +std::string ParserVisitor::ExtractQualifiedName(antlr4::ParserRuleContext* ctx, + const Expr& e) { + if (e == Expr{}) { + return ""; + } + + if (const auto* ident_expr = absl::get_if(&e.kind()); ident_expr) { + return ident_expr->name(); + } + if (const auto* select_expr = absl::get_if(&e.kind()); + select_expr) { + std::string prefix = ExtractQualifiedName(ctx, select_expr->operand()); + if (!prefix.empty()) { + return absl::StrCat(prefix, ".", select_expr->field()); + } + } + factory_.ReportError(factory_.GetSourceRange(e.id()), + "expected a qualified name"); + return ""; +} + +// Replacements for absl::StrReplaceAll for escaping standard whitespace +// characters. +static constexpr auto kStandardReplacements = + std::array, 3>{ + std::make_pair("\n", "\\n"), + std::make_pair("\r", "\\r"), + std::make_pair("\t", "\\t"), + }; + +static constexpr absl::string_view kSingleQuote = "'"; + +// ExprRecursionListener extends the standard ANTLR CelParser to ensure that +// recursive entries into the 'expr' rule are limited to a configurable depth so +// as to prevent stack overflows. +class ExprRecursionListener final : public ParseTreeListener { + public: + explicit ExprRecursionListener( + const int max_recursion_depth = kDefaultMaxRecursionDepth) + : max_recursion_depth_(max_recursion_depth), recursion_depth_(0) {} + ~ExprRecursionListener() override = default; + + void visitTerminal(TerminalNode* node) override {} + void visitErrorNode(ErrorNode* error) override {} + void enterEveryRule(ParserRuleContext* ctx) override; + void exitEveryRule(ParserRuleContext* ctx) override; + + private: + const int max_recursion_depth_; + int recursion_depth_; +}; + +void ExprRecursionListener::enterEveryRule(ParserRuleContext* ctx) { + // Throw a ParseCancellationException since the parsing would otherwise + // continue if this were treated as a syntax error and the problem would + // continue to manifest. + if (ctx->getRuleIndex() == CelParser::RuleExpr) { + if (recursion_depth_ > max_recursion_depth_) { + throw ParseCancellationException( + absl::StrFormat("Expression recursion limit exceeded. limit: %d", + max_recursion_depth_)); + } + recursion_depth_++; + } +} + +void ExprRecursionListener::exitEveryRule(ParserRuleContext* ctx) { + if (ctx->getRuleIndex() == CelParser::RuleExpr) { + recursion_depth_--; + } +} + +class RecoveryLimitErrorStrategy final : public DefaultErrorStrategy { + public: + explicit RecoveryLimitErrorStrategy( + int recovery_limit = kDefaultErrorRecoveryLimit, + int recovery_token_lookahead_limit = + kDefaultErrorRecoveryTokenLookaheadLimit) + : recovery_limit_(recovery_limit), + recovery_attempts_(0), + recovery_token_lookahead_limit_(recovery_token_lookahead_limit) {} + + void recover(Parser* recognizer, std::exception_ptr e) override { + checkRecoveryLimit(recognizer); + DefaultErrorStrategy::recover(recognizer, e); + } + + Token* recoverInline(Parser* recognizer) override { + checkRecoveryLimit(recognizer); + return DefaultErrorStrategy::recoverInline(recognizer); + } + + // Override the ANTLR implementation to introduce a token lookahead limit as + // this prevents pathologically constructed, yet small (< 16kb) inputs from + // consuming inordinate amounts of compute. + // + // This method is only called on error recovery paths. + void consumeUntil(Parser* recognizer, const IntervalSet& set) override { + size_t ttype = recognizer->getInputStream()->LA(1); + int recovery_search_depth = 0; + while (ttype != Token::EOF && !set.contains(ttype) && + recovery_search_depth++ < recovery_token_lookahead_limit_) { + recognizer->consume(); + ttype = recognizer->getInputStream()->LA(1); + } + // Halt all parsing if the lookahead limit is reached during error recovery. + if (recovery_search_depth == recovery_token_lookahead_limit_) { + throw ParseCancellationException("Unable to find a recovery token"); + } + } + + protected: + std::string escapeWSAndQuote(const std::string& s) const override { + std::string result; + result.reserve(s.size() + 2); + absl::StrAppend(&result, kSingleQuote, s, kSingleQuote); + absl::StrReplaceAll(kStandardReplacements, &result); + return result; + } + + private: + void checkRecoveryLimit(Parser* recognizer) { + if (recovery_attempts_++ >= recovery_limit_) { + std::string too_many_errors = + absl::StrFormat("More than %d parse errors.", recovery_limit_); + recognizer->notifyErrorListeners(too_many_errors); + throw ParseCancellationException(too_many_errors); + } + } + + int recovery_limit_; + int recovery_attempts_; + int recovery_token_lookahead_limit_; +}; + +} // namespace + +absl::StatusOr> AntlrParseImpl( + const cel::Source& source, const cel::MacroRegistry& registry, + const ParserOptions& options, std::vector* parse_issues, + cel::EnrichedSourceInfo* enriched_source_info) { + ABSL_DCHECK(!options.enable_pratt_parser); + try { + CodePointStream input(source.content(), source.description()); + if (input.size() > options.expression_size_codepoint_limit) { + return absl::InvalidArgumentError(absl::StrCat( + "expression size exceeds codepoint limit.", " input size: ", + input.size(), ", limit: ", options.expression_size_codepoint_limit)); + } + CelLexer lexer(&input); + CommonTokenStream tokens(&lexer); + CelParser parser(&tokens); + ExprRecursionListener listener(options.max_recursion_depth); + ParserVisitor visitor( + source, options.max_recursion_depth, options.expression_node_limit, + registry, options.add_macro_calls, options.enable_optional_syntax, + options.enable_quoted_identifiers, + options.enable_variadic_logical_operators, + options.fold_unary_operators); + + lexer.removeErrorListeners(); + parser.removeErrorListeners(); + lexer.addErrorListener(&visitor); + parser.addErrorListener(&visitor); + parser.addParseListener(&listener); + + // Limit the number of error recovery attempts to prevent bad expressions + // from consuming lots of cpu / memory. + parser.setErrorHandler(std::make_shared( + options.error_recovery_limit, + options.error_recovery_token_lookahead_limit)); + + Expr expr; + try { + expr = ExprFromAny(visitor.visit(parser.start())); + } catch (const ParseCancellationException& e) { + if (visitor.HasErrored()) { + auto issues = visitor.CollectIssues(); + std::string error_message = FormatIssues(source, issues); + if (parse_issues != nullptr) { + *parse_issues = std::move(issues); + } + return absl::InvalidArgumentError(error_message); + } + return absl::CancelledError(e.what()); + } + + if (visitor.HasErrored()) { + auto issues = visitor.CollectIssues(); + std::string error_message = FormatIssues(source, issues); + if (parse_issues != nullptr) { + *parse_issues = std::move(issues); + } + return absl::InvalidArgumentError(error_message); + } + + if (enriched_source_info != nullptr) { + *enriched_source_info = visitor.enriched_source_info(); + } + + return std::make_unique(std::move(expr), visitor.GetSourceInfo()); + } catch (const std::exception& e) { + return absl::AbortedError(e.what()); + } catch (const char* what) { + // ANTLRv4 has historically thrown C string literals. + return absl::AbortedError(what); + } catch (...) { + // We guarantee to never throw and always return a status. + return absl::UnknownError("An unknown exception occurred"); + } +} + +absl::StatusOr> AntlrParserImpl::ParseImpl( + const cel::Source& source, + std::vector* absl_nullable parse_issues) const { + return AntlrParseImpl(source, macro_registry_, options_, parse_issues); +} + +absl::StatusOr> AntlrParserImpl::PrepareSourceImpl( + absl::string_view input, absl::string_view description) const { + return cel::NewSource( + input, std::string(description), + cel::SourceOptions{.max_codepoint_size = + options_.expression_size_codepoint_limit}); +} + +absl::Status AntlrParserBuilderImpl::AddMacro(const cel::Macro& macro) { + for (const auto& existing_macro : macros_) { + if (existing_macro.key() == macro.key()) { + return absl::AlreadyExistsError( + absl::StrCat("macro already exists: ", macro.key())); + } + } + macros_.push_back(macro); + return absl::OkStatus(); +} + +absl::Status AntlrParserBuilderImpl::AddLibrary(cel::ParserLibrary library) { + if (!library.id.empty()) { + auto [it, inserted] = library_ids_.insert(library.id); + if (!inserted) { + return absl::AlreadyExistsError( + absl::StrCat("parser library already exists: ", library.id)); + } + } + libraries_.push_back(std::move(library)); + return absl::OkStatus(); +} + +absl::Status AntlrParserBuilderImpl::AddLibrarySubset( + cel::ParserLibrarySubset subset) { + if (subset.library_id.empty()) { + return absl::InvalidArgumentError("subset must have a library id"); + } + std::string library_id = subset.library_id; + auto [it, inserted] = + library_subsets_.insert({library_id, std::move(subset)}); + if (!inserted) { + return absl::AlreadyExistsError( + absl::StrCat("parser library subset already exists: ", library_id)); + } + return absl::OkStatus(); +} + +absl::StatusOr> AntlrParserBuilderImpl::Build() { + using std::swap; + // Save the old configured macros so they aren't affected by applying the + // libraries and can be restored if an error occurs. + std::vector individual_macros; + swap(individual_macros, macros_); + absl::Cleanup cleanup([&] { swap(macros_, individual_macros); }); + + cel::MacroRegistry macro_registry; + + for (const auto& library : libraries_) { + CEL_RETURN_IF_ERROR(library.configure(*this)); + if (!library.id.empty()) { + auto it = library_subsets_.find(library.id); + if (it != library_subsets_.end()) { + const cel::ParserLibrarySubset& subset = it->second; + for (const auto& macro : macros_) { + if (subset.should_include_macro(macro)) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(macro)); + } + } + macros_.clear(); + continue; + } + } + + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(macros_)); + macros_.clear(); + } + + absl::flat_hash_set library_ids(library_ids_); + + // Hack to support adding the standard library macros either by option or + // with a library configurer. + if (!options_.disable_standard_macros && !library_ids_.contains("stdlib")) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(Macro::AllMacros())); + library_ids.insert("stdlib"); + } + + if (options_.enable_optional_syntax && !library_ids_.contains("optional")) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptMapMacro())); + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptFlatMapMacro())); + library_ids.insert("optional"); + } + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(individual_macros)); + return std::make_unique(options_, std::move(macro_registry), + std::move(library_ids)); +} + +std::unique_ptr AntlrParserImpl::ToBuilder() const { + auto ins = std::make_unique(options_); + ins->library_ids_ = library_ids_; + ins->macros_ = macro_registry_.ListMacros(); + return ins; +} + +} // namespace cel::parser_internal diff --git a/parser/internal/antlr_parser.h b/parser/internal/antlr_parser.h new file mode 100644 index 000000000..9e0153c6a --- /dev/null +++ b/parser/internal/antlr_parser.h @@ -0,0 +1,106 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_ANTLR_PARSER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_ANTLR_PARSER_H_ + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "common/ast.h" +#include "common/source.h" +#include "parser/macro.h" +#include "parser/macro_registry.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel { +class EnrichedSourceInfo; +} // namespace cel + +namespace cel::parser_internal { + +class AntlrParserImpl final : public cel::Parser { + public: + explicit AntlrParserImpl(const cel::ParserOptions& options, + cel::MacroRegistry macro_registry, + absl::flat_hash_set library_ids) + : options_(options), + macro_registry_(std::move(macro_registry)), + library_ids_(std::move(library_ids)) {} + + ~AntlrParserImpl() override = default; + + absl::StatusOr> ParseImpl( + const cel::Source& source, + std::vector* absl_nullable parse_issues) const override; + + absl::StatusOr> PrepareSourceImpl( + absl::string_view input, absl::string_view description) const override; + + std::unique_ptr ToBuilder() const override; + + private: + cel::ParserOptions options_; + cel::MacroRegistry macro_registry_; + absl::flat_hash_set library_ids_; +}; + +absl::StatusOr> AntlrParseImpl( + const cel::Source& source, const cel::MacroRegistry& registry, + const ParserOptions& options, + std::vector* parse_issues = nullptr, + cel::EnrichedSourceInfo* enriched_source_info = nullptr); + +class AntlrParserBuilderImpl final : public cel::ParserBuilder { + public: + explicit AntlrParserBuilderImpl(const cel::ParserOptions& options) + : options_(options) {} + + cel::ParserOptions& GetOptions() override { return options_; } + + absl::Status AddMacro(const cel::Macro& macro) override; + + absl::Status AddLibrary(cel::ParserLibrary library) override; + + absl::Status AddLibrarySubset(cel::ParserLibrarySubset subset) override; + + absl::StatusOr> Build() override; + + private: + friend class AntlrParserImpl; + + cel::ParserOptions options_; + std::vector macros_; + absl::flat_hash_set library_ids_; + std::vector libraries_; + absl::flat_hash_map library_subsets_; +}; + +inline std::unique_ptr NewAntlrParserBuilder( + const cel::ParserOptions& options = cel::ParserOptions()) { + return std::make_unique(options); +} + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_ANTLR_PARSER_H_ diff --git a/parser/internal/antlr_parser_test.cc b/parser/internal/antlr_parser_test.cc new file mode 100644 index 000000000..4bdafc42b --- /dev/null +++ b/parser/internal/antlr_parser_test.cc @@ -0,0 +1,92 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/antlr_parser.h" + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "common/ast.h" +#include "common/navigable_ast.h" +#include "common/source.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { +namespace { + +using ::absl_testing::IsOk; +using ::testing::HasSubstr; +using ::testing::Not; + +absl::StatusOr> Parse( + absl::string_view expression, absl::string_view description = "", + const cel::ParserOptions& options = cel::ParserOptions(), + std::vector* issues = nullptr) { + std::unique_ptr builder = NewAntlrParserBuilder(options); + CEL_ASSIGN_OR_RETURN(std::unique_ptr parser, builder->Build()); + CEL_ASSIGN_OR_RETURN(std::unique_ptr source, + cel::NewSource(expression, std::string(description))); + return parser->Parse(*source, issues); +} + +TEST(AntlrParserTest, ErrorRecoveryLimits) { + ParserOptions options; + options.error_recovery_limit = 1; + auto result = Parse("......", "", options); + EXPECT_THAT(result, Not(IsOk())); + EXPECT_EQ(result.status().message(), + "ERROR: :1:1: Syntax error: More than 1 parse errors.\n | ......\n " + "| ^\nERROR: :1:2: Syntax error: no viable alternative at input " + "'..'\n | ......\n | .^"); +} + +TEST(AntlrParserTest, RecursionDepthExceeded) { + ParserOptions options; + // AST visitor will recurse a variable amount depending on the terms used in + // the expression. This check occurs in the business logic converting the raw + // Antlr parse tree into an Expr. There is a separate check (via a custom + // listener) for AST depth while running the antlr generated parser. + options.max_recursion_depth = 6; + auto result = Parse("1 + 2 + 3 + 4 + 5 + 6 + 7", "", options); + + EXPECT_THAT(result, Not(IsOk())); + EXPECT_THAT(result.status().message(), + HasSubstr("Exceeded max recursion depth of 6 when parsing.")); +} + +TEST(AntlrParserTest, UnaryOperatorsUnfoldedOption) { + ParserOptions options; + options.fold_unary_operators = false; + + ASSERT_OK_AND_ASSIGN(auto ast, Parse("---a", "", options)); + auto nav_ast = cel::NavigableAst::Build(ast->root_expr()); + EXPECT_EQ(nav_ast.Root().height(), 4); + + for (const auto& node : nav_ast.Root().DescendantsPostorder()) { + if (node.node_kind() == cel::NodeKind::kCall) { + EXPECT_EQ(node.expr()->call_expr().function(), "-_"); + } + } +} + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/internal/ast_factory.cc b/parser/internal/ast_factory.cc new file mode 100644 index 000000000..85e158a8e --- /dev/null +++ b/parser/internal/ast_factory.cc @@ -0,0 +1,320 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/ast_factory.h" + +#include +#include +#include +#include +#include + +#include "absl/functional/function_ref.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "common/expr.h" +#include "internal/status_macros.h" +#include "parser/internal/ast_factory_interface.h" +#include "parser/macro.h" + +namespace cel::parser_internal { + +ListNodeBuilder::ListNodeBuilder(int64_t id) { + expr_.set_id(id); + expr_.mutable_list_expr(); +} + +ListNodeBuilder& ListNodeBuilder::Add(cel::Expr element, + bool optional) { + cel::ListExpr& list_val = expr_.mutable_list_expr(); + cel::ListExprElement expr_element; + expr_element.set_expr(std::move(element)); + expr_element.set_optional(optional); + list_val.mutable_elements().push_back(std::move(expr_element)); + return *this; +} + +cel::Expr ListNodeBuilder::Build() { return std::move(expr_); } + +MapNodeBuilder::MapNodeBuilder(int64_t id) { + expr_.set_id(id); + expr_.mutable_map_expr(); +} + +MapNodeBuilder& MapNodeBuilder::Add(int64_t id, + cel::Expr key, + cel::Expr value, + bool optional) { + cel::MapExpr& map_val = expr_.mutable_map_expr(); + cel::MapExprEntry entry; + entry.set_id(id); + entry.set_key(std::move(key)); + entry.set_value(std::move(value)); + entry.set_optional(optional); + map_val.mutable_entries().push_back(std::move(entry)); + return *this; +} + +cel::Expr MapNodeBuilder::Build() { return std::move(expr_); } + +StructNodeBuilder::StructNodeBuilder(int64_t id, std::string name) { + expr_.set_id(id); + expr_.mutable_struct_expr().set_name(std::move(name)); +} + +StructNodeBuilder& StructNodeBuilder::Add( + int64_t id, std::string name, cel::Expr value, bool optional) { + cel::StructExpr& struct_val = expr_.mutable_struct_expr(); + cel::StructExprField field; + field.set_id(id); + field.set_name(std::move(name)); + field.set_value(std::move(value)); + field.set_optional(optional); + struct_val.mutable_fields().push_back(std::move(field)); + return *this; +} + +cel::Expr StructNodeBuilder::Build() { return std::move(expr_); } + +int64_t AstFactoryInterface::GetId(const cel::Expr& expr) const { + return expr.id(); +} + +bool AstFactoryInterface::IsEmpty(const cel::Expr& expr) const { + return expr.id() == 0; +} + +bool AstFactoryInterface::IsConst(const cel::Expr& expr) const { + return expr.has_const_expr(); +} + +bool AstFactoryInterface::IsIdent(const cel::Expr& expr) const { + return expr.has_ident_expr(); +} + +absl::string_view AstFactoryInterface::GetIdentName( + const cel::Expr& expr) const { + return expr.has_ident_expr() ? absl::string_view(expr.ident_expr().name()) + : absl::string_view(); +} + +bool AstFactoryInterface::IsSelect(const cel::Expr& expr) const { + return expr.has_select_expr(); +} + +bool AstFactoryInterface::IsPresenceTest( + const cel::Expr& expr) const { + return expr.has_select_expr() && expr.select_expr().test_only(); +} + +const cel::Expr* AstFactoryInterface::GetSelectOperand( + const cel::Expr& expr) const { + return expr.has_select_expr() ? &expr.select_expr().operand() : nullptr; +} + +absl::string_view AstFactoryInterface::GetSelectField( + const cel::Expr& expr) const { + return expr.has_select_expr() ? absl::string_view(expr.select_expr().field()) + : absl::string_view(); +} + +absl::StatusOr AstFactoryInterface::CopyAndReplace( + const cel::Expr& expr, + absl::FunctionRef(const cel::Expr&)> replacer, + int max_recursion_depth) const { + if (max_recursion_depth <= 0) { + return absl::InvalidArgumentError("recursion limit exceeded"); + } + std::optional replaced = replacer(expr); + if (replaced.has_value()) { + return std::move(*replaced); + } + + cel::Expr new_expr; + new_expr.set_id(expr.id()); + + switch (expr.kind_case()) { + case cel::ExprKindCase::kUnspecifiedExpr: + break; + case cel::ExprKindCase::kConstant: + new_expr.set_const_expr(expr.const_expr()); + break; + case cel::ExprKindCase::kIdentExpr: + new_expr.set_ident_expr(cel::IdentExpr(expr.ident_expr().name())); + break; + case cel::ExprKindCase::kSelectExpr: { + cel::SelectExpr& select = new_expr.mutable_select_expr(); + select.set_field(expr.select_expr().field()); + select.set_test_only(expr.select_expr().test_only()); + if (expr.select_expr().has_operand()) { + CEL_ASSIGN_OR_RETURN(cel::Expr operand, + CopyAndReplace(expr.select_expr().operand(), + replacer, max_recursion_depth - 1)); + select.set_operand(std::move(operand)); + } + break; + } + case cel::ExprKindCase::kCallExpr: { + cel::CallExpr& call = new_expr.mutable_call_expr(); + call.set_function(expr.call_expr().function()); + if (expr.call_expr().has_target()) { + CEL_ASSIGN_OR_RETURN(cel::Expr target, + CopyAndReplace(expr.call_expr().target(), replacer, + max_recursion_depth - 1)); + call.set_target(std::move(target)); + } + call.mutable_args().reserve(expr.call_expr().args().size()); + for (const auto& arg : expr.call_expr().args()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_arg, + CopyAndReplace(arg, replacer, max_recursion_depth - 1)); + call.mutable_args().push_back(std::move(new_arg)); + } + break; + } + case cel::ExprKindCase::kListExpr: { + cel::ListExpr& list = new_expr.mutable_list_expr(); + list.mutable_elements().reserve(expr.list_expr().elements().size()); + for (const auto& elem : expr.list_expr().elements()) { + cel::ListExprElement new_elem; + new_elem.set_optional(elem.optional()); + if (elem.has_expr()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_child, + CopyAndReplace(elem.expr(), replacer, max_recursion_depth - 1)); + new_elem.set_expr(std::move(new_child)); + } + list.mutable_elements().push_back(std::move(new_elem)); + } + break; + } + case cel::ExprKindCase::kStructExpr: { + cel::StructExpr& str = new_expr.mutable_struct_expr(); + str.set_name(expr.struct_expr().name()); + str.mutable_fields().reserve(expr.struct_expr().fields().size()); + for (const auto& field : expr.struct_expr().fields()) { + cel::StructExprField new_field; + new_field.set_id(field.id()); + new_field.set_name(field.name()); + new_field.set_optional(field.optional()); + if (field.has_value()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_val, + CopyAndReplace(field.value(), replacer, max_recursion_depth - 1)); + new_field.set_value(std::move(new_val)); + } + str.mutable_fields().push_back(std::move(new_field)); + } + break; + } + case cel::ExprKindCase::kMapExpr: { + cel::MapExpr& map = new_expr.mutable_map_expr(); + map.mutable_entries().reserve(expr.map_expr().entries().size()); + for (const auto& entry : expr.map_expr().entries()) { + cel::MapExprEntry new_entry; + new_entry.set_id(entry.id()); + new_entry.set_optional(entry.optional()); + if (entry.has_key()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_key, + CopyAndReplace(entry.key(), replacer, max_recursion_depth - 1)); + new_entry.set_key(std::move(new_key)); + } + if (entry.has_value()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_val, + CopyAndReplace(entry.value(), replacer, max_recursion_depth - 1)); + new_entry.set_value(std::move(new_val)); + } + map.mutable_entries().push_back(std::move(new_entry)); + } + break; + } + case cel::ExprKindCase::kComprehensionExpr: { + cel::ComprehensionExpr& comp = new_expr.mutable_comprehension_expr(); + comp.set_iter_var(expr.comprehension_expr().iter_var()); + comp.set_iter_var2(expr.comprehension_expr().iter_var2()); + comp.set_accu_var(expr.comprehension_expr().accu_var()); + if (expr.comprehension_expr().has_accu_init()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_accu_init, + CopyAndReplace(expr.comprehension_expr().accu_init(), replacer, + max_recursion_depth - 1)); + comp.set_accu_init(std::move(new_accu_init)); + } + if (expr.comprehension_expr().has_iter_range()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_iter_range, + CopyAndReplace(expr.comprehension_expr().iter_range(), replacer, + max_recursion_depth - 1)); + comp.set_iter_range(std::move(new_iter_range)); + } + if (expr.comprehension_expr().has_loop_condition()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_loop_condition, + CopyAndReplace(expr.comprehension_expr().loop_condition(), replacer, + max_recursion_depth - 1)); + comp.set_loop_condition(std::move(new_loop_condition)); + } + if (expr.comprehension_expr().has_loop_step()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_loop_step, + CopyAndReplace(expr.comprehension_expr().loop_step(), replacer, + max_recursion_depth - 1)); + comp.set_loop_step(std::move(new_loop_step)); + } + if (expr.comprehension_expr().has_result()) { + CEL_ASSIGN_OR_RETURN(cel::Expr new_result, + CopyAndReplace(expr.comprehension_expr().result(), + replacer, max_recursion_depth - 1)); + comp.set_result(std::move(new_result)); + } + break; + } + } + return new_expr; +} + +ListNodeBuilder AstFactoryInterface::NewListBuilder( + int64_t id) { + return ListNodeBuilder(id); +} + +StructNodeBuilder AstFactoryInterface::NewStructBuilder( + int64_t id, std::string name) { + return StructNodeBuilder(id, std::move(name)); +} + +MapNodeBuilder AstFactoryInterface::NewMapBuilder( + int64_t id) { + return MapNodeBuilder(id); +} + +std::optional> +AstFactoryInterface::NewMacroExprExpander(std::string_view name, + size_t arg_count, + bool receiver_style) { + if (macro_registry_ == nullptr) { + return std::nullopt; + } + std::optional macro = + macro_registry_->FindMacro(name, arg_count, receiver_style); + if (!macro) { + return std::nullopt; + } + return std::optional>(std::in_place, *macro); +} + +} // namespace cel::parser_internal diff --git a/parser/internal/ast_factory.h b/parser/internal/ast_factory.h new file mode 100644 index 000000000..6240ce527 --- /dev/null +++ b/parser/internal/ast_factory.h @@ -0,0 +1,174 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ + +#include +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/functional/function_ref.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/expr.h" +#include "common/expr_factory.h" +#include "parser/internal/ast_factory_interface.h" +#include "parser/macro.h" +#include "parser/macro_expr_factory.h" +#include "parser/macro_registry.h" + +namespace cel::parser_internal { + +// Explicit specialization of `AstFactoryInterface` for `cel::Expr` AST nodes. + +template <> +class ListNodeBuilder { + public: + explicit ListNodeBuilder(int64_t id); + + ListNodeBuilder& Add(cel::Expr element, bool optional = false); + + cel::Expr Build(); + + private: + cel::Expr expr_; +}; + +template <> +class MapNodeBuilder { + public: + explicit MapNodeBuilder(int64_t id); + + MapNodeBuilder& Add(int64_t id, cel::Expr key, cel::Expr value, + bool optional = false); + + cel::Expr Build(); + + private: + cel::Expr expr_; +}; + +template <> +class StructNodeBuilder { + public: + explicit StructNodeBuilder(int64_t id, std::string name); + + StructNodeBuilder& Add(int64_t id, std::string name, cel::Expr value, + bool optional = false); + + cel::Expr Build(); + + private: + cel::Expr expr_; +}; + +template <> +class AstFactoryInterface; + +template <> +class MacroExprExpanderSupport : public cel::MacroExprFactory {}; + +template <> +class MacroExprExpander { + public: + explicit MacroExprExpander(cel::Macro macro) : macro_(std::move(macro)) {} + + std::optional Expand( + std::optional> target, + absl::Span args, + MacroExprExpanderSupport& support) { + return macro_.Expand(support, target, args); + } + + private: + cel::Macro macro_; +}; + +template <> +class AstFactoryInterface : public cel::ExprFactory { + public: + explicit AstFactoryInterface( + const cel::MacroRegistry* absl_nullable macro_registry = nullptr) + : macro_registry_(macro_registry) {} + + AstFactoryInterface(const AstFactoryInterface&) = delete; + AstFactoryInterface(AstFactoryInterface&&) = delete; + AstFactoryInterface& operator=(const AstFactoryInterface&) = delete; + AstFactoryInterface& operator=(AstFactoryInterface&&) = delete; + + ~AstFactoryInterface() override = default; + + // Node inspection and encapsulation API + int64_t GetId(const cel::Expr& expr) const; + + bool IsEmpty(const cel::Expr& expr) const; + + bool IsConst(const cel::Expr& expr) const; + + bool IsIdent(const cel::Expr& expr) const; + + absl::string_view GetIdentName(const cel::Expr& expr) const; + + bool IsSelect(const cel::Expr& expr) const; + + bool IsPresenceTest(const cel::Expr& expr) const; + + const cel::Expr* GetSelectOperand(const cel::Expr& expr) const; + + absl::string_view GetSelectField(const cel::Expr& expr) const; + + absl::StatusOr CopyAndReplace( + const cel::Expr& expr, + absl::FunctionRef(const cel::Expr&)> replacer, + int max_recursion_depth = 1000) const; + + // Node creation API + using cel::ExprFactory::NewBoolConst; + using cel::ExprFactory::NewBytesConst; + using cel::ExprFactory::NewCall; + using cel::ExprFactory::NewDoubleConst; + using cel::ExprFactory::NewIdent; + using cel::ExprFactory::NewIntConst; + using cel::ExprFactory::NewMemberCall; + using cel::ExprFactory::NewNullConst; + using cel::ExprFactory::NewPresenceTest; + using cel::ExprFactory::NewSelect; + using cel::ExprFactory::NewStringConst; + using cel::ExprFactory::NewUintConst; + using cel::ExprFactory::NewUnspecified; + + ListNodeBuilder NewListBuilder(int64_t id); + + StructNodeBuilder NewStructBuilder(int64_t id, std::string name); + + MapNodeBuilder NewMapBuilder(int64_t id); + + std::optional> NewMacroExprExpander( + std::string_view name, size_t arg_count, bool receiver_style); + + private: + const cel::MacroRegistry* absl_nullable macro_registry_ = nullptr; +}; + +using AstFactory = AstFactoryInterface; + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ diff --git a/parser/internal/ast_factory_interface.h b/parser/internal/ast_factory_interface.h new file mode 100644 index 000000000..dbfa02a0f --- /dev/null +++ b/parser/internal/ast_factory_interface.h @@ -0,0 +1,130 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/functional/function_ref.h" +#include "absl/status/statusor.h" +#include "absl/types/span.h" + +namespace cel::parser_internal { + +template +class ListNodeBuilder { + public: + ListNodeBuilder& Add(ExprNode element, bool optional = false); + ExprNode Build(); +}; + +template +class MapNodeBuilder { + public: + MapNodeBuilder& Add(int64_t id, ExprNode key, ExprNode value, + bool optional = false); + ExprNode Build(); +}; + +template +class StructNodeBuilder { + public: + StructNodeBuilder& Add(int64_t id, std::string name, ExprNode value, + bool optional = false); + ExprNode Build(); +}; + +template +class MacroExprExpanderSupport {}; + +template +class MacroExprExpander { + public: + std::optional Expand( + std::optional> target, + absl::Span args, MacroExprExpanderSupport& support); +}; + +// Interface for decoupling parser logic from the underlying AST node +// data structures. +// +// By parameterizing the parser and factory on `ExprNode`, alternative AST node +// representations (such as `cel::Expr`) can be constructed without modifying +// parser rules. +// +// To implement AST construction using an alternative AST structure: +// 1. Define or specify your custom node type `MyNode`. +// 2. Implement a concrete factory specialization `AstFactoryInterface` +// that provides inspection (`GetId`, `IsSelect`, etc.) and creation +// (`NewCall`, `NewListBuilder`, etc.) operations for `MyNode`. +// 3. Instantiate the parser worker with your node type: +// `PrattParserWorker`. +template +class AstFactoryInterface { + public: + AstFactoryInterface() = default; + AstFactoryInterface(const AstFactoryInterface&) = delete; + AstFactoryInterface(AstFactoryInterface&&) = delete; + AstFactoryInterface& operator=(const AstFactoryInterface&) = delete; + AstFactoryInterface& operator=(AstFactoryInterface&&) = delete; + + int64_t GetId(const ExprNode& expr) const; + bool IsEmpty(const ExprNode& expr) const; + bool IsConst(const ExprNode& expr) const; + bool IsIdent(const ExprNode& expr) const; + std::string_view GetIdentName(const ExprNode& expr) const; + bool IsSelect(const ExprNode& expr) const; + bool IsPresenceTest(const ExprNode& expr) const; + const ExprNode* GetSelectOperand(const ExprNode& expr) const; + std::string_view GetSelectField(const ExprNode& expr) const; + absl::StatusOr CopyAndReplace( + const ExprNode& expr, + absl::FunctionRef(const ExprNode&)> replacer, + int max_recursion_depth = 1000) const; + + ExprNode NewUnspecified(int64_t id); + ExprNode NewNullConst(int64_t id); + ExprNode NewBoolConst(int64_t id, bool value); + ExprNode NewIntConst(int64_t id, int64_t value); + ExprNode NewUintConst(int64_t id, uint64_t value); + ExprNode NewDoubleConst(int64_t id, double value); + ExprNode NewBytesConst(int64_t id, std::string value); + ExprNode NewStringConst(int64_t id, std::string value); + ExprNode NewIdent(int64_t id, std::string name); + ExprNode NewSelect(int64_t id, ExprNode operand, std::string field); + ExprNode NewPresenceTest(int64_t id, ExprNode operand, std::string field); + ExprNode NewCall(int64_t id, std::string function, + std::vector args); + ExprNode NewMemberCall(int64_t id, std::string function, ExprNode target, + std::vector args); + ListNodeBuilder NewListBuilder(int64_t id); + MapNodeBuilder NewMapBuilder(int64_t id); + StructNodeBuilder NewStructBuilder(int64_t id, std::string name); + + // Returns a macro expander for the given macro name, or null if there + // is no registered macro with that name and argument count. + std::optional> NewMacroExprExpander( + std::string_view name, size_t arg_count, bool receiver_style); +}; + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ diff --git a/parser/internal/ast_factory_test.cc b/parser/internal/ast_factory_test.cc new file mode 100644 index 000000000..585c1a269 --- /dev/null +++ b/parser/internal/ast_factory_test.cc @@ -0,0 +1,449 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/ast_factory.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/constant.h" +#include "common/expr.h" +#include "internal/testing.h" +#include "parser/internal/ast_factory_interface.h" +#include "parser/macro.h" +#include "parser/macro_expr_factory.h" +#include "parser/macro_registry.h" + +namespace cel::parser_internal { +namespace { + +using ::absl_testing::IsOk; + +using ::absl_testing::StatusIs; + +TEST(AstFactoryInterfaceTest, AstFactoryUnspecified) { + AstFactory factory; + cel::Expr expr = factory.NewUnspecified(1); + EXPECT_EQ(factory.GetId(expr), 1); + EXPECT_FALSE(factory.IsEmpty(expr)); + EXPECT_FALSE(factory.IsConst(expr)); + EXPECT_FALSE(factory.IsIdent(expr)); + EXPECT_FALSE(factory.IsSelect(expr)); + + cel::Expr empty_expr = factory.NewUnspecified(0); + EXPECT_TRUE(factory.IsEmpty(empty_expr)); +} + +TEST(AstFactoryInterfaceTest, AstFactoryConstNodes) { + AstFactory factory; + + cel::Expr null_expr = factory.NewNullConst(10); + EXPECT_EQ(factory.GetId(null_expr), 10); + EXPECT_TRUE(factory.IsConst(null_expr)); + ASSERT_TRUE(null_expr.has_const_expr()); + EXPECT_TRUE(null_expr.const_expr().has_null_value()); + + cel::Expr bool_expr = factory.NewBoolConst(11, true); + EXPECT_EQ(factory.GetId(bool_expr), 11); + EXPECT_TRUE(factory.IsConst(bool_expr)); + ASSERT_TRUE(bool_expr.has_const_expr()); + EXPECT_TRUE(bool_expr.const_expr().bool_value()); + + cel::Expr int_expr = factory.NewIntConst(12, -42); + EXPECT_EQ(factory.GetId(int_expr), 12); + EXPECT_TRUE(factory.IsConst(int_expr)); + ASSERT_TRUE(int_expr.has_const_expr()); + EXPECT_EQ(int_expr.const_expr().int_value(), -42); + + cel::Expr uint_expr = factory.NewUintConst(13, 100u); + EXPECT_EQ(factory.GetId(uint_expr), 13); + EXPECT_TRUE(factory.IsConst(uint_expr)); + ASSERT_TRUE(uint_expr.has_const_expr()); + EXPECT_EQ(uint_expr.const_expr().uint_value(), 100u); + + cel::Expr double_expr = factory.NewDoubleConst(14, 3.14159); + EXPECT_EQ(factory.GetId(double_expr), 14); + EXPECT_TRUE(factory.IsConst(double_expr)); + ASSERT_TRUE(double_expr.has_const_expr()); + EXPECT_DOUBLE_EQ(double_expr.const_expr().double_value(), 3.14159); + + cel::Expr bytes_expr = factory.NewBytesConst(15, "bytes_val"); + EXPECT_EQ(factory.GetId(bytes_expr), 15); + EXPECT_TRUE(factory.IsConst(bytes_expr)); + ASSERT_TRUE(bytes_expr.has_const_expr()); + EXPECT_EQ(bytes_expr.const_expr().bytes_value(), "bytes_val"); + + cel::Expr string_expr = factory.NewStringConst(16, "string_val"); + EXPECT_EQ(factory.GetId(string_expr), 16); + EXPECT_TRUE(factory.IsConst(string_expr)); + ASSERT_TRUE(string_expr.has_const_expr()); + EXPECT_EQ(string_expr.const_expr().string_value(), "string_val"); +} + +TEST(AstFactoryInterfaceTest, AstFactoryIdentAndSelect) { + AstFactory factory; + + cel::Expr ident_expr = factory.NewIdent(20, "foo"); + EXPECT_EQ(factory.GetId(ident_expr), 20); + EXPECT_TRUE(factory.IsIdent(ident_expr)); + EXPECT_EQ(factory.GetIdentName(ident_expr), "foo"); + + cel::Expr select_expr = + factory.NewSelect(21, factory.NewIdent(20, "foo"), "bar"); + EXPECT_EQ(factory.GetId(select_expr), 21); + EXPECT_TRUE(factory.IsSelect(select_expr)); + EXPECT_FALSE(factory.IsPresenceTest(select_expr)); + EXPECT_EQ(factory.GetSelectField(select_expr), "bar"); + ASSERT_NE(factory.GetSelectOperand(select_expr), nullptr); + EXPECT_EQ(factory.GetIdentName(*factory.GetSelectOperand(select_expr)), + "foo"); + + cel::Expr presence_expr = + factory.NewPresenceTest(22, factory.NewIdent(20, "foo"), "bar"); + EXPECT_EQ(factory.GetId(presence_expr), 22); + EXPECT_TRUE(factory.IsSelect(presence_expr)); + EXPECT_TRUE(factory.IsPresenceTest(presence_expr)); + EXPECT_EQ(factory.GetSelectField(presence_expr), "bar"); +} + +TEST(AstFactoryInterfaceTest, AstFactoryCalls) { + AstFactory factory; + + std::vector call_args; + call_args.push_back(factory.NewIntConst(30, 1)); + call_args.push_back(factory.NewIntConst(31, 2)); + cel::Expr call_expr = factory.NewCall(32, "_+_", std::move(call_args)); + EXPECT_EQ(factory.GetId(call_expr), 32); + ASSERT_TRUE(call_expr.has_call_expr()); + EXPECT_EQ(call_expr.call_expr().function(), "_+_"); + EXPECT_FALSE(call_expr.call_expr().has_target()); + EXPECT_EQ(call_expr.call_expr().args().size(), 2); + + std::vector member_args; + member_args.push_back(factory.NewStringConst(33, "suffix")); + cel::Expr member_call_expr = factory.NewMemberCall( + 34, "endsWith", factory.NewIdent(35, "str_var"), std::move(member_args)); + EXPECT_EQ(factory.GetId(member_call_expr), 34); + ASSERT_TRUE(member_call_expr.has_call_expr()); + EXPECT_EQ(member_call_expr.call_expr().function(), "endsWith"); + EXPECT_TRUE(member_call_expr.call_expr().has_target()); + EXPECT_EQ(member_call_expr.call_expr().target().ident_expr().name(), + "str_var"); + EXPECT_EQ(member_call_expr.call_expr().args().size(), 1); +} + +TEST(AstFactoryInterfaceTest, AstFactoryList) { + AstFactory factory; + + cel::Expr list_expr = factory.NewListBuilder(42) + .Add(factory.NewIntConst(40, 1), false) + .Add(factory.NewIntConst(41, 2), true) + .Build(); + EXPECT_EQ(factory.GetId(list_expr), 42); + ASSERT_TRUE(list_expr.has_list_expr()); + ASSERT_EQ(list_expr.list_expr().elements().size(), 2); + EXPECT_FALSE(list_expr.list_expr().elements()[0].optional()); + EXPECT_EQ(list_expr.list_expr().elements()[0].expr().const_expr().int_value(), + 1); + EXPECT_TRUE(list_expr.list_expr().elements()[1].optional()); + EXPECT_EQ(list_expr.list_expr().elements()[1].expr().const_expr().int_value(), + 2); +} + +TEST(AstFactoryInterfaceTest, AstFactoryStruct) { + AstFactory factory; + + cel::Expr struct_expr = + factory.NewStructBuilder(54, "MyMessage") + .Add(50, "field1", factory.NewIntConst(51, 100), false) + .Add(52, "field2", factory.NewIntConst(53, 200), true) + .Build(); + EXPECT_EQ(factory.GetId(struct_expr), 54); + ASSERT_TRUE(struct_expr.has_struct_expr()); + EXPECT_EQ(struct_expr.struct_expr().name(), "MyMessage"); + ASSERT_EQ(struct_expr.struct_expr().fields().size(), 2); + EXPECT_EQ(struct_expr.struct_expr().fields()[0].id(), 50); + EXPECT_EQ(struct_expr.struct_expr().fields()[0].name(), "field1"); + EXPECT_FALSE(struct_expr.struct_expr().fields()[0].optional()); + EXPECT_EQ(struct_expr.struct_expr().fields()[1].id(), 52); + EXPECT_EQ(struct_expr.struct_expr().fields()[1].name(), "field2"); + EXPECT_TRUE(struct_expr.struct_expr().fields()[1].optional()); +} + +TEST(AstFactoryInterfaceTest, AstFactoryMap) { + AstFactory factory; + + cel::Expr map_expr = factory.NewMapBuilder(66) + .Add(60, factory.NewStringConst(61, "key1"), + factory.NewIntConst(62, 10), false) + .Add(63, factory.NewStringConst(64, "key2"), + factory.NewIntConst(65, 20), true) + .Build(); + EXPECT_EQ(factory.GetId(map_expr), 66); + ASSERT_TRUE(map_expr.has_map_expr()); + ASSERT_EQ(map_expr.map_expr().entries().size(), 2); + EXPECT_EQ(map_expr.map_expr().entries()[0].id(), 60); + EXPECT_EQ(map_expr.map_expr().entries()[0].key().const_expr().string_value(), + "key1"); + EXPECT_EQ(map_expr.map_expr().entries()[0].value().const_expr().int_value(), + 10); + EXPECT_FALSE(map_expr.map_expr().entries()[0].optional()); + EXPECT_EQ(map_expr.map_expr().entries()[1].id(), 63); + EXPECT_TRUE(map_expr.map_expr().entries()[1].optional()); +} + +TEST(AstFactoryInterfaceTest, CopyAndReplace) { + AstFactory factory; + + // x + 1 + std::vector args; + args.push_back(factory.NewIdent(1, "x")); + args.push_back(factory.NewIntConst(2, 1)); + cel::Expr expr = factory.NewCall(3, "_+_", std::move(args)); + + // Transform: x -> y, 1 -> 2 + ASSERT_OK_AND_ASSIGN( + cel::Expr transformed, + factory.CopyAndReplace( + expr, [&](const cel::Expr& e) -> std::optional { + if (e.has_ident_expr() && e.ident_expr().name() == "x") { + return factory.NewIdent(e.id(), "y"); + } + if (e.has_const_expr() && e.const_expr().has_int_value() && + e.const_expr().int_value() == 1) { + return factory.NewIntConst(e.id(), 2); + } + return std::nullopt; + })); + + // Expected: y + 2 + std::vector expected_args; + expected_args.push_back(factory.NewIdent(1, "y")); + expected_args.push_back(factory.NewIntConst(2, 2)); + cel::Expr expected = factory.NewCall(3, "_+_", std::move(expected_args)); + + EXPECT_EQ(transformed, expected); +} + +TEST(AstFactoryInterfaceTest, CopyAndReplaceDeep) { + AstFactory factory; + + auto replacer = [&](const cel::Expr& e) -> std::optional { + if (e.has_const_expr() && e.const_expr().has_double_value()) { + return factory.NewIntConst( + e.id(), static_cast(e.const_expr().double_value())); + } + return std::nullopt; + }; + + cel::Expr expr = factory.NewCall( + 1, "func", + std::vector{ + factory.NewListBuilder(2) + .Add(factory.NewUnspecified(3)) + .Add(factory.NewNullConst(4)) + .Add(factory.NewBoolConst(5, true)) + .Add(factory.NewIntConst(6, 42)) + .Build(), + factory.NewMapBuilder(7) + .Add(991, factory.NewStringConst(8, "k1"), + factory.NewUintConst(9, 100u)) + .Add(992, factory.NewBytesConst(10, "b1"), + factory.NewDoubleConst(11, 3.14159)) + .Build(), + factory.NewStructBuilder(12, "S") + .Add(32, "f1", factory.NewIdent(13, "x")) + .Add( + 33, "f2", + factory.NewSelect(14, factory.NewIdent(15, "y"), "sel_field")) + .Add(34, "f3", + factory.NewPresenceTest(16, factory.NewIdent(17, "z"), + "pres_field")) + .Build(), + factory.NewMemberCall( + 18, "mem_func", factory.NewIdent(19, "target"), + std::vector{[&]() { + cel::Expr comp_expr; + comp_expr.set_id(20); + auto& comp = comp_expr.mutable_comprehension_expr(); + comp.set_iter_var("i"); + comp.set_iter_var2("i2"); + comp.set_accu_var("a"); + comp.set_accu_init(factory.NewDoubleConst(21, 2.71828)); + comp.set_iter_range(factory.NewIdent(22, "range")); + comp.set_loop_condition(factory.NewBoolConst(23, true)); + comp.set_loop_step(factory.NewCall( + 24, "step_func", + std::vector{factory.NewIdent(25, "accu")})); + comp.set_result(factory.NewIdent(26, "result")); + return comp_expr; + }()})}); + + ASSERT_OK_AND_ASSIGN(cel::Expr transformed_expr, + factory.CopyAndReplace(expr, replacer)); + + cel::Expr expected_transformed_expr = factory.NewCall( + 1, "func", + std::vector{ + factory.NewListBuilder(2) + .Add(factory.NewUnspecified(3)) + .Add(factory.NewNullConst(4)) + .Add(factory.NewBoolConst(5, true)) + .Add(factory.NewIntConst(6, 42)) + .Build(), + factory.NewMapBuilder(7) + .Add(991, factory.NewStringConst(8, "k1"), + factory.NewUintConst(9, 100u)) + .Add(992, factory.NewBytesConst(10, "b1"), + factory.NewIntConst(11, 3)) // 3.14159 -> 3 + .Build(), + factory.NewStructBuilder(12, "S") + .Add(32, "f1", factory.NewIdent(13, "x")) + .Add( + 33, "f2", + factory.NewSelect(14, factory.NewIdent(15, "y"), "sel_field")) + .Add(34, "f3", + factory.NewPresenceTest(16, factory.NewIdent(17, "z"), + "pres_field")) + .Build(), + factory.NewMemberCall( + 18, "mem_func", factory.NewIdent(19, "target"), + std::vector{[&]() { + cel::Expr comp_expr; + comp_expr.set_id(20); + auto& comp = comp_expr.mutable_comprehension_expr(); + comp.set_iter_var("i"); + comp.set_iter_var2("i2"); + comp.set_accu_var("a"); + comp.set_accu_init(factory.NewIntConst(21, 2)); // 2.71828 -> 2 + comp.set_iter_range(factory.NewIdent(22, "range")); + comp.set_loop_condition(factory.NewBoolConst(23, true)); + comp.set_loop_step(factory.NewCall( + 24, "step_func", + std::vector{factory.NewIdent(25, "accu")})); + comp.set_result(factory.NewIdent(26, "result")); + return comp_expr; + }()})}); + + EXPECT_EQ(transformed_expr, expected_transformed_expr); +} + +TEST(AstFactoryInterfaceTest, CopyAndReplacePrune) { + AstFactory factory; + + // (x + 1) + 2 + std::vector inner_args; + inner_args.push_back(factory.NewIdent(1, "x")); + inner_args.push_back(factory.NewIntConst(2, 1)); + cel::Expr inner_expr = factory.NewCall(3, "_+_", std::move(inner_args)); + + std::vector args; + args.push_back(std::move(inner_expr)); + args.push_back(factory.NewIntConst(4, 2)); + cel::Expr expr = factory.NewCall(5, "_+_", std::move(args)); + + // Replace the inner call (id 3) with a single ident "y" (id 9), pruning the + // subtree. + ASSERT_OK_AND_ASSIGN( + cel::Expr transformed, + factory.CopyAndReplace( + expr, [&](const cel::Expr& e) -> std::optional { + if (e.id() == 3) { + return factory.NewIdent(9, "y"); + } + return std::nullopt; + })); + + // Expected: y + 2 + std::vector expected_args; + expected_args.push_back(factory.NewIdent(9, "y")); + expected_args.push_back(factory.NewIntConst(4, 2)); + cel::Expr expected = factory.NewCall(5, "_+_", std::move(expected_args)); + + EXPECT_EQ(transformed, expected); +} + +TEST(AstFactoryInterfaceTest, CopyAndReplaceMaxRecursionDepth) { + AstFactory factory; + + cel::Expr expr = factory.NewIdent(1, "x"); + for (int i = 2; i <= 10; ++i) { + std::vector args; + args.push_back(std::move(expr)); + args.push_back(factory.NewIntConst(i, 1)); + expr = factory.NewCall(i, "_+_", std::move(args)); + } + + EXPECT_THAT(factory.CopyAndReplace( + expr, + [](const cel::Expr&) -> std::optional { + return std::nullopt; + }, + /*max_recursion_depth=*/3), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +class TestMacroExprExpanderSupport + : public MacroExprExpanderSupport { + public: + int64_t NextId() override { return 42; } + int64_t CopyId(int64_t id) override { return id; } + cel::Expr ReportError(absl::string_view) override { return cel::Expr(); } + cel::Expr ReportErrorAt(const cel::Expr&, absl::string_view) override { + return cel::Expr(); + } +}; + +TEST(AstFactoryInterfaceTest, MacroExprExpander) { + MacroRegistry macro_registry; + AstFactory factory(¯o_registry); + ASSERT_OK_AND_ASSIGN( + auto foo_macro, + Macro::Global("foo", 1, + [](MacroExprFactory& macro_factory, + absl::Span args) -> std::optional { + return macro_factory.NewCall("my_macro", std::move(args)); + })); + + ASSERT_THAT(macro_registry.RegisterMacro(foo_macro), IsOk()); + + auto expander1 = factory.NewMacroExprExpander("foo", 1, false); + ASSERT_TRUE(expander1.has_value()); + + std::vector expand_args; + expand_args.push_back(factory.NewIdent(1, "x")); + + TestMacroExprExpanderSupport support; + auto result = + expander1->Expand(std::nullopt, absl::MakeSpan(expand_args), support); + ASSERT_TRUE(result.has_value()); + + std::vector expected_args; + expected_args.push_back(factory.NewIdent(1, "x")); + Expr expected = factory.NewCall(42, "my_macro", std::move(expected_args)); + + EXPECT_EQ(*result, expected); + + auto expander2 = factory.NewMacroExprExpander("bar", 1, false); + EXPECT_FALSE(expander2.has_value()); +} + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/internal/lexer.cc b/parser/internal/lexer.cc new file mode 100644 index 000000000..66a9f95bb --- /dev/null +++ b/parser/internal/lexer.cc @@ -0,0 +1,680 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/lexer.h" + +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/base/no_destructor.h" +#include "absl/base/optimization.h" +#include "absl/container/flat_hash_map.h" +#include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" +#include "absl/strings/ascii.h" +#include "absl/strings/string_view.h" + +namespace cel::parser_internal { + +namespace { + +[[nodiscard]] bool IsIdentTrailing(char32_t c) { + return c <= 0x7f && (absl::ascii_isdigit(static_cast(c)) || + absl::ascii_isalpha(static_cast(c)) || c == '_'); +} + +[[nodiscard]] bool IsPlusOrMinus(char32_t c) { return c == '+' || c == '-'; } + +[[nodiscard]] const absl::flat_hash_map& +Keywords() { + static const absl::NoDestructor< + absl::flat_hash_map> + kKeywords({ + {"false", TokenType::kFalse}, + {"true", TokenType::kTrue}, + {"null", TokenType::kNull}, + {"in", TokenType::kIn}, + {"as", TokenType::kReservedWord}, + {"break", TokenType::kReservedWord}, + {"const", TokenType::kReservedWord}, + {"continue", TokenType::kReservedWord}, + {"else", TokenType::kReservedWord}, + {"for", TokenType::kReservedWord}, + {"function", TokenType::kReservedWord}, + {"if", TokenType::kReservedWord}, + {"import", TokenType::kReservedWord}, + {"let", TokenType::kReservedWord}, + {"loop", TokenType::kReservedWord}, + {"package", TokenType::kReservedWord}, + {"namespace", TokenType::kReservedWord}, + {"return", TokenType::kReservedWord}, + {"var", TokenType::kReservedWord}, + {"void", TokenType::kReservedWord}, + {"while", TokenType::kReservedWord}, + }); + return *kKeywords; +} + +} // namespace + +std::string_view TokenTypeToString(TokenType type) { + switch (type) { + case TokenType::kError: + return "error"; + case TokenType::kEnd: + return "end"; + case TokenType::kWhitespace: + return "whitespace"; + case TokenType::kComment: + return "comment"; + case TokenType::kNull: + return "null"; + case TokenType::kFalse: + return "false"; + case TokenType::kTrue: + return "true"; + case TokenType::kIn: + return "in"; + case TokenType::kReservedWord: + return "reserved_word"; + case TokenType::kInt: + return "int"; + case TokenType::kUint: + return "uint"; + case TokenType::kFloat: + return "float"; + case TokenType::kString: + return "string"; + case TokenType::kBytes: + return "bytes"; + case TokenType::kIdent: + return "ident"; + case TokenType::kLeftBracket: + return "["; + case TokenType::kRightBracket: + return "]"; + case TokenType::kLeftBrace: + return "{"; + case TokenType::kRightBrace: + return "}"; + case TokenType::kLeftParen: + return "("; + case TokenType::kRightParen: + return ")"; + case TokenType::kDot: + return "."; + case TokenType::kComma: + return ","; + case TokenType::kMinus: + return "-"; + case TokenType::kPlus: + return "+"; + case TokenType::kAsterisk: + return "*"; + case TokenType::kSlash: + return "/"; + case TokenType::kPercent: + return "%"; + case TokenType::kQuestion: + return "?"; + case TokenType::kColon: + return ":"; + case TokenType::kExclamation: + return "!"; + case TokenType::kEqual: + return "="; + case TokenType::kEqualEqual: + return "=="; + case TokenType::kExclamationEqual: + return "!="; + case TokenType::kLess: + return "<"; + case TokenType::kLessEqual: + return "<="; + case TokenType::kGreater: + return ">"; + case TokenType::kGreaterEqual: + return ">="; + case TokenType::kLogicalAnd: + return "&&"; + case TokenType::kLogicalOr: + return "||"; + default: + return ""; + } +} + +Token Lexer::Lex() { + int32_t start = GetPosition(); + if (ABSL_PREDICT_FALSE(position_ >= content_size_)) { + return MakeToken(TokenType::kEnd, start, start); + } + char32_t c = content_.at(position_); + switch (c) { + case '\f': + ABSL_FALLTHROUGH_INTENDED; + case '\t': + ABSL_FALLTHROUGH_INTENDED; + case '\r': + ABSL_FALLTHROUGH_INTENDED; + case '\n': + ABSL_FALLTHROUGH_INTENDED; + case ' ': { + ConsumeWhitespace(); + return MakeToken(TokenType::kWhitespace, start, GetPosition()); + } + case '.': { + if (position_ + 1 < content_size_ && content_.at(position_ + 1) <= 0x7f && + absl::ascii_isdigit(static_cast(content_.at(position_ + 1)))) { + return ConsumeNumericLiteral(); + } + Advance(1); + return MakeToken(TokenType::kDot, start, GetPosition()); + } + case ',': { + Advance(1); + return MakeToken(TokenType::kComma, start, GetPosition()); + } + case '!': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kExclamationEqual, start, GetPosition()); + } + return MakeToken(TokenType::kExclamation, start, GetPosition()); + } + case '?': { + Advance(1); + return MakeToken(TokenType::kQuestion, start, GetPosition()); + } + case '(': { + Advance(1); + return MakeToken(TokenType::kLeftParen, start, GetPosition()); + } + case ')': { + Advance(1); + return MakeToken(TokenType::kRightParen, start, GetPosition()); + } + case '{': { + Advance(1); + return MakeToken(TokenType::kLeftBrace, start, GetPosition()); + } + case '}': { + Advance(1); + return MakeToken(TokenType::kRightBrace, start, GetPosition()); + } + case '[': { + Advance(1); + return MakeToken(TokenType::kLeftBracket, start, GetPosition()); + } + case ']': { + Advance(1); + return MakeToken(TokenType::kRightBracket, start, GetPosition()); + } + case '=': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kEqualEqual, start, GetPosition()); + } + return MakeToken(TokenType::kEqual, start, GetPosition()); + } + case '<': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kLessEqual, start, GetPosition()); + } + return MakeToken(TokenType::kLess, start, GetPosition()); + } + case '>': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kGreaterEqual, start, GetPosition()); + } + return MakeToken(TokenType::kGreater, start, GetPosition()); + } + case ':': { + Advance(1); + return MakeToken(TokenType::kColon, start, GetPosition()); + } + case '%': { + Advance(1); + return MakeToken(TokenType::kPercent, start, GetPosition()); + } + case '+': { + Advance(1); + return MakeToken(TokenType::kPlus, start, GetPosition()); + } + case '-': { + Advance(1); + return MakeToken(TokenType::kMinus, start, GetPosition()); + } + case '*': { + Advance(1); + return MakeToken(TokenType::kAsterisk, start, GetPosition()); + } + case '/': { + Advance(1); + if (Consume('/')) { + ConsumeLine(); + return MakeToken(TokenType::kComment, start, GetPosition()); + } + return MakeToken(TokenType::kSlash, start, GetPosition()); + } + case '&': { + Advance(1); + if (Consume('&')) { + return MakeToken(TokenType::kLogicalAnd, start, GetPosition()); + } + return SetError(start, GetPosition(), + "unexpected single '&', expected '&&'"); + } + case '|': { + Advance(1); + if (Consume('|')) { + return MakeToken(TokenType::kLogicalOr, start, GetPosition()); + } + return SetError(start, GetPosition(), + "unexpected single '|', expected '||'"); + } + case '_': { + return ConsumeIdent(); + } + case '`': { + return ConsumeQuotedIdent(); + } + case '\'': { + return ConsumeStringLiteral(start, '\''); + } + case '"': { + return ConsumeStringLiteral(start, '"'); + } + case 'r': + ABSL_FALLTHROUGH_INTENDED; + case 'R': + ABSL_FALLTHROUGH_INTENDED; + case 'b': + ABSL_FALLTHROUGH_INTENDED; + case 'B': { + if (auto token = ConsumePrefixedStringLiteral(); token.has_value()) { + return *token; + } + break; + } + default: + break; + } + if (c <= 0x7f && absl::ascii_isdigit(static_cast(c))) { + return ConsumeNumericLiteral(); + } + if (c <= 0x7f && absl::ascii_isalpha(static_cast(c))) { + // Root identifiers (the ones starting with a period) are returned as + // a sequence of kDot and kIdent tokens. + return ConsumeIdent(); + } + Advance(1); + return SetError(start, GetPosition(), "unexpected character"); +} + +// Consumes characters up to and including the first occurrence of character +// `c`. If `is_raw` is false, backslashes are interpreted as escapes. Returns +// true if `c` was found and consumed; false if end of input or an unescaped +// newline was reached. +bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) { + ABSL_DCHECK_NE(c, '\n'); + ABSL_DCHECK_NE(c, '\r'); + int32_t pos = position_; + bool escaped = false; + while (pos < content_size_) { + char32_t cc = content_.at(pos); + if (cc == '\n' || cc == '\r') { + AdvanceProcessingNewLines(pos); + return false; + } + if (!is_raw && cc == '\\') { + escaped = !escaped; + } else { + if (cc == c && (is_raw || !escaped)) { + AdvanceProcessingNewLines(pos + 1); + return true; + } + escaped = false; + } + ++pos; + } + AdvanceProcessingNewLines(content_size_); + return false; +} + +// Consumes characters up to and including the first occurrence of substring `s` +// without interpreting backslashes as escapes (`s` must not contain newlines). +// Returns true if `s` was found and consumed; false if end of input was +// reached. +bool Lexer::ConsumeUntilAfterString(std::u32string_view s) { + ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); + int32_t pos = position_; + while (pos + static_cast(s.size()) <= content_size_) { + bool match = true; + for (size_t i = 0; i < s.size(); ++i) { + if (content_.at(pos + static_cast(i)) != s[i]) { + match = false; + break; + } + } + if (match) { + AdvanceProcessingNewLines(pos + static_cast(s.size())); + return true; + } + ++pos; + } + AdvanceProcessingNewLines(content_size_); + return false; +} + +// Consumes characters up to and including the first occurrence of substring `s` +// where the first character of `s` is not preceded by an odd number of +// backslashes. Returns true if an unescaped `s` was found and consumed; false +// if reached EOF. +bool Lexer::ConsumeUntilAfterUnescapedString(std::u32string_view s) { + ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); + int32_t pos = position_; + bool escaped = false; + while (pos < content_size_) { + char32_t cc = content_.at(pos); + if (cc == '\\') { + escaped = !escaped; + } else { + if (!escaped && pos + static_cast(s.size()) <= content_size_) { + bool match = true; + for (size_t j = 0; j < s.size(); ++j) { + if (content_.at(pos + static_cast(j)) != s[j]) { + match = false; + break; + } + } + if (match) { + AdvanceProcessingNewLines(pos + static_cast(s.size())); + return true; + } + } + escaped = false; + } + ++pos; + } + AdvanceProcessingNewLines(content_size_); + return false; +} + +bool Lexer::MatchString(std::u32string_view s) const { + if (position_ + static_cast(s.size()) > content_size_) { + return false; + } + for (size_t i = 0; i < s.size(); ++i) { + if (content_.at(position_ + static_cast(i)) != s[i]) { + return false; + } + } + return true; +} + +std::optional Lexer::MatchIf( + absl::FunctionRef predicate) const { + if (position_ < content_size_) { + char32_t cp = content_.at(position_); + if (predicate(cp)) { + return cp; + } + } + return std::nullopt; +} + +void Lexer::ConsumeLine() { + while (position_ < content_size_) { + if (content_.at(position_) == '\n') { + Advance(1); + return; + } + Advance(1); + } +} + +void Lexer::ConsumeWhitespace() { + while (position_ < content_size_) { + char32_t c = content_.at(position_); + switch (c) { + case '\f': + ABSL_FALLTHROUGH_INTENDED; + case '\n': + ABSL_FALLTHROUGH_INTENDED; + case ' ': + ABSL_FALLTHROUGH_INTENDED; + case '\r': + ABSL_FALLTHROUGH_INTENDED; + case '\t': + Advance(1); + break; + default: + return; + } + } +} + +bool Lexer::Consume(char32_t c) { + ABSL_DCHECK_NE(c, '\n'); + if (Match(c)) { + Advance(1); + return true; + } + return false; +} + +bool Lexer::ConsumeIgnoreCase(char32_t c) { + ABSL_DCHECK_NE(c, '\n'); + if (MatchIgnoreCase(c)) { + Advance(1); + return true; + } + return false; +} + +bool Lexer::ConsumeString(std::u32string_view s) { + ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); + if (MatchString(s)) { + Advance(s.size()); + return true; + } + return false; +} + +std::optional Lexer::ConsumeIf( + absl::FunctionRef predicate) { + std::optional match = MatchIf(predicate); + if (match.has_value()) { + ABSL_DCHECK_NE(*match, '\n'); + Advance(1); + } + return match; +} + +bool Lexer::ConsumeDigits() { + bool advanced = false; + while (position_ < content_size_) { + char32_t c = content_.at(position_); + if (c > 0x7f || !absl::ascii_isdigit(static_cast(c))) { + break; + } + Advance(1); + advanced = true; + } + return advanced; +} + +bool Lexer::ConsumeHexDigits() { + bool advanced = false; + while (position_ < content_size_) { + char32_t c = content_.at(position_); + if (c > 0x7f || !absl::ascii_isxdigit(static_cast(c))) { + break; + } + Advance(1); + advanced = true; + } + return advanced; +} + +TokenType Lexer::ConsumeIntegralSuffix() { + if (ConsumeIgnoreCase('u')) { + return TokenType::kUint; + } + return TokenType::kInt; +} + +Token Lexer::ConsumeQuotedIdent() { + int32_t start = GetPosition(); + Advance(1); + if (!ConsumeUntilAfter('`', /*is_raw=*/true)) { + return SetError(start, GetPosition(), "unterminated quoted identifier"); + } + return MakeToken(TokenType::kIdent, start, GetPosition()); +} + +Token Lexer::ConsumeStringLiteral(int32_t start, char32_t quote, bool is_bytes, + bool is_raw) { + Advance(1); + std::u32string triple_quote(3, quote); + if (ConsumeString(std::u32string_view(triple_quote.data(), 2))) { + if (is_raw ? !ConsumeUntilAfterString(triple_quote) + : !ConsumeUntilAfterUnescapedString(triple_quote)) { + return SetError(start, GetPosition(), + is_bytes ? "unterminated bytes literal" + : "unterminated string literal"); + } + return MakeToken(is_bytes ? TokenType::kBytes : TokenType::kString, start, + GetPosition()); + } + if (!ConsumeUntilAfter(quote, is_raw)) { + return SetError(start, GetPosition(), + is_bytes ? "unterminated bytes literal" + : "unterminated string literal"); + } + return MakeToken(is_bytes ? TokenType::kBytes : TokenType::kString, start, + GetPosition()); +} + +// Consumes prefixed string and bytes literals. +// Handles the following prefix sequences (case-insensitive for 'r' and 'b'): +// - Raw strings: r"...", r'...', r"""...""", r'''...''' +// - Bytes: b"...", b'...', b"""...""", b'''...''' +// - Raw bytes: br"...", br'...', br"""...""", br'''...''', rb"...", rb'...', +// rb"""...""", rb'''...''' +std::optional Lexer::ConsumePrefixedStringLiteral() { + int32_t start = GetPosition(); + if (position_ >= content_size_) return std::nullopt; + char32_t c = content_.at(position_); + bool is_bytes = (c == 'b' || c == 'B'); + bool is_raw = (c == 'r' || c == 'R'); + size_t lookahead = 1; + if (position_ + 1 < content_size_) { + char32_t c2 = content_.at(position_ + 1); + if ((is_bytes && (c2 == 'r' || c2 == 'R')) || + (!is_bytes && (c2 == 'b' || c2 == 'B'))) { + is_bytes = true; + is_raw = true; + lookahead = 2; + } + } + if (position_ + static_cast(lookahead) < content_size_) { + char32_t quote = content_.at(position_ + static_cast(lookahead)); + if (quote == '"' || quote == '\'') { + Advance(lookahead); + return ConsumeStringLiteral(start, quote, is_bytes, is_raw); + } + } + return std::nullopt; +} + +// Consumes a numeric literal token and returns its TokenType (kInt, kUint, or +// kFloat). Recognizes the following literal formats: +// - Hexadecimal integers (kInt / kUint): 0x1A, 0XFFu, 0x0U +// - Decimal integers (kInt / kUint): 0, 45U, 123456 +// - Floating-point numbers (kFloat): .12345, 1.23, 1e6, 1.5e+10, .5e-3 +Token Lexer::ConsumeNumericLiteral() { + int32_t start = GetPosition(); + char32_t c = content_.at(position_); + bool floating_point = false; + if (c == '.') { + floating_point = true; + Advance(1); + if (!ConsumeDigits()) { + return SetError( + start, GetPosition(), + "floating point literal missing digits after decimal separator"); + } + } else { + Advance(1); + if (c == '0') { + if (ConsumeIgnoreCase('x')) { + if (!ConsumeHexDigits()) { + return SetError( + start, GetPosition(), + "integral literal missing digits after hexadecimal separator"); + } + auto token_type = ConsumeIntegralSuffix(); + return MakeToken(token_type, start, GetPosition()); + } + } + static_cast(ConsumeDigits()); + if (position_ < content_size_ && content_.at(position_) == '.' && + position_ + 1 < content_size_ && content_.at(position_ + 1) <= 0x7f && + absl::ascii_isdigit(static_cast(content_.at(position_ + 1)))) { + floating_point = true; + Advance(1); + static_cast(ConsumeDigits()); + } + } + if (ConsumeIgnoreCase('e')) { + floating_point = true; + static_cast(ConsumeIf(IsPlusOrMinus)); + if (!ConsumeDigits()) { + return SetError( + start, GetPosition(), + "floating point literal missing digits after exponent separator"); + } + } + auto token_type = + floating_point ? TokenType::kFloat : ConsumeIntegralSuffix(); + return MakeToken(token_type, start, GetPosition()); +} + +Token Lexer::ConsumeIdent() { + int32_t start = GetPosition(); + while (position_ < content_size_) { + char32_t c = content_.at(position_); + if (!IsIdentTrailing(c)) { + break; + } + Advance(1); + } + int32_t end = GetPosition(); + std::string word = content_.ToString(start, end); + const auto& keywords = Keywords(); + if (auto it = keywords.find(word); it != keywords.end()) { + return MakeToken(it->second, start, end); + } + return MakeToken(TokenType::kIdent, start, end); +} + +} // namespace cel::parser_internal diff --git a/parser/internal/lexer.h b/parser/internal/lexer.h new file mode 100644 index 000000000..217bc7c41 --- /dev/null +++ b/parser/internal/lexer.h @@ -0,0 +1,287 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/base/optimization.h" +#include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" +#include "absl/strings/ascii.h" +#include "common/source.h" + +namespace cel::parser_internal { + +enum class TokenType { + kError = 0, + kEnd, + kWhitespace, + kComment, + + // Keywords + kNull, + kFalse, + kTrue, + kIn, + kReservedWord, + + // Literals + kInt, + kUint, + kFloat, + kString, + kBytes, + + // Identifiers (standard bare identifiers and backtick-quoted identifiers). + // Note: The lexer does not validate whether a quoted/escaped identifier is + // source-legal or permitted in its syntactic context. Because + // non-source-legal identifiers are used internally in macros and functions, + // the parser must strictly validate the characters inside quoted identifiers + // and verify that they only appear where permitted (e.g., field selections + // and struct field specifiers). + kIdent, + + // Delimiters + kLeftBracket, // [ + kRightBracket, // ] + kLeftBrace, // { + kRightBrace, // } + kLeftParen, // ( + kRightParen, // ) + + // Operators + kDot, // . + kComma, // , + kMinus, // - + kPlus, // + + kAsterisk, // * + kSlash, // / + kPercent, // % + kQuestion, // ? + kColon, // : + kExclamation, // ! + kEqual, // = + kEqualEqual, // == + kExclamationEqual, // != + kLess, // < + kLessEqual, // <= + kGreater, // > + kGreaterEqual, // >= + kLogicalAnd, // && + kLogicalOr, // || +}; + +ABSL_ATTRIBUTE_PURE_FUNCTION std::string_view TokenTypeToString(TokenType type); + +struct Token final { + TokenType type = TokenType::kError; + int32_t start = 0; + int32_t end = 0; +}; + +struct LexerError final { + int32_t start = 0; + int32_t end = 0; + std::string message; +}; + +// Lexer performs fast tokenization of CEL expression source code. +// +// Responsibilities & Parser Expectations: +// This lexer is designed for speed and does not perform comprehensive semantic +// or syntax validation: +// +// 1. String and Bytes Literal Escape Sequences: +// - For standard single- and double-quoted literals ("..." and '...'), the +// lexer recognizes backslash ('\') only to determine whether the closing +// delimiter ('"' or '\'') is escaped (e.g., \" and \' do not terminate the +// literal, whereas \\" terminates it because the backslash is escaped). +// - For triple-quoted literals ("""...""" and '''...''') and raw literals +// (r"...", r'''...'''), backslashes and escape sequences are not processed +// when locating the closing delimiter. +// - The lexer does NOT validate, decode, or check the syntax of any escape +// sequences (e.g., \n, \r, \t, \xHH, \uHHHH, \U00HHHHHH, \0, or invalid +// escapes like \q). All characters and backslashes within the literal +// boundaries are preserved verbatim in the token's text span. +// - The parser/caller is strictly responsible for validating and unescaping +// all escape sequences and reporting syntax errors for invalid escape +// sequences when converting string and bytes tokens during AST +// construction. +// +// 2. Numeric Literals: +// - Performs only general bounds and format matching for integers and +// floating-point numeric literals. The lexer expects the parser to perform +// final validation and numeric conversion when building the AST. +class Lexer final { + public: + explicit Lexer(const cel::Source& source) + : content_(source.content()), + content_size_(static_cast(content_.size())), + position_(0) { + ABSL_DCHECK_LE(content_.size(), static_cast( + std::numeric_limits::max())); + } + + Lexer(const Lexer&) = delete; + Lexer(Lexer&&) = delete; + Lexer& operator=(const Lexer&) = delete; + Lexer& operator=(Lexer&&) = delete; + + // Scans and returns the next token from the source. + [[nodiscard]] ABSL_ATTRIBUTE_NOINLINE Token Lex(); + + // Inspect the error from the last call to `Lex()` that returned an error + // token. The reference is not guaranteed to be valid after further calls to + // `Lex`. + [[nodiscard]] const LexerError& GetError() const + ABSL_ATTRIBUTE_LIFETIME_BOUND { + return error_; + } + + [[nodiscard]] int32_t GetPosition() const { return position_; } + + [[nodiscard]] int32_t SavePosition() const { return position_; } + + void RestorePosition(int32_t position) { + ABSL_DCHECK_GE(position, 0); + ABSL_DCHECK_LE(position, content_size_); + position_ = position; + error_ = LexerError{}; + } + + private: + [[nodiscard]] bool Match(char32_t c) const { + return position_ < content_size_ && content_.at(position_) == c; + } + + [[nodiscard]] bool MatchIgnoreCase(char32_t c) const { + if (position_ >= content_size_) return false; + char32_t cp = content_.at(position_); + return cp <= 0x7f && c <= 0x7f && + absl::ascii_tolower(static_cast(cp)) == + absl::ascii_tolower(static_cast(c)); + } + + void Advance(size_t n) { + ABSL_DCHECK_LE(n, static_cast(content_size_ - position_)); + position_ += static_cast(n); + } + + void AdvanceProcessingNewLines(int32_t end_position) { + ABSL_DCHECK_LE(end_position, content_size_); + ABSL_DCHECK_GE(end_position, position_); + Advance(static_cast(end_position - position_)); + } + + [[nodiscard]] Token MakeToken(TokenType type, int32_t start, int32_t end) { + return Token{.type = type, .start = start, .end = end}; + } + + [[nodiscard]] Token SetError(int32_t start, int32_t end, + std::string message) { + error_ = + LexerError{.start = start, .end = end, .message = std::move(message)}; + return Token{.type = TokenType::kError, .start = start, .end = end}; + } + + // Consumes characters up to and including the first occurrence of character + // `c`. If `is_raw` is false, backslashes are interpreted as escapes. Returns + // true if `c` was found and consumed; false if end of input or an unescaped + // newline was reached. + [[nodiscard]] bool ConsumeUntilAfter(char32_t c, bool is_raw); + + // Consumes characters up to and including the first occurrence of substring + // `s` without interpreting backslashes as escapes (`s` must not contain + // newlines). Returns true if `s` was found and consumed; false if end of + // input was reached. + [[nodiscard]] bool ConsumeUntilAfterString(std::u32string_view s); + + // Consumes characters up to and including the first occurrence of substring + // `s` where the first character of `s` is not preceded by an odd number of + // backslashes. Returns true if an unescaped `s` was found and consumed; false + // if reached EOF. + [[nodiscard]] bool ConsumeUntilAfterUnescapedString(std::u32string_view s); + + [[nodiscard]] bool MatchString(std::u32string_view s) const; + + [[nodiscard]] std::optional MatchIf( + absl::FunctionRef predicate) const; + + void ConsumeLine(); + + void ConsumeWhitespace(); + + [[nodiscard]] bool Consume(char32_t c); + + [[nodiscard]] bool ConsumeIgnoreCase(char32_t c); + + [[nodiscard]] bool ConsumeString(std::u32string_view s); + + [[nodiscard]] std::optional ConsumeIf( + absl::FunctionRef predicate); + + [[nodiscard]] bool ConsumeDigits(); + + [[nodiscard]] bool ConsumeHexDigits(); + + [[nodiscard]] TokenType ConsumeIntegralSuffix(); + + // Consumes a backtick-quoted identifier (`...`) and returns + // TokenType::kIdent. The token text preserves the surrounding backticks so + // the parser can detect quoted identifiers and enforce restrictions on their + // characters and allowed syntactic locations (such as field selections and + // struct field specifiers). + [[nodiscard]] Token ConsumeQuotedIdent(); + + [[nodiscard]] Token ConsumeStringLiteral(int32_t start, char32_t quote, + bool is_bytes = false, + bool is_raw = false); + + // Consumes prefixed string and bytes literals. + // Handles the following prefix sequences (case-insensitive for 'r' and 'b'): + // - Raw strings: r"...", r'...', r"""...""", r'''...''' + // - Bytes: b"...", b'...', b"""...""", b'''...''' + // - Raw bytes: br"...", br'...', br"""...""", br'''...''', rb"...", rb'...', + // rb"""...""", rb'''...''' + [[nodiscard]] std::optional ConsumePrefixedStringLiteral(); + + // Consumes a numeric literal token and returns its TokenType (kInt, kUint, or + // kFloat). Recognizes the following literal formats: + // - Decimal integers (kInt / kUint): 0, 45U, 123456 + // - Hexadecimal integers (kInt / kUint): 0x1A, 0XFFu, 0x0U + // - Floating-point numbers (kFloat): .12345, 1.23, 1e6, 1.5e+10, .5e-3 + [[nodiscard]] Token ConsumeNumericLiteral(); + + // Consumes an identifier token and checks if it matches any reserved + // keywords. + [[nodiscard]] Token ConsumeIdent(); + + cel::SourceContentView content_; + int32_t content_size_ = 0; + int32_t position_ = 0; + LexerError error_; +}; + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ diff --git a/parser/internal/lexer_test.cc b/parser/internal/lexer_test.cc new file mode 100644 index 000000000..8625d8b90 --- /dev/null +++ b/parser/internal/lexer_test.cc @@ -0,0 +1,539 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/lexer.h" + +#include +#include +#include +#include +#include + +#include "common/source.h" +#include "internal/testing.h" + +namespace cel::parser_internal { +namespace { + +MATCHER_P3(IsToken, source, expected_type, expected_text, "") { + if (arg.type != expected_type) { + *result_listener << "type is " << TokenTypeToString(arg.type) + << " (expected " << TokenTypeToString(expected_type) + << ")"; + return false; + } + std::string actual_text = source->content().ToString(arg.start, arg.end); + if (actual_text != expected_text) { + *result_listener << "text is '" << actual_text << "' (expected '" + << expected_text << "')"; + return false; + } + return true; +} + +struct LexerTestCase { + std::string_view name; + std::string_view source; + std::vector> expected_tokens; +}; + +using LexerTest = testing::TestWithParam; + +TEST_P(LexerTest, LexesSuccessTokens) { + const LexerTestCase& test_case = GetParam(); + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(test_case.source)); + Lexer lexer(*source); + + for (const auto& [type, text] : test_case.expected_tokens) { + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), type, text)); + } + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kEnd, "")); +} + +INSTANTIATE_TEST_SUITE_P( + LexerTest, LexerTest, + testing::ValuesIn({ + {"NullSource", std::string_view(nullptr, 0), {}}, + {"Empty", "", {}}, + {"Whitespace", " \n ", {{TokenType::kWhitespace, " \n "}}}, + {"KeywordsAndIdents", + "null false true in as return foo_bar _foo_bar _ `quoted.ident`", + {{TokenType::kNull, "null"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFalse, "false"}, + {TokenType::kWhitespace, " "}, + {TokenType::kTrue, "true"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIn, "in"}, + {TokenType::kWhitespace, " "}, + {TokenType::kReservedWord, "as"}, + {TokenType::kWhitespace, " "}, + {TokenType::kReservedWord, "return"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "foo_bar"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "_foo_bar"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "_"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "`quoted.ident`"}}}, + {"Numbers", + "123 45u 0x1A 3.14 .5 1e6 2.5e-3 45U 0x1Au 0x1AU", + {{TokenType::kInt, "123"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "45u"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "0x1A"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "3.14"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, ".5"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "1e6"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "2.5e-3"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "45U"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0x1Au"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0x1AU"}}}, + {"IntEOF", "123456", {{TokenType::kInt, "123456"}}}, + {"HexIntEOF", "0x1A2B", {{TokenType::kInt, "0x1A2B"}}}, + {"FloatPositiveExponentEOF", "1e+6", {{TokenType::kFloat, "1e+6"}}}, + {"FloatEOF", ".12345", {{TokenType::kFloat, ".12345"}}}, + {"IntDotIdent", + "1.foo", + {{TokenType::kInt, "1"}, + {TokenType::kDot, "."}, + {TokenType::kIdent, "foo"}}}, + {"IntDotWhitespace", + "1. ", + {{TokenType::kInt, "1"}, + {TokenType::kDot, "."}, + {TokenType::kWhitespace, " "}}}, + {"IntDotEOF", "1.", {{TokenType::kInt, "1"}, {TokenType::kDot, "."}}}, + {"DotAtEOFBeforeDigit", + std::string_view(".6", /*length=*/1), + {{TokenType::kDot, "."}}}, + {"DotAtEOFBeforeIdent", + std::string_view(".a", /*length=*/1), + {{TokenType::kDot, "."}}}, + {"ZeroNumbers", + "0 0u 0x0", + {{TokenType::kInt, "0"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0u"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "0x0"}}}, + {"StringsAndBytes", + R"("hello" 'world' """ "allowed!" ""also allowed"" \"""also allowed""\" """ r"raw" b"bytes" rb'\x00' '''multi +single''' R"raw_upper" B"bytes_upper" b'''multi +bytes''' br"raw_bytes" `a.b-c/d e` +"\a\b\f\n\r\t\v\"\'\\\?\` \x1A \u00A0 \U0001F600 \012")", + {{TokenType::kString, "\"hello\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "'world'"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, + "\"\"\" \"allowed!\" \"\"also allowed\"\" \\\"\"\"also " + "allowed\"\"\\\" \"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"raw\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"bytes\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "rb'\\x00'"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "'''multi\nsingle'''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "R\"raw_upper\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "B\"bytes_upper\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b'''multi\nbytes'''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "br\"raw_bytes\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "`a.b-c/d e`"}, + {TokenType::kWhitespace, "\n"}, + {TokenType::kString, + "\"\\a\\b\\f\\n\\r\\t\\v\\\"\\'\\\\\\?\\` \\x1A \\u00A0 \\U0001F600 " + "\\012\""}}}, + {"EmptyStrings", + "\"\" '' \"\"\"\"\"\" '''''' r\"\" r'' r\"\"\"\"\"\" r'''''' b\"\" " + "b'' b\"\"\"\"\"\" b''''''", + {{TokenType::kString, "\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "''''''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r''''''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b''''''"}}}, + {"OperatorsAndDelimiters", + ". , + - * / % == != < <= > >= && || ! ? : [] { } ( )", + {{TokenType::kDot, "."}, + {TokenType::kWhitespace, " "}, + {TokenType::kComma, ","}, + {TokenType::kWhitespace, " "}, + {TokenType::kPlus, "+"}, + {TokenType::kWhitespace, " "}, + {TokenType::kMinus, "-"}, + {TokenType::kWhitespace, " "}, + {TokenType::kAsterisk, "*"}, + {TokenType::kWhitespace, " "}, + {TokenType::kSlash, "/"}, + {TokenType::kWhitespace, " "}, + {TokenType::kPercent, "%"}, + {TokenType::kWhitespace, " "}, + {TokenType::kEqualEqual, "=="}, + {TokenType::kWhitespace, " "}, + {TokenType::kExclamationEqual, "!="}, + {TokenType::kWhitespace, " "}, + {TokenType::kLess, "<"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLessEqual, "<="}, + {TokenType::kWhitespace, " "}, + {TokenType::kGreater, ">"}, + {TokenType::kWhitespace, " "}, + {TokenType::kGreaterEqual, ">="}, + {TokenType::kWhitespace, " "}, + {TokenType::kLogicalAnd, "&&"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLogicalOr, "||"}, + {TokenType::kWhitespace, " "}, + {TokenType::kExclamation, "!"}, + {TokenType::kWhitespace, " "}, + {TokenType::kQuestion, "?"}, + {TokenType::kWhitespace, " "}, + {TokenType::kColon, ":"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBracket, "["}, + {TokenType::kRightBracket, "]"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBrace, "{"}, + {TokenType::kWhitespace, " "}, + {TokenType::kRightBrace, "}"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftParen, "("}, + {TokenType::kWhitespace, " "}, + {TokenType::kRightParen, ")"}}}, + {"Comments", + "a\n// comment\nb", + {{TokenType::kIdent, "a"}, + {TokenType::kWhitespace, "\n"}, + {TokenType::kComment, "// comment\n"}, + {TokenType::kIdent, "b"}}}, + {"CommentWithoutTrailingNewlineEOF", + "// comment without trailing newline", + {{TokenType::kComment, "// comment without trailing newline"}}}, + {"CommentAfterTokenWithoutTrailingNewlineEOF", + "a // comment without trailing newline", + {{TokenType::kIdent, "a"}, + {TokenType::kWhitespace, " "}, + {TokenType::kComment, "// comment without trailing newline"}}}, + {"NestedDelimiters", + "(((1 + 2) * [3, {4: 5}])))", + {{TokenType::kLeftParen, "("}, {TokenType::kLeftParen, "("}, + {TokenType::kLeftParen, "("}, {TokenType::kInt, "1"}, + {TokenType::kWhitespace, " "}, {TokenType::kPlus, "+"}, + {TokenType::kWhitespace, " "}, {TokenType::kInt, "2"}, + {TokenType::kRightParen, ")"}, {TokenType::kWhitespace, " "}, + {TokenType::kAsterisk, "*"}, {TokenType::kWhitespace, " "}, + {TokenType::kLeftBracket, "["}, {TokenType::kInt, "3"}, + {TokenType::kComma, ","}, {TokenType::kWhitespace, " "}, + {TokenType::kLeftBrace, "{"}, {TokenType::kInt, "4"}, + {TokenType::kColon, ":"}, {TokenType::kWhitespace, " "}, + {TokenType::kInt, "5"}, {TokenType::kRightBrace, "}"}, + {TokenType::kRightBracket, "]"}, {TokenType::kRightParen, ")"}, + {TokenType::kRightParen, ")"}, {TokenType::kRightParen, ")"}}}, + {"NestedCommentsInDelimiters", + "(\n // leading comment\n [\n 1, // inline comment\n {2: 3}\n " + " ]\n)", + {{TokenType::kLeftParen, "("}, + {TokenType::kWhitespace, "\n "}, + {TokenType::kComment, "// leading comment\n"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBracket, "["}, + {TokenType::kWhitespace, "\n "}, + {TokenType::kInt, "1"}, + {TokenType::kComma, ","}, + {TokenType::kWhitespace, " "}, + {TokenType::kComment, "// inline comment\n"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBrace, "{"}, + {TokenType::kInt, "2"}, + {TokenType::kColon, ":"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "3"}, + {TokenType::kRightBrace, "}"}, + {TokenType::kWhitespace, "\n "}, + {TokenType::kRightBracket, "]"}, + {TokenType::kWhitespace, "\n"}, + {TokenType::kRightParen, ")"}}}, + {"ComplexNestedLiteralsAndDelimiters", + "`foo.bar`([\"nested\\\"quote\", r'''raw 'quotes' inside'''], " + "b\"bytes\")", + {{TokenType::kIdent, "`foo.bar`"}, + {TokenType::kLeftParen, "("}, + {TokenType::kLeftBracket, "["}, + {TokenType::kString, "\"nested\\\"quote\""}, + {TokenType::kComma, ","}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r'''raw 'quotes' inside'''"}, + {TokenType::kRightBracket, "]"}, + {TokenType::kComma, ","}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"bytes\""}, + {TokenType::kRightParen, ")"}}}, + {"NumericFollowedByIdent", + "0x1A_invalid 123_invalid 1x0 2x 9in-x", + {{TokenType::kInt, "0x1A"}, + {TokenType::kIdent, "_invalid"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "123"}, + {TokenType::kIdent, "_invalid"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "1"}, + {TokenType::kIdent, "x0"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "2"}, + {TokenType::kIdent, "x"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "9"}, + {TokenType::kIn, "in"}, + {TokenType::kMinus, "-"}, + {TokenType::kIdent, "x"}}}, + }), + [](const testing::TestParamInfo& info) { + return std::string(info.param.name); + }); + +TEST(LexerTest, LineOffsets) { + std::string_view source_text = "a\n// comment\nb"; + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(source_text)); + Lexer lexer(*source); + + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kIdent, "a")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kWhitespace, "\n")); + EXPECT_THAT(lexer.Lex(), + IsToken(source.get(), TokenType::kComment, "// comment\n")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kIdent, "b")); + + auto line_offsets = source->line_offsets(); + ASSERT_GE(line_offsets.size(), 2); + EXPECT_EQ(line_offsets[0], 2); + EXPECT_EQ(line_offsets[1], 13); +} + +TEST(LexerTest, LineOffsetsInMultilineStrings) { + std::string_view source_text = "'''multi\nline'''\n\"\"\"another\nline\"\"\""; + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(source_text)); + Lexer lexer(*source); + + EXPECT_THAT(lexer.Lex(), + IsToken(source.get(), TokenType::kString, "'''multi\nline'''")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kWhitespace, "\n")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kString, + "\"\"\"another\nline\"\"\"")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kEnd, "")); + + auto line_offsets = source->line_offsets(); + ASSERT_GE(line_offsets.size(), 3); + EXPECT_EQ(line_offsets[0], 9); + EXPECT_EQ(line_offsets[1], 17); + EXPECT_EQ(line_offsets[2], 28); +} + +struct LexerErrorTestCase { + std::string_view source; + std::string_view expected_error_message; + std::string_view expected_error_location; +}; + +using LexerErrorTest = testing::TestWithParam; + +TEST_P(LexerErrorTest, LexesErrorTokenAndStoresError) { + const LexerErrorTestCase& test_case = GetParam(); + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(test_case.source)); + Lexer lexer(*source); + Token token = lexer.Lex(); + while (token.type != TokenType::kError && token.type != TokenType::kEnd) { + token = lexer.Lex(); + } + EXPECT_EQ(token.type, TokenType::kError); + EXPECT_EQ(lexer.GetError().message, test_case.expected_error_message); + auto location = source->GetLocation(lexer.GetPosition()); + ASSERT_TRUE(location.has_value()); + EXPECT_EQ(source->DisplayErrorLocation(*location), + test_case.expected_error_location); +} + +INSTANTIATE_TEST_SUITE_P( + ErrorCases, LexerErrorTest, + testing::Values( + LexerErrorTestCase{ + .source = "\"unterminated", + .expected_error_message = "unterminated string literal", + .expected_error_location = "\n | \"unterminated" + "\n | .............^", + }, + LexerErrorTestCase{ + .source = "\"another\nline\"", + .expected_error_message = "unterminated string literal", + .expected_error_location = "\n | \"another" + "\n | ........^", + }, + LexerErrorTestCase{ + .source = "0x", + .expected_error_message = + "integral literal missing digits after hexadecimal separator", + .expected_error_location = "\n | 0x" + "\n | ..^", + }, + LexerErrorTestCase{ + .source = "@", + .expected_error_message = "unexpected character", + .expected_error_location = "\n | @" + "\n | .^", + }, + LexerErrorTestCase{ + .source = "1 \v + 2", + .expected_error_message = "unexpected character", + .expected_error_location = "\n | 1 \v + 2" + "\n | ...^", + }, + LexerErrorTestCase{ + .source = "`unterminated quoted", + .expected_error_message = "unterminated quoted identifier", + .expected_error_location = "\n | `unterminated quoted" + "\n | ....................^", + }, + LexerErrorTestCase{ + .source = "`ident\nhere`", + .expected_error_message = "unterminated quoted identifier", + .expected_error_location = "\n | `ident" + "\n | ......^", + }, + LexerErrorTestCase{ + .source = "'''unterminated multi", + .expected_error_message = "unterminated string literal", + .expected_error_location = "\n | '''unterminated multi" + "\n | .....................^", + }, + LexerErrorTestCase{ + .source = "r'unterminated raw", + .expected_error_message = "unterminated string literal", + .expected_error_location = "\n | r'unterminated raw" + "\n | ..................^", + }, + LexerErrorTestCase{ + .source = "b'unterminated bytes", + .expected_error_message = "unterminated bytes literal", + .expected_error_location = "\n | b'unterminated bytes" + "\n | ....................^", + }, + LexerErrorTestCase{ + .source = "1e", + .expected_error_message = + "floating point literal missing digits after exponent " + "separator", + .expected_error_location = "\n | 1e" + "\n | ..^", + }, + LexerErrorTestCase{ + .source = "\"😀😀😀😀😀\" ~error", + .expected_error_message = "unexpected character", + .expected_error_location = "\n | \"😀😀😀😀😀\" ~error" + "\n | .．．．．．...^", + })); + +TEST(LexerErrorRecoveryTest, ResumesAfterError) { + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("1e, {2 3}")); + Lexer lexer(*source); + Token token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kError); + EXPECT_EQ(lexer.GetError().message, + "floating point literal missing digits after exponent separator"); + + token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kComma); + + token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kWhitespace); + + token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kLeftBrace); + + token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kInt); + EXPECT_EQ(token.start, 5); + EXPECT_EQ(token.end, 6); +} + +TEST(LexerPositionTest, SaveAndRestorePosition) { + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("foo + bar * 42")); + Lexer lexer(*source); + + Token tok1 = lexer.Lex(); + EXPECT_EQ(tok1.type, TokenType::kIdent); + + Token tok2 = lexer.Lex(); + EXPECT_EQ(tok2.type, TokenType::kWhitespace); + + // Save position before '+' + int32_t saved = lexer.SavePosition(); + + Token tok3 = lexer.Lex(); + EXPECT_EQ(tok3.type, TokenType::kPlus); + + Token tok4 = lexer.Lex(); + EXPECT_EQ(tok4.type, TokenType::kWhitespace); + + Token tok5 = lexer.Lex(); + EXPECT_EQ(tok5.type, TokenType::kIdent); + + // Restore position to before '+' + lexer.RestorePosition(saved); + + Token tok3_restored = lexer.Lex(); + EXPECT_EQ(tok3_restored.type, TokenType::kPlus); + EXPECT_EQ(tok3_restored.start, tok3.start); + EXPECT_EQ(tok3_restored.end, tok3.end); + + Token tok4_restored = lexer.Lex(); + EXPECT_EQ(tok4_restored.type, TokenType::kWhitespace); + + Token tok5_restored = lexer.Lex(); + EXPECT_EQ(tok5_restored.type, TokenType::kIdent); +} + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/internal/options.h b/parser/internal/options.h index ec2552204..cca9670a5 100644 --- a/parser/internal/options.h +++ b/parser/internal/options.h @@ -15,14 +15,15 @@ #ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ #define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ -namespace cel_parser_internal { +namespace cel::parser_internal { inline constexpr int kDefaultErrorRecoveryLimit = 12; inline constexpr int kDefaultMaxRecursionDepth = 32; inline constexpr int kExpressionSizeCodepointLimit = 100'000; inline constexpr int kDefaultErrorRecoveryTokenLookaheadLimit = 512; +inline constexpr int kDefaultExpressionNodeLimit = 100'000; inline constexpr bool kDefaultAddMacroCalls = false; -} // namespace cel_parser_internal +} // namespace cel::parser_internal #endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ diff --git a/parser/internal/pratt_parser.cc b/parser/internal/pratt_parser.cc new file mode 100644 index 000000000..b5cf10457 --- /dev/null +++ b/parser/internal/pratt_parser.cc @@ -0,0 +1,234 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/pratt_parser.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/base/nullability.h" +#include "absl/cleanup/cleanup.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "common/expr.h" +#include "common/source.h" +#include "internal/status_macros.h" +#include "parser/internal/ast_factory.h" // IWYU pragma: keep +#include "parser/internal/pratt_parser_worker.h" +#include "parser/macro.h" +#include "parser/macro_registry.h" +#include "parser/options.h" +#include "parser/parser_interface.h" +#include "parser/source_factory.h" + +namespace cel::parser_internal { + +namespace { + +std::string DisplayParserError(const cel::Source& source, + SourceLocation location, + std::string_view message) { + int32_t display_column = + location.column >= 0 ? location.column + 1 : location.column; + return absl::StrCat( + absl::StrFormat("ERROR: %s:%d:%d: %s", source.description(), + location.line, display_column, message), + source.DisplayErrorLocation(location)); +} + +std::string FormatIssues(const cel::Source& source, + absl::Span issues) { + return absl::StrJoin( + issues, "\n", [&source](std::string* out, const cel::ParseIssue& issue) { + absl::StrAppend( + out, DisplayParserError(source, issue.location(), issue.message())); + }); +} + +} // namespace + +absl::Status PrattParserBuilderImpl::AddMacro(const cel::Macro& macro) { + for (const cel::Macro& existing_macro : macros_) { + if (existing_macro.key() == macro.key()) { + return absl::AlreadyExistsError( + absl::StrCat("macro already exists: ", macro.key())); + } + } + macros_.push_back(macro); + return absl::OkStatus(); +} + +absl::Status PrattParserBuilderImpl::AddLibrary(cel::ParserLibrary library) { + if (!library.id.empty()) { + auto [it, inserted] = library_ids_.insert(library.id); + if (!inserted) { + return absl::AlreadyExistsError( + absl::StrCat("parser library already exists: ", library.id)); + } + } + libraries_.push_back(std::move(library)); + return absl::OkStatus(); +} + +absl::Status PrattParserBuilderImpl::AddLibrarySubset( + cel::ParserLibrarySubset subset) { + if (subset.library_id.empty()) { + return absl::InvalidArgumentError("subset must have a library id"); + } + std::string library_id = subset.library_id; + auto [it, inserted] = + library_subsets_.insert({library_id, std::move(subset)}); + if (!inserted) { + return absl::AlreadyExistsError( + absl::StrCat("parser library subset already exists: ", library_id)); + } + return absl::OkStatus(); +} + +absl::StatusOr> PrattParserBuilderImpl::Build() { + using std::swap; + std::vector individual_macros; + swap(individual_macros, macros_); + absl::Cleanup cleanup([&] { swap(macros_, individual_macros); }); + + cel::MacroRegistry macro_registry; + + for (const cel::ParserLibrary& library : libraries_) { + CEL_RETURN_IF_ERROR(library.configure(*this)); + if (!library.id.empty()) { + auto it = library_subsets_.find(library.id); + if (it != library_subsets_.end()) { + const cel::ParserLibrarySubset& subset = it->second; + for (const cel::Macro& macro : macros_) { + if (subset.should_include_macro(macro)) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(macro)); + } + } + macros_.clear(); + continue; + } + } + + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(macros_)); + macros_.clear(); + } + + absl::flat_hash_set library_ids(library_ids_); + + if (!options_.disable_standard_macros && !library_ids_.contains("stdlib")) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(Macro::AllMacros())); + library_ids.insert("stdlib"); + } + + if (options_.enable_optional_syntax && !library_ids_.contains("optional")) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptMapMacro())); + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptFlatMapMacro())); + library_ids.insert("optional"); + } + + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(individual_macros)); + return std::make_unique(options_, std::move(macro_registry), + std::move(library_ids)); +} + +template class PrattParserWorker; + +absl::StatusOr> PrattParserImpl::ParseImpl( + const cel::Source& source, + std::vector* absl_nullable parse_issues) const { + return PrattParseImpl(source, macro_registry_, options_, parse_issues); +} + +absl::StatusOr> PrattParserImpl::PrepareSourceImpl( + absl::string_view input, absl::string_view description) const { + return cel::NewSource( + input, std::string(description), + cel::SourceOptions{.max_codepoint_size = + options_.expression_size_codepoint_limit}); +} + +absl::StatusOr> PrattParseImpl( + const cel::Source& source, const cel::MacroRegistry& registry, + const ParserOptions& options, std::vector* parse_issues, + cel::EnrichedSourceInfo* enriched_source_info) { + if (source.content().size() > options.expression_size_codepoint_limit) { + return absl::InvalidArgumentError(absl::StrFormat( + "expression size exceeds codepoint limit. input size: %zu, limit: %d", + source.content().size(), options.expression_size_codepoint_limit)); + } + std::vector issues; + AstFactory factory(®istry); + PrattParserWorker worker( + source, options, &issues, factory, + /*track_node_ranges=*/enriched_source_info != nullptr); + Expr expr = worker.Parse(); + if (worker.is_recursion_limit_exceeded()) { + return absl::CancelledError( + absl::StrFormat("Expression recursion limit exceeded. limit: %d", + options.max_recursion_depth)); + } + if (worker.has_errors()) { + absl::c_stable_sort( + issues, [](const cel::ParseIssue& lhs, const cel::ParseIssue& rhs) { + if (lhs.location().line != rhs.location().line) { + return lhs.location().line < rhs.location().line; + } + return lhs.location().column < rhs.location().column; + }); + std::string err_msg = FormatIssues(source, issues); + if (parse_issues != nullptr) { + parse_issues->swap(issues); + } + return absl::InvalidArgumentError(err_msg); + } + + if (enriched_source_info != nullptr) { + *enriched_source_info = cel::EnrichedSourceInfo(worker.GetNodeRanges()); + } + + cel::SourceInfo source_info; + source_info.set_location(std::string(source.description())); + for (const auto& [id, pos] : worker.GetNodePositions()) { + source_info.mutable_positions().insert({id, pos}); + } + source_info.mutable_line_offsets().reserve(source.line_offsets().size()); + for (int32_t offset : source.line_offsets()) { + source_info.mutable_line_offsets().push_back(offset); + } + source_info.mutable_macro_calls() = worker.ReleaseMacroCalls(); + return std::make_unique(std::move(expr), std::move(source_info)); +} + +std::unique_ptr PrattParserImpl::ToBuilder() const { + auto ins = std::make_unique(options_); + ins->library_ids_ = library_ids_; + ins->macros_ = macro_registry_.ListMacros(); + return ins; +} + +} // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser.h b/parser/internal/pratt_parser.h new file mode 100644 index 000000000..a33ca803b --- /dev/null +++ b/parser/internal/pratt_parser.h @@ -0,0 +1,116 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_H_ + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "common/ast.h" +#include "common/source.h" +#include "parser/macro.h" +#include "parser/macro_registry.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel { +class EnrichedSourceInfo; +} // namespace cel + +namespace cel::parser_internal { + +// PrattParserImpl implements the Pratt parsing algorithm for CEL expressions. +// +// WARNING: Since this implementation uses recursive descent to parse +// expressions, its stack consumption depends on expression nesting and +// recursion limits (`ParserOptions::max_recursion_depth`). Note that: +// In production builds (e.g., inside fibers with small default stacks such as +// 64KB), the available stack space may be much tighter than in default thread +// stacks. +// Consequently, `ParserOptions::max_recursion_depth` may need to be tuned +// depending on the caller environment to prevent stack overflow. +class PrattParserImpl final : public cel::Parser { + public: + explicit PrattParserImpl(const cel::ParserOptions& options, + cel::MacroRegistry macro_registry, + absl::flat_hash_set library_ids) + : options_(options), + macro_registry_(std::move(macro_registry)), + library_ids_(std::move(library_ids)) {} + + ~PrattParserImpl() override = default; + + absl::StatusOr> ParseImpl( + const cel::Source& source, + std::vector* absl_nullable parse_issues) const override; + + absl::StatusOr> PrepareSourceImpl( + absl::string_view input, absl::string_view description) const override; + + std::unique_ptr ToBuilder() const override; + + private: + cel::ParserOptions options_; + cel::MacroRegistry macro_registry_; + absl::flat_hash_set library_ids_; +}; + +absl::StatusOr> PrattParseImpl( + const cel::Source& source, const cel::MacroRegistry& registry, + const ParserOptions& options, + std::vector* parse_issues = nullptr, + cel::EnrichedSourceInfo* enriched_source_info = nullptr); + +class PrattParserBuilderImpl final : public cel::ParserBuilder { + public: + explicit PrattParserBuilderImpl(const cel::ParserOptions& options) + : options_(options) {} + + cel::ParserOptions& GetOptions() override { return options_; } + + absl::Status AddMacro(const cel::Macro& macro) override; + + absl::Status AddLibrary(cel::ParserLibrary library) override; + + absl::Status AddLibrarySubset(cel::ParserLibrarySubset subset) override; + + absl::StatusOr> Build() override; + + private: + friend class PrattParserImpl; + + cel::ParserOptions options_; + std::vector macros_; + absl::flat_hash_set library_ids_; + std::vector libraries_; + absl::flat_hash_map library_subsets_; +}; + +inline std::unique_ptr NewPrattParserBuilder( + const cel::ParserOptions& options = cel::ParserOptions()) { + return std::make_unique(options); +} + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_H_ diff --git a/parser/internal/pratt_parser_benchmark.cc b/parser/internal/pratt_parser_benchmark.cc new file mode 100644 index 000000000..53d451bf1 --- /dev/null +++ b/parser/internal/pratt_parser_benchmark.cc @@ -0,0 +1,482 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "absl/strings/str_cat.h" +#include "common/source.h" +#include "internal/benchmark.h" +#include "parser/internal/pratt_parser.h" +#include "parser/options.h" +#include "parser/parser.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { +namespace { + +enum class ParserImplType { + kPratt, + kAntlr, +}; + +std::unique_ptr CreateParser(ParserImplType type, + cel::ParserOptions options) { + options.max_recursion_depth = 512; + std::unique_ptr builder; + if (type == ParserImplType::kPratt) { + builder = NewPrattParserBuilder(options); + } else { + builder = cel::NewParserBuilder(options); + } + auto parser = builder->Build(); + ABSL_DCHECK_OK(parser.status()); + return std::move(*parser); +} + +// ----------------------------------------------------------------------------- +// Workload 1: Common Representative CEL Expressions +// ----------------------------------------------------------------------------- +void BM_ParseCommon(benchmark::State& state, ParserImplType type) { + cel::ParserOptions options; + options.enable_optional_syntax = true; + options.enable_quoted_identifiers = true; + auto parser = CreateParser(type, options); + + const std::vector test_cases = { + "x * 2 + y / 3", + "foo.bar.baz(1, 2, \"abc\")", + "a > 5 && b < 10 || c == \"xyz\"", + "x ? y : z", + "{\"foo\": 1, \"bar\": [2, 3]}", + "has(foo.bar) && [1, 2, 3].all(x, x > 0)", + }; + + for (auto _ : state) { + for (const auto& expr : test_cases) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK_OK(ast.status()); + benchmark::DoNotOptimize(ast); + } + } +} + +void BM_Pratt_ParseCommon(benchmark::State& state) { + BM_ParseCommon(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseCommon(benchmark::State& state) { + BM_ParseCommon(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseCommon); +BENCHMARK(BM_Antlr_ParseCommon); + +// ----------------------------------------------------------------------------- +// Workload 2: Deep Left-Associative Arithmetic Chains ("a + a + a...") +// ----------------------------------------------------------------------------- +std::string BuildArithmeticChain(int length) { + std::string expr = "a"; + for (int i = 1; i < length; ++i) { + absl::StrAppend(&expr, " + a"); + } + return expr; +} + +void BM_ParseArithmeticChain(benchmark::State& state, ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildArithmeticChain(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK_OK(ast.status()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseArithmeticChain(benchmark::State& state) { + BM_ParseArithmeticChain(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseArithmeticChain(benchmark::State& state) { + BM_ParseArithmeticChain(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseArithmeticChain)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK(BM_Antlr_ParseArithmeticChain)->Arg(10)->Arg(50)->Arg(100); + +// ----------------------------------------------------------------------------- +// Workload 3: Wide Logical Expression Chains ("a || a || a...") +// ----------------------------------------------------------------------------- +std::string BuildLogicalChain(int length) { + std::string expr = "a"; + for (int i = 1; i < length; ++i) { + absl::StrAppend(&expr, " || a"); + } + return expr; +} + +void BM_ParseLogicalChain(benchmark::State& state, ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildLogicalChain(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK_OK(ast.status()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseLogicalChain(benchmark::State& state) { + BM_ParseLogicalChain(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseLogicalChain(benchmark::State& state) { + BM_ParseLogicalChain(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseLogicalChain)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK(BM_Antlr_ParseLogicalChain)->Arg(10)->Arg(50)->Arg(100); + +// ----------------------------------------------------------------------------- +// Workload 4: Deep Member Access Chains ("a.f.f.f...") +// ----------------------------------------------------------------------------- +std::string BuildMemberChain(int length) { + std::string expr = "a"; + for (int i = 1; i < length; ++i) { + absl::StrAppend(&expr, ".f"); + } + return expr; +} + +void BM_ParseMemberChain(benchmark::State& state, ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildMemberChain(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK_OK(ast.status()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseMemberChain(benchmark::State& state) { + BM_ParseMemberChain(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseMemberChain(benchmark::State& state) { + BM_ParseMemberChain(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseMemberChain)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK(BM_Antlr_ParseMemberChain)->Arg(10)->Arg(50)->Arg(100); + +// ----------------------------------------------------------------------------- +// Workload 5: Deeply Nested Parentheses ("(((((a)))))") +// ----------------------------------------------------------------------------- +std::string BuildNestedParentheses(int depth) { + std::string prefix(depth, '('); + std::string suffix(depth, ')'); + return absl::StrCat(prefix, "a", suffix); +} + +void BM_ParseNestedParentheses(benchmark::State& state, ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildNestedParentheses(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK_OK(ast.status()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseNestedParentheses(benchmark::State& state) { + BM_ParseNestedParentheses(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseNestedParentheses(benchmark::State& state) { + BM_ParseNestedParentheses(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseNestedParentheses)->Arg(10)->Arg(50)->Arg(200); +BENCHMARK(BM_Antlr_ParseNestedParentheses)->Arg(10)->Arg(50); + +// ----------------------------------------------------------------------------- +// Workload 5b: Deeply Nested Left Parentheses with Calc ("((((a + 1) + 1))") +// ----------------------------------------------------------------------------- +std::string BuildNestedLeftParenthesesCalc(int depth) { + std::string expr(depth, '('); + absl::StrAppend(&expr, "1 + 2"); + for (int i = 0; i < depth; ++i) { + absl::StrAppend(&expr, ") + 1"); + } + return expr; +} + +void BM_ParseNestedLeftParenthesesCalc(benchmark::State& state, + ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildNestedLeftParenthesesCalc(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK_OK(ast.status()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseNestedLeftParenthesesCalc(benchmark::State& state) { + BM_ParseNestedLeftParenthesesCalc(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseNestedLeftParenthesesCalc(benchmark::State& state) { + BM_ParseNestedLeftParenthesesCalc(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseNestedLeftParenthesesCalc)->Arg(10)->Arg(50)->Arg(200); +BENCHMARK(BM_Antlr_ParseNestedLeftParenthesesCalc)->Arg(10)->Arg(50); + +// ----------------------------------------------------------------------------- +// Workload 6: Common Representative Expressions with Syntax Errors +// ----------------------------------------------------------------------------- +void BM_ParseCommonSyntaxErrors(benchmark::State& state, ParserImplType type) { + cel::ParserOptions options; + options.enable_optional_syntax = true; + options.enable_quoted_identifiers = true; + auto parser = CreateParser(type, options); + + const std::vector test_cases = { + "x * 2 + y /", + "foo.bar.baz(1, 2, \"abc\"", + "a > 5 && && b < 10 || c == \"xyz\"", + "x ? y :", + "{\"foo\": 1, \"bar\": [2, 3", + "has(foo.bar) && [1, 2, 3].all(x, +)", + "a. + b", + "Msg{f 10}", + "(1 + 2 * (3 + )", + }; + + for (auto _ : state) { + for (const auto& expr : test_cases) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK(!ast.ok()); + benchmark::DoNotOptimize(ast); + } + } +} + +void BM_Pratt_ParseCommonSyntaxErrors(benchmark::State& state) { + BM_ParseCommonSyntaxErrors(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseCommonSyntaxErrors(benchmark::State& state) { + BM_ParseCommonSyntaxErrors(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseCommonSyntaxErrors); +BENCHMARK(BM_Antlr_ParseCommonSyntaxErrors); + +// ----------------------------------------------------------------------------- +// Workload 7: Deep Left-Associative Arithmetic Chains with Syntax Error +// ("a + a + ... + ") +// ----------------------------------------------------------------------------- +std::string BuildArithmeticChainSyntaxError(int length) { + std::string expr = BuildArithmeticChain(length); + absl::StrAppend(&expr, " +"); + return expr; +} + +void BM_ParseArithmeticChainSyntaxError(benchmark::State& state, + ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildArithmeticChainSyntaxError(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK(!ast.ok()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseArithmeticChainSyntaxError(benchmark::State& state) { + BM_ParseArithmeticChainSyntaxError(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseArithmeticChainSyntaxError(benchmark::State& state) { + BM_ParseArithmeticChainSyntaxError(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseArithmeticChainSyntaxError)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK(BM_Antlr_ParseArithmeticChainSyntaxError)->Arg(10)->Arg(50)->Arg(100); + +// ----------------------------------------------------------------------------- +// Workload 8: Wide Logical Expression Chains with Syntax Error +// ("a || a || ... || ") +// ----------------------------------------------------------------------------- +std::string BuildLogicalChainSyntaxError(int length) { + std::string expr = BuildLogicalChain(length); + absl::StrAppend(&expr, " ||"); + return expr; +} + +void BM_ParseLogicalChainSyntaxError(benchmark::State& state, + ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildLogicalChainSyntaxError(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK(!ast.ok()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseLogicalChainSyntaxError(benchmark::State& state) { + BM_ParseLogicalChainSyntaxError(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseLogicalChainSyntaxError(benchmark::State& state) { + BM_ParseLogicalChainSyntaxError(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseLogicalChainSyntaxError)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK(BM_Antlr_ParseLogicalChainSyntaxError)->Arg(10)->Arg(50)->Arg(100); + +// ----------------------------------------------------------------------------- +// Workload 9: Deep Member Access Chains with Syntax Error ("a.f.f...f.") +// ----------------------------------------------------------------------------- +std::string BuildMemberChainSyntaxError(int length) { + std::string expr = BuildMemberChain(length); + absl::StrAppend(&expr, "."); + return expr; +} + +void BM_ParseMemberChainSyntaxError(benchmark::State& state, + ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildMemberChainSyntaxError(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK(!ast.ok()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseMemberChainSyntaxError(benchmark::State& state) { + BM_ParseMemberChainSyntaxError(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseMemberChainSyntaxError(benchmark::State& state) { + BM_ParseMemberChainSyntaxError(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseMemberChainSyntaxError)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK(BM_Antlr_ParseMemberChainSyntaxError)->Arg(10)->Arg(50)->Arg(100); + +// ----------------------------------------------------------------------------- +// Workload 10: Deeply Nested Parentheses with Syntax Error ("(((((a") +// ----------------------------------------------------------------------------- +std::string BuildNestedParenthesesSyntaxError(int depth) { + std::string prefix(depth, '('); + return absl::StrCat(prefix, "a"); +} + +void BM_ParseNestedParenthesesSyntaxError(benchmark::State& state, + ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildNestedParenthesesSyntaxError(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK(!ast.ok()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseNestedParenthesesSyntaxError(benchmark::State& state) { + BM_ParseNestedParenthesesSyntaxError(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseNestedParenthesesSyntaxError(benchmark::State& state) { + BM_ParseNestedParenthesesSyntaxError(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseNestedParenthesesSyntaxError)->Arg(10)->Arg(50); +BENCHMARK(BM_Antlr_ParseNestedParenthesesSyntaxError)->Arg(10)->Arg(50); + +// ----------------------------------------------------------------------------- +// Workload 11: Repeated Syntax Errors ("f(*, *, *, ...)") +// ----------------------------------------------------------------------------- +std::string BuildRepeatedSyntaxErrors(int length) { + std::string expr = "f(*"; + for (int i = 1; i < length; ++i) { + absl::StrAppend(&expr, ", *"); + } + absl::StrAppend(&expr, ")"); + return expr; +} + +void BM_ParseRepeatedSyntaxErrors(benchmark::State& state, + ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildRepeatedSyntaxErrors(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK(!ast.ok()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseRepeatedSyntaxErrors(benchmark::State& state) { + BM_ParseRepeatedSyntaxErrors(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseRepeatedSyntaxErrors(benchmark::State& state) { + BM_ParseRepeatedSyntaxErrors(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseRepeatedSyntaxErrors)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK(BM_Antlr_ParseRepeatedSyntaxErrors)->Arg(10)->Arg(50)->Arg(100); + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc new file mode 100644 index 000000000..798bae838 --- /dev/null +++ b/parser/internal/pratt_parser_test.cc @@ -0,0 +1,2273 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/pratt_parser.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "common/constant.h" +#include "common/expr.h" +#include "common/expr_printer.h" +#include "common/source.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "parser/internal/lexer.h" +#include "parser/internal/pratt_parser_worker.h" +#include "parser/macro.h" +#include "parser/macro_expr_factory.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +// Change to 0 to test with the ANTLR parser to check for differences. +#define USE_PRATT_PARSER 1 + +namespace cel::parser_internal { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::IsOkAndHolds; +using ::absl_testing::StatusIs; +using ::testing::Eq; +using ::testing::NotNull; + +template +std::string TestName(const testing::TestParamInfo& test_info) { + std::string name = absl::StrCat(test_info.index, "-", test_info.param.source); + absl::c_replace_if(name, [](char c) { return !absl::ascii_isalnum(c); }, '_'); + return name; +} + +absl::StatusOr> Parse( + absl::string_view expression, + const cel::ParserOptions& options = cel::ParserOptions(), + std::vector* issues = nullptr) { +#if USE_PRATT_PARSER + std::unique_ptr builder = NewPrattParserBuilder(options); +#else + std::unique_ptr builder = cel::NewParserBuilder(options); +#endif + CEL_ASSIGN_OR_RETURN(std::unique_ptr parser, builder->Build()); + CEL_ASSIGN_OR_RETURN(std::unique_ptr source, + cel::NewSource(expression)); + return parser->Parse(*source, issues); +} + +struct TestCase { + absl::string_view source; + absl::string_view expected_ast; + bool enable_optional_syntax = false; + bool enable_variadic_logical_operators = false; +}; + +class PrattParserTest : public testing::TestWithParam {}; + +absl::string_view ConstantKind(const cel::Constant& c) { + switch (c.kind_case()) { + case ConstantKindCase::kBool: + return "bool"; + case ConstantKindCase::kInt: + return "int64"; + case ConstantKindCase::kUint: + return "uint64"; + case ConstantKindCase::kDouble: + return "double"; + case ConstantKindCase::kString: + return "string"; + case ConstantKindCase::kBytes: + return "bytes"; + case ConstantKindCase::kNull: + return "NullValue"; + default: + return "unspecified_constant"; + } +} + +absl::string_view ExprKind(const cel::Expr& e) { + switch (e.kind_case()) { + case ExprKindCase::kConstant: + // special cased, this doesn't appear. + return "Expr.Constant"; + case ExprKindCase::kIdentExpr: + return "Expr.Ident"; + case ExprKindCase::kSelectExpr: + return "Expr.Select"; + case ExprKindCase::kCallExpr: + return "Expr.Call"; + case ExprKindCase::kListExpr: + return "Expr.CreateList"; + case ExprKindCase::kMapExpr: + return "Expr.CreateMap"; + case ExprKindCase::kStructExpr: + return "Expr.CreateStruct"; + case ExprKindCase::kComprehensionExpr: + return "Expr.Comprehension"; + default: + return "unspecified_expr"; + } +} + +class KindAndIdAdorner : public cel::ExpressionAdorner { + public: + std::string Adorn(const cel::Expr& e) const override { + if (e.has_const_expr()) { + const cel::Constant& const_expr = e.const_expr(); + return absl::StrCat("^#", e.id(), ":", ConstantKind(const_expr), "#"); + } else { + return absl::StrCat("^#", e.id(), ":", ExprKind(e), "#"); + } + } + + std::string AdornStructField(const cel::StructExprField& e) const override { + return absl::StrFormat("^#%d:Expr.CreateStruct.Entry#", e.id()); + } + + std::string AdornMapEntry(const cel::MapExprEntry& e) const override { + return absl::StrFormat("^#%d:Expr.CreateStruct.Entry#", e.id()); + } +}; + +std::string Unindent(absl::string_view multiline) { + std::vector unindented_lines; + int indent = -1; + for (absl::string_view line : absl::StrSplit(multiline, '\n')) { + std::size_t pos = line.find_first_not_of(" \t"); + if (pos == absl::string_view::npos) continue; + if (indent == -1) indent = pos; + unindented_lines.push_back(std::string(line.substr(indent))); + } + return absl::StrJoin(unindented_lines, "\n"); +} + +MATCHER_P(AstIs, expected_ast, "") { + KindAndIdAdorner kind_and_id_adorner; + cel::ExprPrinter printer(kind_and_id_adorner); + std::string actual = Unindent(printer.Print(arg)); + std::string expected = Unindent(expected_ast); + if (actual == expected) { + return true; + } + *result_listener << "\n Actual: " << actual + << "\n Expected: " << expected; + return false; +} + +MATCHER_P(AstEq, expected_expr, "") { + KindAndIdAdorner kind_and_id_adorner; + cel::ExprPrinter printer(kind_and_id_adorner); + ParserOptions options; + auto actual_ast = Parse(arg, options); + if (!actual_ast.ok()) { + *result_listener << "\n Actual expression failed to parse: " + << actual_ast.status(); + return false; + } + std::string actual = Unindent(printer.Print((*actual_ast)->root_expr())); + auto expected_ast = Parse(expected_expr, options); + if (!expected_ast.ok()) { + *result_listener << "\n Expected expression failed to parse: " + << expected_ast.status(); + return false; + } + std::string expected = Unindent(printer.Print((*expected_ast)->root_expr())); + if (actual == expected) { + return true; + } + *result_listener << "\n Actual: " << actual + << "\n Expected: " << expected; + return false; +} + +TEST_P(PrattParserTest, Parse) { + const TestCase& test_case = GetParam(); + cel::ParserOptions options; + options.enable_optional_syntax = test_case.enable_optional_syntax; + options.enable_variadic_logical_operators = + test_case.enable_variadic_logical_operators; + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + Parse(test_case.source, options)); + EXPECT_THAT(ast->root_expr(), AstIs(test_case.expected_ast)); +} + +std::vector GetParserTestCases() { + return { + TestCase{ + .source = "null", + .expected_ast = R"( + null^#1:NullValue# + )", + }, + TestCase{ + .source = "true", + .expected_ast = R"( + true^#1:bool# + )", + }, + TestCase{ + .source = "false", + .expected_ast = R"( + false^#1:bool# + )", + }, + TestCase{ + .source = "123", + .expected_ast = R"( + 123^#1:int64# + )", + }, + TestCase{ + .source = "-123", + .expected_ast = R"( + -123^#1:int64# + )", + }, + TestCase{ + .source = "9223372036854775807", + .expected_ast = R"( + 9223372036854775807^#1:int64# + )", + }, + TestCase{ + .source = "-9223372036854775808", + .expected_ast = R"( + -9223372036854775808^#1:int64# + )", + }, + TestCase{ + .source = "0xA", + .expected_ast = R"( + 10^#1:int64# + )", + }, + TestCase{ + .source = "-0x1A", + .expected_ast = R"( + -26^#1:int64# + )", + }, + TestCase{ + .source = "-0X1a", + .expected_ast = R"( + -26^#1:int64# + )", + }, + TestCase{ + .source = "-0x8000000000000000", + .expected_ast = R"( + -9223372036854775808^#1:int64# + )", + }, + TestCase{ + .source = "-0X8000000000000000", + .expected_ast = R"( + -9223372036854775808^#1:int64# + )", + }, + TestCase{ + .source = "42u", + .expected_ast = R"( + 42u^#1:uint64# + )", + }, + TestCase{ + .source = "0u", + .expected_ast = R"( + 0u^#1:uint64# + )", + }, + TestCase{ + .source = "0xAu", + .expected_ast = R"( + 10u^#1:uint64# + )", + }, + TestCase{ + .source = "3.14159", + .expected_ast = R"( + 3.14159^#1:double# + )", + }, + TestCase{ + .source = "-3.14159", + .expected_ast = R"( + -3.14159^#1:double# + )", + }, + TestCase{ + .source = "-5.5e-3", + .expected_ast = R"( + -0.0055^#1:double# + )", + }, + TestCase{ + .source = "b'hello'", + .expected_ast = R"( + b"hello"^#1:bytes# + )", + }, + TestCase{ + .source = "b\"hello\"", + .expected_ast = R"( + b"hello"^#1:bytes# + )", + }, + TestCase{ + .source = "'hello world'", + .expected_ast = R"( + "hello world"^#1:string# + )", + }, + TestCase{ + .source = "\"hello world\"", + .expected_ast = R"( + "hello world"^#1:string# + )", + }, + TestCase{ + .source = "\"\u2764\"", + .expected_ast = "\"\u2764\"^#1:string#", + }, + TestCase{ + .source = "\"\\a\\b\\f\\n\\r\\t\\v'\\\"\\\\\\? Legal escapes\"", + .expected_ast = R"( + "\x07\x08\x0c\n\r\t\x0b'\"\\? Legal escapes"^#1:string# + )", + }, + TestCase{ + .source = "'''hello\nworld'''", + .expected_ast = R"( + "hello\nworld"^#1:string# + )", + }, + TestCase{ + .source = "\"\"\"hello\nworld\"\"\"", + .expected_ast = R"( + "hello\nworld"^#1:string# + )", + }, + TestCase{ + .source = "r\"\"\"hello\nworld\"\"\"", + .expected_ast = R"( + "hello\nworld"^#1:string# + )", + }, + TestCase{ + .source = "\"\"\"hello\\\"\"\"world\"\"\"", + .expected_ast = R"( + "hello\"\"\"world"^#1:string# + )", + }, + TestCase{ + .source = "'''hello\\'''world'''", + .expected_ast = R"( + "hello'''world"^#1:string# + )", + }, + TestCase{ + .source = "a", + .expected_ast = R"( + a^#1:Expr.Ident# + )", + }, + TestCase{ + .source = "(a)", + .expected_ast = R"( + a^#1:Expr.Ident# + )", + }, + TestCase{ + .source = "((a))", + .expected_ast = R"( + a^#1:Expr.Ident# + )", + }, + TestCase{ + .source = "a.b", + .expected_ast = R"( + a^#1:Expr.Ident#.b^#2:Expr.Select# + )", + }, + TestCase{ + .source = "a.?b", + .expected_ast = R"( + _?._( + a^#1:Expr.Ident#, + "b"^#3:string# + )^#2:Expr.Call# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "a.b.c", + .expected_ast = R"( + a^#1:Expr.Ident#.b^#2:Expr.Select#.c^#3:Expr.Select# + )", + }, + TestCase{ + .source = "a.`b`", + .expected_ast = R"( + a^#1:Expr.Ident#.b^#2:Expr.Select# + )", + }, + TestCase{ + .source = "a.`b-c`", + .expected_ast = R"( + a^#1:Expr.Ident#.b-c^#2:Expr.Select# + )", + }, + TestCase{ + .source = "a.?`b`", + .expected_ast = R"( + _?._( + a^#1:Expr.Ident#, + "b"^#3:string# + )^#2:Expr.Call# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "-x", + .expected_ast = R"( + -_( + x^#2:Expr.Ident# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "- -1", + .expected_ast = R"( + 1^#1:int64# + )", + }, + TestCase{ + .source = "-(1 + 2)", + .expected_ast = R"( + -_( + _+_( + 1^#2:int64#, + 2^#4:int64# + )^#3:Expr.Call# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "---a", + .expected_ast = R"( + -_( + a^#2:Expr.Ident# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!false", + .expected_ast = R"( + !_( + false^#2:bool# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!a", + .expected_ast = R"( + !_( + a^#2:Expr.Ident# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "-(!true)", + .expected_ast = R"( + -_( + !_( + true^#3:bool# + )^#2:Expr.Call# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!-42", + .expected_ast = R"( + !_( + -42^#2:int64# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!-4.2", + .expected_ast = R"( + !_( + -4.2^#2:double# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "9in-x", + .expected_ast = R"( + @in( + 9^#1:int64#, + -_( + x^#4:Expr.Ident# + )^#3:Expr.Call# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a.?b.?c", + .expected_ast = R"( + _?._( + _?._( + a^#1:Expr.Ident#, + "b"^#3:string# + )^#2:Expr.Call#, + "c"^#5:string# + )^#4:Expr.Call# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "-.2.V", + .expected_ast = R"( + -0.2^#1:double#.V^#2:Expr.Select# + )", + }, + TestCase{ + .source = "!-.2.V", + .expected_ast = R"( + !_( + -0.2^#2:double#.V^#3:Expr.Select# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!-2.V", + .expected_ast = R"( + !_( + -2^#2:int64#.V^#3:Expr.Select# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!-.2[0]", + .expected_ast = R"( + !_( + _[_]( + -0.2^#2:double#, + 0^#4:int64# + )^#3:Expr.Call# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "-x.foo[0]", + .expected_ast = R"( + -_( + _[_]( + x^#2:Expr.Ident#.foo^#3:Expr.Select#, + 0^#5:int64# + )^#4:Expr.Call# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "-x[0].foo", + .expected_ast = R"( + -_( + _[_]( + x^#2:Expr.Ident#, + 0^#4:int64# + )^#3:Expr.Call#.foo^#5:Expr.Select# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!x[0].foo", + .expected_ast = R"( + !_( + _[_]( + x^#2:Expr.Ident#, + 0^#4:int64# + )^#3:Expr.Call#.foo^#5:Expr.Select# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "a + b", + .expected_ast = R"( + _+_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a - b", + .expected_ast = R"( + _-_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "4--4", + .expected_ast = R"( + _-_( + 4^#1:int64#, + -4^#3:int64# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a * b", + .expected_ast = R"( + _*_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a / b", + .expected_ast = R"( + _/_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a % b", + .expected_ast = R"( + _%_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a in b", + .expected_ast = R"( + @in( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a == b", + .expected_ast = R"( + _==_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a != b", + .expected_ast = R"( + _!=_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a > b", + .expected_ast = R"( + _>_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a >= b", + .expected_ast = R"( + _>=_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a < b", + .expected_ast = R"( + _<_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a <= b", + .expected_ast = R"( + _<=_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a && b", + .expected_ast = R"( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call# + )", + }, + TestCase{ + .source = "a && b && c", + .expected_ast = R"( + _&&_( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + c^#4:Expr.Ident# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a && b && c && d", + .expected_ast = R"( + _&&_( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + _&&_( + c^#4:Expr.Ident#, + d^#6:Expr.Ident# + )^#7:Expr.Call# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a && b && c && d && e", + .expected_ast = R"( + _&&_( + _&&_( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + c^#4:Expr.Ident# + )^#5:Expr.Call#, + _&&_( + d^#6:Expr.Ident#, + e^#8:Expr.Ident# + )^#9:Expr.Call# + )^#7:Expr.Call# + )", + }, + TestCase{ + .source = "a && b && c && d", + .expected_ast = R"( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident#, + c^#4:Expr.Ident#, + d^#6:Expr.Ident# + )^#3:Expr.Call# + )", + .enable_variadic_logical_operators = true, + }, + TestCase{ + .source = "a || b", + .expected_ast = R"( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call# + )", + }, + TestCase{ + .source = "a || b || c", + .expected_ast = R"( + _||_( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + c^#4:Expr.Ident# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a || b || c || d", + .expected_ast = R"( + _||_( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + _||_( + c^#4:Expr.Ident#, + d^#6:Expr.Ident# + )^#7:Expr.Call# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a || b || c || d || e", + .expected_ast = R"( + _||_( + _||_( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + c^#4:Expr.Ident# + )^#5:Expr.Call#, + _||_( + d^#6:Expr.Ident#, + e^#8:Expr.Ident# + )^#9:Expr.Call# + )^#7:Expr.Call# + )", + }, + TestCase{ + .source = "a || b || c || d", + .expected_ast = R"( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident#, + c^#4:Expr.Ident#, + d^#6:Expr.Ident# + )^#3:Expr.Call# + )", + .enable_variadic_logical_operators = true, + }, + TestCase{ + .source = "10 - 3 - 2", + .expected_ast = R"( + _-_( + _-_( + 10^#1:int64#, + 3^#3:int64# + )^#2:Expr.Call#, + 2^#5:int64# + )^#4:Expr.Call# + )", + }, + TestCase{ + .source = "(((10 - 3) - 2))", + .expected_ast = R"( + _-_( + _-_( + 10^#1:int64#, + 3^#3:int64# + )^#2:Expr.Call#, + 2^#5:int64# + )^#4:Expr.Call# + )", + }, + TestCase{ + .source = "1 + 2 * 3 - 1 / 2 == 6 % 1", + .expected_ast = R"( + _==_( + _-_( + _+_( + 1^#1:int64#, + _*_( + 2^#3:int64#, + 3^#5:int64# + )^#4:Expr.Call# + )^#2:Expr.Call#, + _/_( + 1^#7:int64#, + 2^#9:int64# + )^#8:Expr.Call# + )^#6:Expr.Call#, + _%_( + 6^#11:int64#, + 1^#13:int64# + )^#12:Expr.Call# + )^#10:Expr.Call# + )", + }, + TestCase{ + .source = "(1 + (2 * 3) - (1 / 2)) == (6 % 1)", + .expected_ast = R"( + _==_( + _-_( + _+_( + 1^#1:int64#, + _*_( + 2^#3:int64#, + 3^#5:int64# + )^#4:Expr.Call# + )^#2:Expr.Call#, + _/_( + 1^#7:int64#, + 2^#9:int64# + )^#8:Expr.Call# + )^#6:Expr.Call#, + _%_( + 6^#11:int64#, + 1^#13:int64# + )^#12:Expr.Call# + )^#10:Expr.Call# + )", + }, + TestCase{ + .source = "1 + 2 * 3 == 7 && true || false", + .expected_ast = R"( + _||_( + _&&_( + _==_( + _+_( + 1^#1:int64#, + _*_( + 2^#3:int64#, + 3^#5:int64# + )^#4:Expr.Call# + )^#2:Expr.Call#, + 7^#7:int64# + )^#6:Expr.Call#, + true^#8:bool# + )^#9:Expr.Call#, + false^#10:bool# + )^#11:Expr.Call# + )", + }, + TestCase{ + .source = "x > 0 ? 'pos' : 'neg'", + .expected_ast = R"( + _?_:_( + _>_( + x^#1:Expr.Ident#, + 0^#3:int64# + )^#2:Expr.Call#, + "pos"^#5:string#, + "neg"^#6:string# + )^#4:Expr.Call# + )", + }, + TestCase{ + .source = "a?b:c", + .expected_ast = R"( + _?_:_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident#, + c^#4:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a[b]", + .expected_ast = R"( + _[_]( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a[?b]", + .expected_ast = R"( + _[?_]( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "a[3]", + .expected_ast = R"( + _[_]( + a^#1:Expr.Ident#, + 3^#3:int64# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "[1,3,4][0]", + .expected_ast = R"( + _[_]( + [ + 1^#2:int64#, + 3^#3:int64#, + 4^#4:int64# + ]^#1:Expr.CreateList#, + 0^#6:int64# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a()", + .expected_ast = R"( + a()^#1:Expr.Call# + )", + }, + TestCase{ + .source = "a(b)", + .expected_ast = R"( + a( + b^#2:Expr.Ident# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "a(b, c)", + .expected_ast = R"( + a( + b^#2:Expr.Ident#, + c^#3:Expr.Ident# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "a.b()", + .expected_ast = R"( + a^#1:Expr.Ident#.b()^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a.b(1)", + .expected_ast = R"( + a^#1:Expr.Ident#.b( + 1^#3:int64# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a.b(c)", + .expected_ast = R"( + a^#1:Expr.Ident#.b( + c^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "size(x) == x.size()", + .expected_ast = R"( + _==_( + size( + x^#2:Expr.Ident# + )^#1:Expr.Call#, + x^#4:Expr.Ident#.size()^#5:Expr.Call# + )^#3:Expr.Call# + )", + }, + TestCase{ + .source = "\"foo\".size()", + .expected_ast = R"( + "foo"^#1:string#.size()^#2:Expr.Call# + )", + }, + TestCase{ + .source = "[true, true].size() == 2", + .expected_ast = R"( + _==_( + [ + true^#2:bool#, + true^#3:bool# + ]^#1:Expr.CreateList#.size()^#4:Expr.Call#, + 2^#6:int64# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "[]", + .expected_ast = R"( + []^#1:Expr.CreateList# + )", + }, + TestCase{ + .source = "[a]", + .expected_ast = R"( + [ + a^#2:Expr.Ident# + ]^#1:Expr.CreateList# + )", + }, + TestCase{ + .source = "[1, 2, 3]", + .expected_ast = R"( + [ + 1^#2:int64#, + 2^#3:int64#, + 3^#4:int64# + ]^#1:Expr.CreateList# + )", + }, + TestCase{ + .source = "[1, 2, 3,]", + .expected_ast = R"( + [ + 1^#2:int64#, + 2^#3:int64#, + 3^#4:int64# + ]^#1:Expr.CreateList# + )", + }, + TestCase{ + .source = "[?1, 2]", + .expected_ast = R"( + [ + ?1^#2:int64#, + 2^#3:int64# + ]^#1:Expr.CreateList# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "{}", + .expected_ast = R"( + {}^#1:Expr.CreateMap# + )", + }, + TestCase{ + .source = "{'key': 'value', 'num': 42}", + .expected_ast = R"( + { + "key"^#3:string#:"value"^#4:string#^#2:Expr.CreateStruct.Entry#, + "num"^#6:string#:42^#7:int64#^#5:Expr.CreateStruct.Entry# + }^#1:Expr.CreateMap# + )", + }, + TestCase{ + .source = "{?'key': 'value', 'num': 42}", + .expected_ast = R"( + { + ?"key"^#3:string#:"value"^#4:string#^#2:Expr.CreateStruct.Entry#, + "num"^#6:string#:42^#7:int64#^#5:Expr.CreateStruct.Entry# + }^#1:Expr.CreateMap# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "{foo: 5, bar: \"xyz\"}", + .expected_ast = R"( + { + foo^#3:Expr.Ident#:5^#4:int64#^#2:Expr.CreateStruct.Entry#, + bar^#6:Expr.Ident#:"xyz"^#7:string#^#5:Expr.CreateStruct.Entry# + }^#1:Expr.CreateMap# + )", + }, + TestCase{ + .source = "google.protobuf.Empty{}", + .expected_ast = R"( + google.protobuf.Empty{}^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "foo{ }", + .expected_ast = R"( + foo{}^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "foo{ a:b }", + .expected_ast = R"( + foo{ + a:b^#3:Expr.Ident#^#2:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "A{`b`: 1}", + .expected_ast = R"( + A{ + b:1^#3:int64#^#2:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "A{`b-c`: 1}", + .expected_ast = R"( + A{ + b-c:1^#3:int64#^#2:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "Msg{field: 10, other: 'val'}", + .expected_ast = R"( + Msg{ + field:10^#3:int64#^#2:Expr.CreateStruct.Entry#, + other:"val"^#5:string#^#4:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "Msg{?field: 10, other: 'val'}", + .expected_ast = R"( + Msg{ + ?field:10^#3:int64#^#2:Expr.CreateStruct.Entry#, + other:"val"^#5:string#^#4:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "import{}", + .expected_ast = R"( + import{}^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = ".import{}", + .expected_ast = R"( + .import{}^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "import.Foo{}", + .expected_ast = R"( + import.Foo{}^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "Foo.import{}", + .expected_ast = R"( + Foo.import{}^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "(((10 - 3) - 2))", + .expected_ast = R"( + _-_( + _-_( + 10^#1:int64#, + 3^#3:int64# + )^#2:Expr.Call#, + 2^#5:int64# + )^#4:Expr.Call# + )", + }, + }; +} + +INSTANTIATE_TEST_SUITE_P(PrattParserTest, PrattParserTest, + testing::ValuesIn(GetParserTestCases()), + TestName); + +struct ErrorTestCase { + absl::string_view source; + absl::string_view expected_error; + bool enable_optional_syntax = false; + bool enable_quoted_identifiers = false; +}; + +class PrattParserErrorTest : public testing::TestWithParam {}; + +std::string FormatIssues(const cel::Source& source, + const std::vector& issues) { + return absl::StrJoin( + issues, "\n", [&source](std::string* out, const cel::ParseIssue& issue) { + absl::StrAppend( + out, + absl::StrFormat("ERROR: %s:%d:%d: %s", source.description(), + issue.location().line, issue.location().column + 1, + issue.message()), + source.DisplayErrorLocation(issue.location())); + }); +} + +TEST_P(PrattParserErrorTest, ParseSyntaxError) { + const ErrorTestCase& test_case = GetParam(); + cel::ParserOptions options; + options.enable_optional_syntax = test_case.enable_optional_syntax; + options.enable_quoted_identifiers = test_case.enable_quoted_identifiers; + std::vector issues; + absl::StatusOr> result = + Parse(test_case.source, options, &issues); + EXPECT_THAT(result, StatusIs(absl::StatusCode::kInvalidArgument, + Eq(test_case.expected_error))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr source, + cel::NewSource(test_case.source)); + EXPECT_EQ(FormatIssues(*source, issues), test_case.expected_error); +} + +std::vector GetErrorTestCases() { + return { + ErrorTestCase{ + .source = "1 + 2 * 3 4", + .expected_error = + "ERROR: :1:11: Syntax error: unexpected token after " + "expression\n" + " | 1 + 2 * 3 4\n" + " | ..........^", + }, + ErrorTestCase{ + .source = "1{}", + .expected_error = + "ERROR: :1:2: Syntax error: unexpected token after " + "expression\n" + " | 1{}\n" + " | .^", + }, + ErrorTestCase{ + .source = "true ? 1", + .expected_error = + "ERROR: :1:9: Syntax error: expected ':' in conditional " + "expression\n" + " | true ? 1\n" + " | ........^", + }, + ErrorTestCase{ + .source = "a.?b", + .expected_error = "ERROR: :1:2: unsupported syntax '.?'\n" + " | a.?b\n" + " | .^", + }, + ErrorTestCase{ + .source = "a.", + .expected_error = "ERROR: :1:3: Syntax error: expected " + "identifier after '.'\n" + " | a.\n" + " | ..^", + }, + ErrorTestCase{ + .source = "a[?0]", + .expected_error = "ERROR: :1:2: unsupported syntax '?'\n" + " | a[?0]\n" + " | .^", + }, + ErrorTestCase{ + .source = ". *", + .expected_error = + "ERROR: :1:3: Syntax error: expected identifier\n" + " | . *\n" + " | ..^", + }, + ErrorTestCase{ + .source = ".as", + .expected_error = "ERROR: :1:2: reserved identifier: as\n" + " | .as\n" + " | .^", + }, + ErrorTestCase{ + .source = "* 2", + .expected_error = + "ERROR: :1:1: Syntax error: unexpected token\n" + " | * 2\n" + " | ^\n" + "ERROR: :1:3: Syntax error: unexpected token after " + "expression\n" + " | * 2\n" + " | ..^", + }, + ErrorTestCase{ + .source = "(1 + 2", + .expected_error = + "ERROR: :1:7: Syntax error: mismatched input " + "expecting ')'\n" + " | (1 + 2\n" + " | ......^", + }, + ErrorTestCase{ + .source = "[?1]", + .expected_error = "ERROR: :1:2: unsupported syntax '?'\n" + " | [?1]\n" + " | .^", + }, + ErrorTestCase{ + .source = "[1, 2", + .expected_error = "ERROR: :1:6: Syntax error: expected ']'\n" + " | [1, 2\n" + " | .....^", + }, + ErrorTestCase{ + .source = "{?'k': 'v'}", + .expected_error = "ERROR: :1:2: unsupported syntax '?'\n" + " | {?'k': 'v'}\n" + " | .^", + }, + ErrorTestCase{ + .source = "{'k' 'v'}", + .expected_error = + "ERROR: :1:6: Syntax error: expected ':' in map entry\n" + " | {'k' 'v'}\n" + " | .....^", + }, + ErrorTestCase{ + .source = "{'k': 'v'", + .expected_error = "ERROR: :1:10: Syntax error: expected '}'\n" + " | {'k': 'v'\n" + " | .........^", + }, + ErrorTestCase{ + .source = "Msg{?f: 1}", + .expected_error = "ERROR: :1:5: unsupported syntax '?'\n" + " | Msg{?f: 1}\n" + " | ....^", + }, + ErrorTestCase{ + .source = "Msg{1: 2}", + .expected_error = + "ERROR: :1:5: Syntax error: expected struct field name\n" + " | Msg{1: 2}\n" + " | ....^", + }, + ErrorTestCase{ + .source = "Msg{f 10}", + .expected_error = + "ERROR: :1:7: Syntax error: expected ':' in struct field\n" + " | Msg{f 10}\n" + " | ......^", + }, + ErrorTestCase{ + .source = "Msg{f: 10", + .expected_error = "ERROR: :1:10: Syntax error: expected '}'\n" + " | Msg{f: 10\n" + " | .........^", + }, + ErrorTestCase{ + .source = "f(1, 2", + .expected_error = + "ERROR: :1:7: Syntax error: mismatched input " + "expecting ')'\n" + " | f(1, 2\n" + " | ......^", + }, + ErrorTestCase{ + .source = "foo(a,b,)", + .expected_error = "ERROR: :1:9: unexpected token\n" + " | foo(a,b,)\n" + " | ........^", + }, + ErrorTestCase{ + .source = "999999999999999999999999999999999999999", + .expected_error = + "ERROR: :1:1: Syntax error: invalid int literal\n" + " | 999999999999999999999999999999999999999\n" + " | ^", + }, + ErrorTestCase{ + .source = "999999999999999999999999999999999999999u", + .expected_error = + "ERROR: :1:1: Syntax error: invalid uint literal\n" + " | 999999999999999999999999999999999999999u\n" + " | ^", + }, + ErrorTestCase{ + .source = "1e", + .expected_error = + "ERROR: :1:1: Syntax error: floating point literal " + "missing digits after exponent separator\n" + " | 1e\n" + " | ^", + }, + ErrorTestCase{ + .source = "\"unterminated", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"unterminated\n" + " | ^", + }, + ErrorTestCase{ + .source = "\"\"\"hello\nworld", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"\"\"hello\n" + " | ^", + }, + ErrorTestCase{ + .source = "'''hello\nworld", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | '''hello\n" + " | ^", + }, + ErrorTestCase{ + .source = "r\"\"\"hello\nworld", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | r\"\"\"hello\n" + " | ^", + }, + ErrorTestCase{ + .source = "\"hello\nworld\"", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after " + "expression\n" + " | world\"\n" + " | ^", + }, + ErrorTestCase{ + .source = "'hello\nworld'", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | 'hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after " + "expression\n" + " | world'\n" + " | ^", + }, + ErrorTestCase{ + .source = "r\"hello\nworld\"", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | r\"hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after " + "expression\n" + " | world\"\n" + " | ^", + }, + ErrorTestCase{ + .source = "`hello\nworld`", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated quoted " + "identifier\n" + " | `hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after " + "expression\n" + " | world`\n" + " | ^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "\"hello\rworld\"", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"hello\rworld\"\n" + " | ^\n" + "ERROR: :1:8: Syntax error: unexpected token after " + "expression\n" + " | \"hello\rworld\"\n" + " | .......^", + }, + ErrorTestCase{ + .source = "b\"unterminated", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated bytes literal\n" + " | b\"unterminated\n" + " | ^", + }, + ErrorTestCase{ + .source = "a.?`foo`", + .expected_error = "ERROR: :1:4: unsupported syntax '`'\n" + " | a.?`foo`\n" + " | ...^", + .enable_optional_syntax = true, + .enable_quoted_identifiers = false, + }, + ErrorTestCase{ + .source = "a.`foo`()", + .expected_error = "ERROR: :1:3: unexpected quoted identifier\n" + " | a.`foo`()\n" + " | ..^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "`foo`", + .expected_error = "ERROR: :1:1: unexpected quoted identifier\n" + " | `foo`\n" + " | ^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "`foo`()", + .expected_error = "ERROR: :1:1: unexpected quoted identifier\n" + " | `foo`()\n" + " | ^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "a.`b@c`", + .expected_error = "ERROR: :1:3: unexpected quoted identifier\n" + " | a.`b@c`\n" + " | ..^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "a.``", + .expected_error = "ERROR: :1:3: unexpected quoted identifier\n" + " | a.``\n" + " | ..^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "a.`foo`", + .expected_error = "ERROR: :1:3: unsupported syntax '`'\n" + " | a.`foo`\n" + " | ..^", + .enable_quoted_identifiers = false, + }, + ErrorTestCase{ + .source = "`foo", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated quoted " + "identifier\n" + " | `foo\n" + " | ^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "f(*, 1e, {2 3})", + .expected_error = + "ERROR: :1:3: Syntax error: unexpected token\n" + " | f(*, 1e, {2 3})\n" + " | ..^\n" + "ERROR: :1:6: Syntax error: floating point literal " + "missing digits after exponent separator\n" + " | f(*, 1e, {2 3})\n" + " | .....^\n" + "ERROR: :1:13: Syntax error: expected ':' in map entry\n" + " | f(*, 1e, {2 3})\n" + " | ............^", + }, + ErrorTestCase{ + .source = "(1 + *) + 2", + .expected_error = + "ERROR: :1:6: Syntax error: unexpected token\n" + " | (1 + *) + 2\n" + " | .....^", + }, + ErrorTestCase{ + .source = "f(1 + *, 2)", + .expected_error = + "ERROR: :1:7: Syntax error: unexpected token\n" + " | f(1 + *, 2)\n" + " | ......^", + }, + ErrorTestCase{ + .source = "(a. + 1)", + .expected_error = + "ERROR: :1:5: Syntax error: expected identifier after " + "'.'\n" + " | (a. + 1)\n" + " | ....^", + }, + ErrorTestCase{ + .source = "f(a., 1)", + .expected_error = + "ERROR: :1:5: Syntax error: expected identifier after " + "'.'\n" + " | f(a., 1)\n" + " | ....^", + }, + ErrorTestCase{ + .source = "[a., 1]", + .expected_error = + "ERROR: :1:4: Syntax error: expected identifier after " + "'.'\n" + " | [a., 1]\n" + " | ...^", + }, + ErrorTestCase{ + .source = "-0x8000000000000001", + .expected_error = + "ERROR: :1:2: Syntax error: invalid int literal\n" + " | -0x8000000000000001\n" + " | .^", + }, + ErrorTestCase{ + .source = "-0x10000000000000000", + .expected_error = + "ERROR: :1:2: Syntax error: invalid int literal\n" + " | -0x10000000000000000\n" + " | .^", + }, + ErrorTestCase{ + .source = "-9223372036854775809", + .expected_error = + "ERROR: :1:2: Syntax error: invalid int literal\n" + " | -9223372036854775809\n" + " | .^", + }, + ErrorTestCase{ + .source = "-999999999999999999999999999999999999999", + .expected_error = + "ERROR: :1:2: Syntax error: invalid int literal\n" + " | -999999999999999999999999999999999999999\n" + " | .^", + }, + ErrorTestCase{ + .source = "-", + .expected_error = + "ERROR: :1:2: Syntax error: mismatched input '' " + "expecting expression\n" + " | -\n" + " | .^", + }, + ErrorTestCase{ + .source = "- *", + .expected_error = + "ERROR: :1:3: Syntax error: unexpected token\n" + " | - *\n" + " | ..^", + }, + ErrorTestCase{ + .source = "\"😀😀😀😀😀\" ~error", + .expected_error = + "ERROR: :1:9: Syntax error: unexpected character\n" + " | \"😀😀😀😀😀\" ~error\n" + " | .．．．．．..^", + }, + ErrorTestCase{ + .source = "-!x", + .expected_error = + "ERROR: :1:2: Syntax error: unexpected token\n" + " | -!x\n" + " | .^\n" + "ERROR: :1:3: Syntax error: unexpected token after " + "expression\n" + " | -!x\n" + " | ..^", + }, + ErrorTestCase{ + .source = "!-x", + .expected_error = "ERROR: :1:2: Syntax error: unexpected '-'\n" + " | !-x\n" + " | .^", + }, + ErrorTestCase{ + .source = "a.in", + .expected_error = + "ERROR: :1:3: Syntax error: expected identifier after " + "'.'\n" + " | a.in\n" + " | ..^", + }, + ErrorTestCase{ + .source = "1 \v + 2", + .expected_error = + "ERROR: :1:3: Syntax error: unexpected character\n" + " | 1 \v + 2\n" + " | ..^", + }, + ErrorTestCase{ + .source = "has(a.`$b`)", + .expected_error = "ERROR: :1:7: unexpected quoted identifier\n" + " | has(a.`$b`)\n" + " | ......^", + .enable_quoted_identifiers = true, + }, + // Parentheses are ignored for AST construction, not message creation + ErrorTestCase{ + .source = "(a){}", + .expected_error = + "ERROR: :1:4: Syntax error: unexpected token after " + "expression\n" + " | (a){}\n" + " | ...^", + }, + ErrorTestCase{ + .source = "(a.b){}", + .expected_error = + "ERROR: :1:6: Syntax error: unexpected token after " + "expression\n" + " | (a.b){}\n" + " | .....^", + }, + ErrorTestCase{ + .source = "(a).b{}", + .expected_error = + "ERROR: :1:6: Syntax error: unexpected token after " + "expression\n" + " | (a).b{}\n" + " | .....^", + }, + // Backtick quoted identifiers should also be rejected on message creation + ErrorTestCase{ + .source = "a.`b-c`{}", + .expected_error = + "ERROR: :1:8: Syntax error: unexpected token after " + "expression\n" + " | a.`b-c`{}\n" + " | .......^", + .enable_quoted_identifiers = true, + }, + // Invalid field names are disallowed for message creation + ErrorTestCase{ + .source = "Msg{`$b`: 1}", + .expected_error = "ERROR: :1:5: unexpected quoted identifier\n" + " | Msg{`$b`: 1}\n" + " | ....^", + .enable_quoted_identifiers = true, + }, + }; +} + +INSTANTIATE_TEST_SUITE_P(PrattParserErrorTest, PrattParserErrorTest, + testing::ValuesIn(GetErrorTestCases()), + TestName); + +TEST(PrattParserTest, SourceInfoPositionsPopulated) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, Parse("a + b")); + const cel::SourceInfo& source_info = ast->source_info(); + EXPECT_FALSE(source_info.positions().empty()); + + const cel::Expr& root = ast->root_expr(); + EXPECT_EQ(source_info.positions().at(root.id()), 2); + ASSERT_TRUE(root.has_call_expr()); + ASSERT_EQ(root.call_expr().args().size(), 2); + EXPECT_EQ(source_info.positions().at(root.call_expr().args()[0].id()), 0); + EXPECT_EQ(source_info.positions().at(root.call_expr().args()[1].id()), 4); + + cel::ParserOptions opt_options; + opt_options.enable_optional_syntax = true; + ASSERT_OK_AND_ASSIGN(std::unique_ptr opt_ast, + Parse("a.?b.?c", opt_options)); + const auto& opt_pos = opt_ast->source_info().positions(); + EXPECT_EQ(opt_pos.at(1), 0); + EXPECT_EQ(opt_pos.at(2), 1); + EXPECT_EQ(opt_pos.at(3), 0); + EXPECT_EQ(opt_pos.at(4), 4); + EXPECT_EQ(opt_pos.at(5), 0); +} + +TEST(PrattParserRecursionDepthTest, ParseRecursionDepth) { + cel::ParserOptions options; + options.max_recursion_depth = 5; + EXPECT_THAT(Parse("((((1))))", options), IsOkAndHolds(NotNull())); + EXPECT_THAT(Parse("[[[[[[1]]]]]]", options), + StatusIs(absl::StatusCode::kCancelled)); +} + +// Parentheses are unwound iteratively, so nesting them does not grow the C++ +// call stack. The recursion limit is raised above the nesting level here to +// exercise that: each '(' still consumes one unit of the recursion budget. +TEST(PrattParserRecursionDepthTest, DeeplyNestedParens) { + cel::ParserOptions options; + options.max_recursion_depth = 2100; + std::string literal_expr = + std::string(1000, '(') + "42" + std::string(1000, ')'); + EXPECT_THAT(Parse(literal_expr, options), IsOkAndHolds(NotNull())); + + std::string binary_expr = + std::string(1000, '(') + "1 + 2" + std::string(1000, ')'); + EXPECT_THAT(Parse(binary_expr, options), IsOkAndHolds(NotNull())); + + std::string left_nested_calc_expr = std::string(1000, '(') + "1 + 2"; + for (int i = 0; i < 1000; ++i) { + left_nested_calc_expr += ") + 1"; + } + EXPECT_THAT(Parse(left_nested_calc_expr, options), IsOkAndHolds(NotNull())); +} + +TEST(PrattParserRecursionDepthTest, NestedAndGroupingParensCombinations) { + EXPECT_THAT("(( (1) + 2 ))", AstEq("1 + 2")); + EXPECT_THAT("((1 + 2) * (3 + 4))", AstEq("(1 + 2) * (3 + 4)")); + EXPECT_THAT("((((1)) + ((2))))", AstEq("1 + 2")); + EXPECT_THAT("(((1 + 2) * 3) + 4)", AstEq("(1 + 2) * 3 + 4")); + EXPECT_THAT("f((((1))), (((2))))", AstEq("f(1, 2)")); + EXPECT_THAT("[{((1)): ((2))}]", AstEq("[{1: 2}]")); + EXPECT_THAT("(((a))).b[0]", AstEq("a.b[0]")); + EXPECT_THAT("((((a).b[0]) + 1) ? 2 : 3)", AstEq("(a.b[0] + 1) ? 2 : 3")); + EXPECT_THAT("a ? ((((b)))) : c", AstEq("a ? b : c")); + EXPECT_THAT("((((7) + 1) + 1) + 1)", AstEq("7 + 1 + 1 + 1")); + EXPECT_THAT("(true) || (true || (true || false))", + AstEq("true || (true || (true || false))")); + EXPECT_THAT("(1 + 1 + 1) + 1", AstEq("1 + 1 + 1 + 1")); + EXPECT_THAT("((1 + 1) + 1) + 1", AstEq("1 + 1 + 1 + 1")); + EXPECT_THAT("(((1 + 1 + 1))) + 1", AstEq("1 + 1 + 1 + 1")); + EXPECT_THAT("1 + (1 + 1 + 1)", AstEq("1 + (1 + 1 + 1)")); + EXPECT_THAT("((1 + ((7))))", AstEq("1 + 7")); + EXPECT_THAT("(a.b.c).d.e.f", AstEq("a.b.c.d.e.f")); + EXPECT_THAT("((a && b.c.d).e)", AstEq("(a && b.c.d).e")); + EXPECT_THAT("((a && b && c.d.e).f)", AstEq("(a && b && c.d.e).f")); + EXPECT_THAT("((a ? b : c).d[0] ? (e ? f : g) : h) + ((x).y)", + AstEq("((a ? b : c).d[0] ? (e ? f : g) : h) + x.y")); +} + +TEST(PrattParserRecursionDepthTest, MismatchedParensStillReportErrors) { + cel::ParserOptions options; + EXPECT_THAT(Parse("((((1))", options), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(Parse("(((1 + 2]", options), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(Parse("(( [ 1 ) ] ))", options), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +TEST(PrattParserRecursionDepthTest, SequentialScopesDoNotAccumulateDepth) { + cel::ParserOptions options; + options.max_recursion_depth = 2; + EXPECT_THAT(Parse("[1] + [2] + [3]", options), IsOkAndHolds(NotNull())); +} + +TEST(PrattParserRecursionDepthTest, DeeplyNestedTernary) { + cel::ParserOptions options; + options.max_recursion_depth = 5; + EXPECT_THAT(Parse("a ? b : a ? b : a ? b : a ? b : c", options), + IsOkAndHolds(NotNull())); + EXPECT_THAT(Parse("a ? b : a ? b : a ? b : a ? b : a ? b : c", options), + StatusIs(absl::StatusCode::kCancelled)); +} + +TEST(PrattParserRecursionDepthTest, NestedParensWithTernaryAndSelectors) { + EXPECT_THAT(Parse("((a ? b : c).d[0] ? (e ? f : g) : h) + ((x).y)"), + IsOkAndHolds(NotNull())); +} + +class TestParserWorker : public ParserWorker { + // Expose the protected constructor and methods for testing. + public: + using ParserWorker::GetTokenText; + using ParserWorker::ParserWorker; +}; + +TEST(ParserWorkerTest, GetTokenTextBoundsChecking) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr source, + cel::NewSource("hello world")); + cel::ParserOptions options; + TestParserWorker worker(*source, options, nullptr); + EXPECT_EQ(worker.GetTokenText( + Token{.type = TokenType::kIdent, .start = 0, .end = 5}), + "hello"); + EXPECT_EQ(worker.GetTokenText( + Token{.type = TokenType::kIdent, .start = -1, .end = 5}), + ""); + EXPECT_EQ(worker.GetTokenText( + Token{.type = TokenType::kIdent, .start = 5, .end = 2}), + ""); + EXPECT_EQ(worker.GetTokenText( + Token{.type = TokenType::kIdent, .start = 0, .end = 100}), + ""); +} + +struct MacroTestCase { + absl::string_view source; + absl::string_view expected_ast; +}; + +class PrattParserMacroTest : public testing::TestWithParam {}; + +TEST_P(PrattParserMacroTest, MacroExprExpander) { + const MacroTestCase& test_case = GetParam(); + auto builder = NewPrattParserBuilder(); + ASSERT_OK_AND_ASSIGN( + auto global_macro, + Macro::Global("foo", 1, + [](MacroExprFactory& macro_factory, + absl::Span args) -> std::optional { + return macro_factory.NewCall("my_macro", std::move(args)); + })); + + ASSERT_OK_AND_ASSIGN( + auto receiver_macro, + Macro::Receiver("bar", 2, + [](MacroExprFactory& macro_factory, Expr& target, + absl::Span args) -> std::optional { + return macro_factory.NewMemberCall( + "my_bar", std::move(target), std::move(args)); + })); + + ASSERT_THAT(builder->AddMacro(global_macro), IsOk()); + ASSERT_THAT(builder->AddMacro(receiver_macro), IsOk()); + ASSERT_OK_AND_ASSIGN(auto parser, builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(test_case.source)); + ASSERT_OK_AND_ASSIGN(auto ast, parser->Parse(*source)); + + EXPECT_THAT(ast->root_expr(), AstIs(test_case.expected_ast)); +} + +std::vector GetMacroTestCases() { + return { + MacroTestCase{ + .source = "foo(x)", + .expected_ast = R"( + my_macro( + x^#2:Expr.Ident# + )^#3:Expr.Call# + )", + }, + MacroTestCase{ + .source = "x.bar(y, z)", + .expected_ast = R"( + x^#1:Expr.Ident#.my_bar( + y^#3:Expr.Ident#, + z^#4:Expr.Ident# + )^#5:Expr.Call# + )", + }, + // Number of args doesn't match the macro definition + MacroTestCase{ + .source = "foo(x, y)", + .expected_ast = R"( + foo( + x^#2:Expr.Ident#, + y^#3:Expr.Ident# + )^#1:Expr.Call# + )", + }, + // Number of args doesn't match the macro definition + MacroTestCase{ + .source = "x.bar(y)", + .expected_ast = R"( + x^#1:Expr.Ident#.bar( + y^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + // No target provided for receiver macro + MacroTestCase{ + .source = "bar(x, y)", + .expected_ast = R"( + bar( + x^#2:Expr.Ident#, + y^#3:Expr.Ident# + )^#1:Expr.Call# + )", + }, + // Target provided for global macro + MacroTestCase{ + .source = "x.foo(y)", + .expected_ast = R"( + x^#1:Expr.Ident#.foo( + y^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + // Global macro not registered + MacroTestCase{ + .source = "baz(x)", + .expected_ast = R"( + baz( + x^#2:Expr.Ident# + )^#1:Expr.Call# + )", + }, + // Receiver macro not registered + MacroTestCase{ + .source = "x.baz(x)", + .expected_ast = R"( + x^#1:Expr.Ident#.baz( + x^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + // has() macro + MacroTestCase{ + .source = "has(message.field)", + .expected_ast = R"( + message^#2:Expr.Ident#.field~test-only~^#4:Expr.Select# + )", + }, + }; +} + +INSTANTIATE_TEST_SUITE_P(PrattParserMacroTest, PrattParserMacroTest, + testing::ValuesIn(GetMacroTestCases()), + TestName); + +TEST(PrattParserMacroCallsTest, MacroCallsDisabledByDefault) { + cel::ParserOptions options; + options.add_macro_calls = false; + ASSERT_OK_AND_ASSIGN(auto ast, Parse("has(a.b)", options)); + EXPECT_TRUE(ast->source_info().macro_calls().empty()); +} + +TEST(PrattParserMacroCallsTest, GlobalMacroCallRecorded) { + cel::ParserOptions options; + options.add_macro_calls = true; + ASSERT_OK_AND_ASSIGN(auto ast, Parse("has(a.b)", options)); + + const auto& macro_calls = ast->source_info().macro_calls(); + EXPECT_FALSE(macro_calls.empty()); + EXPECT_TRUE(macro_calls.contains(ast->root_expr().id())); + + const auto& macro_call = macro_calls.at(ast->root_expr().id()); + EXPECT_THAT(macro_call, AstIs(R"( + has( + a^#2:Expr.Ident#.b^#3:Expr.Select# + )^#0:Expr.Call# + )")); +} + +TEST(PrattParserMacroCallsTest, ReceiverMacroCallRecorded) { + cel::ParserOptions options; + options.add_macro_calls = true; + ASSERT_OK_AND_ASSIGN(auto ast, Parse("[1, 2].exists(x, x > 0)", options)); + + const auto& macro_calls = ast->source_info().macro_calls(); + EXPECT_FALSE(macro_calls.empty()); + EXPECT_TRUE(macro_calls.contains(ast->root_expr().id())); + + const auto& exists_macro_call = macro_calls.at(ast->root_expr().id()); + EXPECT_THAT(exists_macro_call, AstIs(R"( + [ + 1^#2:int64#, + 2^#3:int64# + ]^#1:Expr.CreateList#.exists( + x^#5:Expr.Ident#, + _>_( + x^#6:Expr.Ident#, + 0^#8:int64# + )^#7:Expr.Call# + )^#0:Expr.Call# + )")); +} + +TEST(PrattParserMacroCallsTest, NestedMacroCallsUseCopyAndReplaceReplacer) { + cel::ParserOptions options; + options.add_macro_calls = true; + ASSERT_OK_AND_ASSIGN(auto ast, Parse("[1, 2].all(x, has(x.b))", options)); + + const auto& root_expr = ast->root_expr(); + EXPECT_THAT(root_expr, AstIs(R"( + __comprehension__( + // Variable + x, + // Target + [ + 1^#2:int64#, + 2^#3:int64# + ]^#1:Expr.CreateList#, + // Accumulator + @result, + // Init + true^#10:bool#, + // LoopCondition + @not_strictly_false( + @result^#11:Expr.Ident# + )^#12:Expr.Call#, + // LoopStep + _&&_( + @result^#13:Expr.Ident#, + x^#7:Expr.Ident#.b~test-only~^#9:Expr.Select# + )^#14:Expr.Call#, + // Result + @result^#15:Expr.Ident#)^#16:Expr.Comprehension# + )")); + + const auto& macro_calls = ast->source_info().macro_calls(); + // There should be 2 recorded macro calls: one for 'all', one for 'has'. + EXPECT_EQ(macro_calls.size(), 2); + + int64_t all_macro_id = root_expr.id(); + EXPECT_TRUE(macro_calls.contains(all_macro_id)); + + const auto& all_macro_call = macro_calls.at(all_macro_id); + // The second argument of 'all' is the inner 'has(x.b)' call. + // Because 'has(x.b)' was already expanded and recorded in macro_calls + // it is represented as an UnspecifiedExpr holding the inner macro's ID. + EXPECT_THAT(all_macro_call, AstIs(R"( + [ + 1^#2:int64#, + 2^#3:int64# + ]^#1:Expr.CreateList#.all( + x^#5:Expr.Ident#, + ^#9:unspecified_expr# + )^#0:Expr.Call# + )")); + + const auto& has_call = all_macro_call.call_expr().args()[1]; + int64_t has_macro_id = has_call.id(); + EXPECT_EQ(has_macro_id, 9); // ^#9:unspecified_expr# + EXPECT_TRUE(macro_calls.contains(has_macro_id)); + + const auto& has_macro_call = macro_calls.at(has_macro_id); + EXPECT_THAT(has_macro_call, AstIs(R"( + has( + x^#7:Expr.Ident#.b^#8:Expr.Select# + )^#0:Expr.Call# + )")); +} + +TEST(PrattParserMacroErrorTest, ReportError) { + auto builder = NewPrattParserBuilder(); + ASSERT_OK_AND_ASSIGN( + auto error_macro, + Macro::Global("bad_macro", 1, + [](MacroExprFactory& macro_factory, + absl::Span args) -> std::optional { + return macro_factory.ReportError("custom macro error"); + })); + + ASSERT_THAT(builder->AddMacro(error_macro), IsOk()); + ASSERT_OK_AND_ASSIGN(auto parser, builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("42 + bad_macro(x)")); + std::vector issues; + auto ast = parser->Parse(*source, &issues); + EXPECT_THAT(ast, StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_EQ(FormatIssues(*source, issues), + "ERROR: :1:15: custom macro error\n" + " | 42 + bad_macro(x)\n" + " | ..............^"); +} + +TEST(PrattParserMacroErrorTest, ReportErrorAt) { + auto builder = NewPrattParserBuilder(); + ASSERT_OK_AND_ASSIGN( + auto error_at_macro, + Macro::Global("bad_macro_at", 1, + [](MacroExprFactory& macro_factory, + absl::Span args) -> std::optional { + return macro_factory.ReportErrorAt(args[0], + "custom error at arg"); + })); + + ASSERT_THAT(builder->AddMacro(error_at_macro), IsOk()); + ASSERT_OK_AND_ASSIGN(auto parser, builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("bad_macro_at(x)")); + std::vector issues; + auto ast = parser->Parse(*source, &issues); + EXPECT_THAT(ast, StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_EQ(FormatIssues(*source, issues), + "ERROR: :1:14: custom error at arg\n" + " | bad_macro_at(x)\n" + " | .............^"); +} + +TEST(PrattParserErrorRecoveryTest, ErrorRecoveryLimitZero) { + cel::ParserOptions options; + options.error_recovery_limit = 0; + std::vector issues; + auto result = Parse("......", options, &issues); + EXPECT_THAT(result, StatusIs(absl::StatusCode::kInvalidArgument)); + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("......")); + EXPECT_EQ(FormatIssues(*source, issues), + "ERROR: :-1:0: Error recovery limit (0) exceeded"); +} + +TEST(PrattParserErrorRecoveryTest, ErrorRecoveryLimitOne) { + cel::ParserOptions options; + options.error_recovery_limit = 1; + std::vector issues; + auto result = Parse("......", options, &issues); + EXPECT_THAT(result, StatusIs(absl::StatusCode::kInvalidArgument)); + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("......")); + EXPECT_EQ(FormatIssues(*source, issues), + "ERROR: :-1:0: Error recovery limit (1) exceeded\n" + "ERROR: :1:2: Syntax error: expected identifier\n" + " | ......\n" + " | .^"); +} + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc new file mode 100644 index 000000000..84d1d6ed6 --- /dev/null +++ b/parser/internal/pratt_parser_worker.cc @@ -0,0 +1,318 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/pratt_parser_worker.h" + +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/base/optimization.h" +#include "absl/cleanup/cleanup.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "common/operators.h" +#include "common/source.h" +#include "parser/internal/lexer.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { +namespace { + +using CelOperator = ::google::api::expr::common::CelOperator; + +const BinaryOpInfo kLogicalOr = {1, CelOperator::LOGICAL_OR, true, + TokenType::kLogicalOr}; +const BinaryOpInfo kLogicalAnd = {2, CelOperator::LOGICAL_AND, true, + TokenType::kLogicalAnd}; +const BinaryOpInfo kLess = {3, CelOperator::LESS, false, TokenType::kLess}; +const BinaryOpInfo kLessEqual = {3, CelOperator::LESS_EQUALS, false, + TokenType::kLessEqual}; +const BinaryOpInfo kGreater = {3, CelOperator::GREATER, false, + TokenType::kGreater}; +const BinaryOpInfo kGreaterEqual = {3, CelOperator::GREATER_EQUALS, false, + TokenType::kGreaterEqual}; +const BinaryOpInfo kEqualEqual = {3, CelOperator::EQUALS, false, + TokenType::kEqualEqual}; +const BinaryOpInfo kExclamationEqual = {3, CelOperator::NOT_EQUALS, false, + TokenType::kExclamationEqual}; +#pragma push_macro("IN") +#undef IN +const BinaryOpInfo kIn = {3, CelOperator::IN, false, TokenType::kIn}; +#pragma pop_macro("IN") +const BinaryOpInfo kPlus = {4, CelOperator::ADD, false, TokenType::kPlus}; +const BinaryOpInfo kMinus = {4, CelOperator::SUBTRACT, false, + TokenType::kMinus}; +const BinaryOpInfo kAsterisk = {5, CelOperator::MULTIPLY, false, + TokenType::kAsterisk}; +const BinaryOpInfo kSlash = {5, CelOperator::DIVIDE, false, TokenType::kSlash}; +const BinaryOpInfo kPercent = {5, CelOperator::MODULO, false, + TokenType::kPercent}; +const BinaryOpInfo kDefaultOpInfo = {0, "", false, TokenType::kError}; + +} // namespace + +const BinaryOpInfo& GetBinaryOpInfo(TokenType type) { + switch (type) { + case TokenType::kLogicalOr: + return kLogicalOr; + case TokenType::kLogicalAnd: + return kLogicalAnd; + case TokenType::kLess: + return kLess; + case TokenType::kLessEqual: + return kLessEqual; + case TokenType::kGreater: + return kGreater; + case TokenType::kGreaterEqual: + return kGreaterEqual; + case TokenType::kEqualEqual: + return kEqualEqual; + case TokenType::kExclamationEqual: + return kExclamationEqual; + case TokenType::kIn: + return kIn; + case TokenType::kPlus: + return kPlus; + case TokenType::kMinus: + return kMinus; + case TokenType::kAsterisk: + return kAsterisk; + case TokenType::kSlash: + return kSlash; + case TokenType::kPercent: + return kPercent; + default: + return kDefaultOpInfo; + } +} + +ParserWorker::ParserWorker( + const cel::Source& source, const cel::ParserOptions& options, + std::vector* absl_nullable parse_issues, + bool track_node_ranges) + : source_(source), + options_(options), + lexer_(source_), + parse_issues_(parse_issues), + track_node_ranges_(track_node_ranges) {} + +void ParserWorker::InitTokenStream() { + current_token_ = Token{.type = TokenType::kError, .start = 0, .end = 0}; + peek_token_ = NextSignificantToken(); +} + +std::string ParserWorker::GetTokenText(const Token& tok) const { + if (tok.start >= 0 && tok.end >= tok.start && + tok.end <= static_cast(source_.content().size())) { + return source_.content().ToString(tok.start, tok.end); + } + return ""; +} + +Token ParserWorker::NextSignificantToken(bool report_error) { + if (is_recovery_limit_exceeded()) { + return Token{.type = TokenType::kEnd, .start = 0, .end = 0}; + } + while (true) { + Token tok = lexer_.Lex(); + if (tok.type == TokenType::kWhitespace || tok.type == TokenType::kComment) { + continue; + } + if (tok.type == TokenType::kError && report_error) { + ReportSyntaxError(tok, lexer_.GetError().message); + if (is_recovery_limit_exceeded()) { + return Token{.type = TokenType::kEnd, .start = 0, .end = 0}; + } + } + return tok; + } +} + +Token ParserWorker::NextToken() { + current_token_ = peek_token_; + if (is_recovery_limit_exceeded()) { + peek_token_ = Token{.type = TokenType::kEnd, .start = 0, .end = 0}; + return current_token_; + } + if (peek_token_.type != TokenType::kEnd) { + peek_token_ = NextSignificantToken(); + } + return current_token_; +} + +bool ParserWorker::Expect(TokenType type, absl::string_view msg) { + if (peek_token_.type == type) { + NextToken(); + return true; + } + if (recursion_limit_exceeded_ || is_recovery_limit_exceeded()) { + return false; + } + if (peek_token_.type != TokenType::kError) { + std::string err_msg; + if (msg.empty()) { + std::string tok_text = GetTokenText(peek_token_); + std::string formatted_tok; + if (peek_token_.type == TokenType::kEnd) { + formatted_tok = ""; + } else { + formatted_tok = absl::StrCat("'", tok_text, "'"); + } + err_msg = absl::StrCat("mismatched input ", formatted_tok, " expecting '", + TokenTypeToString(type), "'"); + } else { + err_msg = std::string(msg); + } + ReportSyntaxError(peek_token_, err_msg); + } + SynchronizeOnDelimiter(); + return false; +} + +void ParserWorker::SynchronizeOnDelimiter() { + if (recursion_limit_exceeded_ || is_recovery_limit_exceeded()) { + peek_token_ = Token{.type = TokenType::kEnd, .start = 0, .end = 0}; + return; + } + while (peek_token_.type != TokenType::kEnd) { + if (peek_token_.type == TokenType::kComma || + peek_token_.type == TokenType::kRightParen || + peek_token_.type == TokenType::kRightBracket || + peek_token_.type == TokenType::kRightBrace) { + break; + } + NextToken(); + } +} + +// Checks whether the current identifier is the root of a struct/message +// creation expression (`CreateMessage` in grammar: `'.'? IDENTIFIER ('.' +// IDENTIFIER)* '{' ... '}'`). Unlike standalone identifiers or selector +// chains, reserved identifiers (e.g. `import{}` or `import.Foo{}`) are +// permitted in message type names. +bool ParserWorker::IsStructCreationAhead() { + if (peek_token_.type == TokenType::kLeftBrace) { + return true; + } + if (peek_token_.type != TokenType::kDot) { + return false; + } + const int32_t saved_pos = lexer_.SavePosition(); + auto restore_lexer = absl::MakeCleanup( + [this, saved_pos] { lexer_.RestorePosition(saved_pos); }); + Token tok = peek_token_; + while (tok.type == TokenType::kDot) { + tok = NextSignificantToken(/*report_error=*/false); + if (tok.type != TokenType::kIdent && tok.type != TokenType::kReservedWord) { + return false; + } + // quoted identifiers are not allowed in struct creation expressions + if (tok.start < tok.end && source_.content().at(tok.start) == '`') { + return false; + } + tok = NextSignificantToken(/*report_error=*/false); + } + return tok.type == TokenType::kLeftBrace; +} + +int64_t ParserWorker::NextId(int32_t position) { + int64_t id = next_id_++; + if (id > options_.expression_node_limit && !node_limit_exceeded_) { + ReportError(position, "expression node limit exceeded"); + node_limit_exceeded_ = true; + } + if (position >= 0) { + positions_.insert({id, position}); + if (ABSL_PREDICT_FALSE(track_node_ranges_)) { + node_ranges_.insert({id, {position, position}}); + } + } + return id; +} + +int64_t ParserWorker::NextId() { return NextId(-1); } + +void ParserWorker::SetPosition(int64_t id, const Token& token) { + if (token.start >= 0) { + positions_[id] = token.start; + SetNodeRange(id, token.start, + token.end > token.start ? token.end - 1 : token.start); + } +} + +int64_t ParserWorker::CopyId(int64_t id) { + if (id == 0) { + return 0; + } + int32_t pos = 0; + if (auto it = positions_.find(id); it != positions_.end()) { + pos = it->second; + } + int64_t new_id = NextId(pos); + if (ABSL_PREDICT_FALSE(track_node_ranges_)) { + if (auto it = node_ranges_.find(id); it != node_ranges_.end()) { + node_ranges_[new_id] = it->second; + } + } + return new_id; +} + +void ParserWorker::EraseId(int64_t id) { + positions_.erase(id); + if (ABSL_PREDICT_FALSE(track_node_ranges_)) { + node_ranges_.erase(id); + } + if (next_id_ == id + 1) { + --next_id_; + } +} + +void ParserWorker::ReportError(int32_t position, absl::string_view msg) { + cel::SourceLocation loc; + if (auto found = source_.GetLocation(position); found.has_value()) { + loc = *found; + } + ReportError(loc, msg); +} + +void ParserWorker::ReportError(const SourceLocation& loc, + absl::string_view msg) { + if (error_count_ > options_.error_recovery_limit) { + return; + } + error_count_++; + if (error_count_ == options_.error_recovery_limit + 1) { + if (parse_issues_ != nullptr) { + parse_issues_->push_back( + cel::ParseIssue(absl::StrFormat("Error recovery limit (%d) exceeded", + options_.error_recovery_limit))); + } + peek_token_ = Token{.type = TokenType::kEnd, .start = 0, .end = 0}; + } + if (parse_issues_ != nullptr && + error_count_ <= options_.error_recovery_limit) { + parse_issues_->push_back(cel::ParseIssue(loc, std::string(msg))); + } +} + +void ParserWorker::ReportSyntaxError(const Token& token, + absl::string_view msg) { + ReportError(token.start, absl::StrCat("Syntax error: ", msg)); +} + +} // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h new file mode 100644 index 000000000..0aac95710 --- /dev/null +++ b/parser/internal/pratt_parser_worker.h @@ -0,0 +1,1372 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/base/optimization.h" +#include "absl/container/flat_hash_map.h" +#include "absl/status/statusor.h" +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" +#include "common/operators.h" +#include "common/source.h" +#include "internal/lexis.h" +#include "internal/strings.h" +#include "parser/internal/ast_factory_interface.h" +#include "parser/internal/lexer.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { + +class ParserWorker { + public: + ParserWorker(const cel::Source& source, const cel::ParserOptions& options, + std::vector* absl_nullable parse_issues, + bool track_node_ranges = false); + + const absl::flat_hash_map& GetNodePositions() const { + return positions_; + } + const absl::flat_hash_map>& + GetNodeRanges() const { + return node_ranges_; + } + absl::Span GetLineOffsets() const { + return source_.line_offsets(); + } + bool has_errors() const { return error_count_ > 0; } + bool is_recursion_limit_exceeded() const { return recursion_limit_exceeded_; } + + protected: + const cel::Source& source() const { return source_; } + const cel::ParserOptions& options() const { return options_; } + // Token stream management + void InitTokenStream(); + Token NextSignificantToken(bool report_error = true); + Token NextToken(); + bool Expect(TokenType type, absl::string_view msg = ""); + std::string GetTokenText(const Token& tok) const; + void SynchronizeOnDelimiter(); + // Returns true if the tokens following the current identifier form the + // remainder of a struct/message creation type name (e.g. `.Foo.Bar{...}`). + // Used to disambiguate between selectors and struct creation expressions + // based on the grammar rule, which permits CEL reserved words in message type + // names. + bool IsStructCreationAhead(); + + // ID and Position tracking + int64_t NextId(int32_t position); + int64_t NextId(const Token& token) { + int64_t id = NextId(token.start); + SetNodeRange(id, token.start, + token.end > token.start ? token.end - 1 : token.start); + return id; + } + int64_t NextId(); + void SetPosition(int64_t id, const Token& token); + int64_t CopyId(int64_t id); + void EraseId(int64_t id); + void SetNodeRange(int64_t id, int32_t begin, int32_t end) { + if (ABSL_PREDICT_FALSE(track_node_ranges_)) { + if (id != 0 && begin >= 0 && end >= begin) { + node_ranges_[id] = {begin, end}; + } + } + } + + // Recursion budget accounting. + // + // `recursion_depth_` tracks the levels currently held by live parse frames. + // Chains that are parsed iteratively (operator chains, selector chains, runs + // of parentheses or unary operators) do not add frames, so they report the + // levels they consume through `chain_depth` arguments and through + // `last_parsed_depth_` instead. Counting them keeps `max_recursion_depth` a + // bound on the depth of the resulting AST, as it is for the ANTLR parser. + // + // Returns true and trips the recursion limit if parsing `chain_depth` further + // levels on top of the current depth would exceed the configured limit. + bool CheckRecursion(int chain_depth) { + if (ABSL_PREDICT_FALSE(recursion_depth_ + chain_depth > + options_.max_recursion_depth)) { + recursion_limit_exceeded_ = true; + peek_token_ = Token{.type = TokenType::kEnd, .start = 0, .end = 0}; + return true; + } + return false; + } + + // Error reporting and recovery + bool is_recovery_limit_exceeded() const { + return error_count_ > options_.error_recovery_limit; + } + void ReportError(int32_t position, absl::string_view msg); + void ReportError(const SourceLocation& loc, absl::string_view msg); + void ReportError(const Token& token, absl::string_view msg) { + ReportError(token.start, msg); + } + void ReportSyntaxError(const Token& token, absl::string_view msg); + + const cel::Source& source_; + cel::ParserOptions options_; + Lexer lexer_; + Token current_token_; + Token peek_token_; + int recursion_depth_ = 0; + // Depth of the subtree parsed most recently, reported upward so that callers + // can fold it into their own chain depth. + int last_parsed_depth_ = 0; + int64_t next_id_ = 1; + bool node_limit_exceeded_ = false; + absl::flat_hash_map positions_; + absl::flat_hash_map> node_ranges_; + std::vector* absl_nullable parse_issues_; + int error_count_ = 0; + bool lexer_error_reported_ = false; + bool recursion_limit_exceeded_ = false; + bool track_node_ranges_ = false; +}; + +struct BinaryOpInfo { + int precedence = 0; + absl::string_view name; + bool is_logical = false; + TokenType type = TokenType::kError; +}; + +// Generic Pratt parser implementation parameterized by the AST node type +// (`ExprNode`). +// +// This class implements the core recursive-descent and operator-precedence +// parsing logic for CEL without depending on a concrete expression node data +// structure. All inspection and construction of AST nodes is performed through +// `AstFactoryInterface`. +// +// See `AstFactoryInterface` documentation in `ast_factory_interface.h` for +// details on how to use this generic parser with alternative AST structures. +template +class PrattParserWorker : public ParserWorker { + public: + using ParserWorker::NextId; + using ParserWorker::SetPosition; + + explicit PrattParserWorker( + const cel::Source& source, const cel::ParserOptions& options, + std::vector* absl_nullable parse_issues, + AstFactoryInterface& ast_factory, + bool track_node_ranges = false) + : ParserWorker(source, options, parse_issues, track_node_ranges), + ast_factory_(ast_factory) { + this->InitTokenStream(); + } + + ExprNode Parse(); + + absl::flat_hash_map ReleaseMacroCalls() { + return std::move(macro_calls_); + } + + private: + class MacroExpanderSupport : public MacroExprExpanderSupport { + public: + MacroExpanderSupport(PrattParserWorker& worker, int32_t macro_position) + : worker_(worker), macro_position_(macro_position) {} + + int64_t NextId() { + return macro_position_ >= 0 ? worker_.NextId(macro_position_) + : worker_.NextId(); + } + + int64_t CopyId(int64_t id) { return worker_.CopyId(id); } + + ExprNode ReportError(absl::string_view message) { + if (macro_position_ >= 0) { + worker_.ReportError(macro_position_, message); + } else { + worker_.ReportError(worker_.current_token_.start, message); + } + return ExprNode(); + } + + ExprNode ReportErrorAt(const ExprNode& expr, absl::string_view message) { + int32_t pos = 0; + auto it = + worker_.GetNodePositions().find(worker_.ast_factory_.GetId(expr)); + if (it != worker_.GetNodePositions().end()) { + pos = it->second; + } + worker_.ReportError(pos, message); + return ExprNode(); + } + + private: + PrattParserWorker& worker_; + int32_t macro_position_; + }; + + using CelOperator = ::google::api::expr::common::CelOperator; + + ExprNode ParseExpr(); + + // Parses one element of a delimited construct (list, map, struct or argument + // list), accumulating `last_parsed_depth_` to the deepest element seen so + // far. A construct is as deep as its deepest element, not its last one, so + // callers must reset `last_parsed_depth_` to 0 before the first element. + ExprNode ParseElementExpr(); + // Parses binary operator expressions and ternary conditional expressions + // (`? :`) using operator-precedence (Pratt) parsing. Consumes operators from + // the token stream whose binding precedence is greater than or equal to + // `min_prec`. + // + // Example (`min_prec = 0` on `a + b * c`): Parses prefix/postfix expression + // `a`, encounters `+` (precedence 1 >= 0), recurses with `ParseBinary(2)` to + // parse `b * c` at higher precedence, and builds the `+` call node. + // + // Example (`min_prec = 0` on `a ? b : c`): Parses condition `a`, encounters + // `?`, consumes `?`, and recurses with `ParseBinary(1)` for true branch `b` + // and `ParseBinary(0)` for false branch `c`. + ExprNode ParseBinaryAndTernary(int min_prec); + // `initial_chain_depth` is the depth already accumulated by `lhs`, which the + // operators consumed by this loop keep accumulating on top of. + void ParseBinaryAndTernaryFromLhs(ExprNode& lhs, int min_prec, + int initial_chain_depth); + + // Parses ternary conditional expressions (`condition ? true_expr : + // false_expr`). + void ParseTernary(ExprNode& lhs); + + // Helper method for parsing a contiguous chain of same-precedence logical + // operators (`&&` or `||`) iteratively into a list of terms and operator IDs. + // Constructs either a balanced binary AST (`(a && b) && c`) or a single + // variadic call (`_&&_(a, b, c)`) via `BalanceLogical`. This avoids deep C++ + // recursion when processing expressions with dozens of chained logical terms. + // + // Example (`a && b && c && d`): Iteratively collects terms `[a, b, c, d]` and + // builds `((a && b) && c) && d` without ascending/descending C++ stack + // frames for each term. + void ParseBalancedLogicalChain(ExprNode& lhs, const BinaryOpInfo& op_info); + + // Parses prefix unary operators (`!`, `-`) via `ParseUnaryOps()`, or + // delegates to `ParseMember()` to parse a primary expression and any + // trailing postfix operations. + // + // Example (`!a.b[0].c(x)`): `ParseUnaryOps()` consumes `!` and calls + // `ParseMember()` for `a.b[0].c(x)`. + ExprNode ParseSelectorChain(); + + // Parses a primary expression (`ParsePrimary()`) followed by any trailing + // postfix member, indexing, method call, or struct initializer operations + // (`ParseSelectorChainTail()`), capturing the start offset of the member + // expression for source position tracking in optional field selections. + // + // Example (`a.b[0].c(x)`): Calls `ParsePrimary()` to obtain `a`, then calls + // `ParseSelectorChainTail()` to loop iteratively through `.b`, `[0]`, and + // `.c(x)`. + ExprNode ParseMember(); + + // Handler for postfix member, index, receiver method, and + // struct initializer operations (`.field`, `[index]`, `.method(args)`, + // `Type{field: val}`). + // + // Processes continuous postfix operation chains iteratively. + // + // `initial_chain_depth` is the depth already accumulated by `lhs`: a + // parenthesized primary such as `(a.b.c)` contributes its own depth, which + // the selectors trailing the closing `)` continue to accumulate on top of. + void ParseSelectorChainTail(ExprNode& lhs, int32_t member_start_position, + bool can_be_struct_name, int initial_chain_depth); + + // Parses prefix unary operators (`!`, `-`). + ExprNode ParseUnaryOps(); + + // Parses primary leaf expressions (`nud` atomic atoms), including + // parenthesized expressions (`(expr)`), literal constants (`null`, `true`, + // `false`, numbers, strings, bytes), collection initializers (`[list]`, + // `{map}`), identifiers + // (`foo`), and global or namespace function/macro calls (`foo(args)`, + // `.pkg.func(args)`). + // + // For parenthesized expressions and collection literals, delegates back to + // `ParseExpr()` or collection subroutines (`ParseList()`, `ParseMap()`). + // For global function calls (`foo(args)`), attempts macro expansion before + // falling back to standard call node construction. + // + // Example (`(a + b)`): Consumes `(`, recurses to `ParseExpr()`, and expects + // `)`. Example (`has(x.y)`): Consumes `has`, parses arguments `(x.y)`, and + // expands the `has` macro. + ExprNode ParsePrimary(bool* absl_nullable can_be_struct_name = nullptr); + + ExprNode ParseList(); + ExprNode ParseMap(); + ExprNode ParseStruct(int64_t obj_id, absl::string_view struct_name); + std::vector ParseArguments(TokenType close_token); + ExprNode ParseIntLiteral(); + ExprNode ParseNegativeIntLiteral(int64_t node_id); + ExprNode ParseUintLiteral(); + ExprNode ParseDoubleLiteral(); + ExprNode ParseNegativeDoubleLiteral(int64_t node_id); + ExprNode ParseStringLiteral(); + ExprNode ParseBytesLiteral(); + void BuildBinaryCall(int64_t op_id, absl::string_view op_name, ExprNode& lhs, + ExprNode rhs); + ExprNode ParseIdentOrCall(bool* absl_nullable can_be_struct_name = nullptr); + std::string NormalizeIdent(const Token& tok, bool allow_quoted, + bool* absl_nullable is_quoted = nullptr); + std::optional ExtractStructName(const ExprNode& expr); + int32_t GetLeftmostPosition(const ExprNode& expr); + ExprNode BalancedTree(absl::string_view op, std::vector& terms, + const std::vector& ops, int lo, int hi); + ExprNode BalanceLogical(absl::string_view op, std::vector terms, + std::vector ops, bool enable_variadic); + + std::optional TryExpandMacro(int64_t expr_id, + absl::string_view function, + ExprNode* target, + std::vector& args); + + // Save the original structure of the expression before macro expansion. + void RecordMacroCall(int64_t macro_id, absl::string_view function, + std::optional target, + std::vector arguments); + + AstFactoryInterface& ast_factory_; + absl::flat_hash_map macro_calls_; +}; + +template +ExprNode PrattParserWorker::Parse() { + ExprNode expr = ParseExpr(); + if (is_recursion_limit_exceeded() || is_recovery_limit_exceeded()) { + return expr; + } + if (peek_token_.type != TokenType::kEnd && + peek_token_.type != TokenType::kError) { + ReportSyntaxError(peek_token_, "unexpected token after expression"); + } + return expr; +} + +template +ExprNode PrattParserWorker::ParseExpr() { + if (recursion_limit_exceeded_ || is_recovery_limit_exceeded()) { + return ExprNode(); + } + if (CheckRecursion(1)) { + return ExprNode(); + } + recursion_depth_++; + ExprNode expr = ParseBinaryAndTernary(0); + recursion_depth_--; + return expr; +} + +template +ExprNode PrattParserWorker::ParseElementExpr() { + int max_depth = last_parsed_depth_; + ExprNode expr = ParseExpr(); + last_parsed_depth_ = std::max(max_depth, last_parsed_depth_); + return expr; +} + +template +void PrattParserWorker::ParseTernary(ExprNode& lhs) { + int64_t op_id = NextId(NextToken()); + std::vector args; + args.reserve(3); + args.push_back(std::move(lhs)); + args.push_back(ParseBinaryAndTernary(1)); + if (!Expect(TokenType::kColon, "expected ':' in conditional expression")) { + return; + } + args.push_back(ParseExpr()); + lhs = ast_factory_.NewCall(op_id, CelOperator::CONDITIONAL, std::move(args)); +} + +const BinaryOpInfo& GetBinaryOpInfo(TokenType type); + +template +void PrattParserWorker::BuildBinaryCall(int64_t op_id, + absl::string_view op_name, + ExprNode& lhs, ExprNode rhs) { + std::vector args; + args.reserve(2); + args.push_back(std::move(lhs)); + args.push_back(std::move(rhs)); + lhs = ast_factory_.NewCall(op_id, std::string(op_name), std::move(args)); +} + +// Parses binary operator expressions and ternary conditional expressions +// (`? :`) using Pratt operator-precedence parsing (e.g., `a + b * c`). +template +ExprNode PrattParserWorker::ParseBinaryAndTernary(int min_prec) { + ExprNode lhs = ParseSelectorChain(); + ParseBinaryAndTernaryFromLhs(lhs, min_prec, last_parsed_depth_); + return lhs; +} + +template +void PrattParserWorker::ParseBinaryAndTernaryFromLhs( + ExprNode& lhs, int min_prec, int initial_chain_depth) { + int chain_depth = initial_chain_depth; + while (!recursion_limit_exceeded_ && !is_recovery_limit_exceeded()) { + TokenType tok = peek_token_.type; + if (tok == TokenType::kQuestion && min_prec <= 0) { + ParseTernary(lhs); + continue; + } + + const BinaryOpInfo& op_info = GetBinaryOpInfo(tok); + if (op_info.precedence < min_prec || op_info.precedence == 0) break; + + if (op_info.is_logical) { + ParseBalancedLogicalChain(lhs, op_info); + continue; + } + + Token op_tok = NextToken(); + if (CheckRecursion(chain_depth)) { + return; + } + chain_depth++; + int64_t op_id = NextId(op_tok); + BuildBinaryCall(op_id, op_info.name, lhs, + ParseBinaryAndTernary(op_info.precedence + 1)); + // `last_parsed_depth_` is the depth of the rhs just parsed. It hangs one + // level below this operator, while `chain_depth` already covers the lhs, so + // the operator node is as deep as whichever side is deeper: `x + a.b.c.d` + // reaches 4 through its rhs and `a.b.c.d + x` reaches 4 through its lhs. + chain_depth = std::max(chain_depth, last_parsed_depth_ + 1); + last_parsed_depth_ = chain_depth; + } +} + +// Parses continuous chains of logical operators (`&&`, `||`) iteratively +// (e.g., `a && b && c`) and constructs a balanced or variadic AST. +template +void PrattParserWorker::ParseBalancedLogicalChain( + ExprNode& lhs, const BinaryOpInfo& op_info) { + std::vector terms; + std::vector ops; + terms.push_back(std::move(lhs)); + while (peek_token_.type == op_info.type) { + Token op_tok = NextToken(); + ExprNode rhs = ParseBinaryAndTernary(op_info.precedence + 1); + ops.push_back(NextId(op_tok)); + terms.push_back(std::move(rhs)); + } + lhs = BalanceLogical(op_info.name, std::move(terms), std::move(ops), + options_.enable_variadic_logical_operators); + last_parsed_depth_ = 0; +} + +template +ExprNode PrattParserWorker::ParseSelectorChain() { + last_parsed_depth_ = 0; + TokenType tok = peek_token_.type; + if (tok == TokenType::kExclamation || tok == TokenType::kMinus) { + return ParseUnaryOps(); + } + return ParseMember(); +} + +template +ExprNode PrattParserWorker::ParseMember() { + last_parsed_depth_ = 0; + int32_t member_start = peek_token_.start; + bool can_be_struct_name = false; + ExprNode lhs = ParsePrimary(&can_be_struct_name); + TokenType tok = peek_token_.type; + if (tok == TokenType::kDot || tok == TokenType::kLeftBracket || + tok == TokenType::kLeftBrace) { + ParseSelectorChainTail(lhs, member_start, can_be_struct_name, + last_parsed_depth_); + } + return lhs; +} + +// Parses prefix and postfix member/indexing operations iteratively +// (e.g., `!a.b[0].c(x)`). +template +void PrattParserWorker::ParseSelectorChainTail( + ExprNode& lhs, int32_t member_start_position, bool can_be_struct_name, + int initial_chain_depth) { + int chain_depth = initial_chain_depth; + while (true) { + TokenType tok = peek_token_.type; + if (tok == TokenType::kDot) { + if (CheckRecursion(chain_depth)) { + return; + } + chain_depth++; + Token dot_tok = NextToken(); + bool optional = false; + if (peek_token_.type == TokenType::kQuestion) { + NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(dot_tok, "unsupported syntax '.?'"); + } + } + Token id_tok = NextToken(); + if (id_tok.type != TokenType::kIdent && + id_tok.type != TokenType::kReservedWord) { + if (id_tok.type != TokenType::kError) { + ReportSyntaxError(id_tok, "expected identifier after '.'"); + } + SynchronizeOnDelimiter(); + last_parsed_depth_ = chain_depth; + return; + } + bool is_member_call = peek_token_.type == TokenType::kLeftParen; + bool is_quoted = false; + std::string id_text = + NormalizeIdent(id_tok, /*allow_quoted=*/!is_member_call, &is_quoted); + if (optional) { + int64_t op_id = NextId(dot_tok); + std::vector args; + args.reserve(2); + args.push_back(std::move(lhs)); + int64_t field_id = NextId(member_start_position); + SetNodeRange(field_id, member_start_position, + id_tok.end > member_start_position + ? id_tok.end - 1 + : member_start_position); + args.push_back( + ast_factory_.NewStringConst(field_id, std::move(id_text))); + lhs = ast_factory_.NewCall(op_id, CelOperator::OPT_SELECT, + std::move(args)); + can_be_struct_name = false; + } else if (peek_token_.type == TokenType::kLeftParen) { + Token lparen = NextToken(); + int64_t call_id = NextId(lparen); + std::vector args = ParseArguments(TokenType::kRightParen); + if (std::optional expanded = + TryExpandMacro(call_id, id_text, &lhs, args); + expanded.has_value()) { + lhs = std::move(*expanded); + } else { + lhs = ast_factory_.NewMemberCall(call_id, id_text, std::move(lhs), + std::move(args)); + } + // `ParseArguments` leaves `last_parsed_depth_` at the deepest argument. + // Arguments hang one level below the call node, so `a.f(b.c.d.e)` is 4 + // deep. The max preserves the selectors already walked when the + // arguments are shallower, as in `a.b.c.f(1)`. + chain_depth = std::max(chain_depth, last_parsed_depth_ + 1); + can_be_struct_name = false; + } else { + lhs = ast_factory_.NewSelect(NextId(dot_tok), std::move(lhs), id_text); + can_be_struct_name = can_be_struct_name && !is_quoted; + } + } else if (tok == TokenType::kLeftBracket) { + if (CheckRecursion(chain_depth)) { + return; + } + chain_depth++; + Token bracket_tok = NextToken(); + int64_t op_id = NextId(bracket_tok); + bool optional = false; + if (peek_token_.type == TokenType::kQuestion) { + NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(bracket_tok, "unsupported syntax '?'"); + } + } + ExprNode index = ParseExpr(); + Expect(TokenType::kRightBracket, "expected ']'"); + std::vector args; + args.reserve(2); + args.push_back(std::move(lhs)); + args.push_back(std::move(index)); + lhs = ast_factory_.NewCall( + op_id, optional ? CelOperator::OPT_INDEX : CelOperator::INDEX, + std::move(args)); + // `last_parsed_depth_` is the depth of the index expression just parsed. + // It hangs one level below the index node, so `a[b.c.d.e]` is 4 deep. The + // max preserves the selectors already walked when the index is shallower, + // as in `a.b.c[0]`. + chain_depth = std::max(chain_depth, last_parsed_depth_ + 1); + can_be_struct_name = false; + } else if (tok == TokenType::kLeftBrace) { + if (!can_be_struct_name) { + break; + } + int32_t struct_pos = GetLeftmostPosition(lhs); + if (auto struct_name = ExtractStructName(lhs); struct_name.has_value()) { + lhs = ParseStruct(NextId(struct_pos), *struct_name); + // `ParseStruct` leaves `last_parsed_depth_` at the deepest field value, + // which carries through unchanged: `Msg{f: a.b.c.d}` is 3 deep. There + // is no `+ 1` here because struct creation is a primary rather than a + // chain link, so it adds no level of its own. The max preserves the + // selectors already walked when the fields are shallower, as in + // `a.b.Msg{f: 1}`. + chain_depth = std::max(chain_depth, last_parsed_depth_); + can_be_struct_name = false; + } else { + break; + } + } else { + break; + } + } + last_parsed_depth_ = chain_depth; +} + +template +ExprNode PrattParserWorker::ParseUnaryOps() { + struct UnaryOp { + Token token; + int64_t id = 0; + }; + Token first_op = NextToken(); + TokenType op_type = first_op.type; + std::vector ops; + ops.push_back({first_op}); + while (peek_token_.type == op_type) { + ops.push_back({NextToken()}); + } + + if (op_type == TokenType::kMinus && ops.size() == 1 && + (peek_token_.type == TokenType::kInt || + peek_token_.type == TokenType::kFloat)) { + ExprNode lhs = (peek_token_.type == TokenType::kInt) + ? ParseNegativeIntLiteral(NextId(first_op)) + : ParseNegativeDoubleLiteral(NextId(first_op)); + TokenType tok = peek_token_.type; + if (tok == TokenType::kDot || tok == TokenType::kLeftBracket || + tok == TokenType::kLeftBrace) { + ParseSelectorChainTail(lhs, first_op.start, + /*can_be_struct_name=*/false, + /*initial_chain_depth=*/0); + } + return lhs; + } + + if (options_.fold_unary_operators) { + if (ops.size() % 2 == 0) { + ops.clear(); + } else { + ops.resize(1); + } + } + + for (auto& op : ops) { + op.id = NextId(op.token); + } + + // Every retained operator wraps the operand in one more call node, so the run + // costs as many levels as it has operators even though it is parsed by a + // loop. The outermost one is the deepest, so checking it covers the rest. + if (!ops.empty() && CheckRecursion(static_cast(ops.size()) - 1)) { + return ExprNode(); + } + recursion_depth_ += static_cast(ops.size()); + + ExprNode operand; + if (op_type == TokenType::kExclamation && + peek_token_.type == TokenType::kMinus) { + Token minus_tok = NextToken(); + if (peek_token_.type == TokenType::kInt) { + operand = ParseNegativeIntLiteral(NextId(minus_tok)); + ParseSelectorChainTail(operand, minus_tok.start, + /*can_be_struct_name=*/false, + /*initial_chain_depth=*/0); + } else if (peek_token_.type == TokenType::kFloat) { + operand = ParseNegativeDoubleLiteral(NextId(minus_tok)); + ParseSelectorChainTail(operand, minus_tok.start, + /*can_be_struct_name=*/false, + /*initial_chain_depth=*/0); + } else { + ReportSyntaxError(minus_tok, "unexpected '-'"); + operand = ParseMember(); + } + } else { + operand = ParseMember(); + } + recursion_depth_ -= static_cast(ops.size()); + if (recursion_limit_exceeded_) { + return ExprNode(); + } + + for (int i = static_cast(ops.size()) - 1; i >= 0; --i) { + std::vector args; + args.push_back(std::move(operand)); + absl::string_view op_name = (ops[i].token.type == TokenType::kExclamation) + ? CelOperator::LOGICAL_NOT + : CelOperator::NEGATE; + operand = + ast_factory_.NewCall(ops[i].id, std::string(op_name), std::move(args)); + } + + return operand; +} + +// Parses identifiers (e.g., `foo`, `.foo`) and global function or macro calls +// (e.g., `foo(args)`, `has(x.y)`). +template +ExprNode PrattParserWorker::ParseIdentOrCall( + bool* absl_nullable can_be_struct_name) { + TokenType tok_type = peek_token_.type; + bool leading_dot = false; + Token first_tok = peek_token_; + if (tok_type == TokenType::kDot) { + NextToken(); + leading_dot = true; + } + Token id_tok = NextToken(); + if (id_tok.type != TokenType::kIdent && + id_tok.type != TokenType::kReservedWord) { + if (id_tok.type != TokenType::kError) { + ReportSyntaxError(id_tok, "expected identifier"); + } + return ast_factory_.NewUnspecified(NextId(id_tok)); + } + bool is_quoted = false; + std::string id_text = + NormalizeIdent(id_tok, /*allow_quoted=*/false, &is_quoted); + if (ABSL_PREDICT_FALSE(id_tok.type == TokenType::kReservedWord)) { + if (cel::internal::LexisIsReserved(id_text) && !IsStructCreationAhead()) { + ReportError(id_tok, absl::StrFormat("reserved identifier: %s", id_text)); + } + } + std::string name = + leading_dot ? absl::StrCat(".", id_text) : std::string(id_text); + if (peek_token_.type == TokenType::kLeftParen) { + Token lparen = NextToken(); + int64_t call_id = NextId(lparen); + std::vector args = ParseArguments(TokenType::kRightParen); + if (auto expanded = TryExpandMacro(call_id, name, nullptr, args); + expanded.has_value()) { + return std::move(*expanded); + } + return ast_factory_.NewCall(call_id, name, std::move(args)); + } + if (can_be_struct_name != nullptr) { + *can_be_struct_name = !is_quoted; + } + int64_t id = NextId(leading_dot ? first_tok : id_tok); + return ast_factory_.NewIdent(id, std::move(name)); +} + +// Parses primary leaf expressions, including parenthesized expressions +// (`(expr)`), literals (`null`, `true`, numbers, strings, bytes), collection +// initializers +// (`[...]`, `{...}`), and identifiers/global function calls (`foo`, +// `has(x.y)`). +template +ExprNode PrattParserWorker::ParsePrimary( + bool* absl_nullable can_be_struct_name) { + if (can_be_struct_name != nullptr) { + *can_be_struct_name = false; + } + switch (peek_token_.type) { + case TokenType::kLeftParen: { + if (recursion_limit_exceeded_ || is_recovery_limit_exceeded()) { + return ExprNode(); + } + // To avoid deep call-stack recursion on heavily nested parentheses (e.g. + // "((((a))))" or "((((a + 1) + 1) + 1))"), consume all consecutive + // leading '(' tokens upfront, parse the innermost expression once, and + // then iteratively unwind each enclosing '(' from innermost to outermost. + // After consuming each matching ')', if more enclosing '(' remain open + // and the next token is not another ')', continue parsing any trailing + // selectors or binary/ternary operators belonging to that enclosing + // parenthesized level using the already-parsed inner expression as the + // LHS. + int open_parens = 1; + NextToken(); + std::vector extra_paren_starts; + while (peek_token_.type == TokenType::kLeftParen) { + extra_paren_starts.push_back(peek_token_.start); + open_parens++; + NextToken(); + } + // Every '(' is a nesting level and costs one unit of recursion budget, so + // charge all of them here. `recursion_depth_` is already 1 for the + // enclosing expression (as in ParseUnaryOpsChain), so `open_parens` + // parentheses reach depth `recursion_depth_ + open_parens - 1`. + // `recursion_depth_` itself only advances by 1 because the parens are + // unwound iteratively and entering ParseBinaryAndTernary() below adds + // just one stack frame. + if (CheckRecursion(open_parens - 1)) { + return ExprNode(); + } + recursion_depth_++; + ExprNode expr = ParseBinaryAndTernary(0); + int chain_depth = last_parsed_depth_; + for (int i = 0; i < open_parens; ++i) { + Expect(TokenType::kRightParen); + if (i < open_parens - 1 && peek_token_.type != TokenType::kRightParen) { + TokenType tok = peek_token_.type; + if (tok == TokenType::kDot || tok == TokenType::kLeftBracket) { + ParseSelectorChainTail(expr, + extra_paren_starts[open_parens - 2 - i], + /*can_be_struct_name=*/false, chain_depth); + } + ParseBinaryAndTernaryFromLhs(expr, 0, last_parsed_depth_); + chain_depth = last_parsed_depth_; + } + } + recursion_depth_--; + last_parsed_depth_ = chain_depth; + return expr; + } + case TokenType::kNull: + return ast_factory_.NewNullConst(NextId(NextToken())); + case TokenType::kTrue: + case TokenType::kFalse: { + Token tok = NextToken(); + return ast_factory_.NewBoolConst(NextId(tok), + tok.type == TokenType::kTrue); + } + case TokenType::kInt: + return ParseIntLiteral(); + case TokenType::kUint: + return ParseUintLiteral(); + case TokenType::kFloat: + return ParseDoubleLiteral(); + case TokenType::kString: + return ParseStringLiteral(); + case TokenType::kBytes: + return ParseBytesLiteral(); + case TokenType::kLeftBracket: + return ParseList(); + case TokenType::kLeftBrace: + return ParseMap(); + case TokenType::kDot: + case TokenType::kIdent: + case TokenType::kReservedWord: + return ParseIdentOrCall(can_be_struct_name); + default: { + Token bad_tok = NextToken(); + if (bad_tok.type != TokenType::kError) { + if (bad_tok.type == TokenType::kEnd) { + ReportSyntaxError(bad_tok, + "mismatched input '' expecting expression"); + } else { + ReportSyntaxError(bad_tok, "unexpected token"); + } + } + return ast_factory_.NewUnspecified(NextId(bad_tok)); + } + } +} + +// Parses list creation literals (e.g., `[1, 2, ?3]`). +template +ExprNode PrattParserWorker::ParseList() { + Token open_tok = NextToken(); + int64_t list_id = NextId(open_tok); + ListNodeBuilder builder = ast_factory_.NewListBuilder(list_id); + last_parsed_depth_ = 0; + while (peek_token_.type != TokenType::kRightBracket && + peek_token_.type != TokenType::kEnd) { + bool optional = false; + if (peek_token_.type == TokenType::kQuestion) { + Token q = NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(q, "unsupported syntax '?'"); + } + } + builder.Add(ParseElementExpr(), optional); + if (peek_token_.type == TokenType::kComma) { + NextToken(); + } else { + break; + } + } + if (Expect(TokenType::kRightBracket, "expected ']'")) { + SetNodeRange(list_id, open_tok.start, current_token_.end - 1); + } + return builder.Build(); +} + +// Parses map creation literals (e.g., `{"key": "value", ?"opt_key": 42}`). +template +ExprNode PrattParserWorker::ParseMap() { + Token open_tok = NextToken(); + int64_t map_id = NextId(open_tok); + MapNodeBuilder builder = ast_factory_.NewMapBuilder(map_id); + last_parsed_depth_ = 0; + while (peek_token_.type != TokenType::kRightBrace && + peek_token_.type != TokenType::kEnd) { + bool optional = false; + Token key_start = peek_token_; + if (key_start.type == TokenType::kQuestion) { + Token q = NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(q, "unsupported syntax '?'"); + } + key_start = peek_token_; + } + int64_t entry_id = NextId(); + ExprNode key = ParseElementExpr(); + Token colon = peek_token_; + if (!Expect(TokenType::kColon, "expected ':' in map entry")) { + break; + } + SetPosition(entry_id, colon); + builder.Add(entry_id, std::move(key), ParseElementExpr(), optional); + if (peek_token_.type == TokenType::kComma) { + NextToken(); + } else { + break; + } + } + if (Expect(TokenType::kRightBrace, "expected '}'")) { + SetNodeRange(map_id, open_tok.start, current_token_.end - 1); + } + return builder.Build(); +} + +// Parses struct literas for a type name (e.g., `Msg{field: 10, ?opt: "val"}`). +template +ExprNode PrattParserWorker::ParseStruct( + int64_t obj_id, absl::string_view struct_name) { + Token open_tok = NextToken(); + StructNodeBuilder builder = + ast_factory_.NewStructBuilder(obj_id, std::string(struct_name)); + last_parsed_depth_ = 0; + while (peek_token_.type != TokenType::kRightBrace && + peek_token_.type != TokenType::kEnd) { + bool optional = false; + if (peek_token_.type == TokenType::kQuestion) { + Token q = NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(q, "unsupported syntax '?'"); + } + } + Token field_tok = NextToken(); + if (field_tok.type != TokenType::kIdent && + field_tok.type != TokenType::kReservedWord) { + ReportSyntaxError(field_tok, "expected struct field name"); + SynchronizeOnDelimiter(); + break; + } + std::string field_name = NormalizeIdent(field_tok, /*allow_quoted=*/true); + Token colon = peek_token_; + if (!Expect(TokenType::kColon, "expected ':' in struct field")) { + break; + } + int64_t field_id = NextId(colon); + builder.Add(field_id, std::move(field_name), ParseElementExpr(), optional); + if (peek_token_.type == TokenType::kComma) { + NextToken(); + } else { + break; + } + } + if (Expect(TokenType::kRightBrace, "expected '}'")) { + int32_t start_pos = open_tok.start; + auto it = positions_.find(obj_id); + if (it != positions_.end()) { + start_pos = it->second; + } + SetNodeRange(obj_id, start_pos, current_token_.end - 1); + } + return builder.Build(); +} + +// Parses comma-separated arguments (e.g., `(arg1, arg2)` in call). +template +std::vector PrattParserWorker::ParseArguments( + TokenType close_token) { + std::vector args; + last_parsed_depth_ = 0; + if (peek_token_.type != close_token && peek_token_.type != TokenType::kEnd) { + while (true) { + args.push_back(ParseElementExpr()); + if (peek_token_.type == TokenType::kComma) { + NextToken(); + if (peek_token_.type == close_token) { + ReportError(peek_token_, "unexpected token"); + break; + } + continue; + } + break; + } + } + Expect(close_token); + return args; +} + +// Parses decimal & hexadecimal ints (e.g., `42`, `0x1A`). +template +ExprNode PrattParserWorker::ParseIntLiteral() { + Token tok = NextToken(); + std::string value(GetTokenText(tok)); + int64_t int_val = 0; + if (absl::StartsWith(value, "0x") || absl::StartsWith(value, "0X")) { + if (absl::SimpleHexAtoi(value, &int_val)) { + return ast_factory_.NewIntConst(NextId(tok), int_val); + } + } else if (absl::SimpleAtoi(value, &int_val)) { + return ast_factory_.NewIntConst(NextId(tok), int_val); + } + ReportSyntaxError(tok, "invalid int literal"); + return ast_factory_.NewUnspecified(NextId(tok)); +} + +template +ExprNode PrattParserWorker::ParseNegativeIntLiteral(int64_t node_id) { + Token lit_tok = NextToken(); + std::string text = GetTokenText(lit_tok); + int64_t int_val = 0; + bool success = false; + if (absl::StartsWith(text, "0x") || absl::StartsWith(text, "0X")) { + uint64_t uint_val = 0; + if (absl::SimpleHexAtoi(text, &uint_val)) { + if (uint_val <= uint64_t{0x8000000000000000}) { + if (uint_val == uint64_t{0x8000000000000000}) { + int_val = std::numeric_limits::min(); + } else { + int_val = -static_cast(uint_val); + } + success = true; + } + } + } else if (absl::SimpleAtoi(text, &int_val)) { + int_val = -int_val; + success = true; + } else { + // Separately handle -2^63, which is not representable as -(2^63) + std::string val = absl::StrCat("-", text); + success = absl::SimpleAtoi(val, &int_val); + } + + if (success) { + return ast_factory_.NewIntConst(node_id, int_val); + } + ReportSyntaxError(lit_tok, "invalid int literal"); + return ast_factory_.NewUnspecified(NextId(lit_tok)); +} + +// Parses unsigned ints (e.g., `42u`, `0x1Au`). +template +ExprNode PrattParserWorker::ParseUintLiteral() { + Token tok = NextToken(); + std::string value(GetTokenText(tok)); + if (!value.empty() && (value.back() == 'u' || value.back() == 'U')) { + value.pop_back(); + } + uint64_t uint_val = 0; + if (absl::StartsWith(value, "0x") || absl::StartsWith(value, "0X")) { + if (absl::SimpleHexAtoi(value, &uint_val)) { + return ast_factory_.NewUintConst(NextId(tok), uint_val); + } + } else if (absl::SimpleAtoi(value, &uint_val)) { + return ast_factory_.NewUintConst(NextId(tok), uint_val); + } + ReportSyntaxError(tok, "invalid uint literal"); + return ast_factory_.NewUnspecified(NextId(tok)); +} + +// Parses floating-point numbers (e.g., `3.14159`, `1e-10`). +template +ExprNode PrattParserWorker::ParseDoubleLiteral() { + Token tok = NextToken(); + std::string value(GetTokenText(tok)); + double double_val = 0.0; + if (absl::SimpleAtod(value, &double_val)) { + return ast_factory_.NewDoubleConst(NextId(tok), double_val); + } + ReportSyntaxError(tok, "invalid double literal"); + return ast_factory_.NewUnspecified(NextId(tok)); +} + +template +ExprNode PrattParserWorker::ParseNegativeDoubleLiteral( + int64_t node_id) { + Token lit_tok = NextToken(); + double double_val = 0.0; + if (absl::SimpleAtod(GetTokenText(lit_tok), &double_val)) { + return ast_factory_.NewDoubleConst(node_id, -double_val); + } + ReportSyntaxError(lit_tok, "invalid double literal"); + return ast_factory_.NewUnspecified(NextId(lit_tok)); +} + +// Parses string literals (e.g., `"hello"`, `'world'`, `"""multi"""`). +template +ExprNode PrattParserWorker::ParseStringLiteral() { + Token tok = NextToken(); + absl::StatusOr status_or_val = + cel::internal::ParseStringLiteral(GetTokenText(tok)); + if (!status_or_val.ok()) { + ReportError(tok, status_or_val.status().message()); + return ast_factory_.NewUnspecified(NextId(tok)); + } + return ast_factory_.NewStringConst(NextId(tok), std::move(*status_or_val)); +} + +// Parses byte sequence literals (e.g., `b"hello"`, `b'world'`). +template +ExprNode PrattParserWorker::ParseBytesLiteral() { + Token tok = NextToken(); + absl::StatusOr status_or_val = + cel::internal::ParseBytesLiteral(GetTokenText(tok)); + if (!status_or_val.ok()) { + ReportError(tok, status_or_val.status().message()); + return ast_factory_.NewUnspecified(NextId(tok)); + } + return ast_factory_.NewBytesConst(NextId(tok), std::move(*status_or_val)); +} + +// Normalizes regular & quoted identifiers (e.g., `foo`, `` `quoted.ident` ``). +template +std::string PrattParserWorker::NormalizeIdent( + const Token& tok, bool allow_quoted, bool* absl_nullable is_quoted) { + std::string text = GetTokenText(tok); + if (text.empty()) { + if (is_quoted != nullptr) *is_quoted = false; + return ""; + } + if (text.front() == '`') { + if (is_quoted != nullptr) *is_quoted = true; + if (!allow_quoted) { + ReportError(tok, "unexpected quoted identifier"); + return ""; + } + if (!options_.enable_quoted_identifiers) { + ReportError(tok, "unsupported syntax '`'"); + } + if (text.size() < 2 || text.back() != '`') { + ReportError(tok, "unterminated quoted identifier"); + return ""; + } + // Validate the quoted identifier syntax: + // ESC_IDENTIFIER : '`' (LETTER | DIGIT | '_' | '.' | '-' | '/' | ' ')+ '`'; + std::string_view inner(text.data() + 1, text.size() - 2); + if (inner.empty()) { + ReportError(tok, "unexpected quoted identifier"); + return ""; + } + for (char c : inner) { + if (!absl::ascii_isalnum(static_cast(c)) && c != '_' && + c != '.' && c != '-' && c != '/' && c != ' ') { + ReportError(tok, "unexpected quoted identifier"); + return ""; + } + } + return std::string(inner); + } + if (is_quoted != nullptr) *is_quoted = false; + return std::string(text); +} + +// Extracts qualified type names (e.g., `a.b.Msg` from `a.b.Msg{}`). +template +std::optional PrattParserWorker::ExtractStructName( + const ExprNode& expr) { + if (ast_factory_.IsConst(expr)) { + return std::nullopt; + } + if (ast_factory_.IsIdent(expr)) { + std::string name(ast_factory_.GetIdentName(expr)); + EraseId(ast_factory_.GetId(expr)); + return name; + } + if (ast_factory_.IsSelect(expr)) { + if (ast_factory_.IsPresenceTest(expr)) return std::nullopt; + const ExprNode* operand = ast_factory_.GetSelectOperand(expr); + if (operand == nullptr) return std::nullopt; + EraseId(ast_factory_.GetId(expr)); + absl::optional prefix = ExtractStructName(*operand); + if (!prefix) return std::nullopt; + std::string name = + absl::StrCat(*prefix, ".", ast_factory_.GetSelectField(expr)); + return name; + } + return std::nullopt; +} + +template +int32_t PrattParserWorker::GetLeftmostPosition(const ExprNode& expr) { + if (ast_factory_.IsIdent(expr)) { + auto it = positions_.find(ast_factory_.GetId(expr)); + return it != positions_.end() ? it->second : 0; + } + if (ast_factory_.IsSelect(expr)) { + return GetLeftmostPosition(*ast_factory_.GetSelectOperand(expr)); + } + auto it = positions_.find(ast_factory_.GetId(expr)); + return it != positions_.end() ? it->second : 0; +} + +// Recursively constructs a balanced binary AST tree for chained associative +// operators (e.g., chains of `+` or `*`). To prevent deep recursion and +// stack overflow during evaluation of expressions like `a + b + c + d`, this +// method splits the operand terms in half at midpoint `(lo + hi + 1) / 2` and +// combines the left and right subtrees with binary call nodes. +template +ExprNode PrattParserWorker::BalancedTree( + absl::string_view op, std::vector& terms, + const std::vector& ops, int lo, int hi) { + int mid = (lo + hi + 1) / 2; + std::vector arguments; + arguments.reserve(2); + if (mid == lo) { + arguments.push_back(std::move(terms[mid])); + } else { + arguments.push_back(BalancedTree(op, terms, ops, lo, mid - 1)); + } + if (mid == hi) { + arguments.push_back(std::move(terms[mid + 1])); + } else { + arguments.push_back(BalancedTree(op, terms, ops, mid + 1, hi)); + } + return ast_factory_.NewCall(ops[mid], std::string(op), std::move(arguments)); +} + +// Constructs the AST representation for chained logical operators (e.g., +// `a && b && c` or `a || b || c`). When variadic logical ops are enabled +// (`enable_variadic = true`), it creates a single N-ary function call node +// containing all terms as arguments (e.g., `_&&_(a, b, c)`). Otherwise, it +// delegates to `BalancedTree` to produce a balanced binary tree of logical +// operations `(a && b) && c`. +template +ExprNode PrattParserWorker::BalanceLogical( + absl::string_view op, std::vector terms, std::vector ops, + bool enable_variadic) { + if (terms.size() == 1) { + return std::move(terms[0]); + } + if (enable_variadic) { + return ast_factory_.NewCall(ops[0], std::string(op), std::move(terms)); + } + return BalancedTree(op, terms, ops, 0, ops.size() - 1); +} + +template +std::optional PrattParserWorker::TryExpandMacro( + int64_t expr_id, absl::string_view function, ExprNode* target, + std::vector& args) { + bool is_receiver = target != nullptr; + auto expander = + ast_factory_.NewMacroExprExpander(function, args.size(), is_receiver); + if (!expander) { + return std::nullopt; + } + if (node_limit_exceeded_) { + ReportError(expr_id, + "could not expand macro: expression node limit exceeded"); + return std::nullopt; + } + + std::vector macro_args; + ExprNode macro_target; + bool add_macro_calls = options_.add_macro_calls; + // We must build the copies of the macro arguments before calling Expand, + // because Expand is allowed to mutate the arguments in-place upon success. + // However, we only record the macro call if Expand actually succeeds. + if (add_macro_calls) { + auto build_macro_call_arg = [&](const ExprNode& expr) -> ExprNode { + absl::StatusOr copy_or = ast_factory_.CopyAndReplace( + expr, + [this](const ExprNode& e) -> std::optional { + if (auto it = macro_calls_.find(ast_factory_.GetId(e)); + it != macro_calls_.end()) { + return ast_factory_.NewUnspecified(ast_factory_.GetId(e)); + } + return std::nullopt; + }, + options_.max_recursion_depth - recursion_depth_); + if (!copy_or.ok()) { + int32_t macro_position = 0; + if (auto it = positions_.find(expr_id); it != positions_.end()) { + macro_position = it->second; + } + ReportError(macro_position, copy_or.status().message()); + return ast_factory_.NewUnspecified(0); + } + return *std::move(copy_or); + }; + macro_args.reserve(args.size()); + for (const auto& arg : args) { + macro_args.push_back(build_macro_call_arg(arg)); + } + if (target != nullptr) { + macro_target = build_macro_call_arg(*target); + } + } + + absl::optional> target_ref; + if (target != nullptr) { + target_ref = *target; + } + + int32_t macro_position = 0; + if (auto it = positions_.find(expr_id); it != positions_.end()) { + macro_position = it->second; + } + MacroExpanderSupport support(*this, macro_position); + std::optional expanded_expr = + expander->Expand(target_ref, absl::MakeSpan(args), support); + + if (expanded_expr) { + if (add_macro_calls) { + RecordMacroCall(ast_factory_.GetId(*expanded_expr), function, + target != nullptr + ? std::make_optional(std::move(macro_target)) + : std::nullopt, + std::move(macro_args)); + } + EraseId(expr_id); + return expanded_expr; + } + + return std::nullopt; +} + +template +void PrattParserWorker::RecordMacroCall( + int64_t macro_id, absl::string_view function, + std::optional target, std::vector arguments) { + ExprNode call_expr; + if (target.has_value()) { + call_expr = ast_factory_.NewMemberCall( + 0, std::string(function), std::move(*target), std::move(arguments)); + } else { + call_expr = + ast_factory_.NewCall(0, std::string(function), std::move(arguments)); + } + macro_calls_.insert({macro_id, std::move(call_expr)}); +} + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ diff --git a/parser/macro.cc b/parser/macro.cc index eaa1ebd1a..057787e8d 100644 --- a/parser/macro.cc +++ b/parser/macro.cc @@ -25,6 +25,7 @@ #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -38,8 +39,15 @@ namespace cel { namespace { +constexpr absl::string_view kOptionalMapVar = "@target"; + using google::api::expr::common::CelOperator; +bool IsSimpleIdentifier(const Expr& expr) { + return expr.has_ident_expr() && !expr.ident_expr().name().empty() && + !absl::StartsWith(expr.ident_expr().name(), "."); +} + inline MacroExpander ToMacroExpander(GlobalMacroExpander expander) { ABSL_DCHECK(expander); return [expander = std::move(expander)]( @@ -87,14 +95,14 @@ absl::optional ExpandAllMacro(MacroExprFactory& factory, Expr& target, if (args.size() != 2) { return factory.ReportError("all() requires 2 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "all() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { - return factory.ReportErrorAt(args[1], - absl::StrCat("all() variable name cannot be ", - kAccumulatorVariableName)); + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { + return factory.ReportErrorAt( + args[1], absl::StrCat("all() variable name cannot be ", + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewBoolConst(true); auto condition = @@ -119,14 +127,14 @@ absl::optional ExpandExistsMacro(MacroExprFactory& factory, Expr& target, if (args.size() != 2) { return factory.ReportError("exists() requires 2 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "exists() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("exists() variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewBoolConst(false); auto condition = factory.NewCall( @@ -153,14 +161,14 @@ absl::optional ExpandExistsOneMacro(MacroExprFactory& factory, if (args.size() != 2) { return factory.ReportError("exists_one() requires 2 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "exists_one() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("exists_one() variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewIntConst(0); auto condition = factory.NewBoolConst(true); @@ -192,14 +200,14 @@ absl::optional ExpandMap2Macro(MacroExprFactory& factory, Expr& target, if (args.size() != 2) { return factory.ReportError("map() requires 2 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "map() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { - return factory.ReportErrorAt(args[1], - absl::StrCat("map() variable name cannot be ", - kAccumulatorVariableName)); + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { + return factory.ReportErrorAt( + args[1], absl::StrCat("map() variable name cannot be ", + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewList(); auto condition = factory.NewBoolConst(true); @@ -225,14 +233,14 @@ absl::optional ExpandMap3Macro(MacroExprFactory& factory, Expr& target, if (args.size() != 3) { return factory.ReportError("map() requires 3 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "map() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { - return factory.ReportErrorAt(args[1], - absl::StrCat("map() variable name cannot be ", - kAccumulatorVariableName)); + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { + return factory.ReportErrorAt( + args[1], absl::StrCat("map() variable name cannot be ", + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewList(); auto condition = factory.NewBoolConst(true); @@ -260,14 +268,14 @@ absl::optional ExpandFilterMacro(MacroExprFactory& factory, Expr& target, if (args.size() != 2) { return factory.ReportError("filter() requires 2 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "filter() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("filter() variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto name = args[0].ident_expr().name(); @@ -298,31 +306,61 @@ absl::optional ExpandOptMapMacro(MacroExprFactory& factory, Expr& target, if (args.size() != 2) { return factory.ReportError("optMap() requires 2 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "optMap() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("optMap() variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto var_name = args[0].ident_expr().name(); - auto target_copy = factory.Copy(target); - std::vector call_args; - call_args.reserve(3); - call_args.push_back(factory.NewMemberCall("hasValue", std::move(target))); + if (target.has_ident_expr()) { + auto target_copy = factory.Copy(target); + std::vector call_args; + call_args.reserve(3); + call_args.push_back(factory.NewMemberCall("hasValue", std::move(target))); + auto iter_range = factory.NewList(); + auto accu_init = factory.NewMemberCall("value", std::move(target_copy)); + auto condition = factory.NewBoolConst(false); + auto fold = factory.NewComprehension( + "#unused", std::move(iter_range), std::move(var_name), + std::move(accu_init), std::move(condition), std::move(args[0]), + std::move(args[1])); + call_args.push_back(factory.NewCall("optional.of", std::move(fold))); + call_args.push_back(factory.NewCall("optional.none")); + return factory.NewCall(CelOperator::CONDITIONAL, std::move(call_args)); + } + + // If the target is complex, use an internal bind expression to avoid + // repeating it and blowing up the AST in the expansion + auto tmp = factory.NewIdent(kOptionalMapVar); + auto tmp_copy = factory.Copy(tmp); + auto iter_range = factory.NewList(); - auto accu_init = factory.NewMemberCall("value", std::move(target_copy)); + auto accu_init = factory.NewMemberCall("value", std::move(tmp_copy)); auto condition = factory.NewBoolConst(false); + auto loop_step = std::move(args[0]); auto fold = factory.NewComprehension( "#unused", std::move(iter_range), std::move(var_name), - std::move(accu_init), std::move(condition), std::move(args[0]), + std::move(accu_init), std::move(condition), std::move(loop_step), std::move(args[1])); + std::vector call_args; + call_args.reserve(3); + call_args.push_back(factory.NewMemberCall("hasValue", std::move(tmp))); call_args.push_back(factory.NewCall("optional.of", std::move(fold))); call_args.push_back(factory.NewCall("optional.none")); - return factory.NewCall(CelOperator::CONDITIONAL, std::move(call_args)); + auto result = factory.NewCall(CelOperator::CONDITIONAL, std::move(call_args)); + + iter_range = factory.NewList(); + accu_init = std::move(target); + condition = factory.NewBoolConst(false); + loop_step = factory.NewIdent(kOptionalMapVar); + return factory.NewComprehension( + "#unused", std::move(iter_range), kOptionalMapVar, std::move(accu_init), + std::move(condition), loop_step, std::move(result)); } Macro MakeOptMapMacro() { @@ -337,30 +375,58 @@ absl::optional ExpandOptFlatMapMacro(MacroExprFactory& factory, if (args.size() != 2) { return factory.ReportError("optFlatMap() requires 2 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "optFlatMap() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("optFlatMap() variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto var_name = args[0].ident_expr().name(); - auto target_copy = factory.Copy(target); - std::vector call_args; - call_args.reserve(3); - call_args.push_back(factory.NewMemberCall("hasValue", std::move(target))); + if (target.has_ident_expr()) { + auto target_copy = factory.Copy(target); + std::vector call_args; + call_args.reserve(3); + call_args.push_back(factory.NewMemberCall("hasValue", std::move(target))); + auto iter_range = factory.NewList(); + auto accu_init = factory.NewMemberCall("value", std::move(target_copy)); + auto condition = factory.NewBoolConst(false); + call_args.push_back(factory.NewComprehension( + "#unused", std::move(iter_range), std::move(var_name), + std::move(accu_init), std::move(condition), std::move(args[0]), + std::move(args[1]))); + call_args.push_back(factory.NewCall("optional.none")); + return factory.NewCall(CelOperator::CONDITIONAL, std::move(call_args)); + } + + auto tmp = factory.NewIdent(kOptionalMapVar); + auto tmp_copy = factory.Copy(tmp); + auto iter_range = factory.NewList(); - auto accu_init = factory.NewMemberCall("value", std::move(target_copy)); + auto accu_init = factory.NewMemberCall("value", std::move(tmp_copy)); auto condition = factory.NewBoolConst(false); - call_args.push_back(factory.NewComprehension( + auto loop_step = std::move(args[0]); + auto inner = factory.NewComprehension( "#unused", std::move(iter_range), std::move(var_name), - std::move(accu_init), std::move(condition), std::move(args[0]), - std::move(args[1]))); + std::move(accu_init), std::move(condition), std::move(loop_step), + std::move(args[1])); + std::vector call_args; + call_args.reserve(3); + call_args.push_back(factory.NewMemberCall("hasValue", std::move(tmp))); + call_args.push_back(std::move(inner)); call_args.push_back(factory.NewCall("optional.none")); - return factory.NewCall(CelOperator::CONDITIONAL, std::move(call_args)); + auto result = factory.NewCall(CelOperator::CONDITIONAL, std::move(call_args)); + + iter_range = factory.NewList(); + accu_init = std::move(target); + condition = factory.NewBoolConst(false); + loop_step = factory.NewIdent(kOptionalMapVar); + return factory.NewComprehension( + "#unused", std::move(iter_range), kOptionalMapVar, std::move(accu_init), + std::move(condition), loop_step, std::move(result)); } Macro MakeOptFlatMapMacro() { diff --git a/parser/macro_expr_factory.h b/parser/macro_expr_factory.h index ffba5e2f2..75f8d1d74 100644 --- a/parser/macro_expr_factory.h +++ b/parser/macro_expr_factory.h @@ -33,6 +33,11 @@ namespace cel { class ParserMacroExprFactory; class TestMacroExprFactory; +namespace parser_internal { +template +class MacroExprExpanderSupport; +} // namespace parser_internal + // `MacroExprFactory` is a specialization of `ExprFactory` for `MacroExpander` // which disallows explicitly specifying IDs. class MacroExprFactory : protected ExprFactory { @@ -318,9 +323,9 @@ class MacroExprFactory : protected ExprFactory { private: friend class ParserMacroExprFactory; friend class TestMacroExprFactory; + friend class parser_internal::MacroExprExpanderSupport; - explicit MacroExprFactory(absl::string_view accu_var) - : ExprFactory(accu_var) {} + explicit MacroExprFactory() = default; }; } // namespace cel diff --git a/parser/macro_expr_factory_test.cc b/parser/macro_expr_factory_test.cc index 04705eec6..b95cbe16f 100644 --- a/parser/macro_expr_factory_test.cc +++ b/parser/macro_expr_factory_test.cc @@ -15,6 +15,7 @@ #include "parser/macro_expr_factory.h" #include +#include #include #include "absl/strings/string_view.h" @@ -27,7 +28,7 @@ namespace cel { class TestMacroExprFactory final : public MacroExprFactory { public: - TestMacroExprFactory() : MacroExprFactory(kAccumulatorVariableName) {} + TestMacroExprFactory() = default; ExprId id() const { return id_; } @@ -39,6 +40,7 @@ class TestMacroExprFactory final : public MacroExprFactory { return NewUnspecified(NextId()); } + using MacroExprFactory::NewBind; using MacroExprFactory::NewBoolConst; using MacroExprFactory::NewCall; using MacroExprFactory::NewComprehension; @@ -69,6 +71,8 @@ class TestMacroExprFactory final : public MacroExprFactory { namespace { +using ::testing::IsEmpty; + TEST(MacroExprFactory, CopyUnspecified) { TestMacroExprFactory factory; EXPECT_EQ(factory.Copy(factory.NewUnspecified()), factory.NewUnspecified(2)); @@ -147,5 +151,52 @@ TEST(MacroExprFactory, CopyComprehension) { factory.NewIdent(11, "foo"), factory.NewIdent(12, "bar"))); } +TEST(MacroExprFactory, NewBind) { + TestMacroExprFactory factory; + Expr bind_expr = factory.NewIdent(10, "x"); + Expr rest_expr = factory.NewIdent(20, "y"); + + auto next_id = [id = 100]() mutable { return id++; }; + + Expr expr = + factory.NewBind(next_id, "a", std::move(bind_expr), std::move(rest_expr)); + + EXPECT_EQ(expr.id(), 100); + ASSERT_TRUE(expr.has_comprehension_expr()); + + const auto& comp = expr.comprehension_expr(); + EXPECT_EQ(comp.iter_var(), "#unused"); + + ASSERT_TRUE(comp.has_iter_range()); + EXPECT_EQ(comp.iter_range().id(), 101); + EXPECT_EQ(comp.iter_range().kind_case(), ExprKindCase::kListExpr); + EXPECT_THAT(comp.iter_range().list_expr().elements(), IsEmpty()); + + EXPECT_EQ(comp.accu_var(), "a"); + + ASSERT_TRUE(comp.has_accu_init()); + Expr expected_bind_expr; + expected_bind_expr.set_id(10); + expected_bind_expr.mutable_ident_expr().set_name("x"); + EXPECT_EQ(comp.accu_init(), expected_bind_expr); + + ASSERT_TRUE(comp.has_loop_condition()); + EXPECT_EQ(comp.loop_condition().id(), 102); + EXPECT_EQ(comp.loop_condition().kind_case(), ExprKindCase::kConstant); + EXPECT_TRUE(comp.loop_condition().const_expr().has_bool_value()); + EXPECT_FALSE(comp.loop_condition().const_expr().bool_value()); + + ASSERT_TRUE(comp.has_loop_step()); + EXPECT_EQ(comp.loop_step().id(), 103); + EXPECT_EQ(comp.loop_step().kind_case(), ExprKindCase::kIdentExpr); + EXPECT_EQ(comp.loop_step().ident_expr().name(), "a"); + + ASSERT_TRUE(comp.has_result()); + Expr expected_rest_expr; + expected_rest_expr.set_id(20); + expected_rest_expr.mutable_ident_expr().set_name("y"); + EXPECT_EQ(comp.result(), expected_rest_expr); +} + } // namespace } // namespace cel diff --git a/parser/macro_registry.cc b/parser/macro_registry.cc index 3a816b10e..d36761e87 100644 --- a/parser/macro_registry.cc +++ b/parser/macro_registry.cc @@ -55,7 +55,7 @@ absl::optional MacroRegistry::FindMacro(absl::string_view name, bool receiver_style) const { // :: if (name.empty() || absl::StrContains(name, ':')) { - return absl::nullopt; + return std::nullopt; } // Try argument count specific key first. auto key = absl::StrCat(name, ":", arg_count, ":", @@ -68,7 +68,7 @@ absl::optional MacroRegistry::FindMacro(absl::string_view name, if (auto it = macros_.find(key); it != macros_.end()) { return it->second; } - return absl::nullopt; + return std::nullopt; } std::vector MacroRegistry::ListMacros() const { diff --git a/parser/macro_registry_test.cc b/parser/macro_registry_test.cc index 9e6da87a4..db8a99ab2 100644 --- a/parser/macro_registry_test.cc +++ b/parser/macro_registry_test.cc @@ -30,14 +30,14 @@ using ::testing::Ne; TEST(MacroRegistry, RegisterAndFind) { MacroRegistry macros; EXPECT_THAT(macros.RegisterMacro(HasMacro()), IsOk()); - EXPECT_THAT(macros.FindMacro("has", 1, false), Ne(absl::nullopt)); + EXPECT_THAT(macros.FindMacro("has", 1, false), Ne(std::nullopt)); } TEST(MacroRegistry, RegisterRollsback) { MacroRegistry macros; EXPECT_THAT(macros.RegisterMacros({HasMacro(), AllMacro(), AllMacro()}), StatusIs(absl::StatusCode::kAlreadyExists)); - EXPECT_THAT(macros.FindMacro("has", 1, false), Eq(absl::nullopt)); + EXPECT_THAT(macros.FindMacro("has", 1, false), Eq(std::nullopt)); } } // namespace diff --git a/parser/options.h b/parser/options.h index a41d16104..1ddce71ac 100644 --- a/parser/options.h +++ b/parser/options.h @@ -25,25 +25,33 @@ struct ParserOptions final { // Limit of the number of error recovery attempts made by the ANTLR parser // when processing an input. This limit, when reached, will halt further // parsing of the expression. - int error_recovery_limit = ::cel_parser_internal::kDefaultErrorRecoveryLimit; + int error_recovery_limit = ::cel::parser_internal::kDefaultErrorRecoveryLimit; // Limit on the amount of recursive parse instructions permitted when building // the abstract syntax tree for the expression. This prevents pathological // inputs from causing stack overflows. - int max_recursion_depth = ::cel_parser_internal::kDefaultMaxRecursionDepth; + int max_recursion_depth = ::cel::parser_internal::kDefaultMaxRecursionDepth; // Limit on the number of codepoints in the input string which the parser will // attempt to parse. int expression_size_codepoint_limit = - ::cel_parser_internal::kExpressionSizeCodepointLimit; + ::cel::parser_internal::kExpressionSizeCodepointLimit; // Limit on the number of lookahead tokens to consume when attempting to // recover from an error. int error_recovery_token_lookahead_limit = - ::cel_parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; + ::cel::parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; + + // Limit on the number of expression nodes in the abstract syntax tree for the + // expression. This prevents cases where macro expansion results in an AST + // that is larger than expected from the source expression. Once exceeded, + // the parser will record an error and stop expanding macros but continue + // parsing to report other errors. + int expression_node_limit = + ::cel::parser_internal::kDefaultExpressionNodeLimit; // Add macro calls to macro_calls list in source_info. - bool add_macro_calls = ::cel_parser_internal::kDefaultAddMacroCalls; + bool add_macro_calls = ::cel::parser_internal::kDefaultAddMacroCalls; // Enable support for optional syntax. bool enable_optional_syntax = false; @@ -51,7 +59,9 @@ struct ParserOptions final { // Disable standard macros (has, all, exists, exists_one, filter, map). bool disable_standard_macros = false; - // Enable hidden accumulator variable '@result' for builtin comprehensions. + // Deprecated: The builtin and extension macros now always use the new + // accumulator variable name. + // This option has no effect. bool enable_hidden_accumulator_var = true; // Enables support for identifier quoting syntax: @@ -60,6 +70,37 @@ struct ParserOptions final { // Limited to field specifiers in select and message creation, // enabled by default bool enable_quoted_identifiers = true; + + // Enables parsing logical AND & OR operators as a single flat variadic call + // instead of a balanced/nested binary AST structure. + bool enable_variadic_logical_operators = false; + + // Enable the manual (Pratt) parser instead of the ANTLR parser. + // + // CAUTION: Pratt Parser is work in progress. Don't use unless you are aware + // of the risks. There is currently no support whatsoever available for + // clients of this option. + // + // This option is temporary and should not be used by new code outside of the + // early testing of the Pratt parser. + // TODO(b/527638023): Remove this option once the ANTLR parser is removed. + bool enable_pratt_parser = false; + + // Folds repeated unary operators (!, -). + // + // If the operator appears repeatedly, the parser will ignore every contiguous + // pair. + // + // This makes it possible to parse some semantically invalid expressions as + // valid ones, though they are not particularly harmful. + // + // Examples that parse to the same AST: + // + // `---1` : `-(1)` + // `!!!!!true` : !`true` + // `--0u` : `0u` + // `!!"hello"` : `"hello"`). + bool fold_unary_operators = true; }; } // namespace cel @@ -70,20 +111,20 @@ using ParserOptions = ::cel::ParserOptions; ABSL_DEPRECATED("Use ParserOptions().error_recovery_limit instead.") inline constexpr int kDefaultErrorRecoveryLimit = - ::cel_parser_internal::kDefaultErrorRecoveryLimit; + ::cel::parser_internal::kDefaultErrorRecoveryLimit; ABSL_DEPRECATED("Use ParserOptions().max_recursion_depth instead.") inline constexpr int kDefaultMaxRecursionDepth = - ::cel_parser_internal::kDefaultMaxRecursionDepth; + ::cel::parser_internal::kDefaultMaxRecursionDepth; ABSL_DEPRECATED("Use ParserOptions().expression_size_codepoint_limit instead.") inline constexpr int kExpressionSizeCodepointLimit = - ::cel_parser_internal::kExpressionSizeCodepointLimit; + ::cel::parser_internal::kExpressionSizeCodepointLimit; ABSL_DEPRECATED( "Use ParserOptions().error_recovery_token_lookahead_limit instead.") inline constexpr int kDefaultErrorRecoveryTokenLookaheadLimit = - ::cel_parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; + ::cel::parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; ABSL_DEPRECATED("Use ParserOptions().add_macro_calls instead.") inline constexpr bool kDefaultAddMacroCalls = - ::cel_parser_internal::kDefaultAddMacroCalls; + ::cel::parser_internal::kDefaultAddMacroCalls; } // namespace google::api::expr::parser diff --git a/parser/parser.cc b/parser/parser.cc index d9f74e712..37c74365d 100644 --- a/parser/parser.cc +++ b/parser/parser.cc @@ -14,1833 +14,89 @@ #include "parser/parser.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include #include -#include #include #include #include "cel/expr/syntax.pb.h" -#include "absl/base/macros.h" -#include "absl/base/optimization.h" -#include "absl/cleanup/cleanup.h" -#include "absl/container/btree_map.h" -#include "absl/container/flat_hash_map.h" -#include "absl/container/flat_hash_set.h" -#include "absl/functional/overload.h" -#include "absl/log/absl_check.h" -#include "absl/memory/memory.h" -#include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/strings/match.h" -#include "absl/strings/numbers.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" -#include "absl/strings/str_join.h" -#include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "absl/types/span.h" -#include "absl/types/variant.h" -#include "antlr4-runtime.h" #include "common/ast.h" #include "common/ast/expr_proto.h" #include "common/ast/source_info_proto.h" -#include "common/constant.h" -#include "common/expr_factory.h" -#include "common/operators.h" #include "common/source.h" -#include "internal/lexis.h" #include "internal/status_macros.h" -#include "internal/strings.h" -#include "internal/utf8.h" -#pragma push_macro("IN") -#undef IN -#include "parser/internal/CelBaseVisitor.h" -#include "parser/internal/CelLexer.h" -#include "parser/internal/CelParser.h" -#pragma pop_macro("IN") +#ifndef EXCLUDE_CEL_ANTLR_PARSER +#include "parser/internal/antlr_parser.h" +#endif +#include "parser/internal/pratt_parser.h" #include "parser/macro.h" -#include "parser/macro_expr_factory.h" #include "parser/macro_registry.h" #include "parser/options.h" #include "parser/parser_interface.h" #include "parser/source_factory.h" namespace google::api::expr::parser { -namespace { -class ParserVisitor; -} -} // namespace google::api::expr::parser - -namespace cel { - -namespace { - -constexpr const char kHiddenAccumulatorVariableName[] = "@result"; - -std::any ExprPtrToAny(std::unique_ptr&& expr) { - return std::make_any(expr.release()); -} - -std::any ExprToAny(Expr&& expr) { - return ExprPtrToAny(std::make_unique(std::move(expr))); -} - -std::unique_ptr ExprPtrFromAny(std::any&& any) { - return absl::WrapUnique(std::any_cast(std::move(any))); -} - -Expr ExprFromAny(std::any&& any) { - auto expr = ExprPtrFromAny(std::move(any)); - return std::move(*expr); -} - -struct ParserError { - std::string message; - SourceRange range; -}; - -std::string DisplayParserError(const cel::Source& source, - const ParserError& error) { - auto location = - source.GetLocation(error.range.begin).value_or(SourceLocation{}); - return absl::StrCat(absl::StrFormat("ERROR: %s:%zu:%zu: %s", - source.description(), location.line, - // add one to the 0-based column - location.column + 1, error.message), - source.DisplayErrorLocation(location)); -} - -int32_t PositiveOrMax(int32_t value) { - return value >= 0 ? value : std::numeric_limits::max(); -} - -SourceRange SourceRangeFromToken(const antlr4::Token* token) { - SourceRange range; - if (token != nullptr) { - if (auto start = token->getStartIndex(); start != INVALID_INDEX) { - range.begin = static_cast(start); - } - if (auto end = token->getStopIndex(); end != INVALID_INDEX) { - range.end = static_cast(end + 1); - } - } - return range; -} - -SourceRange SourceRangeFromParserRuleContext( - const antlr4::ParserRuleContext* context) { - SourceRange range; - if (context != nullptr) { - if (auto start = context->getStart() != nullptr - ? context->getStart()->getStartIndex() - : INVALID_INDEX; - start != INVALID_INDEX) { - range.begin = static_cast(start); - } - if (auto end = context->getStop() != nullptr - ? context->getStop()->getStopIndex() - : INVALID_INDEX; - end != INVALID_INDEX) { - range.end = static_cast(end + 1); - } - } - return range; -} - -} // namespace - -class ParserMacroExprFactory final : public MacroExprFactory { - public: - explicit ParserMacroExprFactory(const cel::Source& source, - absl::string_view accu_var) - : MacroExprFactory(accu_var), source_(source) {} - - void BeginMacro(SourceRange macro_position) { - macro_position_ = macro_position; - } - - void EndMacro() { macro_position_ = SourceRange{}; } - - Expr ReportError(absl::string_view message) override { - return ReportError(macro_position_, message); - } - - Expr ReportError(int64_t expr_id, absl::string_view message) { - return ReportError(GetSourceRange(expr_id), message); - } - - Expr ReportError(SourceRange range, absl::string_view message) { - ++error_count_; - if (errors_.size() <= 100) { - errors_.push_back(ParserError{std::string(message), range}); - } - return NewUnspecified(NextId(range)); - } - - Expr ReportErrorAt(const Expr& expr, absl::string_view message) override { - return ReportError(GetSourceRange(expr.id()), message); - } - - SourceRange GetSourceRange(int64_t id) const { - if (auto it = positions_.find(id); it != positions_.end()) { - return it->second; - } - return SourceRange{}; - } - - int64_t NextId(const SourceRange& range) { - auto id = expr_id_++; - if (range.begin != -1 || range.end != -1) { - positions_.insert(std::pair{id, range}); - } - return id; - } - - bool HasErrors() const { return error_count_ != 0; } - - std::string ErrorMessage() { - // Errors are collected as they are encountered, not by their location - // within the source. To have a more stable error message as implementation - // details change, we sort the collected errors by their source location - // first. - std::stable_sort( - errors_.begin(), errors_.end(), - [](const ParserError& lhs, const ParserError& rhs) -> bool { - auto lhs_begin = PositiveOrMax(lhs.range.begin); - auto lhs_end = PositiveOrMax(lhs.range.end); - auto rhs_begin = PositiveOrMax(rhs.range.begin); - auto rhs_end = PositiveOrMax(rhs.range.end); - return lhs_begin < rhs_begin || - (lhs_begin == rhs_begin && lhs_end < rhs_end); - }); - // Build the summary error message using the sorted errors. - bool errors_truncated = error_count_ > 100; - std::vector messages; - messages.reserve( - errors_.size() + - errors_truncated); // Reserve space for the transform and an - // additional element when truncation occurs. - std::transform(errors_.begin(), errors_.end(), std::back_inserter(messages), - [this](const ParserError& error) { - return cel::DisplayParserError(source_, error); - }); - if (errors_truncated) { - messages.emplace_back( - absl::StrCat(error_count_ - 100, " more errors were truncated.")); - } - return absl::StrJoin(messages, "\n"); - } - - void AddMacroCall(int64_t macro_id, absl::string_view function, - absl::optional target, std::vector arguments) { - macro_calls_.insert( - {macro_id, target.has_value() - ? NewMemberCall(0, function, std::move(*target), - std::move(arguments)) - : NewCall(0, function, std::move(arguments))}); - } - - Expr BuildMacroCallArg(const Expr& expr) { - if (auto it = macro_calls_.find(expr.id()); it != macro_calls_.end()) { - return NewUnspecified(expr.id()); - } - return absl::visit( - absl::Overload( - [this, &expr](const UnspecifiedExpr&) -> Expr { - return NewUnspecified(expr.id()); - }, - [this, &expr](const Constant& const_expr) -> Expr { - return NewConst(expr.id(), const_expr); - }, - [this, &expr](const IdentExpr& ident_expr) -> Expr { - return NewIdent(expr.id(), ident_expr.name()); - }, - [this, &expr](const SelectExpr& select_expr) -> Expr { - return select_expr.test_only() - ? NewPresenceTest( - expr.id(), - BuildMacroCallArg(select_expr.operand()), - select_expr.field()) - : NewSelect(expr.id(), - BuildMacroCallArg(select_expr.operand()), - select_expr.field()); - }, - [this, &expr](const CallExpr& call_expr) -> Expr { - std::vector macro_arguments; - macro_arguments.reserve(call_expr.args().size()); - for (const auto& argument : call_expr.args()) { - macro_arguments.push_back(BuildMacroCallArg(argument)); - } - absl::optional macro_target; - if (call_expr.has_target()) { - macro_target = BuildMacroCallArg(call_expr.target()); - } - return macro_target.has_value() - ? NewMemberCall(expr.id(), call_expr.function(), - std::move(*macro_target), - std::move(macro_arguments)) - : NewCall(expr.id(), call_expr.function(), - std::move(macro_arguments)); - }, - [this, &expr](const ListExpr& list_expr) -> Expr { - std::vector macro_elements; - macro_elements.reserve(list_expr.elements().size()); - for (const auto& element : list_expr.elements()) { - auto& cloned_element = macro_elements.emplace_back(); - if (element.has_expr()) { - cloned_element.set_expr(BuildMacroCallArg(element.expr())); - } - cloned_element.set_optional(element.optional()); - } - return NewList(expr.id(), std::move(macro_elements)); - }, - [this, &expr](const StructExpr& struct_expr) -> Expr { - std::vector macro_fields; - macro_fields.reserve(struct_expr.fields().size()); - for (const auto& field : struct_expr.fields()) { - auto& macro_field = macro_fields.emplace_back(); - macro_field.set_id(field.id()); - macro_field.set_name(field.name()); - macro_field.set_value(BuildMacroCallArg(field.value())); - macro_field.set_optional(field.optional()); - } - return NewStruct(expr.id(), struct_expr.name(), - std::move(macro_fields)); - }, - [this, &expr](const MapExpr& map_expr) -> Expr { - std::vector macro_entries; - macro_entries.reserve(map_expr.entries().size()); - for (const auto& entry : map_expr.entries()) { - auto& macro_entry = macro_entries.emplace_back(); - macro_entry.set_id(entry.id()); - macro_entry.set_key(BuildMacroCallArg(entry.key())); - macro_entry.set_value(BuildMacroCallArg(entry.value())); - macro_entry.set_optional(entry.optional()); - } - return NewMap(expr.id(), std::move(macro_entries)); - }, - [this, &expr](const ComprehensionExpr& comprehension_expr) -> Expr { - return NewComprehension( - expr.id(), comprehension_expr.iter_var(), - BuildMacroCallArg(comprehension_expr.iter_range()), - comprehension_expr.accu_var(), - BuildMacroCallArg(comprehension_expr.accu_init()), - BuildMacroCallArg(comprehension_expr.loop_condition()), - BuildMacroCallArg(comprehension_expr.loop_step()), - BuildMacroCallArg(comprehension_expr.result())); - }), - expr.kind()); - } - - using ExprFactory::NewBoolConst; - using ExprFactory::NewBytesConst; - using ExprFactory::NewCall; - using ExprFactory::NewComprehension; - using ExprFactory::NewConst; - using ExprFactory::NewDoubleConst; - using ExprFactory::NewIdent; - using ExprFactory::NewIntConst; - using ExprFactory::NewList; - using ExprFactory::NewListElement; - using ExprFactory::NewMap; - using ExprFactory::NewMapEntry; - using ExprFactory::NewMemberCall; - using ExprFactory::NewNullConst; - using ExprFactory::NewPresenceTest; - using ExprFactory::NewSelect; - using ExprFactory::NewStringConst; - using ExprFactory::NewStruct; - using ExprFactory::NewStructField; - using ExprFactory::NewUintConst; - using ExprFactory::NewUnspecified; - - const absl::btree_map& positions() const { - return positions_; - } - - const absl::flat_hash_map& macro_calls() const { - return macro_calls_; - } - - absl::flat_hash_map release_macro_calls() { - using std::swap; - absl::flat_hash_map result; - swap(result, macro_calls_); - return result; - } - - void EraseId(ExprId id) { - positions_.erase(id); - if (expr_id_ == id + 1) { - --expr_id_; - } - } - - protected: - int64_t NextId() override { return NextId(macro_position_); } - - int64_t CopyId(int64_t id) override { - if (id == 0) { - return 0; - } - return NextId(GetSourceRange(id)); - } - - private: - int64_t expr_id_ = 1; - absl::btree_map positions_; - absl::flat_hash_map macro_calls_; - std::vector errors_; - size_t error_count_ = 0; - const Source& source_; - SourceRange macro_position_; -}; - -} // namespace cel -namespace google::api::expr::parser { - -namespace { - -using ::antlr4::CharStream; -using ::antlr4::CommonTokenStream; -using ::antlr4::DefaultErrorStrategy; -using ::antlr4::ParseCancellationException; -using ::antlr4::Parser; -using ::antlr4::ParserRuleContext; -using ::antlr4::Token; -using ::antlr4::misc::IntervalSet; -using ::antlr4::tree::ErrorNode; -using ::antlr4::tree::ParseTreeListener; -using ::antlr4::tree::TerminalNode; -using ::cel::Expr; -using ::cel::ExprFromAny; -using ::cel::ExprKind; -using ::cel::ExprToAny; -using ::cel::IdentExpr; -using ::cel::ListExprElement; -using ::cel::MapExprEntry; -using ::cel::SelectExpr; -using ::cel::SourceRangeFromParserRuleContext; -using ::cel::SourceRangeFromToken; -using ::cel::StructExprField; -using ::cel_parser_internal::CelBaseVisitor; -using ::cel_parser_internal::CelLexer; -using ::cel_parser_internal::CelParser; -using common::CelOperator; -using common::ReverseLookupOperator; +using ::cel::Macro; +using ::cel::ParserOptions; using ::cel::expr::ParsedExpr; +using ::google::api::expr::parser::EnrichedSourceInfo; +using ::google::api::expr::parser::VerboseParsedExpr; -class CodePointStream final : public CharStream { - public: - CodePointStream(cel::SourceContentView buffer, absl::string_view source_name) - : buffer_(buffer), - source_name_(source_name), - size_(buffer_.size()), - index_(0) {} - - void consume() override { - if (ABSL_PREDICT_FALSE(index_ >= size_)) { - ABSL_ASSERT(LA(1) == IntStream::EOF); - throw antlr4::IllegalStateException("cannot consume EOF"); - } - index_++; - } - - size_t LA(ptrdiff_t i) override { - if (ABSL_PREDICT_FALSE(i == 0)) { - return 0; - } - auto p = static_cast(index_); - if (i < 0) { - i++; - if (p + i - 1 < 0) { - return IntStream::EOF; - } - } - if (p + i - 1 >= static_cast(size_)) { - return IntStream::EOF; - } - return buffer_.at(static_cast(p + i - 1)); - } - - ptrdiff_t mark() override { return -1; } - - void release(ptrdiff_t marker) override {} - - size_t index() override { return index_; } - - void seek(size_t index) override { index_ = std::min(index, size_); } - - size_t size() override { return size_; } - - std::string getSourceName() const override { - return source_name_.empty() ? IntStream::UNKNOWN_SOURCE_NAME - : std::string(source_name_); - } - - std::string getText(const antlr4::misc::Interval& interval) override { - if (ABSL_PREDICT_FALSE(interval.a < 0 || interval.b < 0)) { - return std::string(); - } - size_t start = static_cast(interval.a); - if (ABSL_PREDICT_FALSE(start >= size_)) { - return std::string(); - } - size_t stop = static_cast(interval.b); - if (ABSL_PREDICT_FALSE(stop >= size_)) { - stop = size_ - 1; - } - return buffer_.ToString(static_cast(start), - static_cast(stop) + 1); - } - - std::string toString() const override { return buffer_.ToString(); } - - private: - cel::SourceContentView const buffer_; - const absl::string_view source_name_; - const size_t size_; - size_t index_; -}; - -// Scoped helper for incrementing the parse recursion count. -// Increments on creation, decrements on destruction (stack unwind). -class ScopedIncrement final { - public: - explicit ScopedIncrement(int& recursion_depth) - : recursion_depth_(recursion_depth) { - ++recursion_depth_; - } - - ~ScopedIncrement() { --recursion_depth_; } - - private: - int& recursion_depth_; -}; - -// balancer performs tree balancing on operators whose arguments are of equal -// precedence. -// -// The purpose of the balancer is to ensure a compact serialization format for -// the logical &&, || operators which have a tendency to create long DAGs which -// are skewed in one direction. Since the operators are commutative re-ordering -// the terms *must not* affect the evaluation result. -// -// Based on code from //third_party/cel/go/parser/helper.go -class ExpressionBalancer final { - public: - ExpressionBalancer(cel::ParserMacroExprFactory& factory, std::string function, - Expr expr); - - // addTerm adds an operation identifier and term to the set of terms to be - // balanced. - void AddTerm(int64_t op, Expr term); - - // balance creates a balanced tree from the sub-terms and returns the final - // Expr value. - Expr Balance(); - - private: - // balancedTree recursively balances the terms provided to a commutative - // operator. - Expr BalancedTree(int lo, int hi); - - private: - cel::ParserMacroExprFactory& factory_; - std::string function_; - std::vector terms_; - std::vector ops_; -}; - -ExpressionBalancer::ExpressionBalancer(cel::ParserMacroExprFactory& factory, - std::string function, Expr expr) - : factory_(factory), function_(std::move(function)) { - terms_.push_back(std::move(expr)); -} - -void ExpressionBalancer::AddTerm(int64_t op, Expr term) { - terms_.push_back(std::move(term)); - ops_.push_back(op); -} - -Expr ExpressionBalancer::Balance() { - if (terms_.size() == 1) { - return std::move(terms_[0]); - } - return BalancedTree(0, ops_.size() - 1); -} - -Expr ExpressionBalancer::BalancedTree(int lo, int hi) { - int mid = (lo + hi + 1) / 2; - - std::vector arguments; - arguments.reserve(2); - - if (mid == lo) { - arguments.push_back(std::move(terms_[mid])); - } else { - arguments.push_back(BalancedTree(lo, mid - 1)); - } - - if (mid == hi) { - arguments.push_back(std::move(terms_[mid + 1])); - } else { - arguments.push_back(BalancedTree(mid + 1, hi)); - } - return factory_.NewCall(ops_[mid], function_, std::move(arguments)); -} - -class ParserVisitor final : public CelBaseVisitor, - public antlr4::BaseErrorListener { - public: - ParserVisitor(const cel::Source& source, int max_recursion_depth, - absl::string_view accu_var, - const cel::MacroRegistry& macro_registry, - bool add_macro_calls = false, - bool enable_optional_syntax = false, - bool enable_quoted_identifiers = false) - : source_(source), - factory_(source_, accu_var), - macro_registry_(macro_registry), - recursion_depth_(0), - max_recursion_depth_(max_recursion_depth), - add_macro_calls_(add_macro_calls), - enable_optional_syntax_(enable_optional_syntax), - enable_quoted_identifiers_(enable_quoted_identifiers) {} - - ~ParserVisitor() override = default; - - std::any visit(antlr4::tree::ParseTree* tree) override; - - std::any visitStart(CelParser::StartContext* ctx) override; - std::any visitExpr(CelParser::ExprContext* ctx) override; - std::any visitConditionalOr(CelParser::ConditionalOrContext* ctx) override; - std::any visitConditionalAnd(CelParser::ConditionalAndContext* ctx) override; - std::any visitRelation(CelParser::RelationContext* ctx) override; - std::any visitCalc(CelParser::CalcContext* ctx) override; - std::any visitUnary(CelParser::UnaryContext* ctx); - std::any visitLogicalNot(CelParser::LogicalNotContext* ctx) override; - std::any visitNegate(CelParser::NegateContext* ctx) override; - std::any visitSelect(CelParser::SelectContext* ctx) override; - std::any visitMemberCall(CelParser::MemberCallContext* ctx) override; - std::any visitIndex(CelParser::IndexContext* ctx) override; - std::any visitCreateMessage(CelParser::CreateMessageContext* ctx) override; - std::any visitFieldInitializerList( - CelParser::FieldInitializerListContext* ctx) override; - std::vector visitFields( - CelParser::FieldInitializerListContext* ctx); - std::any visitGlobalCall(CelParser::GlobalCallContext* ctx) override; - std::any visitIdent(CelParser::IdentContext* ctx) override; - std::any visitNested(CelParser::NestedContext* ctx) override; - std::any visitCreateList(CelParser::CreateListContext* ctx) override; - std::vector visitList(CelParser::ListInitContext* ctx); - std::vector visitList(CelParser::ExprListContext* ctx); - std::any visitCreateMap(CelParser::CreateMapContext* ctx) override; - std::any visitConstantLiteral( - CelParser::ConstantLiteralContext* ctx) override; - std::any visitPrimaryExpr(CelParser::PrimaryExprContext* ctx) override; - std::any visitMemberExpr(CelParser::MemberExprContext* ctx) override; - - std::any visitMapInitializerList( - CelParser::MapInitializerListContext* ctx) override; - std::vector visitEntries( - CelParser::MapInitializerListContext* ctx); - std::any visitInt(CelParser::IntContext* ctx) override; - std::any visitUint(CelParser::UintContext* ctx) override; - std::any visitDouble(CelParser::DoubleContext* ctx) override; - std::any visitString(CelParser::StringContext* ctx) override; - std::any visitBytes(CelParser::BytesContext* ctx) override; - std::any visitBoolTrue(CelParser::BoolTrueContext* ctx) override; - std::any visitBoolFalse(CelParser::BoolFalseContext* ctx) override; - std::any visitNull(CelParser::NullContext* ctx) override; - // Note: this is destructive and intended to be called after the parse is - // finished. - cel::SourceInfo GetSourceInfo(); - EnrichedSourceInfo enriched_source_info() const; - void syntaxError(antlr4::Recognizer* recognizer, - antlr4::Token* offending_symbol, size_t line, size_t col, - const std::string& msg, std::exception_ptr e) override; - bool HasErrored() const; - - std::string ErrorMessage(); - - private: - template - Expr GlobalCallOrMacro(int64_t expr_id, absl::string_view function, - Args&&... args) { - std::vector arguments; - arguments.reserve(sizeof...(Args)); - (arguments.push_back(std::forward(args)), ...); - return GlobalCallOrMacroImpl(expr_id, function, std::move(arguments)); - } - - Expr GlobalCallOrMacroImpl(int64_t expr_id, absl::string_view function, - std::vector args); - Expr ReceiverCallOrMacroImpl(int64_t expr_id, absl::string_view function, - Expr target, std::vector args); - std::string ExtractQualifiedName(antlr4::ParserRuleContext* ctx, - const Expr& e); - - std::string NormalizeIdentifier(CelParser::EscapeIdentContext* ctx); - // Attempt to unnest parse context. - // - // Walk the parse tree to the first complex term to reduce recursive depth in - // the visit* calls. - antlr4::tree::ParseTree* UnnestContext(antlr4::tree::ParseTree* tree); - - private: - const cel::Source& source_; - cel::ParserMacroExprFactory factory_; - const cel::MacroRegistry& macro_registry_; - int recursion_depth_; - const int max_recursion_depth_; - const bool add_macro_calls_; - const bool enable_optional_syntax_; - const bool enable_quoted_identifiers_; -}; - -template ::value>> -T* tree_as(antlr4::tree::ParseTree* tree) { - return dynamic_cast(tree); -} - -std::any ParserVisitor::visit(antlr4::tree::ParseTree* tree) { - ScopedIncrement inc(recursion_depth_); - if (recursion_depth_ > max_recursion_depth_) { - return ExprToAny(factory_.ReportError( - absl::StrFormat("Exceeded max recursion depth of %d when parsing.", - max_recursion_depth_))); - } - tree = UnnestContext(tree); - if (auto* ctx = tree_as(tree)) { - return visitStart(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitExpr(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitConditionalAnd(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitConditionalOr(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitRelation(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitCalc(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitLogicalNot(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitPrimaryExpr(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitMemberExpr(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitSelect(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitMemberCall(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitMapInitializerList(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitNegate(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitIndex(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitUnary(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitCreateList(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitCreateMessage(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitCreateMap(ctx); - } - - if (tree) { - return ExprToAny( - factory_.ReportError(SourceRangeFromParserRuleContext( - tree_as(tree)), - "unknown parsetree type")); - } - return ExprToAny(factory_.ReportError("<> parsetree")); -} - -std::any ParserVisitor::visitPrimaryExpr(CelParser::PrimaryExprContext* pctx) { - CelParser::PrimaryContext* primary = pctx->primary(); - if (auto* ctx = tree_as(primary)) { - return visitNested(ctx); - } else if (auto* ctx = tree_as(primary)) { - return visitIdent(ctx); - } else if (auto* ctx = tree_as(primary)) { - return visitGlobalCall(ctx); - } else if (auto* ctx = tree_as(primary)) { - return visitCreateList(ctx); - } else if (auto* ctx = tree_as(primary)) { - return visitCreateMap(ctx); - } else if (auto* ctx = tree_as(primary)) { - return visitCreateMessage(ctx); - } else if (auto* ctx = tree_as(primary)) { - return visitConstantLiteral(ctx); - } - if (factory_.HasErrors()) { - // ANTLR creates PrimaryContext rather than a derived class during certain - // error conditions. This is odd, but we ignore it as we already have errors - // that occurred. - return ExprToAny(factory_.NewUnspecified(factory_.NextId({}))); - } - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(pctx), - "invalid primary expression")); -} - -std::any ParserVisitor::visitMemberExpr(CelParser::MemberExprContext* mctx) { - CelParser::MemberContext* member = mctx->member(); - if (auto* ctx = tree_as(member)) { - return visitPrimaryExpr(ctx); - } else if (auto* ctx = tree_as(member)) { - return visitSelect(ctx); - } else if (auto* ctx = tree_as(member)) { - return visitMemberCall(ctx); - } else if (auto* ctx = tree_as(member)) { - return visitIndex(ctx); - } - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(mctx), - "unsupported simple expression")); -} - -std::any ParserVisitor::visitStart(CelParser::StartContext* ctx) { - return visit(ctx->expr()); -} - -antlr4::tree::ParseTree* ParserVisitor::UnnestContext( - antlr4::tree::ParseTree* tree) { - antlr4::tree::ParseTree* last = nullptr; - while (tree != last) { - last = tree; - - if (auto* ctx = tree_as(tree)) { - tree = ctx->expr(); - } - - if (auto* ctx = tree_as(tree)) { - if (ctx->op != nullptr) { - return ctx; - } - tree = ctx->e; - } - - if (auto* ctx = tree_as(tree)) { - if (!ctx->ops.empty()) { - return ctx; - } - tree = ctx->e; - } - - if (auto* ctx = tree_as(tree)) { - if (!ctx->ops.empty()) { - return ctx; - } - tree = ctx->e; - } - - if (auto* ctx = tree_as(tree)) { - if (ctx->calc() == nullptr) { - return ctx; - } - tree = ctx->calc(); - } - - if (auto* ctx = tree_as(tree)) { - if (ctx->unary() == nullptr) { - return ctx; - } - tree = ctx->unary(); - } - - if (auto* ctx = tree_as(tree)) { - tree = ctx->member(); - } - - if (auto* ctx = tree_as(tree)) { - if (auto* nested = tree_as(ctx->primary())) { - tree = nested->e; - } else { - return ctx; - } - } - } - - return tree; -} - -std::any ParserVisitor::visitExpr(CelParser::ExprContext* ctx) { - auto result = ExprFromAny(visit(ctx->e)); - if (!ctx->op) { - return ExprToAny(std::move(result)); - } - std::vector arguments; - arguments.reserve(3); - arguments.push_back(std::move(result)); - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - arguments.push_back(ExprFromAny(visit(ctx->e1))); - arguments.push_back(ExprFromAny(visit(ctx->e2))); - - return ExprToAny( - factory_.NewCall(op_id, CelOperator::CONDITIONAL, std::move(arguments))); -} - -std::any ParserVisitor::visitConditionalOr( - CelParser::ConditionalOrContext* ctx) { - auto result = ExprFromAny(visit(ctx->e)); - if (ctx->ops.empty()) { - return ExprToAny(std::move(result)); - } - ExpressionBalancer b(factory_, CelOperator::LOGICAL_OR, std::move(result)); - for (size_t i = 0; i < ctx->ops.size(); ++i) { - auto op = ctx->ops[i]; - if (i >= ctx->e1.size()) { - return ExprToAny( - factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "unexpected character, wanted '||'")); - } - auto next = ExprFromAny(visit(ctx->e1[i])); - int64_t op_id = factory_.NextId(SourceRangeFromToken(op)); - b.AddTerm(op_id, std::move(next)); - } - return ExprToAny(b.Balance()); -} - -std::any ParserVisitor::visitConditionalAnd( - CelParser::ConditionalAndContext* ctx) { - auto result = ExprFromAny(visit(ctx->e)); - if (ctx->ops.empty()) { - return ExprToAny(std::move(result)); - } - ExpressionBalancer b(factory_, CelOperator::LOGICAL_AND, std::move(result)); - for (size_t i = 0; i < ctx->ops.size(); ++i) { - auto op = ctx->ops[i]; - if (i >= ctx->e1.size()) { - return ExprToAny( - factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "unexpected character, wanted '&&'")); - } - auto next = ExprFromAny(visit(ctx->e1[i])); - int64_t op_id = factory_.NextId(SourceRangeFromToken(op)); - b.AddTerm(op_id, std::move(next)); - } - return ExprToAny(b.Balance()); -} - -std::any ParserVisitor::visitRelation(CelParser::RelationContext* ctx) { - if (ctx->calc()) { - return visit(ctx->calc()); - } - std::string op_text; - if (ctx->op) { - op_text = ctx->op->getText(); - } - auto op = ReverseLookupOperator(op_text); - if (op) { - auto lhs = ExprFromAny(visit(ctx->relation(0))); - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - auto rhs = ExprFromAny(visit(ctx->relation(1))); - return ExprToAny( - GlobalCallOrMacro(op_id, *op, std::move(lhs), std::move(rhs))); - } - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "operator not found")); -} - -std::any ParserVisitor::visitCalc(CelParser::CalcContext* ctx) { - if (ctx->unary()) { - return visit(ctx->unary()); - } - std::string op_text; - if (ctx->op) { - op_text = ctx->op->getText(); - } - auto op = ReverseLookupOperator(op_text); - if (op) { - auto lhs = ExprFromAny(visit(ctx->calc(0))); - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - auto rhs = ExprFromAny(visit(ctx->calc(1))); - return ExprToAny( - GlobalCallOrMacro(op_id, *op, std::move(lhs), std::move(rhs))); - } - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "operator not found")); -} - -std::any ParserVisitor::visitUnary(CelParser::UnaryContext* ctx) { - return ExprToAny(factory_.NewStringConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), "<>")); -} - -std::any ParserVisitor::visitLogicalNot(CelParser::LogicalNotContext* ctx) { - if (ctx->ops.size() % 2 == 0) { - return visit(ctx->member()); - } - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->ops[0])); - auto target = ExprFromAny(visit(ctx->member())); - return ExprToAny( - GlobalCallOrMacro(op_id, CelOperator::LOGICAL_NOT, std::move(target))); -} - -std::any ParserVisitor::visitNegate(CelParser::NegateContext* ctx) { - if (ctx->ops.size() % 2 == 0) { - return visit(ctx->member()); - } - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->ops[0])); - auto target = ExprFromAny(visit(ctx->member())); - return ExprToAny( - GlobalCallOrMacro(op_id, CelOperator::NEGATE, std::move(target))); -} - -std::string ParserVisitor::NormalizeIdentifier( - CelParser::EscapeIdentContext* ctx) { - if (auto* raw_id = tree_as(ctx); raw_id) { - return raw_id->id->getText(); - } - if (auto* escaped_id = tree_as(ctx); - escaped_id) { - if (!enable_quoted_identifiers_) { - factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "unsupported syntax '`'"); - } - auto escaped_id_text = escaped_id->id->getText(); - return escaped_id_text.substr(1, escaped_id_text.size() - 2); - } - - // Fallthrough might occur if the parser is in an error state. - return ""; -} - -std::any ParserVisitor::visitSelect(CelParser::SelectContext* ctx) { - auto operand = ExprFromAny(visit(ctx->member())); - // Handle the error case where no valid identifier is specified. - if (!ctx->id || !ctx->op) { - return ExprToAny(factory_.NewUnspecified( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); - } - auto id = NormalizeIdentifier(ctx->id); - if (ctx->opt != nullptr) { - if (!enable_optional_syntax_) { - return ExprToAny(factory_.ReportError( - SourceRangeFromParserRuleContext(ctx), "unsupported syntax '.?'")); - } - auto op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - std::vector arguments; - arguments.reserve(2); - arguments.push_back(std::move(operand)); - arguments.push_back(factory_.NewStringConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), std::move(id))); - return ExprToAny(factory_.NewCall(op_id, "_?._", std::move(arguments))); - } - return ExprToAny( - factory_.NewSelect(factory_.NextId(SourceRangeFromToken(ctx->op)), - std::move(operand), std::move(id))); -} - -std::any ParserVisitor::visitMemberCall(CelParser::MemberCallContext* ctx) { - auto operand = ExprFromAny(visit(ctx->member())); - // Handle the error case where no valid identifier is specified. - if (!ctx->id) { - return ExprToAny(factory_.NewUnspecified( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); - } - auto id = ctx->id->getText(); - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->open)); - auto args = visitList(ctx->args); - return ExprToAny( - ReceiverCallOrMacroImpl(op_id, id, std::move(operand), std::move(args))); -} - -std::any ParserVisitor::visitIndex(CelParser::IndexContext* ctx) { - auto target = ExprFromAny(visit(ctx->member())); - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - auto index = ExprFromAny(visit(ctx->index)); - if (!enable_optional_syntax_ && ctx->opt != nullptr) { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "unsupported syntax '.?'")); - } - return ExprToAny(GlobalCallOrMacro( - op_id, ctx->opt != nullptr ? "_[?_]" : CelOperator::INDEX, - std::move(target), std::move(index))); -} - -std::any ParserVisitor::visitCreateMessage( - CelParser::CreateMessageContext* ctx) { - std::vector parts; - parts.reserve(ctx->ids.size()); - for (const auto* id : ctx->ids) { - parts.push_back(id->getText()); - } - std::string name; - if (ctx->leadingDot) { - name.push_back('.'); - name.append(absl::StrJoin(parts, ".")); - } else { - name = absl::StrJoin(parts, "."); - } - int64_t obj_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - std::vector fields; - if (ctx->entries) { - fields = visitFields(ctx->entries); - } - return ExprToAny( - factory_.NewStruct(obj_id, std::move(name), std::move(fields))); -} - -std::any ParserVisitor::visitFieldInitializerList( - CelParser::FieldInitializerListContext* ctx) { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "<>")); -} - -std::vector ParserVisitor::visitFields( - CelParser::FieldInitializerListContext* ctx) { - std::vector res; - if (!ctx || ctx->fields.empty()) { - return res; - } - - res.reserve(ctx->fields.size()); - for (size_t i = 0; i < ctx->fields.size(); ++i) { - if (i >= ctx->cols.size() || i >= ctx->values.size()) { - // This is the result of a syntax error detected elsewhere. - return res; - } - auto* f = ctx->fields[i]; - if (!f->escapeIdent()) { - ABSL_DCHECK(HasErrored()); - // This is the result of a syntax error detected elsewhere. - return res; - } - - std::string id = NormalizeIdentifier(f->escapeIdent()); - - int64_t init_id = factory_.NextId(SourceRangeFromToken(ctx->cols[i])); - if (!enable_optional_syntax_ && f->opt) { - factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "unsupported syntax '?'"); - continue; - } - auto value = ExprFromAny(visit(ctx->values[i])); - res.push_back(factory_.NewStructField(init_id, std::move(id), - std::move(value), f->opt != nullptr)); - } - - return res; -} - -std::any ParserVisitor::visitIdent(CelParser::IdentContext* ctx) { - std::string ident_name; - if (ctx->leadingDot) { - ident_name = "."; - } - if (!ctx->id) { - return ExprToAny(factory_.NewUnspecified( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); - } - // check if ID is in reserved identifiers - if (cel::internal::LexisIsReserved(ctx->id->getText())) { - return ExprToAny(factory_.ReportError( - SourceRangeFromParserRuleContext(ctx), - absl::StrFormat("reserved identifier: %s", ctx->id->getText()))); - } - - ident_name += ctx->id->getText(); - - return ExprToAny(factory_.NewIdent( - factory_.NextId(SourceRangeFromToken(ctx->id)), std::move(ident_name))); -} - -std::any ParserVisitor::visitGlobalCall(CelParser::GlobalCallContext* ctx) { - std::string ident_name; - if (ctx->leadingDot) { - ident_name = "."; - } - if (!ctx->id || !ctx->op) { - return ExprToAny(factory_.NewUnspecified( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); - } - // check if ID is in reserved identifiers - if (cel::internal::LexisIsReserved(ctx->id->getText())) { - return ExprToAny(factory_.ReportError( - SourceRangeFromParserRuleContext(ctx), - absl::StrFormat("reserved identifier: %s", ctx->id->getText()))); - } - - ident_name += ctx->id->getText(); - - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - auto args = visitList(ctx->args); - return ExprToAny( - GlobalCallOrMacroImpl(op_id, std::move(ident_name), std::move(args))); -} - -std::any ParserVisitor::visitNested(CelParser::NestedContext* ctx) { - return visit(ctx->e); -} - -std::any ParserVisitor::visitCreateList(CelParser::CreateListContext* ctx) { - int64_t list_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - auto elems = visitList(ctx->elems); - return ExprToAny(factory_.NewList(list_id, std::move(elems))); -} - -std::vector ParserVisitor::visitList( - CelParser::ListInitContext* ctx) { - std::vector rv; - if (!ctx) return rv; - rv.reserve(ctx->elems.size()); - for (size_t i = 0; i < ctx->elems.size(); ++i) { - auto* expr_ctx = ctx->elems[i]; - if (expr_ctx == nullptr) { - return rv; - } - if (!enable_optional_syntax_ && expr_ctx->opt != nullptr) { - factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "unsupported syntax '?'"); - rv.push_back(factory_.NewListElement(factory_.NewUnspecified(0), false)); - continue; - } - rv.push_back(factory_.NewListElement(ExprFromAny(visitExpr(expr_ctx->e)), - expr_ctx->opt != nullptr)); - } - return rv; -} - -std::vector ParserVisitor::visitList(CelParser::ExprListContext* ctx) { - std::vector rv; - if (!ctx) return rv; - std::transform(ctx->e.begin(), ctx->e.end(), std::back_inserter(rv), - [this](CelParser::ExprContext* expr_ctx) { - return ExprFromAny(visitExpr(expr_ctx)); - }); - return rv; -} - -std::any ParserVisitor::visitCreateMap(CelParser::CreateMapContext* ctx) { - int64_t struct_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - std::vector entries; - if (ctx->entries) { - entries = visitEntries(ctx->entries); - } - return ExprToAny(factory_.NewMap(struct_id, std::move(entries))); -} - -std::any ParserVisitor::visitConstantLiteral( - CelParser::ConstantLiteralContext* clctx) { - CelParser::LiteralContext* literal = clctx->literal(); - if (auto* ctx = tree_as(literal)) { - return visitInt(ctx); - } else if (auto* ctx = tree_as(literal)) { - return visitUint(ctx); - } else if (auto* ctx = tree_as(literal)) { - return visitDouble(ctx); - } else if (auto* ctx = tree_as(literal)) { - return visitString(ctx); - } else if (auto* ctx = tree_as(literal)) { - return visitBytes(ctx); - } else if (auto* ctx = tree_as(literal)) { - return visitBoolFalse(ctx); - } else if (auto* ctx = tree_as(literal)) { - return visitBoolTrue(ctx); - } else if (auto* ctx = tree_as(literal)) { - return visitNull(ctx); - } - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(clctx), - "invalid constant literal expression")); -} - -std::any ParserVisitor::visitMapInitializerList( - CelParser::MapInitializerListContext* ctx) { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "<>")); -} - -std::vector ParserVisitor::visitEntries( - CelParser::MapInitializerListContext* ctx) { - std::vector res; - if (!ctx || ctx->keys.empty()) { - return res; - } - - res.reserve(ctx->cols.size()); - for (size_t i = 0; i < ctx->cols.size(); ++i) { - auto id = factory_.NextId(SourceRangeFromToken(ctx->cols[i])); - if (!enable_optional_syntax_ && ctx->keys[i]->opt) { - factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "unsupported syntax '?'"); - res.push_back(factory_.NewMapEntry(0, factory_.NewUnspecified(0), - factory_.NewUnspecified(0), false)); - continue; - } - auto key = ExprFromAny(visit(ctx->keys[i]->e)); - auto value = ExprFromAny(visit(ctx->values[i])); - res.push_back(factory_.NewMapEntry(id, std::move(key), std::move(value), - ctx->keys[i]->opt != nullptr)); - } - return res; -} - -std::any ParserVisitor::visitInt(CelParser::IntContext* ctx) { - std::string value; - if (ctx->sign) { - value = ctx->sign->getText(); - } - value += ctx->tok->getText(); - int64_t int_value; - if (absl::StartsWith(ctx->tok->getText(), "0x")) { - if (absl::SimpleHexAtoi(value, &int_value)) { - return ExprToAny(factory_.NewIntConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), int_value)); - } else { - return ExprToAny(factory_.ReportError( - SourceRangeFromParserRuleContext(ctx), "invalid hex int literal")); - } - } - if (absl::SimpleAtoi(value, &int_value)) { - return ExprToAny(factory_.NewIntConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), int_value)); - } else { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "invalid int literal")); - } -} - -std::any ParserVisitor::visitUint(CelParser::UintContext* ctx) { - std::string value = ctx->tok->getText(); - // trim the 'u' designator included in the uint literal. - if (!value.empty()) { - value.resize(value.size() - 1); - } - uint64_t uint_value; - if (absl::StartsWith(ctx->tok->getText(), "0x")) { - if (absl::SimpleHexAtoi(value, &uint_value)) { - return ExprToAny(factory_.NewUintConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), uint_value)); - } else { - return ExprToAny(factory_.ReportError( - SourceRangeFromParserRuleContext(ctx), "invalid hex uint literal")); - } - } - if (absl::SimpleAtoi(value, &uint_value)) { - return ExprToAny(factory_.NewUintConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), uint_value)); - } else { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "invalid uint literal")); - } -} - -std::any ParserVisitor::visitDouble(CelParser::DoubleContext* ctx) { - std::string value; - if (ctx->sign) { - value = ctx->sign->getText(); - } - value += ctx->tok->getText(); - double double_value; - if (absl::SimpleAtod(value, &double_value)) { - return ExprToAny(factory_.NewDoubleConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), double_value)); +absl::StatusOr EnrichedParse( + const cel::Source& source, const cel::MacroRegistry& registry, + const ParserOptions& options) { + ParsedExpr parsed_expr; + EnrichedSourceInfo enriched_source_info; + std::unique_ptr ast; +#ifdef EXCLUDE_CEL_ANTLR_PARSER + CEL_ASSIGN_OR_RETURN( + ast, cel::parser_internal::PrattParseImpl(source, registry, options, + /*parse_issues=*/nullptr, + &enriched_source_info)); +#else + if (options.enable_pratt_parser) { + CEL_ASSIGN_OR_RETURN( + ast, cel::parser_internal::PrattParseImpl(source, registry, options, + /*parse_issues=*/nullptr, + &enriched_source_info)); } else { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "invalid double literal")); - } -} - -std::any ParserVisitor::visitString(CelParser::StringContext* ctx) { - auto status_or_value = cel::internal::ParseStringLiteral(ctx->tok->getText()); - if (!status_or_value.ok()) { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - status_or_value.status().message())); - } - return ExprToAny(factory_.NewStringConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), - std::move(status_or_value).value())); -} - -std::any ParserVisitor::visitBytes(CelParser::BytesContext* ctx) { - auto status_or_value = cel::internal::ParseBytesLiteral(ctx->tok->getText()); - if (!status_or_value.ok()) { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - status_or_value.status().message())); - } - return ExprToAny(factory_.NewBytesConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), - std::move(status_or_value).value())); -} - -std::any ParserVisitor::visitBoolTrue(CelParser::BoolTrueContext* ctx) { - return ExprToAny(factory_.NewBoolConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), true)); -} - -std::any ParserVisitor::visitBoolFalse(CelParser::BoolFalseContext* ctx) { - return ExprToAny(factory_.NewBoolConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), false)); -} - -std::any ParserVisitor::visitNull(CelParser::NullContext* ctx) { - return ExprToAny(factory_.NewNullConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); -} - -cel::SourceInfo ParserVisitor::GetSourceInfo() { - cel::SourceInfo source_info; - source_info.set_location(std::string(source_.description())); - for (const auto& positions : factory_.positions()) { - source_info.mutable_positions().insert( - std::pair{positions.first, positions.second.begin}); - } - source_info.mutable_line_offsets().reserve(source_.line_offsets().size()); - for (const auto& line_offset : source_.line_offsets()) { - source_info.mutable_line_offsets().push_back(line_offset); - } - - source_info.mutable_macro_calls() = factory_.release_macro_calls(); - return source_info; -} - -EnrichedSourceInfo ParserVisitor::enriched_source_info() const { - std::map> offsets; - for (const auto& positions : factory_.positions()) { - offsets.insert( - std::pair{positions.first, - std::pair{positions.second.begin, positions.second.end - 1}}); - } - return EnrichedSourceInfo(std::move(offsets)); -} - -void ParserVisitor::syntaxError(antlr4::Recognizer* recognizer, - antlr4::Token* offending_symbol, size_t line, - size_t col, const std::string& msg, - std::exception_ptr e) { - cel::SourceRange range; - if (auto position = source_.GetPosition(cel::SourceLocation{ - static_cast(line), static_cast(col)}); - position) { - range.begin = *position; - } - factory_.ReportError(range, absl::StrCat("Syntax error: ", msg)); -} - -bool ParserVisitor::HasErrored() const { return factory_.HasErrors(); } - -std::string ParserVisitor::ErrorMessage() { return factory_.ErrorMessage(); } - -Expr ParserVisitor::GlobalCallOrMacroImpl(int64_t expr_id, - absl::string_view function, - std::vector args) { - if (auto macro = macro_registry_.FindMacro(function, args.size(), false); - macro) { - std::vector macro_args; - if (add_macro_calls_) { - macro_args.reserve(args.size()); - for (const auto& arg : args) { - macro_args.push_back(factory_.BuildMacroCallArg(arg)); - } - } - factory_.BeginMacro(factory_.GetSourceRange(expr_id)); - auto expr = macro->Expand(factory_, absl::nullopt, absl::MakeSpan(args)); - factory_.EndMacro(); - if (expr) { - if (add_macro_calls_) { - factory_.AddMacroCall(expr->id(), function, absl::nullopt, - std::move(macro_args)); - } - // We did not end up using `expr_id`. Delete metadata. - factory_.EraseId(expr_id); - return std::move(*expr); - } - } - - return factory_.NewCall(expr_id, function, std::move(args)); -} - -Expr ParserVisitor::ReceiverCallOrMacroImpl(int64_t expr_id, - absl::string_view function, - Expr target, - std::vector args) { - if (auto macro = macro_registry_.FindMacro(function, args.size(), true); - macro) { - Expr macro_target; - std::vector macro_args; - if (add_macro_calls_) { - macro_args.reserve(args.size()); - macro_target = factory_.BuildMacroCallArg(target); - for (const auto& arg : args) { - macro_args.push_back(factory_.BuildMacroCallArg(arg)); - } - } - factory_.BeginMacro(factory_.GetSourceRange(expr_id)); - auto expr = macro->Expand(factory_, std::ref(target), absl::MakeSpan(args)); - factory_.EndMacro(); - if (expr) { - if (add_macro_calls_) { - factory_.AddMacroCall(expr->id(), function, std::move(macro_target), - std::move(macro_args)); - } - // We did not end up using `expr_id`. Delete metadata. - factory_.EraseId(expr_id); - return std::move(*expr); - } - } - return factory_.NewMemberCall(expr_id, function, std::move(target), - std::move(args)); -} - -std::string ParserVisitor::ExtractQualifiedName(antlr4::ParserRuleContext* ctx, - const Expr& e) { - if (e == Expr{}) { - return ""; - } - - if (const auto* ident_expr = absl::get_if(&e.kind()); ident_expr) { - return ident_expr->name(); - } - if (const auto* select_expr = absl::get_if(&e.kind()); - select_expr) { - std::string prefix = ExtractQualifiedName(ctx, select_expr->operand()); - if (!prefix.empty()) { - return absl::StrCat(prefix, ".", select_expr->field()); - } - } - factory_.ReportError(factory_.GetSourceRange(e.id()), - "expected a qualified name"); - return ""; -} - -// Replacements for absl::StrReplaceAll for escaping standard whitespace -// characters. -static constexpr auto kStandardReplacements = - std::array, 3>{ - std::make_pair("\n", "\\n"), - std::make_pair("\r", "\\r"), - std::make_pair("\t", "\\t"), - }; - -static constexpr absl::string_view kSingleQuote = "'"; - -// ExprRecursionListener extends the standard ANTLR CelParser to ensure that -// recursive entries into the 'expr' rule are limited to a configurable depth so -// as to prevent stack overflows. -class ExprRecursionListener final : public ParseTreeListener { - public: - explicit ExprRecursionListener( - const int max_recursion_depth = kDefaultMaxRecursionDepth) - : max_recursion_depth_(max_recursion_depth), recursion_depth_(0) {} - ~ExprRecursionListener() override {} - - void visitTerminal(TerminalNode* node) override {}; - void visitErrorNode(ErrorNode* error) override {}; - void enterEveryRule(ParserRuleContext* ctx) override; - void exitEveryRule(ParserRuleContext* ctx) override; - - private: - const int max_recursion_depth_; - int recursion_depth_; -}; - -void ExprRecursionListener::enterEveryRule(ParserRuleContext* ctx) { - // Throw a ParseCancellationException since the parsing would otherwise - // continue if this were treated as a syntax error and the problem would - // continue to manifest. - if (ctx->getRuleIndex() == CelParser::RuleExpr) { - if (recursion_depth_ > max_recursion_depth_) { - throw ParseCancellationException( - absl::StrFormat("Expression recursion limit exceeded. limit: %d", - max_recursion_depth_)); - } - recursion_depth_++; + CEL_ASSIGN_OR_RETURN( + ast, cel::parser_internal::AntlrParseImpl(source, registry, options, + /*parse_issues=*/nullptr, + &enriched_source_info)); } -} +#endif -void ExprRecursionListener::exitEveryRule(ParserRuleContext* ctx) { - if (ctx->getRuleIndex() == CelParser::RuleExpr) { - recursion_depth_--; - } + CEL_RETURN_IF_ERROR(cel::ast_internal::ExprToProto( + ast->root_expr(), parsed_expr.mutable_expr())); + CEL_RETURN_IF_ERROR(cel::ast_internal::SourceInfoToProto( + ast->source_info(), parsed_expr.mutable_source_info())); + return VerboseParsedExpr(std::move(parsed_expr), + std::move(enriched_source_info)); } -class RecoveryLimitErrorStrategy final : public DefaultErrorStrategy { - public: - explicit RecoveryLimitErrorStrategy( - int recovery_limit = kDefaultErrorRecoveryLimit, - int recovery_token_lookahead_limit = - kDefaultErrorRecoveryTokenLookaheadLimit) - : recovery_limit_(recovery_limit), - recovery_attempts_(0), - recovery_token_lookahead_limit_(recovery_token_lookahead_limit) {} - - void recover(Parser* recognizer, std::exception_ptr e) override { - checkRecoveryLimit(recognizer); - DefaultErrorStrategy::recover(recognizer, e); - } - - Token* recoverInline(Parser* recognizer) override { - checkRecoveryLimit(recognizer); - return DefaultErrorStrategy::recoverInline(recognizer); - } - - // Override the ANTLR implementation to introduce a token lookahead limit as - // this prevents pathologically constructed, yet small (< 16kb) inputs from - // consuming inordinate amounts of compute. - // - // This method is only called on error recovery paths. - void consumeUntil(Parser* recognizer, const IntervalSet& set) override { - size_t ttype = recognizer->getInputStream()->LA(1); - int recovery_search_depth = 0; - while (ttype != Token::EOF && !set.contains(ttype) && - recovery_search_depth++ < recovery_token_lookahead_limit_) { - recognizer->consume(); - ttype = recognizer->getInputStream()->LA(1); - } - // Halt all parsing if the lookahead limit is reached during error recovery. - if (recovery_search_depth == recovery_token_lookahead_limit_) { - throw ParseCancellationException("Unable to find a recovery token"); - } - } - - protected: - std::string escapeWSAndQuote(const std::string& s) const override { - std::string result; - result.reserve(s.size() + 2); - absl::StrAppend(&result, kSingleQuote, s, kSingleQuote); - absl::StrReplaceAll(kStandardReplacements, &result); - return result; - } - - private: - void checkRecoveryLimit(Parser* recognizer) { - if (recovery_attempts_++ >= recovery_limit_) { - std::string too_many_errors = - absl::StrFormat("More than %d parse errors.", recovery_limit_); - recognizer->notifyErrorListeners(too_many_errors); - throw ParseCancellationException(too_many_errors); - } - } - - int recovery_limit_; - int recovery_attempts_; - int recovery_token_lookahead_limit_; -}; - -struct ParseResult { - cel::Expr expr; - cel::SourceInfo source_info; - EnrichedSourceInfo enriched_source_info; -}; - -absl::StatusOr ParseImpl(const cel::Source& source, - const cel::MacroRegistry& registry, - const ParserOptions& options) { - try { - CodePointStream input(source.content(), source.description()); - if (input.size() > options.expression_size_codepoint_limit) { - return absl::InvalidArgumentError(absl::StrCat( - "expression size exceeds codepoint limit.", " input size: ", - input.size(), ", limit: ", options.expression_size_codepoint_limit)); - } - CelLexer lexer(&input); - CommonTokenStream tokens(&lexer); - CelParser parser(&tokens); - ExprRecursionListener listener(options.max_recursion_depth); - absl::string_view accu_var = cel::kAccumulatorVariableName; - if (options.enable_hidden_accumulator_var) { - accu_var = cel::kHiddenAccumulatorVariableName; - } - ParserVisitor visitor(source, options.max_recursion_depth, accu_var, - registry, options.add_macro_calls, - options.enable_optional_syntax, - options.enable_quoted_identifiers); - - lexer.removeErrorListeners(); - parser.removeErrorListeners(); - lexer.addErrorListener(&visitor); - parser.addErrorListener(&visitor); - parser.addParseListener(&listener); - - // Limit the number of error recovery attempts to prevent bad expressions - // from consuming lots of cpu / memory. - parser.setErrorHandler(std::make_shared( - options.error_recovery_limit, - options.error_recovery_token_lookahead_limit)); - - Expr expr; - try { - expr = ExprFromAny(visitor.visit(parser.start())); - } catch (const ParseCancellationException& e) { - if (visitor.HasErrored()) { - return absl::InvalidArgumentError(visitor.ErrorMessage()); - } - return absl::CancelledError(e.what()); - } - - if (visitor.HasErrored()) { - return absl::InvalidArgumentError(visitor.ErrorMessage()); - } - - return { - ParseResult{.expr = std::move(expr), - .source_info = visitor.GetSourceInfo(), - .enriched_source_info = visitor.enriched_source_info()}}; - } catch (const std::exception& e) { - return absl::AbortedError(e.what()); - } catch (const char* what) { - // ANTLRv4 has historically thrown C string literals. - return absl::AbortedError(what); - } catch (...) { - // We guarantee to never throw and always return a status. - return absl::UnknownError("An unknown exception occurred"); - } +absl::StatusOr EnrichedParse( + absl::string_view expression, const std::vector& macros, + absl::string_view description, const ParserOptions& options) { + CEL_ASSIGN_OR_RETURN(auto source, + cel::NewSource(expression, std::string(description))); + cel::MacroRegistry macro_registry; + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(macros)); + return EnrichedParse(*source, macro_registry, options); } -class ParserImpl : public cel::Parser { - public: - explicit ParserImpl(const ParserOptions& options, - cel::MacroRegistry macro_registry, - absl::flat_hash_set library_ids) - : options_(options), - macro_registry_(std::move(macro_registry)), - library_ids_(std::move(library_ids)) {} - - absl::StatusOr> Parse( - const cel::Source& source) const override { - CEL_ASSIGN_OR_RETURN(auto parse_result, - ParseImpl(source, macro_registry_, options_)); - return std::make_unique(std::move(parse_result.expr), - std::move(parse_result.source_info)); - } - - std::unique_ptr ToBuilder() const override; - - private: - const ParserOptions options_; - const cel::MacroRegistry macro_registry_; - absl::flat_hash_set library_ids_; -}; - -class ParserBuilderImpl : public cel::ParserBuilder { - public: - explicit ParserBuilderImpl(const ParserOptions& options) - : options_(options) {} - - ParserOptions& GetOptions() override { return options_; } - - absl::Status AddMacro(const cel::Macro& macro) override { - for (const auto& existing_macro : macros_) { - if (existing_macro.key() == macro.key()) { - return absl::AlreadyExistsError( - absl::StrCat("macro already exists: ", macro.key())); - } - } - macros_.push_back(macro); - return absl::OkStatus(); - } - - absl::Status AddLibrary(cel::ParserLibrary library) override { - if (!library.id.empty()) { - auto [it, inserted] = library_ids_.insert(library.id); - if (!inserted) { - return absl::AlreadyExistsError( - absl::StrCat("parser library already exists: ", library.id)); - } - } - libraries_.push_back(std::move(library)); - return absl::OkStatus(); - } - - absl::Status AddLibrarySubset(cel::ParserLibrarySubset subset) override { - if (subset.library_id.empty()) { - return absl::InvalidArgumentError("subset must have a library id"); - } - std::string library_id = subset.library_id; - auto [it, inserted] = - library_subsets_.insert({library_id, std::move(subset)}); - if (!inserted) { - return absl::AlreadyExistsError( - absl::StrCat("parser library subset already exists: ", library_id)); - } - return absl::OkStatus(); - } - - absl::StatusOr> Build() override { - using std::swap; - // Save the old configured macros so they aren't affected by applying the - // libraries and can be restored if an error occurs. - std::vector individual_macros; - swap(individual_macros, macros_); - absl::Cleanup cleanup([&] { swap(macros_, individual_macros); }); - - cel::MacroRegistry macro_registry; - - for (const auto& library : libraries_) { - CEL_RETURN_IF_ERROR(library.configure(*this)); - if (!library.id.empty()) { - auto it = library_subsets_.find(library.id); - if (it != library_subsets_.end()) { - const cel::ParserLibrarySubset& subset = it->second; - for (const auto& macro : macros_) { - if (subset.should_include_macro(macro)) { - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(macro)); - } - } - macros_.clear(); - continue; - } - } - - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(macros_)); - macros_.clear(); - } - - absl::flat_hash_set library_ids(library_ids_); - - // Hack to support adding the standard library macros either by option or - // with a library configurer. - if (!options_.disable_standard_macros && !library_ids_.contains("stdlib")) { - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(Macro::AllMacros())); - library_ids.insert("stdlib"); - } - - if (options_.enable_optional_syntax && !library_ids_.contains("optional")) { - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptMapMacro())); - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptFlatMapMacro())); - library_ids.insert("optional"); - } - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(individual_macros)); - return std::make_unique(options_, std::move(macro_registry), - std::move(library_ids)); - } - - private: - friend class ParserImpl; - - ParserOptions options_; - std::vector macros_; - absl::flat_hash_set library_ids_; - std::vector libraries_; - absl::flat_hash_map library_subsets_; -}; - -std::unique_ptr ParserImpl::ToBuilder() const { - auto ins = std::make_unique(options_); - ins->library_ids_ = library_ids_; - ins->macros_ = macro_registry_.ListMacros(); - return ins; +absl::StatusOr ParseWithMacros(absl::string_view expression, + const std::vector& macros, + absl::string_view description, + const ParserOptions& options) { + CEL_ASSIGN_OR_RETURN(auto verbose_parsed_expr, + EnrichedParse(expression, macros, description, options)); + return verbose_parsed_expr.parsed_expr(); } -} // namespace - absl::StatusOr Parse(absl::string_view expression, absl::string_view description, const ParserOptions& options) { @@ -1855,41 +111,7 @@ absl::StatusOr Parse(absl::string_view expression, return ParseWithMacros(expression, macros, description, options); } -absl::StatusOr ParseWithMacros(absl::string_view expression, - const std::vector& macros, - absl::string_view description, - const ParserOptions& options) { - CEL_ASSIGN_OR_RETURN(auto verbose_parsed_expr, - EnrichedParse(expression, macros, description, options)); - return verbose_parsed_expr.parsed_expr(); -} - -absl::StatusOr EnrichedParse( - absl::string_view expression, const std::vector& macros, - absl::string_view description, const ParserOptions& options) { - CEL_ASSIGN_OR_RETURN(auto source, - cel::NewSource(expression, std::string(description))); - cel::MacroRegistry macro_registry; - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(macros)); - return EnrichedParse(*source, macro_registry, options); -} - -absl::StatusOr EnrichedParse( - const cel::Source& source, const cel::MacroRegistry& registry, - const ParserOptions& options) { - CEL_ASSIGN_OR_RETURN(ParseResult parse_result, - ParseImpl(source, registry, options)); - ParsedExpr parsed_expr; - CEL_RETURN_IF_ERROR(cel::ast_internal::ExprToProto( - parse_result.expr, parsed_expr.mutable_expr())); - - CEL_RETURN_IF_ERROR(cel::ast_internal::SourceInfoToProto( - parse_result.source_info, parsed_expr.mutable_source_info())); - return VerboseParsedExpr(std::move(parsed_expr), - std::move(parse_result.enriched_source_info)); -} - -absl::StatusOr Parse( +absl::StatusOr<::cel::expr::ParsedExpr> Parse( const cel::Source& source, const cel::MacroRegistry& registry, const ParserOptions& options) { CEL_ASSIGN_OR_RETURN(auto verbose_expr, @@ -1901,13 +123,15 @@ absl::StatusOr Parse( namespace cel { -// Creates a new parser builder. -// -// Intended for use with the Compiler class, most users should prefer the free -// functions above for independent parsing of expressions. std::unique_ptr NewParserBuilder(const ParserOptions& options) { - return std::make_unique( - options); +#ifdef EXCLUDE_CEL_ANTLR_PARSER + return parser_internal::NewPrattParserBuilder(options); +#else + if (options.enable_pratt_parser) { + return parser_internal::NewPrattParserBuilder(options); + } + return parser_internal::NewAntlrParserBuilder(options); +#endif } } // namespace cel diff --git a/parser/parser_interface.h b/parser/parser_interface.h index 7cc21ff26..eb902c041 100644 --- a/parser/parser_interface.h +++ b/parser/parser_interface.h @@ -16,10 +16,14 @@ #include #include +#include +#include +#include "absl/base/nullability.h" #include "absl/functional/any_invocable.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/string_view.h" #include "common/ast.h" #include "common/source.h" #include "parser/macro.h" @@ -73,6 +77,26 @@ class ParserBuilder { virtual absl::StatusOr> Build() = 0; }; +// Information about a parse failure. +class ParseIssue { + public: + explicit ParseIssue(std::string message) : message_(std::move(message)) {} + ParseIssue(SourceLocation location, std::string message) + : location_(location), message_(std::move(message)) {} + + ParseIssue(const ParseIssue& other) = default; + ParseIssue& operator=(const ParseIssue& other) = default; + ParseIssue(ParseIssue&& other) = default; + ParseIssue& operator=(ParseIssue&& other) = default; + + SourceLocation location() const { return location_; } + absl::string_view message() const { return message_; } + + private: + SourceLocation location_; + std::string message_; +}; + // Interface for stateful CEL parser objects for use with a `Compiler` // (bundled parse and type check). This is not needed for most users: // prefer using the free functions in `parser.h` for more flexibility. @@ -81,13 +105,56 @@ class Parser { virtual ~Parser() = default; // Parses the given source into a CEL AST. - virtual absl::StatusOr> Parse( - const cel::Source& source) const = 0; + absl::StatusOr> Parse( + const cel::Source& source) const; + + // Parses the given source into a CEL AST, collecting parse errors in + // `issues`. If `issues` is non-null, it will be cleared and all parse + // issues will be appended to it. + absl::StatusOr> Parse( + const cel::Source& source, std::vector* issues) const; + + // Returns a Source object from the given input. + // Validates that the input is well-formed utf-8 and within the configured + // source limits. + absl::StatusOr> PrepareSource( + absl::string_view input, absl::string_view description) const; + absl::StatusOr> PrepareSource( + absl::string_view input) const; // Returns a builder initialized with the configuration of this parser. virtual std::unique_ptr ToBuilder() const = 0; + + protected: + virtual absl::StatusOr> ParseImpl( + const cel::Source& source, + std::vector* absl_nullable parse_issues) const = 0; + + virtual absl::StatusOr> PrepareSourceImpl( + absl::string_view input, absl::string_view description) const = 0; }; +inline absl::StatusOr> Parser::Parse( + const cel::Source& source) const { + return ParseImpl(source, nullptr); +} + +inline absl::StatusOr> Parser::Parse( + const cel::Source& source, std::vector* issues) const { + if (issues != nullptr) issues->clear(); + return ParseImpl(source, issues); +} + +inline absl::StatusOr> Parser::PrepareSource( + absl::string_view input, absl::string_view description) const { + return PrepareSourceImpl(input, description); +} + +inline absl::StatusOr> Parser::PrepareSource( + absl::string_view input) const { + return PrepareSourceImpl(input, ""); +} + } // namespace cel #endif // THIRD_PARTY_CEL_CPP_PARSER_PARSER_INTERFACE_H_ diff --git a/parser/parser_test.cc b/parser/parser_test.cc index 3659fd8fd..90838cc65 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -16,7 +16,7 @@ #include #include -#include +#include #include #include @@ -32,13 +32,13 @@ #include "absl/types/optional.h" #include "common/constant.h" #include "common/expr.h" +#include "common/expr_printer.h" #include "common/source.h" #include "internal/testing.h" #include "parser/macro.h" #include "parser/options.h" #include "parser/parser_interface.h" #include "parser/source_factory.h" -#include "testutil/expr_printer.h" namespace google::api::expr::parser { @@ -48,16 +48,26 @@ using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::ConstantKindCase; using ::cel::ExprKindCase; -using ::cel::test::ExprPrinter; +using ::cel::ExprPrinter; using ::cel::expr::Expr; +using ::testing::AnyOf; using ::testing::HasSubstr; using ::testing::Not; +#ifdef EXCLUDE_CEL_ANTLR_PARSER +// Only test pratt parser. +#define TestedParserImpls testing::Values(true) +#else +// Test both pratt and antlr parsers. +#define TestedParserImpls testing::Values(true, false) +#endif + struct TestInfo { TestInfo(const std::string& I, const std::string& P, const std::string& E = "", const std::string& L = "", - const std::string& R = "", const std::string& M = "") - : I(I), P(P), E(E), L(L), R(R), M(M) {} + const std::string& R = "", const std::string& M = "", + const std::string& E_PRATT = "") + : I(I), P(P), E(E), L(L), R(R), M(M), E_PRATT(E_PRATT) {} // I contains the input expression to be parsed. std::string I; @@ -77,6 +87,9 @@ struct TestInfo { // M contains the expected macro call output of hte expression tree. std::string M; + + // E_PRATT contains alternative error output when using pratt parser. + std::string E_PRATT; }; std::vector test_cases = { @@ -115,6 +128,27 @@ std::vector test_cases = { " a^#1:Expr.Ident#,\n" " 3^#3:int64#\n" ")^#2:Expr.Call#"}, + {"-x.foo[0]", + "-_(\n" + " _[_](\n" + " x^#2:Expr.Ident#.foo^#3:Expr.Select#,\n" + " 0^#5:int64#\n" + " )^#4:Expr.Call#\n" + ")^#1:Expr.Call#"}, + {"-x[0].foo", + "-_(\n" + " _[_](\n" + " x^#2:Expr.Ident#,\n" + " 0^#4:int64#\n" + " )^#3:Expr.Call#.foo^#5:Expr.Select#\n" + ")^#1:Expr.Call#"}, + {"!x[0].foo", + "!_(\n" + " _[_](\n" + " x^#2:Expr.Ident#,\n" + " 0^#4:int64#\n" + " )^#3:Expr.Call#.foo^#5:Expr.Select#\n" + ")^#1:Expr.Call#"}, {"SomeMessage{foo: 5, bar: \"xyz\"}", "SomeMessage{\n" " foo:5^#3:int64#^#2:Expr.CreateStruct.Entry#,\n" @@ -131,6 +165,16 @@ std::vector test_cases = { " foo^#3:Expr.Ident#:5^#4:int64#^#2:Expr.CreateStruct.Entry#,\n" " bar^#6:Expr.Ident#:\"xyz\"^#7:string#^#5:Expr.CreateStruct.Entry#\n" "}^#1:Expr.CreateStruct#"}, + {"{\"foo\": 5, \"bar\": \"xyz\"}", + "{\n" + " \"foo\"^#3:string#:5^#4:int64#^#2:Expr.CreateStruct.Entry#,\n" + " \"bar\"^#6:string#:\"xyz\"^#7:string#^#5:Expr.CreateStruct.Entry#\n" + "}^#1:Expr.CreateStruct#"}, + {"{'a': 1, 'b': 2}", + "{\n" + " \"a\"^#3:string#:1^#4:int64#^#2:Expr.CreateStruct.Entry#,\n" + " \"b\"^#6:string#:2^#7:int64#^#5:Expr.CreateStruct.Entry#\n" + "}^#1:Expr.CreateStruct#"}, {"a > 5 && a < 10", "_&&_(\n" " _>_(\n" @@ -160,6 +204,11 @@ std::vector test_cases = { "NUM_FLOAT, " "NUM_INT, " "NUM_UINT, STRING, BYTES, IDENTIFIER}\n | {\n" + " | .^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:2: Syntax error: expected '}'\n" + " | {\n" " | .^"}, // test cases from Go @@ -417,14 +466,27 @@ std::vector test_cases = { " | ....^\n" "ERROR: :1:7: Syntax error: extraneous input 'b' expecting \n" " | *@a | b\n" - " | ......^"}, + " | ......^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:1: Syntax error: unexpected token\n" + " | *@a | b\n" + " | ^\n" + "ERROR: :1:2: Syntax error: unexpected character\n" + " | *@a | b\n" + " | .^"}, {"a | b", "", "ERROR: :1:3: Syntax error: token recognition error at: '| '\n" " | a | b\n" " | ..^\n" "ERROR: :1:5: Syntax error: extraneous input 'b' expecting \n" " | a | b\n" - " | ....^"}, + " | ....^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:3: Syntax error: unexpected single '|', expected '||'\n" + " | a | b\n" + " | ..^"}, {"?", "", "ERROR: :1:1: Syntax error: mismatched input '?' expecting " "{'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, " @@ -432,13 +494,34 @@ std::vector test_cases = { "ERROR: :1:2: Syntax error: mismatched input '' expecting " "{'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, " "NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n | ?\n | .^\n" - "ERROR: :4294967295:0: <> parsetree"}, + "ERROR: :4294967295:0: <> parsetree", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:1: Syntax error: unexpected token\n" + " | ?\n" + " | ^"}, {"t{>C}", "", "ERROR: :1:3: Syntax error: extraneous input '>' expecting {'}', " "',', '\\u003F', IDENTIFIER, ESC_IDENTIFIER}\n | t{>C}\n | ..^\nERROR: " ":1:5: " "Syntax error: " - "mismatched input '}' expecting ':'\n | t{>C}\n | ....^"}, + "mismatched input '}' expecting ':'\n | t{>C}\n | ....^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:3: Syntax error: expected struct field name\n" + " | t{>C}\n" + " | ..^"}, + {"foo(a,b,)", "", + "ERROR: :1:9: Syntax error: mismatched input ')' expecting " + "{'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, " + "NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" + " | foo(a,b,)\n" + " | ........^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:9: unexpected token\n" + " | foo(a,b,)\n" + " | ........^"}, // Macro tests {"has(m.f)", "m^#2:Expr.Ident#.f~test-only~^#4:Expr.Select#", "", @@ -593,6 +676,11 @@ std::vector test_cases = { {"TestAllTypes(){single_int32: 1, single_int64: 2}", "", "ERROR: :1:15: Syntax error: mismatched input '{' expecting \n" " | TestAllTypes(){single_int32: 1, single_int64: 2}\n" + " | ..............^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:15: Syntax error: unexpected token after expression\n" + " | TestAllTypes(){single_int32: 1, single_int64: 2}\n" " | ..............^"}, {"size(x) == x.size()", "_==_(\n" @@ -610,12 +698,22 @@ std::vector test_cases = { "'{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, " "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | 1 + $\n" - " | .....^"}, + " | .....^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:5: Syntax error: unexpected character\n" + " | 1 + $\n" + " | ....^"}, {"1 + 2\n" "3 +", "", "ERROR: :2:1: Syntax error: mismatched input '3' expecting \n" " | 3 +\n" + " | ^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :2:1: Syntax error: unexpected token after expression\n" + " | 3 +\n" " | ^"}, {"\"\\\"\"", "\"\\\"\"^#1:string#"}, {"[1,3,4][0]", @@ -631,6 +729,53 @@ std::vector test_cases = { "ERROR: :1:7: all() variable name must be a simple identifier\n" " | 1.all(2, 3)\n" " | ......^"}, + {"[].all(.x, x)", "", + "ERROR: :1:9: all() variable name must be a simple identifier\n" + " | [].all(.x, x)\n" + " | ........^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:8: all() variable name must be a simple identifier\n" + " | [].all(.x, x)\n" + " | .......^"}, + {"[].exists(.x, x)", "", + "ERROR: :1:12: exists() variable name must be a simple identifier\n" + " | [].exists(.x, x)\n" + " | ...........^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:11: exists() variable name must be a simple identifier\n" + " | [].exists(.x, x)\n" + " | ..........^"}, + {"[].exists_one(.x, x)", "", + "ERROR: :1:16: exists_one() variable name must be a simple " + "identifier\n" + " | [].exists_one(.x, x)\n" + " | ...............^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:15: exists_one() variable name must be a simple " + "identifier\n" + " | [].exists_one(.x, x)\n" + " | ..............^"}, + {"[].map(.x, x, x)", "", + "ERROR: :1:9: map() variable name must be a simple identifier\n" + " | [].map(.x, x, x)\n" + " | ........^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:8: map() variable name must be a simple identifier\n" + " | [].map(.x, x, x)\n" + " | .......^"}, + {"[].filter(.x, x)", "", + "ERROR: :1:12: filter() variable name must be a simple identifier\n" + " | [].filter(.x, x)\n" + " | ...........^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:11: filter() variable name must be a simple identifier\n" + " | [].filter(.x, x)\n" + " | ..........^"}, {"x[\"a\"].single_int32 == 23", "_==_(\n" " _[_](\n" @@ -700,7 +845,12 @@ std::vector test_cases = { "'{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, " "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | 1 + +\n" - " | .....^"}, + " | .....^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:5: Syntax error: unexpected token\n" + " | 1 + +\n" + " | ....^"}, {"\"abc\" + \"def\"", "_+_(\n" " \"abc\"^#1:string#,\n" @@ -710,6 +860,11 @@ std::vector test_cases = { "ERROR: :1:10: Syntax error: no viable alternative at input " "'.\"a\"'\n" " | {\"a\": 1}.\"a\"\n" + " | .........^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:10: Syntax error: expected identifier after '.'\n" + " | {\"a\": 1}.\"a\"\n" " | .........^"}, {"\"\\xC3\\XBF\"", "\"ÿ\"^#1:string#"}, {"\"\\303\\277\"", "\"ÿ\"^#1:string#"}, @@ -729,7 +884,13 @@ std::vector test_cases = { "'{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, " "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | \"\\xFh\"\n" - " | ......^"}, + " | ......^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:1: Invalid string literal: Illegal escape sequence: Hex " + "escape must be followed by 2 hex digits but saw: \\xFh\n" + " | \"\\xFh\"\n" + " | ^"}, {"\"\\a\\b\\f\\n\\r\\t\\v\\'\\\"\\\\\\? Illegal escape \\>\"", "", "ERROR: :1:1: Syntax error: token recognition error at: " "'\"\\a\\b\\f\\n\\r\\t\\v\\'\\\"\\\\\\? Illegal escape \\>'\n" @@ -743,7 +904,13 @@ std::vector test_cases = { " '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, " "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | \"\\a\\b\\f\\n\\r\\t\\v\\'\\\"\\\\\\? Illegal escape \\>\"\n" - " | ..........................................^"}, + " | ..........................................^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:1: Invalid string literal: Illegal escape sequence: " + "\\>\n" + " | \"\\a\\b\\f\\n\\r\\t\\v\\'\\\"\\\\\\? Illegal escape \\>\"\n" + " | ^"}, {"'😁' in ['😁', '😑', '😦']", "@in(\n" " \"😁\"^#1:string#,\n" @@ -793,7 +960,15 @@ std::vector test_cases = { " | .........＾\n" "ERROR: :2:11: Syntax error: no viable alternative at input '.'\n" " | && in.😁\n" - " | .........．^"}, + " | .........．^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :2:7: Syntax error: unexpected token\n" + " | && in.😁\n" + " | ......^\n" + "ERROR: :2:10: Syntax error: unexpected character\n" + " | && in.😁\n" + " | .........＾"}, {"as", "", "ERROR: :1:1: reserved identifier: as\n" " | as\n" @@ -841,7 +1016,12 @@ std::vector test_cases = { "'{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, " "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | in\n" - " | ..^"}, + " | ..^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:1: Syntax error: unexpected token\n" + " | in\n" + " | ^"}, {"let", "", "ERROR: :1:1: reserved identifier: let\n" " | let\n" @@ -886,6 +1066,17 @@ std::vector test_cases = { " | ...................^\n" "ERROR: :1:26: reserved identifier: var\n" " | [1, 2, 3].map(var, var * var)\n" + " | .........................^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:15: reserved identifier: var\n" + " | [1, 2, 3].map(var, var * var)\n" + " | ..............^\n" + "ERROR: :1:20: reserved identifier: var\n" + " | [1, 2, 3].map(var, var * var)\n" + " | ...................^\n" + "ERROR: :1:26: reserved identifier: var\n" + " | [1, 2, 3].map(var, var * var)\n" " | .........................^"}, {"[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[" "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[" @@ -896,6 +1087,12 @@ std::vector test_cases = { "]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]" "]]]]]]", "", "Expression recursion limit exceeded. limit: 32", "", "", ""}, + {"a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : " + "a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : " + "a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : " + "a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : " + "a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : c", + "", "Expression recursion limit exceeded. limit: 32", "", "", ""}, { // Note, the ANTLR parse stack may recurse much more deeply and permit // more detailed expressions than the visitor can recurse over in @@ -908,14 +1105,17 @@ std::vector test_cases = { "", "", }, - { - "[\n\t\r[\n\t\r[\n\t\r]\n\t\r]\n\t\r", - "", // parse output not validated as it is too large. - "ERROR: :6:3: Syntax error: mismatched input '' expecting " - "{']', ','}\n" - " | \r\n" - " | ..^", - }, + {"[\n\t\r[\n\t\r[\n\t\r]\n\t\r]\n\t\r", "", + // parse output not validated as it is too large. + "ERROR: :6:3: Syntax error: mismatched input '' expecting " + "{']', ','}\n" + " | \r\n" + " | ..^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :6:3: Syntax error: expected ']'\n" + " | \r\n" + " | ..^"}, // Identifier quoting syntax tests. {"a.`b`", "a^#1:Expr.Ident#.b^#2:Expr.Select#"}, @@ -956,21 +1156,36 @@ std::vector test_cases = { " | ..^\n" "ERROR: :1:7: Syntax error: token recognition error at: '`'\n" " | a.`b c`\n" - " | ......^"}, + " | ......^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:3: unexpected quoted identifier\n" + " | a.`b c`\n" + " | ..^"}, {"a.`@foo`", "", "ERROR: :1:3: Syntax error: token recognition error at: '`@'\n" " | a.`@foo`\n" " | ..^\n" "ERROR: :1:8: Syntax error: token recognition error at: '`'\n" " | a.`@foo`\n" - " | .......^"}, + " | .......^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:3: unexpected quoted identifier\n" + " | a.`@foo`\n" + " | ..^"}, {"a.`$foo`", "", "ERROR: :1:3: Syntax error: token recognition error at: '`$'\n" " | a.`$foo`\n" " | ..^\n" "ERROR: :1:8: Syntax error: token recognition error at: '`'\n" " | a.`$foo`\n" - " | .......^"}, + " | .......^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:3: unexpected quoted identifier\n" + " | a.`$foo`\n" + " | ..^"}, {"`a.b`", "", "ERROR: :1:1: Syntax error: mismatched input '`a.b`' expecting " "{'[', '{', " @@ -978,6 +1193,11 @@ std::vector test_cases = { "NUM_UINT, STRING, " "BYTES, IDENTIFIER}\n" " | `a.b`\n" + " | ^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:1: unexpected quoted identifier\n" + " | `a.b`\n" " | ^"}, {"`a.b`()", "", "ERROR: :1:1: Syntax error: extraneous input '`a.b`' expecting " @@ -989,11 +1209,21 @@ std::vector test_cases = { "'{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM" "_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | `a.b`()\n" - " | ......^"}, + " | ......^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:1: unexpected quoted identifier\n" + " | `a.b`()\n" + " | ^"}, {"foo.`a.b`()", "", "ERROR: :1:10: Syntax error: mismatched input '(' expecting \n" " | foo.`a.b`()\n" - " | .........^"}, + " | .........^", + "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:5: unexpected quoted identifier\n" + " | foo.`a.b`()\n" + " | ....^"}, // Macro calls tests {"x.filter(y, y.filter(z, z > 0))", @@ -1046,8 +1276,7 @@ std::vector test_cases = { " @result^#24:Expr.Ident#\n" " )^#25:Expr.Call#,\n" " // Result\n" - " @result^#26:Expr.Ident#)^#27:Expr.Comprehension#" - "", + " @result^#26:Expr.Ident#)^#27:Expr.Comprehension#", "", "", "", "x^#1:Expr.Ident#.filter(\n" " y^#3:Expr.Ident#,\n" @@ -1279,6 +1508,10 @@ std::vector test_cases = { "Msg{\n " "?field:value^#3:Expr.Ident#^#2:Expr.CreateStruct.Entry#\n}^#1:Expr." "CreateStruct#"}, + {"import{}", "import{}^#1:Expr.CreateStruct#"}, + {".import{}", ".import{}^#1:Expr.CreateStruct#"}, + {"import.Foo{}", "import.Foo{}^#1:Expr.CreateStruct#"}, + {"Foo.import{}", "Foo.import{}^#1:Expr.CreateStruct#"}, {"m.optMap(v, f)", "_?_:_(\n m^#1:Expr.Ident#.hasValue()^#6:Expr.Call#,\n optional.of(\n " " __comprehension__(\n // Variable\n #unused,\n // " @@ -1295,7 +1528,274 @@ std::vector test_cases = { "m^#5:Expr.Ident#.value()^#8:Expr.Call#,\n // LoopCondition\n " "false^#9:bool#,\n // LoopStep\n v^#3:Expr.Ident#,\n // Result\n " " f^#4:Expr.Ident#)^#10:Expr.Comprehension#,\n " - "optional.none()^#11:Expr.Call#\n)^#12:Expr.Call#"}}; + "optional.none()^#11:Expr.Call#\n)^#12:Expr.Call#"}, + + // Multiline and string literal tests + {"'''hello\nworld'''", "\"hello\\nworld\"^#1:string#"}, + {"\"\"\"hello\nworld\"\"\"", "\"hello\\nworld\"^#1:string#"}, + {"r\"\"\"hello\nworld\"\"\"", "\"hello\\nworld\"^#1:string#"}, + {"\"\"\"hello\\\"\"\"world\"\"\"", "\"hello\\\"\\\"\\\"world\"^#1:string#"}, + {"'''hello\\'''world'''", "\"hello'''world\"^#1:string#"}, + {"\"\"\"hello\nworld", "", + "ERROR: :1:3: Syntax error: token recognition error at: " + "'\"hello\\n'\n" + " | \"\"\"hello\n" + " | ..^\n" + "ERROR: :2:1: Syntax error: extraneous input 'world' expecting " + "\n" + " | world\n" + " | ^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"\"\"hello\n" + " | ^"}, + {"'''hello\nworld", "", + "ERROR: :1:3: Syntax error: token recognition error at: " + "''hello\\n'\n" + " | '''hello\n" + " | ..^\n" + "ERROR: :2:1: Syntax error: extraneous input 'world' expecting " + "\n" + " | world\n" + " | ^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | '''hello\n" + " | ^"}, + {"r\"\"\"hello\nworld", "", + "ERROR: :1:4: Syntax error: token recognition error at: " + "'\"hello\\n'\n" + " | r\"\"\"hello\n" + " | ...^\n" + "ERROR: :2:1: Syntax error: extraneous input 'world' expecting " + "\n" + " | world\n" + " | ^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | r\"\"\"hello\n" + " | ^"}, + {"\"hello\nworld\"", "", + "ERROR: :1:1: Syntax error: token recognition error at: " + "'\"hello\\n'\n" + " | \"hello\n" + " | ^\n" + "ERROR: :2:6: Syntax error: token recognition error at: '\"'\n" + " | world\"\n" + " | .....^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after expression\n" + " | world\"\n" + " | ^"}, + {"'hello\nworld'", "", + "ERROR: :1:1: Syntax error: token recognition error at: " + "''hello\\n'\n" + " | 'hello\n" + " | ^\n" + "ERROR: :2:6: Syntax error: token recognition error at: '''\n" + " | world'\n" + " | .....^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | 'hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after expression\n" + " | world'\n" + " | ^"}, + {"r\"hello\nworld\"", "", + "ERROR: :1:2: Syntax error: token recognition error at: " + "'\"hello\\n'\n" + " | r\"hello\n" + " | .^\n" + "ERROR: :2:1: Syntax error: extraneous input 'world' expecting " + "\n" + " | world\"\n" + " | ^\n" + "ERROR: :2:6: Syntax error: token recognition error at: '\"'\n" + " | world\"\n" + " | .....^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | r\"hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after expression\n" + " | world\"\n" + " | ^"}, + {"`hello\nworld`", "", + "ERROR: :1:1: Syntax error: token recognition error at: " + "'`hello\\n'\n" + " | `hello\n" + " | ^\n" + "ERROR: :2:6: Syntax error: token recognition error at: '`'\n" + " | world`\n" + " | .....^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated quoted identifier\n" + " | `hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after expression\n" + " | world`\n" + " | ^"}, + {"\"hello\rworld\"", "", + "ERROR: :1:1: Syntax error: token recognition error at: " + "'\"hello\\r'\n" + " | \"hello\rworld\"\n" + " | ^\n" + "ERROR: :1:13: Syntax error: token recognition error at: '\"'\n" + " | \"hello\rworld\"\n" + " | ............^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"hello\rworld\"\n" + " | ^\n" + "ERROR: :1:8: Syntax error: unexpected token after expression\n" + " | \"hello\rworld\"\n" + " | .......^"}, + {"a.?b.?c", + "_?._(\n" + " _?._(\n" + " a^#1:Expr.Ident#,\n" + " \"b\"^#3:string#\n" + " )^#2:Expr.Call#,\n" + " \"c\"^#5:string#\n" + ")^#4:Expr.Call#", + "", + "_?._(\n" + " _?._(\n" + " a^#1[1,0]#,\n" + " \"b\"^#3[1,0]#\n" + " )^#2[1,1]#,\n" + " \"c\"^#5[1,0]#\n" + ")^#4[1,4]#"}, + {"!-42", + "!_(\n" + " -42^#2:int64#\n" + ")^#1:Expr.Call#", + "", + "!_(\n" + " -42^#2[1,1]#\n" + ")^#1[1,0]#"}, + {"!-4.2", + "!_(\n" + " -4.2^#2:double#\n" + ")^#1:Expr.Call#", + "", + "!_(\n" + " -4.2^#2[1,1]#\n" + ")^#1[1,0]#"}, + {"9in-x", + "@in(\n" + " 9^#1:int64#,\n" + " -_(\n" + " x^#4:Expr.Ident#\n" + " )^#3:Expr.Call#\n" + ")^#2:Expr.Call#"}, + {"1 \v + 2", "", + "ERROR: :1:3: Syntax error: token recognition error at: '\v'\n" + " | 1 \v + 2\n" + " | ..^", + "", "", "", + "ERROR: :1:3: Syntax error: unexpected character\n" + " | 1 \v + 2\n" + " | ..^"}, + {"-!x", "", + "ERROR: :1:2: Syntax error: no viable alternative at input '-!'\n" + " | -!x\n" + " | .^", + "", "", "", + "ERROR: :1:2: Syntax error: unexpected token\n" + " | -!x\n" + " | .^\n" + "ERROR: :1:3: Syntax error: unexpected token after expression\n" + " | -!x\n" + " | ..^"}, + {"!-x", "", + "ERROR: :1:2: invalid constant literal expression\n" + " | !-x\n" + " | .^\n" + "ERROR: :1:3: Syntax error: no viable alternative at input '-x'\n" + " | !-x\n" + " | ..^", + "", "", "", + "ERROR: :1:2: Syntax error: unexpected '-'\n" + " | !-x\n" + " | .^"}, + {"a.in", "", + "ERROR: :1:3: Syntax error: no viable alternative at input '.in'\n" + " | a.in\n" + " | ..^\n" + "ERROR: :1:5: Syntax error: mismatched input '' expecting " + "{'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, " + "NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" + " | a.in\n" + " | ....^", + "", "", "", + "ERROR: :1:3: Syntax error: expected identifier after '.'\n" + " | a.in\n" + " | ..^"}, + {"has(a.`$b`)", "", + "ERROR: :1:7: Syntax error: token recognition error at: '`$'\n" + " | has(a.`$b`)\n" + " | ......^\n" + "ERROR: :1:10: Syntax error: token recognition error at: '`)'\n" + " | has(a.`$b`)\n" + " | .........^\n" + "ERROR: :1:12: Syntax error: missing ')' at ''\n" + " | has(a.`$b`)\n" + " | ...........^", + "", "", "", + "ERROR: :1:7: unexpected quoted identifier\n" + " | has(a.`$b`)\n" + " | ......^"}, + {"(a){}", "", + "ERROR: :1:4: Syntax error: mismatched input '{' expecting \n" + " | (a){}\n" + " | ...^", + "", "", "", + "ERROR: :1:4: Syntax error: unexpected token after expression\n" + " | (a){}\n" + " | ...^"}, + {"(a.b){}", "", + "ERROR: :1:6: Syntax error: mismatched input '{' expecting \n" + " | (a.b){}\n" + " | .....^", + "", "", "", + "ERROR: :1:6: Syntax error: unexpected token after expression\n" + " | (a.b){}\n" + " | .....^"}, + {"(a).b{}", "", + "ERROR: :1:6: Syntax error: mismatched input '{' expecting \n" + " | (a).b{}\n" + " | .....^", + "", "", "", + "ERROR: :1:6: Syntax error: unexpected token after expression\n" + " | (a).b{}\n" + " | .....^"}, + {"a.`b-c`{}", "", + "ERROR: :1:8: Syntax error: mismatched input '{' expecting \n" + " | a.`b-c`{}\n" + " | .......^", + "", "", "", + "ERROR: :1:8: Syntax error: unexpected token after expression\n" + " | a.`b-c`{}\n" + " | .......^"}, + {"Msg{`$b`: 1}", "", + "ERROR: :1:5: Syntax error: token recognition error at: '`$'\n" + " | Msg{`$b`: 1}\n" + " | ....^\n" + "ERROR: :1:8: Syntax error: token recognition error at: '`:'\n" + " | Msg{`$b`: 1}\n" + " | .......^\n" + "ERROR: :1:11: Syntax error: missing ':' at '1'\n" + " | Msg{`$b`: 1}\n" + " | ..........^", + "", "", "", + "ERROR: :1:5: unexpected quoted identifier\n" + " | Msg{`$b`: 1}\n" + " | ....^"}}; absl::string_view ConstantKind(const cel::Constant& c) { switch (c.kind_case()) { @@ -1341,7 +1841,7 @@ absl::string_view ExprKind(const cel::Expr& e) { } } -class KindAndIdAdorner : public cel::test::ExpressionAdorner { +class KindAndIdAdorner : public cel::ExpressionAdorner { public: // Use default source_info constructor to make source_info "optional". This // will prevent macro_calls lookups from interfering with adorning expressions @@ -1380,7 +1880,7 @@ class KindAndIdAdorner : public cel::test::ExpressionAdorner { const cel::expr::SourceInfo& source_info_; }; -class LocationAdorner : public cel::test::ExpressionAdorner { +class LocationAdorner : public cel::ExpressionAdorner { public: explicit LocationAdorner(const cel::expr::SourceInfo& source_info) : source_info_(source_info) {} @@ -1435,8 +1935,13 @@ class LocationAdorner : public cel::test::ExpressionAdorner { std::string ConvertEnrichedSourceInfoToString( const EnrichedSourceInfo& enriched_source_info) { + std::vector>> sorted_offsets( + enriched_source_info.offsets().begin(), + enriched_source_info.offsets().end()); + absl::c_sort(sorted_offsets); std::vector offsets; - for (const auto& offset : enriched_source_info.offsets()) { + offsets.reserve(sorted_offsets.size()); + for (const auto& offset : sorted_offsets) { offsets.push_back(absl::StrFormat( "[%d,%d,%d]", offset.first, offset.second.first, offset.second.second)); } @@ -1468,27 +1973,35 @@ std::string ConvertMacroCallsToString( return result.substr(0, result.size() - 3); } -class ExpressionTest : public testing::TestWithParam {}; +class ExpressionTest + : public testing::TestWithParam> { + protected: + ExpressionTest() { options_.enable_pratt_parser = std::get<1>(GetParam()); } + + ParserOptions options_; +}; TEST_P(ExpressionTest, Parse) { - const TestInfo& test_info = GetParam(); - ParserOptions options; - options.enable_hidden_accumulator_var = true; + const TestInfo& test_info = std::get<0>(GetParam()); if (!test_info.M.empty()) { - options.add_macro_calls = true; + options_.add_macro_calls = true; } - options.enable_optional_syntax = true; - options.enable_quoted_identifiers = true; + options_.enable_optional_syntax = true; + options_.enable_quoted_identifiers = true; std::vector macros = Macro::AllMacros(); macros.push_back(cel::OptMapMacro()); macros.push_back(cel::OptFlatMapMacro()); - auto result = EnrichedParse(test_info.I, macros, "", options); + auto result = EnrichedParse(test_info.I, macros, "", options_); if (test_info.E.empty()) { ASSERT_THAT(result, IsOk()); } else { EXPECT_THAT(result, Not(IsOk())); - EXPECT_EQ(test_info.E, result.status().message()); + if (options_.enable_pratt_parser && !test_info.E_PRATT.empty()) { + EXPECT_EQ(test_info.E_PRATT, result.status().message()); + } else { + EXPECT_EQ(test_info.E, result.status().message()); + } } if (!test_info.P.empty()) { @@ -1505,10 +2018,9 @@ TEST_P(ExpressionTest, Parse) { std::string adorned_string = w.PrintProto(result->parsed_expr().expr()); EXPECT_EQ(test_info.L, adorned_string) << result->parsed_expr().ShortDebugString(); - ; } - if (!test_info.R.empty()) { + if (!options_.enable_pratt_parser && !test_info.R.empty()) { EXPECT_EQ(test_info.R, ConvertEnrichedSourceInfoToString( result->enriched_source_info())); } @@ -1517,11 +2029,39 @@ TEST_P(ExpressionTest, Parse) { EXPECT_EQ(test_info.M, ConvertMacroCallsToString( result.value().parsed_expr().source_info())) << result->parsed_expr().ShortDebugString(); - ; } } -TEST(ExpressionTest, TsanOom) { +TEST_P(ExpressionTest, CompositeExpressionOffsets) { + std::vector macros = Macro::AllMacros(); + + std::string list_expr = "[1, 2]"; + auto list_result = EnrichedParse(list_expr, macros, "", options_); + ASSERT_THAT(list_result, IsOk()); + auto list_offsets = list_result->enriched_source_info().offsets(); + EXPECT_EQ(list_offsets.at(1), std::make_pair(0, 5)); + + std::string map_expr = "{'a': 1}"; + auto map_result = EnrichedParse(map_expr, macros, "", options_); + ASSERT_THAT(map_result, IsOk()); + auto map_offsets = map_result->enriched_source_info().offsets(); + EXPECT_EQ(map_offsets.at(1), std::make_pair(0, 7)); + + std::string msg_expr = "Msg{f: 1}"; + auto msg_result = EnrichedParse(msg_expr, macros, "", options_); + ASSERT_THAT(msg_result, IsOk()); + auto msg_offsets = msg_result->enriched_source_info().offsets(); + EXPECT_EQ(msg_offsets.at(1), std::make_pair(0, 8)); +} + +class ExpressionImplTest : public testing::TestWithParam { + protected: + ExpressionImplTest() { options_.enable_pratt_parser = GetParam(); } + + ParserOptions options_; +}; + +TEST_P(ExpressionImplTest, TsanOom) { Parse( "[[a([[???[a[[??[a([[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[" "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[" @@ -1537,33 +2077,21 @@ TEST(ExpressionTest, TsanOom) { "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[" "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[" "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[???[" - "a([[????") + "a([[????", + "", options_) .IgnoreError(); } -TEST(ExpressionTest, ErrorRecoveryLimits) { - ParserOptions options; - options.error_recovery_limit = 1; - auto result = Parse("......", "", options); - EXPECT_THAT(result, Not(IsOk())); - EXPECT_EQ(result.status().message(), - "ERROR: :1:1: Syntax error: More than 1 parse errors.\n | ......\n " - "| ^\nERROR: :1:2: Syntax error: no viable alternative at input " - "'..'\n | ......\n | .^"); -} - -TEST(ExpressionTest, ExpressionSizeLimit) { - ParserOptions options; - options.expression_size_codepoint_limit = 10; - auto result = Parse("...............", "", options); +TEST_P(ExpressionImplTest, ExpressionSizeLimit) { + options_.expression_size_codepoint_limit = 10; + auto result = Parse("...............", "", options_); EXPECT_THAT(result, Not(IsOk())); EXPECT_EQ( result.status().message(), "expression size exceeds codepoint limit. input size: 15, limit: 10"); } -TEST(ExpressionTest, RecursionDepthLongArgList) { - ParserOptions options; +TEST_P(ExpressionImplTest, RecursionDepthLongArgList) { // The particular number here is an implementation detail: the underlying // visitor will recurse up to 8 times before branching to the create list or // const steps. The call graph looks something like: @@ -1572,29 +2100,14 @@ TEST(ExpressionTest, RecursionDepthLongArgList) { // ->visitCreateList->visit[arg]->visitExpr... // The expected max depth for create list with an arbitrary number of elements // is 15. - options.max_recursion_depth = 16; + options_.max_recursion_depth = 16; - EXPECT_THAT(Parse("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]", "", options), IsOk()); + EXPECT_THAT(Parse("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]", "", options_), IsOk()); } -TEST(ExpressionTest, RecursionDepthExceeded) { - ParserOptions options; - // AST visitor will recurse a variable amount depending on the terms used in - // the expression. This check occurs in the business logic converting the raw - // Antlr parse tree into an Expr. There is a separate check (via a custom - // listener) for AST depth while running the antlr generated parser. - options.max_recursion_depth = 6; - auto result = Parse("1 + 2 + 3 + 4 + 5 + 6 + 7", "", options); - - EXPECT_THAT(result, Not(IsOk())); - EXPECT_THAT(result.status().message(), - HasSubstr("Exceeded max recursion depth of 6 when parsing.")); -} - -TEST(ExpressionTest, DisableQuotedIdentifiers) { - ParserOptions options; - options.enable_quoted_identifiers = false; - auto result = Parse("foo.`bar`", "", options); +TEST_P(ExpressionImplTest, DisableQuotedIdentifiers) { + options_.enable_quoted_identifiers = false; + auto result = Parse("foo.`bar`", "", options_); EXPECT_THAT(result, Not(IsOk())); EXPECT_THAT(result.status().message(), @@ -1603,11 +2116,10 @@ TEST(ExpressionTest, DisableQuotedIdentifiers) { " | ....^")); } -TEST(ExpressionTest, DisableStandardMacros) { - ParserOptions options; - options.disable_standard_macros = true; +TEST_P(ExpressionImplTest, DisableStandardMacros) { + options_.disable_standard_macros = true; - auto result = Parse("has(foo.bar)", "", options); + auto result = Parse("has(foo.bar)", "", options_); ASSERT_THAT(result, IsOk()); KindAndIdAdorner kind_and_id_adorner; @@ -1620,281 +2132,15 @@ TEST(ExpressionTest, DisableStandardMacros) { << adorned_string; } -TEST(ExpressionTest, RecursionDepthIgnoresParentheses) { - ParserOptions options; - options.max_recursion_depth = 6; - auto result = Parse("(((1 + 2 + 3 + 4 + (5 + 6))))", "", options); - - EXPECT_THAT(result, IsOk()); -} - -const std::vector& UpdatedAccuVarTestCases() { - static const std::vector* kInstance = new std::vector{ - {"[].exists(x, x > 0)", - "__comprehension__(\n" - " // Variable\n" - " x,\n" - " // Target\n" - " []^#1:Expr.CreateList#,\n" - " // Accumulator\n" - " __result__,\n" - " // Init\n" - " false^#7:bool#,\n" - " // LoopCondition\n" - " @not_strictly_false(\n" - " !_(\n" - " __result__^#8:Expr.Ident#\n" - " )^#9:Expr.Call#\n" - " )^#10:Expr.Call#,\n" - " // LoopStep\n" - " _||_(\n" - " __result__^#11:Expr.Ident#,\n" - " _>_(\n" - " x^#4:Expr.Ident#,\n" - " 0^#6:int64#\n" - " )^#5:Expr.Call#\n" - " )^#12:Expr.Call#,\n" - " // Result\n" - " __result__^#13:Expr.Ident#)^#14:Expr.Comprehension#"}, - {"[].exists_one(x, x > 0)", - "__comprehension__(\n" - " // Variable\n" - " x,\n" - " // Target\n" - " []^#1:Expr.CreateList#,\n" - " // Accumulator\n" - " __result__,\n" - " // Init\n" - " 0^#7:int64#,\n" - " // LoopCondition\n" - " true^#8:bool#,\n" - " // LoopStep\n" - " _?_:_(\n" - " _>_(\n" - " x^#4:Expr.Ident#,\n" - " 0^#6:int64#\n" - " )^#5:Expr.Call#,\n" - " _+_(\n" - " __result__^#9:Expr.Ident#,\n" - " 1^#10:int64#\n" - " )^#11:Expr.Call#,\n" - " __result__^#12:Expr.Ident#\n" - " )^#13:Expr.Call#,\n" - " // Result\n" - " _==_(\n" - " __result__^#14:Expr.Ident#,\n" - " 1^#15:int64#\n" - " )^#16:Expr.Call#)^#17:Expr.Comprehension#"}, - {"[].all(x, x > 0)", - "__comprehension__(\n" - " // Variable\n" - " x,\n" - " // Target\n" - " []^#1:Expr.CreateList#,\n" - " // Accumulator\n" - " __result__,\n" - " // Init\n" - " true^#7:bool#,\n" - " // LoopCondition\n" - " @not_strictly_false(\n" - " __result__^#8:Expr.Ident#\n" - " )^#9:Expr.Call#,\n" - " // LoopStep\n" - " _&&_(\n" - " __result__^#10:Expr.Ident#,\n" - " _>_(\n" - " x^#4:Expr.Ident#,\n" - " 0^#6:int64#\n" - " )^#5:Expr.Call#\n" - " )^#11:Expr.Call#,\n" - " // Result\n" - " __result__^#12:Expr.Ident#)^#13:Expr.Comprehension#"}, - {"[].map(x, x + 1)", - "__comprehension__(\n" - " // Variable\n" - " x,\n" - " // Target\n" - " []^#1:Expr.CreateList#,\n" - " // Accumulator\n" - " __result__,\n" - " // Init\n" - " []^#7:Expr.CreateList#,\n" - " // LoopCondition\n" - " true^#8:bool#,\n" - " // LoopStep\n" - " _+_(\n" - " __result__^#9:Expr.Ident#,\n" - " [\n" - " _+_(\n" - " x^#4:Expr.Ident#,\n" - " 1^#6:int64#\n" - " )^#5:Expr.Call#\n" - " ]^#10:Expr.CreateList#\n" - " )^#11:Expr.Call#,\n" - " // Result\n" - " __result__^#12:Expr.Ident#)^#13:Expr.Comprehension#"}, - {"[].map(x, x > 0, x + 1)", - "__comprehension__(\n" - " // Variable\n" - " x,\n" - " // Target\n" - " []^#1:Expr.CreateList#,\n" - " // Accumulator\n" - " __result__,\n" - " // Init\n" - " []^#10:Expr.CreateList#,\n" - " // LoopCondition\n" - " true^#11:bool#,\n" - " // LoopStep\n" - " _?_:_(\n" - " _>_(\n" - " x^#4:Expr.Ident#,\n" - " 0^#6:int64#\n" - " )^#5:Expr.Call#,\n" - " _+_(\n" - " __result__^#12:Expr.Ident#,\n" - " [\n" - " _+_(\n" - " x^#7:Expr.Ident#,\n" - " 1^#9:int64#\n" - " )^#8:Expr.Call#\n" - " ]^#13:Expr.CreateList#\n" - " )^#14:Expr.Call#,\n" - " __result__^#15:Expr.Ident#\n" - " )^#16:Expr.Call#,\n" - " // Result\n" - " __result__^#17:Expr.Ident#)^#18:Expr.Comprehension#"}, - {"[].filter(x, x > 0)", - "__comprehension__(\n" - " // Variable\n" - " x,\n" - " // Target\n" - " []^#1:Expr.CreateList#,\n" - " // Accumulator\n" - " __result__,\n" - " // Init\n" - " []^#7:Expr.CreateList#,\n" - " // LoopCondition\n" - " true^#8:bool#,\n" - " // LoopStep\n" - " _?_:_(\n" - " _>_(\n" - " x^#4:Expr.Ident#,\n" - " 0^#6:int64#\n" - " )^#5:Expr.Call#,\n" - " _+_(\n" - " __result__^#9:Expr.Ident#,\n" - " [\n" - " x^#3:Expr.Ident#\n" - " ]^#10:Expr.CreateList#\n" - " )^#11:Expr.Call#,\n" - " __result__^#12:Expr.Ident#\n" - " )^#13:Expr.Call#,\n" - " // Result\n" - " __result__^#14:Expr.Ident#)^#15:Expr.Comprehension#"}, - // Maintain restriction on '__result__' variable name until the default is - // changed everywhere. - { - "[].map(__result__, true)", - /*.P=*/"", - /*.E=*/ - "ERROR: :1:20: map() variable name cannot be __result__\n" - " | [].map(__result__, true)\n" - " | ...................^", - }, - { - "[].map(__result__, true, false)", - /*.P=*/"", - /*.E=*/ - "ERROR: :1:20: map() variable name cannot be __result__\n" - " | [].map(__result__, true, false)\n" - " | ...................^", - }, - { - "[].filter(__result__, true)", - /*.P=*/"", - /*.E=*/ - "ERROR: :1:23: filter() variable name cannot be __result__\n" - " | [].filter(__result__, true)\n" - " | ......................^", - }, - { - "[].exists(__result__, true)", - /*.P=*/"", - /*.E=*/ - "ERROR: :1:23: exists() variable name cannot be __result__\n" - " | [].exists(__result__, true)\n" - " | ......................^", - }, - { - "[].all(__result__, true)", - /*.P=*/"", - /*.E=*/ - "ERROR: :1:20: all() variable name cannot be __result__\n" - " | [].all(__result__, true)\n" - " | ...................^", - }, - { - "[].exists_one(__result__, true)", - /*.P=*/"", - /*.E=*/ - "ERROR: :1:27: exists_one() variable name cannot be " - "__result__\n" - " | [].exists_one(__result__, true)\n" - " | ..........................^", - }}; - return *kInstance; -} - -class UpdatedAccuVarDisabledTest : public testing::TestWithParam {}; - -TEST_P(UpdatedAccuVarDisabledTest, Parse) { - const TestInfo& test_info = GetParam(); - ParserOptions options; - options.enable_hidden_accumulator_var = false; - if (!test_info.M.empty()) { - options.add_macro_calls = true; - } - - auto result = - EnrichedParse(test_info.I, Macro::AllMacros(), "", options); - if (test_info.E.empty()) { - EXPECT_THAT(result, IsOk()); - } else { - EXPECT_THAT(result, Not(IsOk())); - EXPECT_EQ(test_info.E, result.status().message()); - } - - if (!test_info.P.empty()) { - KindAndIdAdorner kind_and_id_adorner; - ExprPrinter w(kind_and_id_adorner); - std::string adorned_string = w.PrintProto(result->parsed_expr().expr()); - EXPECT_EQ(test_info.P, adorned_string) - << result->parsed_expr().ShortDebugString(); - } - - if (!test_info.L.empty()) { - LocationAdorner location_adorner(result->parsed_expr().source_info()); - ExprPrinter w(location_adorner); - std::string adorned_string = w.PrintProto(result->parsed_expr().expr()); - EXPECT_EQ(test_info.L, adorned_string) - << result->parsed_expr().ShortDebugString(); - } +class NewParserBuilderTest : public testing::TestWithParam { + protected: + NewParserBuilderTest() { options_.enable_pratt_parser = GetParam(); } - if (!test_info.R.empty()) { - EXPECT_EQ(test_info.R, ConvertEnrichedSourceInfoToString( - result->enriched_source_info())); - } - - if (!test_info.M.empty()) { - EXPECT_EQ(test_info.M, ConvertMacroCallsToString( - result.value().parsed_expr().source_info())) - << result->parsed_expr().ShortDebugString(); - } -} + ParserOptions options_; +}; -TEST(NewParserBuilderTest, Defaults) { - auto builder = cel::NewParserBuilder(); +TEST_P(NewParserBuilderTest, Defaults) { + auto builder = cel::NewParserBuilder(options_); ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); ASSERT_OK_AND_ASSIGN(auto source, @@ -1904,8 +2150,8 @@ TEST(NewParserBuilderTest, Defaults) { EXPECT_FALSE(ast->IsChecked()); } -TEST(NewParserBuilderTest, CustomMacros) { - auto builder = cel::NewParserBuilder(); +TEST_P(NewParserBuilderTest, CustomMacros) { + auto builder = cel::NewParserBuilder(options_); builder->GetOptions().disable_standard_macros = true; ASSERT_THAT(builder->AddMacro(cel::HasMacro()), IsOk()); ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); @@ -1927,8 +2173,8 @@ TEST(NewParserBuilderTest, CustomMacros) { ")^#9:Expr.Call#"); } -TEST(NewParserBuilderTest, StandardMacrosNotAddedWithStdlib) { - auto builder = cel::NewParserBuilder(); +TEST_P(NewParserBuilderTest, StandardMacrosNotAddedWithStdlib) { + auto builder = cel::NewParserBuilder(options_); builder->GetOptions().disable_standard_macros = false; // Add a fake stdlib to check that we don't try to add the standard macros // again. Emulates what happens when we add support for subsetting stdlib by @@ -1957,15 +2203,15 @@ TEST(NewParserBuilderTest, StandardMacrosNotAddedWithStdlib) { ")^#9:Expr.Call#"); } -TEST(NewParserBuilderTest, ForwardsOptions) { - auto builder = cel::NewParserBuilder(); +TEST_P(NewParserBuilderTest, ForwardsOptions) { + auto builder = cel::NewParserBuilder(options_); builder->GetOptions().enable_optional_syntax = true; ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("a.?b")); ASSERT_OK_AND_ASSIGN(auto ast, parser->Parse(*source)); EXPECT_FALSE(ast->IsChecked()); - builder = cel::NewParserBuilder(); + builder = cel::NewParserBuilder(options_); builder->GetOptions().enable_optional_syntax = false; ASSERT_OK_AND_ASSIGN(parser, std::move(*builder).Build()); ASSERT_OK_AND_ASSIGN(source, cel::NewSource("a.?b")); @@ -1973,8 +2219,8 @@ TEST(NewParserBuilderTest, ForwardsOptions) { StatusIs(absl::StatusCode::kInvalidArgument)); } -TEST(NewParserBuilderTest, ToBuilderCopiesConfig) { - auto builder = cel::NewParserBuilder(); +TEST_P(NewParserBuilderTest, ToBuilderCopiesConfig) { + auto builder = cel::NewParserBuilder(options_); builder->GetOptions().enable_optional_syntax = true; builder->GetOptions().disable_standard_macros = true; ASSERT_THAT(builder->AddLibrary({"custom_lib", @@ -1995,8 +2241,8 @@ TEST(NewParserBuilderTest, ToBuilderCopiesConfig) { EXPECT_FALSE(ast->IsChecked()); } -TEST(NewParserBuilderTest, ToBuilderHandlesStdlibAndOptionalByLibrary) { - auto builder = cel::NewParserBuilder(); +TEST_P(NewParserBuilderTest, ToBuilderHandlesStdlibAndOptionalByLibrary) { + auto builder = cel::NewParserBuilder(options_); builder->GetOptions().disable_standard_macros = true; builder->GetOptions().enable_optional_syntax = false; @@ -2031,8 +2277,8 @@ TEST(NewParserBuilderTest, ToBuilderHandlesStdlibAndOptionalByLibrary) { ")^#1:Expr.Call#"); } -TEST(NewParserBuilderTest, ToBuilderPreservesStdlibAndOptionalFromOptions) { - auto builder = cel::NewParserBuilder(); +TEST_P(NewParserBuilderTest, ToBuilderPreservesStdlibAndOptionalFromOptions) { + auto builder = cel::NewParserBuilder(options_); builder->GetOptions().disable_standard_macros = false; builder->GetOptions().enable_optional_syntax = true; @@ -2048,19 +2294,494 @@ TEST(NewParserBuilderTest, ToBuilderPreservesStdlibAndOptionalFromOptions) { EXPECT_FALSE(ast->IsChecked()); } -std::string TestName(const testing::TestParamInfo& test_info) { - std::string name = absl::StrCat(test_info.index, "-", test_info.param.I); +struct VariadicLogicalOperatorsTestCase { + std::string input; + std::string expected_adorned_string; +}; + +class VariadicLogicalOperatorsTest + : public testing::TestWithParam< + std::tuple> { + protected: + VariadicLogicalOperatorsTest() { + options_.enable_pratt_parser = std::get<1>(GetParam()); + } + + ParserOptions options_; +}; + +TEST_P(VariadicLogicalOperatorsTest, Parse) { + const auto& test_case = std::get<0>(GetParam()); + auto builder = cel::NewParserBuilder(options_); + builder->GetOptions().enable_variadic_logical_operators = true; + ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(test_case.input)); + ASSERT_OK_AND_ASSIGN(auto ast, parser->Parse(*source)); + + KindAndIdAdorner kind_and_id_adorner; + ExprPrinter w(kind_and_id_adorner); + std::string adorned_string = w.Print(ast->root_expr()); + EXPECT_EQ(adorned_string, test_case.expected_adorned_string); +} + +std::string VariadicLogicalOperatorsTestName( + const testing::TestParamInfo< + std::tuple>& test_info) { + bool enable_pratt = std::get<1>(test_info.param); + return absl::StrCat(test_info.index, "_", enable_pratt ? "Pratt" : "Legacy"); +} + +INSTANTIATE_TEST_SUITE_P( + VariadicLogicalOperators, VariadicLogicalOperatorsTest, + testing::Combine( + testing::Values( + VariadicLogicalOperatorsTestCase{ + .input = "a && b && c && d", + .expected_adorned_string = "_&&_(\n" + " a^#1:Expr.Ident#,\n" + " b^#2:Expr.Ident#,\n" + " c^#4:Expr.Ident#,\n" + " d^#6:Expr.Ident#\n" + ")^#3:Expr.Call#"}, + VariadicLogicalOperatorsTestCase{ + .input = "a || b || c || d", + .expected_adorned_string = "_||_(\n" + " a^#1:Expr.Ident#,\n" + " b^#2:Expr.Ident#,\n" + " c^#4:Expr.Ident#,\n" + " d^#6:Expr.Ident#\n" + ")^#3:Expr.Call#"}, + VariadicLogicalOperatorsTestCase{ + .input = "a && b && (c || d || e)", + .expected_adorned_string = "_&&_(\n" + " a^#1:Expr.Ident#,\n" + " b^#2:Expr.Ident#,\n" + " _||_(\n" + " c^#4:Expr.Ident#,\n" + " d^#5:Expr.Ident#,\n" + " e^#7:Expr.Ident#\n" + " )^#6:Expr.Call#\n" + ")^#3:Expr.Call#"}), + TestedParserImpls), + VariadicLogicalOperatorsTestName); + +class ParserTest : public testing::TestWithParam { + protected: + ParserTest() { options_.enable_pratt_parser = GetParam(); } + + ParserOptions options_; +}; + +TEST_P(ParserTest, ParseFailurePopulatesIssues) { + auto builder = cel::NewParserBuilder(options_); + ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); + + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("a +", "test.cel")); + std::vector issues; + auto ast_result = parser->Parse(*source, &issues); + EXPECT_THAT(ast_result, Not(IsOk())); + ASSERT_THAT(issues, testing::SizeIs(1)); + EXPECT_THAT(ast_result.status().message(), + HasSubstr("ERROR: test.cel:1:4: Syntax error: mismatched input " + "'' expecting")); + EXPECT_THAT(issues[0].message(), + HasSubstr("Syntax error: mismatched input '' expecting")); + EXPECT_EQ(issues[0].location().line, 1); + // 0-based, but adjusted to 1-based in error message. + EXPECT_EQ(issues[0].location().column, 3); +} + +TEST_P(ParserTest, ExpressionNodeLimitExceeded) { + auto builder = cel::NewParserBuilder(options_); + builder->GetOptions().expression_node_limit = 2; + ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); + + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("a + b + c", "test.cel")); + std::vector issues; + auto ast_result = parser->Parse(*source, &issues); + EXPECT_THAT(ast_result, Not(IsOk())); + ASSERT_THAT(issues, testing::Not(testing::IsEmpty())); + EXPECT_THAT(ast_result.status().message(), + HasSubstr("expression node limit exceeded")); + EXPECT_THAT(issues[0].message(), HasSubstr("expression node limit exceeded")); +} + +TEST_P(ParserTest, MacroExpansionNodeLimitExceeded) { + auto builder = cel::NewParserBuilder(options_); + builder->GetOptions().expression_node_limit = 5; + ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); + + ASSERT_OK_AND_ASSIGN( + auto source, cel::NewSource("[1, 2, 3, 4, 5].map(x, x * 2)", "test.cel")); + std::vector issues; + auto ast_result = parser->Parse(*source, &issues); + EXPECT_THAT(ast_result, Not(IsOk())); + ASSERT_THAT(issues, testing::Not(testing::IsEmpty())); + EXPECT_THAT(ast_result.status().message(), + HasSubstr("expression node limit exceeded")); + EXPECT_THAT( + issues, + testing::Contains(testing::Property( + &cel::ParseIssue::message, + HasSubstr( + "could not expand macro: expression node limit exceeded")))); +} + +TEST_P(ParserTest, MacroExpansionNodeLimitNotExceeded) { + auto builder = cel::NewParserBuilder(options_); + builder->GetOptions().expression_node_limit = 100; + ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); + + ASSERT_OK_AND_ASSIGN( + auto source, cel::NewSource("[1, 2, 3, 4, 5].map(x, x * 2)", "test.cel")); + std::vector issues; + auto ast_result = parser->Parse(*source, &issues); + ASSERT_THAT(ast_result, IsOk()); + EXPECT_THAT(issues, testing::IsEmpty()); +} + +TEST_P(ParserTest, PrepareSourceForwardsCodepointLimit) { + auto builder = cel::NewParserBuilder(options_); + builder->GetOptions().expression_size_codepoint_limit = 10; + ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); + + EXPECT_THAT(parser->PrepareSource("123456789012345", "test.cel"), + absl_testing::StatusIs( + absl::StatusCode::kInvalidArgument, + HasSubstr("expression is larger than codepoint limit 10"))); + + ASSERT_OK_AND_ASSIGN(auto source, + parser->PrepareSource("1234567890", "test.cel")); + EXPECT_EQ(source->description(), "test.cel"); + EXPECT_EQ(source->content().ToString(), "1234567890"); + + ASSERT_OK_AND_ASSIGN(auto source_default_desc, + parser->PrepareSource("1234567890")); + EXPECT_EQ(source_default_desc->description(), ""); +} + +// Test case pinning down the recursion depth that the parsers charge for +// `source`. +// +// `antlr_min_recursion_depth` and `pratt_min_recursion_depth` are the smallest +// `max_recursion_depth` values that still parse `source` with the respective +// parser; parsing with a smaller limit is expected to be rejected. Both parsers +// charge a level for every nesting level of the resulting AST, so the two +// columns only differ by a constant: the ANTLR parser additionally recurses +// through the grammar rules that wrap an expression, while the Pratt parser +// parses operator chains, selector chains and runs of parentheses iteratively +// and charges them against the depth they reach instead. +struct RecursionDepthTestCase { + std::string name; + std::string source; + int antlr_min_recursion_depth; + int pratt_min_recursion_depth; +}; + +std::string Repeat(absl::string_view part, int count) { + std::string result; + for (int i = 0; i < count; ++i) { + absl::StrAppend(&result, part); + } + return result; +} + +std::string WrapInParens(absl::string_view inner, int count) { + return absl::StrCat(Repeat("(", count), inner, Repeat(")", count)); +} + +std::vector GetRecursionDepthTestCases() { + // Nesting level used by the generated expressions below. + constexpr int kDepth = 32; + + return { + // `1 + 2 + ... + 34`: a flat chain of 33 binary operators. + {.name = "LargeCalc", + .source = + [] { + std::vector terms; + terms.reserve(kDepth + 2); + for (int i = 1; i <= kDepth + 2; ++i) { + terms.push_back(absl::StrCat(i)); + } + return absl::StrJoin(terms, " + "); + }(), + .antlr_min_recursion_depth = kDepth + 2, + .pratt_min_recursion_depth = kDepth + 1}, + // Runs of parentheses are unwound iteratively by the Pratt parser, but + // each one is still a nesting level and is charged as such. + {.name = "NestedParens", + .source = WrapInParens("7", kDepth), + .antlr_min_recursion_depth = kDepth, + .pratt_min_recursion_depth = kDepth}, + {.name = "NestedParensWithCalc", + .source = absl::StrCat(WrapInParens("7", kDepth), " + ", + WrapInParens("7", kDepth - 1)), + .antlr_min_recursion_depth = kDepth, + .pratt_min_recursion_depth = kDepth}, + // `a.f0.f1 ... .f32`: a flat chain of 33 field selections. + {.name = "FieldSelections", + .source = + [] { + std::string source = "a"; + for (int i = 0; i <= kDepth; ++i) { + absl::StrAppend(&source, ".f", i); + } + return source; + }(), + .antlr_min_recursion_depth = kDepth + 2, + .pratt_min_recursion_depth = kDepth + 1}, + // `a[1][2] ... [33]`: a flat chain of 33 index operations, each of which + // parses its index expression one level deeper. + {.name = "IndexOperations", + .source = + [] { + std::string source = "a"; + for (int i = 1; i <= kDepth + 1; ++i) { + absl::StrAppend(&source, "[", i, "]"); + } + return source; + }(), + .antlr_min_recursion_depth = kDepth + 2, + .pratt_min_recursion_depth = kDepth + 1}, + // `a < 1 < 2 < ... < 33`: a flat chain of 33 relation operators. + {.name = "RelationOperators", + .source = + [] { + std::string source = "a"; + for (int i = 1; i <= kDepth + 1; ++i) { + absl::StrAppend(&source, " < ", i); + } + return source; + }(), + .antlr_min_recursion_depth = kDepth + 2, + .pratt_min_recursion_depth = kDepth + 1}, + // 14 operands with 20 index operations each, joined by relation + // operators. + {.name = "IndexRelationOperators", + .source = + [] { + std::string operand = "a"; + for (int i = 1; i <= 20; ++i) { + absl::StrAppend(&operand, "[", i, "]"); + } + return absl::StrJoin(std::vector(14, operand), + " != "); + }(), + .antlr_min_recursion_depth = kDepth + 2, + .pratt_min_recursion_depth = kDepth + 1}, + // `a ? b : a ? b : ... : c`: the false branch of a ternary is parsed + // recursively, so each link costs one level. + {.name = "Ternary", + .source = absl::StrCat(Repeat("a ? b : ", kDepth + 1), "c"), + .antlr_min_recursion_depth = kDepth + 2, + .pratt_min_recursion_depth = kDepth + 2}, + {.name = "TernaryTrueBranchParens", + .source = absl::StrCat("a ? ", WrapInParens("b", kDepth), " : c"), + .antlr_min_recursion_depth = kDepth, + .pratt_min_recursion_depth = kDepth}, + // `((((7) + 1) + 1) ... + 1)`: parentheses nested to the left, each level + // adding a binary operator. + {.name = "NestedLeftParensWithCalc", + .source = + absl::StrCat(Repeat("(", kDepth), "7", Repeat(") + 1", kDepth)), + .antlr_min_recursion_depth = kDepth + 1, + .pratt_min_recursion_depth = kDepth}, + // `(true) || (true || ( ... || false) ... )`: parentheses nested to the + // right, so each level is parsed recursively as the right hand side + // operand. + {.name = "NestedRightParensWithLogicalOr", + .source = absl::StrCat("(true) || ", Repeat("(true || ", kDepth), + "false", Repeat(")", kDepth)), + .antlr_min_recursion_depth = kDepth + 2, + .pratt_min_recursion_depth = kDepth}, + // Parens nested inside an operand are charged against the depth reached + // so far rather than accumulating on top of the enclosing parens. + {.name = "GroupingParensAroundCalc", + .source = "((1 + ((7))))", + .antlr_min_recursion_depth = 4, + .pratt_min_recursion_depth = 3}, + {.name = "GroupingParensAroundCalcChain", + .source = "(((1 + 2 + 3 + 4 + (5 + 6))))", + .antlr_min_recursion_depth = 5, + .pratt_min_recursion_depth = 5}, + {.name = "ParenthesizedLhsCalc", + .source = "(1 + 1 + 1) + 1 + 1 + 1", + .antlr_min_recursion_depth = 6, + .pratt_min_recursion_depth = 5}, + {.name = "NestedLeftParensLhsCalc", + .source = "((1 + 1) + 1) + 1 + 1 + 1", + .antlr_min_recursion_depth = 6, + .pratt_min_recursion_depth = 5}, + {.name = "GroupingParensLhsCalc", + .source = "(((1 + 1 + 1))) + 1 + 1 + 1", + .antlr_min_recursion_depth = 6, + .pratt_min_recursion_depth = 5}, + {.name = "ParenthesizedRhsCalc", + .source = "1 + (1 + 1 + 1 + 1 + 1)", + .antlr_min_recursion_depth = 6, + .pratt_min_recursion_depth = 5}, + {.name = "ParenthesizedFieldSelections", + .source = "(a.b.c).d.e.f", + .antlr_min_recursion_depth = 6, + .pratt_min_recursion_depth = 5}, + // Depth accumulated inside a call argument does not carry into the + // enclosing selector and operator chains that wrap the call. + {.name = "CallArgumentFieldSelections", + .source = "f(a.b.c.d).e", + .antlr_min_recursion_depth = 6, + .pratt_min_recursion_depth = 4}, + {.name = "CallArgumentFieldSelectionsWithCalc", + .source = "x + f(a.b.c.d) + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 5}, + {.name = "CallArgumentCalcWithCalc", + .source = "x + f(a + b + c + d) + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 5}, + {.name = "IndexFieldSelectionsWithCalc", + .source = "x + a[b.c.d.e] + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 6}, + {.name = "MemberCallArgumentFieldSelectionsWithCalc", + .source = "x + a.f(b.c.d.e) + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 6}, + {.name = "StructFieldSelectionsWithCalc", + .source = "x + Msg{f: a.b.c.d} + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 5}, + // A delimited construct contributes its deepest element, not its last + // one. + {.name = "CallArgumentDeepestNotLast", + .source = "x + f(a.b.c.d, 1) + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 5}, + {.name = "MemberCallArgumentDeepestNotLast", + .source = "x + a.f(b.c.d.e, 1) + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 6}, + {.name = "ListElementDeepestNotLast", + .source = "x + [a.b.c.d, 1][0] + y", + .antlr_min_recursion_depth = 8, + .pratt_min_recursion_depth = 6}, + {.name = "MapValueDeepestNotLast", + .source = "x + {'k': a.b.c.d, 'j': 1}['k'] + y", + .antlr_min_recursion_depth = 8, + .pratt_min_recursion_depth = 6}, + {.name = "MapKeyDeepestNotLast", + .source = "x + {a.b.c.d.e: 1, 'j': 2}['j'] + y", + .antlr_min_recursion_depth = 9, + .pratt_min_recursion_depth = 7}, + {.name = "StructFieldDeepestNotLast", + .source = "x + Msg{f: a.b.c.d, g: 1} + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 5}, + {.name = "NestedListElementDeepestNotLast", + .source = "x + [[a.b.c.d, 1], 1][0] + y", + .antlr_min_recursion_depth = 9, + .pratt_min_recursion_depth = 6}, + {.name = "ParenthesizedLogicalAndFieldSelection", + .source = "((a && b.c.d).e)", + .antlr_min_recursion_depth = 5, + .pratt_min_recursion_depth = 3}, + {.name = "ParenthesizedLogicalAndChainFieldSelection", + .source = "((a && b && c.d.e).f)", + .antlr_min_recursion_depth = 5, + .pratt_min_recursion_depth = 3}, + {.name = "FieldSelectionsWithCalc", + .source = "a.b.c.d.e + f.g", + .antlr_min_recursion_depth = 6, + .pratt_min_recursion_depth = 5}, + }; +} + +class RecursionDepthTest + : public testing::TestWithParam> { + protected: + RecursionDepthTest() { + options_.enable_pratt_parser = std::get<1>(GetParam()); + } + + const RecursionDepthTestCase& test_case() const { + return std::get<0>(GetParam()); + } + + int min_recursion_depth() const { + return options_.enable_pratt_parser ? test_case().pratt_min_recursion_depth + : test_case().antlr_min_recursion_depth; + } + + ParserOptions options_; +}; + +TEST_P(RecursionDepthTest, AtMinRecursionDepthSucceeds) { + options_.max_recursion_depth = min_recursion_depth(); + + EXPECT_THAT(Parse(test_case().source, "", options_), IsOk()); +} + +TEST_P(RecursionDepthTest, BelowMinRecursionDepthFails) { + if (min_recursion_depth() == 0) { + GTEST_SKIP() << "expression is parsed without recursing"; + } + int limit = min_recursion_depth() - 1; + options_.max_recursion_depth = limit; + + auto result = Parse(test_case().source, "", options_); + + ASSERT_THAT(result, Not(IsOk())); + // The ANTLR parser reports the limit either from the grammar rule listener + // or, if that one does not trip first, from the AST building visitor. + EXPECT_THAT( + result.status().message(), + AnyOf(HasSubstr(absl::StrFormat( + "Expression recursion limit exceeded. limit: %d", limit)), + HasSubstr(absl::StrFormat( + "Exceeded max recursion depth of %d when parsing", limit)))); +} + +std::string RecursionDepthTestName( + const testing::TestParamInfo>& + test_info) { + const RecursionDepthTestCase& test_case = std::get<0>(test_info.param); + bool enable_pratt = std::get<1>(test_info.param); + return absl::StrCat(test_info.index, "_", enable_pratt ? "Pratt_" : "Legacy_", + test_case.name); +} + +INSTANTIATE_TEST_SUITE_P( + CelParserTest, RecursionDepthTest, + testing::Combine(testing::ValuesIn(GetRecursionDepthTestCases()), + TestedParserImpls), + RecursionDepthTestName); + +std::string ExpressionTestName( + const testing::TestParamInfo>& test_info) { + const TestInfo& info = std::get<0>(test_info.param); + bool enable_pratt = std::get<1>(test_info.param); + std::string name = absl::StrCat(test_info.index, "_", + enable_pratt ? "Pratt_" : "Legacy_", info.I); absl::c_replace_if(name, [](char c) { return !absl::ascii_isalnum(c); }, '_'); return name; - return name; +} + +std::string ParserImplTestName(const testing::TestParamInfo& info) { + return info.param ? "Pratt" : "Legacy"; } INSTANTIATE_TEST_SUITE_P(CelParserTest, ExpressionTest, - testing::ValuesIn(test_cases), TestName); + testing::Combine(testing::ValuesIn(test_cases), + TestedParserImpls), + ExpressionTestName); + +INSTANTIATE_TEST_SUITE_P(CelParserTest, ExpressionImplTest, TestedParserImpls, + ParserImplTestName); -INSTANTIATE_TEST_SUITE_P(UpdatedAccuVarTest, UpdatedAccuVarDisabledTest, - testing::ValuesIn(UpdatedAccuVarTestCases()), - TestName); +INSTANTIATE_TEST_SUITE_P(CelParserTest, NewParserBuilderTest, TestedParserImpls, + ParserImplTestName); +INSTANTIATE_TEST_SUITE_P(CelParserTest, ParserTest, TestedParserImpls, + ParserImplTestName); } // namespace } // namespace google::api::expr::parser diff --git a/parser/source_factory.h b/parser/source_factory.h index 501e1017a..11f64be6c 100644 --- a/parser/source_factory.h +++ b/parser/source_factory.h @@ -16,15 +16,16 @@ #define THIRD_PARTY_CEL_CPP_PARSER_SOURCE_FACTORY_H_ #include -#include #include -namespace google::api::expr::parser { +#include "absl/container/flat_hash_map.h" + +namespace cel { class EnrichedSourceInfo { public: explicit EnrichedSourceInfo( - std::map> offsets) + absl::flat_hash_map> offsets) : offsets_(std::move(offsets)) {} EnrichedSourceInfo() = default; @@ -33,15 +34,22 @@ class EnrichedSourceInfo { EnrichedSourceInfo(EnrichedSourceInfo&& other) = default; EnrichedSourceInfo& operator=(EnrichedSourceInfo&& other) = default; - const std::map>& offsets() const { + const absl::flat_hash_map>& offsets() + const { return offsets_; } private: // A map between node_id and pair of start position and end position - std::map> offsets_; + absl::flat_hash_map> offsets_; }; +} // namespace cel + +namespace google::api::expr::parser { + +using EnrichedSourceInfo = ::cel::EnrichedSourceInfo; + } // namespace google::api::expr::parser #endif // THIRD_PARTY_CEL_CPP_PARSER_SOURCE_FACTORY_H_ diff --git a/policy/BUILD b/policy/BUILD new file mode 100644 index 000000000..f14d39f73 --- /dev/null +++ b/policy/BUILD @@ -0,0 +1,246 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_test.bzl", "cc_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "cel_policy", + srcs = [ + "cel_policy.cc", + ], + hdrs = [ + "cel_policy.h", + ], + deps = [ + "//common:source", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings", + ], +) + +cc_test( + name = "cel_policy_test", + srcs = ["cel_policy_test.cc"], + deps = [ + ":cel_policy", + "//common:source", + "//internal:testing", + "@com_google_absl//absl/strings", + ], +) + +cc_library( + name = "cel_policy_parser", + srcs = [ + "cel_policy_parse_context.cc", + "cel_policy_parse_result.cc", + ], + hdrs = [ + "cel_policy_parse_context.h", + "cel_policy_parse_result.h", + "cel_policy_parser.h", + ], + deps = [ + ":cel_policy", + "//common:source", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", + ], +) + +cc_library( + name = "yaml_policy_parser", + srcs = [ + "yaml_policy_parser.cc", + ], + hdrs = ["yaml_policy_parser.h"], + copts = ["-fexceptions"], + features = ["-use_header_modules"], + deps = [ + ":cel_policy", + ":cel_policy_parser", + "//common:source", + "//internal:status_macros", + "//policy/internal:yaml_string_element_scanner", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@yaml-cpp", + ], +) + +cc_library( + name = "cel_policy_validation_result", + srcs = [ + "cel_policy_validation_result.cc", + ], + hdrs = [ + "cel_policy_validation_result.h", + ], + deps = [ + ":cel_policy", + ":cel_policy_parser", + "//common:ast", + "//common:decl", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", + ], +) + +cc_library( + name = "compiler", + srcs = ["compiler.cc"], + hdrs = ["compiler.h"], + deps = [ + ":cel_policy", + ":cel_policy_parser", + ":cel_policy_validation_result", + "//checker:type_check_issue", + "//checker:type_checker", + "//checker:validation_result", + "//common:ast", + "//common:ast_rewrite", + "//common:constant", + "//common:container", + "//common:decl", + "//common:expr", + "//common:format_type_name", + "//common:navigable_ast", + "//common:source", + "//common:type", + "//common:type_kind", + "//compiler", + "//internal:status_macros", + "//policy/internal:issue_reporter", + "//policy/internal:optimizer_expr_factory", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "yaml_policy_parser_test", + srcs = [ + "test_custom_yaml_policy_parser.cc", + "yaml_policy_parser_test.cc", + ], + data = [ + "//policy/testdata:policy_testdata", + ], + deps = [ + ":cel_policy", + ":cel_policy_parser", + ":yaml_policy_parser", + "//common:source", + "//internal:runfiles", + "//internal:status_macros", + "//internal:testing", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@yaml-cpp", + ], +) + +cc_test( + name = "compiler_test", + srcs = ["compiler_test.cc"], + data = [ + "//policy/testdata:policy_testdata", + ], + deps = [ + ":cel_policy", + ":cel_policy_parser", + ":cel_policy_validation_result", + ":compiler", + ":yaml_policy_parser", + "//common:ast", + "//common:ast_proto", + "//common:decl", + "//common:expr", + "//common:navigable_ast", + "//common:source", + "//common:type", + "//common:value", + "//common:value_testing", + "//compiler", + "//compiler:compiler_factory", + "//compiler:optional", + "//compiler:standard_library", + "//extensions:bindings_ext", + "//internal:runfiles", + "//internal:status_macros", + "//internal:testing", + "//internal:testing_descriptor_pool", + "//internal:testing_message_factory", + "//runtime", + "//runtime:activation", + "//runtime:optional_types", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", + "//tools:cel_unparser", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_protobuf//:protobuf", + ], +) + +cc_library( + name = "test_util", + srcs = ["test_util.cc"], + hdrs = ["test_util.h"], + copts = ["-fexceptions"], + features = ["-use_header_modules"], + deps = [ + "//internal:status_macros", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_cel_spec//proto/cel/expr:value_cc_proto", + "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto", + "@com_google_protobuf//:struct_cc_proto", + "@yaml-cpp", + ], +) diff --git a/policy/cel_policy.cc b/policy/cel_policy.cc new file mode 100644 index 000000000..6e08c1e10 --- /dev/null +++ b/policy/cel_policy.cc @@ -0,0 +1,306 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/cel_policy.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "common/source.h" + +namespace cel { + +namespace { + +std::string IdDebugString(CelPolicyElementId id) { + if (id == -1) { + return ""; + } + return absl::StrCat("#", id, "> "); +} + +std::string IndentBlock(absl::string_view text) { + if (text.empty()) { + return ""; + } + std::vector lines; + for (absl::string_view line : absl::StrSplit(text, '\n')) { + if (line.empty()) { + lines.push_back(""); + } else { + lines.push_back(absl::StrCat(" ", line)); + } + } + return absl::StrJoin(lines, "\n"); +} + +} // namespace + +void CelPolicySource::NoteSourcePosition(CelPolicyElementId id, + SourcePosition position) { + source_info_[id].position = position; +} + +void CelPolicySource::NoteSourceRange(CelPolicyElementId id, + std::optional range, + bool quoted) { + ElementSourceInfo& info = source_info_[id]; + info.range = range; + info.quoted = quoted; + if (range.has_value() && info.position == -1) { + info.position = range->begin; + } +} + +std::optional CelPolicySource::GetSourcePosition( + CelPolicyElementId id) const { + auto it = source_info_.find(id); + if (it == source_info_.end() || it->second.position == -1) { + return std::nullopt; + } + return it->second.position; +} + +std::optional CelPolicySource::GetSourceRange( + CelPolicyElementId id) const { + auto it = source_info_.find(id); + if (it == source_info_.end() || !it->second.range.has_value()) { + return std::nullopt; + } + return it->second.range; +} + +std::optional CelPolicySource::IsQuoted(CelPolicyElementId id) const { + auto it = source_info_.find(id); + if (it == source_info_.end() || !it->second.range.has_value()) { + return std::nullopt; + } + return it->second.quoted; +} + +std::optional CelPolicySource::GetSourceLocation( + CelPolicyElementId id) const { + auto it = source_info_.find(id); + if (it == source_info_.end() || it->second.position == -1) { + return std::nullopt; + } + return policy_source_->GetLocation(it->second.position); +} + +std::string CelPolicySource::DebugString() const { + std::string result; + + // Sort the source elements in descending order of position + std::vector> sorted_positions; + for (const auto& [id, info] : source_info_) { + if (info.position != -1) { + sorted_positions.push_back({id, info.position}); + } + } + std::sort(sorted_positions.begin(), sorted_positions.end(), + [](const auto& a, const auto& b) { + if (a.second == b.second) { + return a.first < b.first; + } + return a.second > b.second; + }); + + result = policy_source_->content().ToString(); + for (const auto& [id, position] : sorted_positions) { + result.insert(position, IdDebugString(id)); + } + return result; +} + +std::string ValueString::DebugString() const { + return absl::StrCat(IdDebugString(id_), "\"", value_, "\""); +} + +std::string Import::DebugString() const { + std::string result; + absl::StrAppend(&result, IdDebugString(id_), "name: ", name_.DebugString()); + return result; +} + +std::string OutputBlock::DebugString() const { + std::string result; + absl::StrAppend(&result, "output: ", output_.DebugString()); + if (explanation_.has_value()) { + absl::StrAppend(&result, "\nexplanation: ", explanation_->DebugString()); + } + return result; +} + +Match::Match(const Match& other) + : id_(other.id_), condition_(other.condition_) { + if (std::holds_alternative(other.result_)) { + result_ = std::get(other.result_); + } else if (std::holds_alternative(other.result_)) { + result_ = std::get(other.result_); + } else { + result_ = + std::make_unique(*std::get>(other.result_)); + } +} + +Match& Match::operator=(const Match& other) { + if (this != &other) { + id_ = other.id_; + condition_ = other.condition_; + if (std::holds_alternative(other.result_)) { + result_ = std::get(other.result_); + } else if (std::holds_alternative(other.result_)) { + result_ = std::get(other.result_); + } else { + result_ = std::make_unique( + *std::get>(other.result_)); + } + } + return *this; +} + +std::string Match::DebugString() const { + std::string result; + absl::StrAppend(&result, IdDebugString(id_), "match: {\n"); + if (condition_.has_value()) { + absl::StrAppend(&result, " condition: ", condition_->DebugString(), "\n"); + } + if (has_rule()) { + absl::StrAppend(&result, " result:\n", + IndentBlock(IndentBlock(rule().DebugString())), "\n"); + } else { + absl::StrAppend(&result, " result: {\n", + IndentBlock(IndentBlock(output_block().DebugString())), + "\n }\n"); + } + absl::StrAppend(&result, "}"); + return result; +} + +std::string Variable::DebugString() const { + std::string result; + absl::StrAppend(&result, "variable: {\n"); + absl::StrAppend(&result, " name: ", name_.DebugString(), "\n"); + absl::StrAppend(&result, " expression: ", expression_.DebugString(), "\n"); + if (description_.has_value()) { + absl::StrAppend(&result, " description: ", description_->DebugString(), + "\n"); + } + if (display_name_.has_value()) { + absl::StrAppend(&result, " display_name: ", display_name_->DebugString(), + "\n"); + } + absl::StrAppend(&result, "}"); + return result; +} + +std::string Rule::DebugString() const { + std::string result; + absl::StrAppend(&result, IdDebugString(id_), "rule: {\n"); + if (rule_id_.has_value()) { + absl::StrAppend(&result, " rule_id: ", rule_id_->DebugString(), "\n"); + } + if (description_.has_value()) { + absl::StrAppend(&result, " description: ", description_->DebugString(), + "\n"); + } + if (is_aggregate_) { + absl::StrAppend(&result, " is_aggregate: true\n"); + } + for (const Variable& variable : variables_) { + absl::StrAppend(&result, IndentBlock(variable.DebugString()), "\n"); + } + for (const Match& match : matches_) { + absl::StrAppend(&result, IndentBlock(match.DebugString()), "\n"); + } + absl::StrAppend(&result, "}"); + return result; +} + +std::string MetadataValueDebugString(std::any value) { + if (value.type() == typeid(std::monostate)) { + return "null"; + } + if (value.type() == typeid(ValueString)) { + return std::any_cast(value).DebugString(); + } + if (value.type() == typeid(bool)) { + return std::any_cast(value) ? "true" : "false"; + } + if (value.type() == typeid(int)) { + return absl::StrCat(std::any_cast(value)); + } + if (value.type() == typeid(std::string)) { + return std::any_cast(value); + } + return absl::StrCat("typeid: ", value.type().name()); +} + +std::string CelPolicy::DebugString() const { + std::string result; + absl::StrAppend(&result, "CelPolicy{\n"); + absl::StrAppend( + &result, + " ===========================================================\n"); + absl::StrAppend(&result, IndentBlock(IndentBlock(source_->DebugString())), + "\n"); + absl::StrAppend( + &result, + " ===========================================================\n"); + absl::StrAppend(&result, " name: ", name_.DebugString(), "\n"); + if (description_.has_value()) { + absl::StrAppend(&result, " description: ", description_->DebugString(), + "\n"); + } + if (display_name_.has_value()) { + absl::StrAppend(&result, " display_name: ", display_name_->DebugString(), + "\n"); + } + if (!metadata_.empty()) { + std::vector sorted_keys; + for (const auto& [key, _] : metadata_) { + sorted_keys.push_back(key); + } + std::sort(sorted_keys.begin(), sorted_keys.end()); + + absl::StrAppend(&result, " metadata: {\n"); + for (const auto& key : sorted_keys) { + const auto& value = metadata_.at(key); + absl::StrAppend(&result, " ", key, ": ", + MetadataValueDebugString(value), "\n"); + } + absl::StrAppend(&result, " }\n"); + } + if (!imports_.empty()) { + absl::StrAppend(&result, " imports:\n"); + for (const Import& import : imports_) { + absl::StrAppend(&result, " ", import.DebugString(), "\n"); + } + } + absl::StrAppend(&result, IndentBlock(rule_.DebugString()), "\n"); + absl::StrAppend(&result, "}"); + return result; +} + +} // namespace cel diff --git a/policy/cel_policy.h b/policy/cel_policy.h new file mode 100644 index 000000000..25c9372c8 --- /dev/null +++ b/policy/cel_policy.h @@ -0,0 +1,346 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" +#include "absl/log/absl_check.h" +#include "absl/strings/string_view.h" +#include "common/source.h" + +namespace cel { + +using CelPolicyElementId = int32_t; + +struct ElementSourceInfo { + SourcePosition position = -1; + std::optional range; + bool quoted = false; +}; + +class CelPolicySource { + public: + explicit CelPolicySource(cel::SourcePtr policy_source) + : policy_source_(std::move(policy_source)) {} + + const Source* absl_nonnull content() const { return policy_source_.get(); } + + void NoteSourcePosition(CelPolicyElementId id, SourcePosition position); + + void NoteSourceRange(CelPolicyElementId id, std::optional range, + bool quoted); + + std::optional GetSourcePosition(CelPolicyElementId id) const; + + std::optional GetSourceRange(CelPolicyElementId id) const; + + std::optional IsQuoted(CelPolicyElementId id) const; + + std::optional GetSourceLocation(CelPolicyElementId id) const; + + std::string DebugString() const; + + private: + cel::SourcePtr policy_source_; + absl::flat_hash_map source_info_; +}; + +class ValueString { + public: + ValueString() : id_(-1) {} + + explicit ValueString(CelPolicyElementId id, absl::string_view value) + : id_(id), value_(value) {} + + CelPolicyElementId id() const { return id_; } + absl::string_view value() const { return value_; } + + std::string DebugString() const; + + private: + CelPolicyElementId id_; + std::string value_; +}; + +class Import { + public: + Import(CelPolicyElementId id, ValueString name) + : id_(id), name_(std::move(name)) {} + CelPolicyElementId id() const { return id_; } + const ValueString& name() const { return name_; } + + std::string DebugString() const; + + private: + CelPolicyElementId id_; + ValueString name_; +}; + +// Defines a variable that can be used in CEL expressions within the policy. +// Variables are evaluated once and stored in the activation context. +class Variable { + public: + const ValueString& name() const { return name_; } + void set_name(ValueString name) { name_ = std::move(name); } + + const ValueString& expression() const { return expression_; } + void set_expression(ValueString expression) { + expression_ = std::move(expression); + } + + std::optional description() const { return description_; } + void set_description(ValueString description) { + description_ = std::move(description); + } + + std::optional display_name() const { return display_name_; } + void set_display_name(ValueString display_name) { + display_name_ = std::move(display_name); + } + + std::string DebugString() const; + + private: + ValueString name_; + ValueString expression_; + std::optional description_; + std::optional display_name_; +}; + +class Rule; + +class OutputBlock { + public: + OutputBlock() = default; + OutputBlock(ValueString output, std::optional explanation) + : output_(std::move(output)), explanation_(std::move(explanation)) {} + + const ValueString& output() const { return output_; } + void set_output(ValueString output) { output_ = std::move(output); } + + const std::optional& explanation() const { return explanation_; } + void set_explanation(ValueString explanation) { + explanation_ = std::move(explanation); + } + + std::string DebugString() const; + + private: + ValueString output_; + std::optional explanation_; +}; + +// Defines a match condition and result. +// If the result is a Rule, it is considered a sub-rule and will be evaluated +// only if the match condition evaluates to true. +class Match { + public: + Match() = default; + Match(const Match& other); + Match& operator=(const Match& other); + + CelPolicyElementId id() const; + void set_id(CelPolicyElementId id); + + bool has_condition() const; + std::optional condition() const; + void set_condition(ValueString condition); + + bool has_output_block() const; + const OutputBlock& output_block() const; + OutputBlock& mutable_output_block(); + + bool has_rule() const; + const Rule& rule() const; + Rule& mutable_rule(); + + void set_result(OutputBlock result); + void set_result(std::unique_ptr result); + + std::string DebugString() const; + + private: + CelPolicyElementId id_ = -1; + std::optional condition_; + std::variant> result_; +}; + +// Rule is the body of the policy and contains a list of variables and matches. +// Variables are evaluated once and stored in the evaluation context. +// +// Rule kinds: +// +// First-match rule (default): +// Matches are evaluated in order and the first match is returned. If the +// match contains a sub-rule, the sub-rule is evaluated only if the match +// condition evaluates to true. +// +// Aggregate rule: +// All matches are evaluated and the results are aggregated into a list. If the +// match condition holds, it is added to the aggregate result. +// There can only be one aggregate rule per policy. +class Rule { + public: + Rule() = default; + Rule(const Rule& other) = default; + + CelPolicyElementId id() const { return id_; } + void set_id(CelPolicyElementId id) { id_ = id; } + + const std::optional& rule_id() const { return rule_id_; } + void set_rule_id(ValueString rule_id) { rule_id_ = std::move(rule_id); } + + const std::optional& description() const { return description_; } + void set_description(ValueString description) { + description_ = std::move(description); + } + + const std::vector& variables() const { return variables_; } + std::vector& mutable_variables() { return variables_; } + + const std::vector& matches() const { return matches_; } + std::vector& mutable_matches() { return matches_; } + + bool is_aggregate() const { return is_aggregate_; } + void set_is_aggregate(bool is_aggregate) { is_aggregate_ = is_aggregate; } + + std::string DebugString() const; + + private: + CelPolicyElementId id_ = -1; + std::optional rule_id_; + std::optional description_; + std::vector variables_; + std::vector matches_; + bool is_aggregate_ = false; +}; + +// CelPolicy is the top-level policy object. +// It contains a source, name, description, display name, imports, and a rule. +// The source is the CEL policy source code. +// The name, description, and display name are metadata about the policy. +// The rule is the main body of the policy. +class CelPolicy { + public: + explicit CelPolicy(std::shared_ptr source) + : source_(std::move(source)) {} + + CelPolicy(const CelPolicy& other) = default; + CelPolicy& operator=(const CelPolicy& other) = default; + + const CelPolicySource* absl_nullable source() const { return source_.get(); } + const std::shared_ptr& source_ptr() const { return source_; } + + const ValueString& name() const { return name_; } + void set_name(ValueString name) { name_ = std::move(name); } + + std::optional description() const { return description_; } + void set_description(ValueString description) { + description_ = std::move(description); + } + std::optional display_name() const { return display_name_; } + void set_display_name(ValueString display_name) { + display_name_ = std::move(display_name); + } + const absl::flat_hash_map& metadata() const { + return metadata_; + } + absl::flat_hash_map& mutable_metadata() { + return metadata_; + } + const std::vector& imports() const { return imports_; } + std::vector& mutable_imports() { return imports_; } + + const Rule& rule() const { return rule_; } + Rule& mutable_rule() { return rule_; } + + std::string DebugString() const; + + private: + std::shared_ptr source_; + ValueString name_; + std::optional description_; + std::optional display_name_; + absl::flat_hash_map metadata_; + std::vector imports_; + Rule rule_; +}; + +// Implementation details. + +inline CelPolicyElementId Match::id() const { return id_; } +inline void Match::set_id(CelPolicyElementId id) { id_ = id; } + +inline bool Match::has_condition() const { return condition_.has_value(); } + +inline std::optional Match::condition() const { + return condition_; +} + +inline void Match::set_condition(ValueString condition) { + condition_ = std::move(condition); +} + +inline bool Match::has_output_block() const { + return std::holds_alternative(result_); +} + +inline const OutputBlock& Match::output_block() const { + ABSL_DCHECK(std::holds_alternative(result_)); + return std::get(result_); +} + +inline OutputBlock& Match::mutable_output_block() { + if (!std::holds_alternative(result_)) { + result_ = OutputBlock(); + } + return std::get(result_); +} + +inline bool Match::has_rule() const { + return std::holds_alternative>(result_); +} + +inline const Rule& Match::rule() const { + ABSL_DCHECK(std::holds_alternative>(result_)); + return *std::get>(result_); +} + +inline Rule& Match::mutable_rule() { + ABSL_DCHECK(std::holds_alternative>(result_)); + return *std::get>(result_); +} + +inline void Match::set_result(OutputBlock result) { + result_ = std::move(result); +} + +inline void Match::set_result(std::unique_ptr result) { + result_ = std::move(result); +} + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_H_ diff --git a/policy/cel_policy_parse_context.cc b/policy/cel_policy_parse_context.cc new file mode 100644 index 000000000..0c12c7f0c --- /dev/null +++ b/policy/cel_policy_parse_context.cc @@ -0,0 +1,48 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/cel_policy_parse_context.h" + +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" + +namespace cel { + +CelPolicy& CelPolicyParseContext::policy() const { + ABSL_CHECK(policy_ != nullptr) + << "CelPolicyParseContext::policy() called after GetResult()"; + return *policy_; +} + +CelPolicyParseResult CelPolicyParseContext::GetResult() { + if (policy_ != nullptr && issues_.empty()) { + return CelPolicyParseResult(std::move(policy_source_), std::move(policy_), + std::move(issues_)); + } + policy_.reset(); + return CelPolicyParseResult(std::move(policy_source_), nullptr, + std::move(issues_)); +} + +void CelPolicyParseContext::ReportError(CelPolicyElementId element_id, + std::string_view message) { + issues_.push_back(CelPolicyIssue(element_id, message)); +} + +} // namespace cel diff --git a/policy/cel_policy_parse_context.h b/policy/cel_policy_parse_context.h new file mode 100644 index 000000000..6482fa1ae --- /dev/null +++ b/policy/cel_policy_parse_context.h @@ -0,0 +1,65 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSE_CONTEXT_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSE_CONTEXT_H_ + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" + +namespace cel { + +// A mutable context for parsing a CelPolicy. An instance of this class is +// created for each policy parse and is passed to the parser, which is meant to +// be stateless. +// +// Parsers call methods on this class to report issues and populate the policy +// being parsed. Call GetResult() to obtain the resulting CelPolicyParseResult, +// which takes ownership of the parsed policy. Do not use the context after +// calling GetResult(). +class CelPolicyParseContext { + public: + explicit CelPolicyParseContext(std::shared_ptr policy_source) + : policy_source_(std::move(policy_source)), + policy_(std::make_unique(policy_source_)) {} + + CelPolicySource& policy_source() const { return *policy_source_; } + + // Returns the policy being parsed. It should not be used after + // calling GetResult(). + CelPolicy& policy() const; + + // The context should not be used after calling GetResult(). + CelPolicyParseResult GetResult(); + + // Reports an error for the given element with the given error message. + void ReportError(CelPolicyElementId id, std::string_view message); + + CelPolicyElementId next_element_id() { return next_element_id_++; } + + private: + std::shared_ptr policy_source_; + CelPolicyElementId next_element_id_ = 0; + std::vector issues_; + std::unique_ptr policy_; +}; + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSE_CONTEXT_H_ diff --git a/policy/cel_policy_parse_result.cc b/policy/cel_policy_parse_result.cc new file mode 100644 index 000000000..d8c9bb2b0 --- /dev/null +++ b/policy/cel_policy_parse_result.cc @@ -0,0 +1,95 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/cel_policy_parse_result.h" + +#include +#include + +#include "absl/base/nullability.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "common/source.h" +#include "policy/cel_policy.h" + +namespace cel { +namespace { + +absl::string_view SeverityString(CelPolicyIssue::Severity severity) { + switch (severity) { + case CelPolicyIssue::Severity::kInformation: + return "INFORMATION"; + case CelPolicyIssue::Severity::kWarning: + return "WARNING"; + case CelPolicyIssue::Severity::kError: + return "ERROR"; + case CelPolicyIssue::Severity::kDeprecated: + return "DEPRECATED"; + default: + return "SEVERITY_UNSPECIFIED"; + } +} + +} // namespace + +std::string CelPolicyIssue::ToDisplayString( + const CelPolicySource* absl_nullable source) const { + SourceLocation location; + std::string description; + std::string snippet; + if (source != nullptr) { + if (relative_position_) { + std::optional range = source->GetSourceRange(element_id_); + std::optional base = + source->GetSourcePosition(element_id_); + if (range.has_value()) { + base = range->begin; + } + if (element_id_ == -1) { + base.emplace(0); + } + if (base) { + location = source->content() + ->GetLocation(*base + *relative_position_) + .value_or(SourceLocation{}); + } + } else { + location = + source->GetSourceLocation(element_id_).value_or(SourceLocation{}); + } + description = std::string(source->content()->description()); + snippet = source->content()->DisplayErrorLocation(location); + } + + const int display_column = location.column >= 0 ? location.column + 1 : -1; + + return absl::StrFormat("%s: %s:%d:%d: %s%s", SeverityString(severity_), + description, location.line, display_column, message_, + snippet); +} + +std::string CelPolicyParseResult::FormattedIssues() const { + std::string formatted_issues; + for (const CelPolicyIssue& issue : issues_) { + if (!formatted_issues.empty()) { + absl::StrAppend(&formatted_issues, "\n"); + } + absl::StrAppend(&formatted_issues, issue.ToDisplayString(*policy_source_)); + } + return formatted_issues; +} + +} // namespace cel diff --git a/policy/cel_policy_parse_result.h b/policy/cel_policy_parse_result.h new file mode 100644 index 000000000..2bf80b1ce --- /dev/null +++ b/policy/cel_policy_parse_result.h @@ -0,0 +1,105 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSE_RESULT_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSE_RESULT_H_ + +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/source.h" +#include "policy/cel_policy.h" + +namespace cel { + +class CelPolicyIssue { + public: + enum class Severity { kInformation, kDeprecated, kWarning, kError }; + + CelPolicyIssue(CelPolicyElementId element_id, absl::string_view message) + : element_id_(element_id), message_(message) {} + CelPolicyIssue(CelPolicyElementId element_id, Severity severity, + absl::string_view message) + : element_id_(element_id), severity_(severity), message_(message) {} + CelPolicyIssue(CelPolicyElementId element_id, + SourcePosition relative_position, absl::string_view message) + : element_id_(element_id), + relative_position_(relative_position), + message_(message) {} + CelPolicyIssue(CelPolicyElementId element_id, + SourcePosition relative_position, Severity severity, + absl::string_view message) + : element_id_(element_id), + relative_position_(relative_position), + severity_(severity), + message_(message) {} + + std::string ToDisplayString( + const CelPolicySource* absl_nullable source) const; + std::string ToDisplayString(const CelPolicySource& source) const { + return ToDisplayString(&source); + } + + Severity severity() const { return severity_; } + absl::string_view message() const { return message_; } + + private: + CelPolicyElementId element_id_; + std::optional relative_position_; + Severity severity_ = Severity::kError; + std::string message_; +}; + +class CelPolicyParseResult { + public: + explicit CelPolicyParseResult(std::shared_ptr policy_source, + std::unique_ptr policy, + std::vector issues) + : policy_source_(std::move(policy_source)), + policy_(std::move(policy)), + issues_(std::move(issues)) {} + + bool IsValid() const { return policy_ != nullptr; } + + const CelPolicy* absl_nullable GetPolicy() const { return policy_.get(); } + + absl::StatusOr> ReleasePolicy() { + if (policy_ == nullptr) { + return absl::FailedPreconditionError( + "CelPolicyParseResult is empty. Check for Issues."); + } + return std::move(policy_); + } + + absl::Span GetIssues() const { return issues_; } + + std::string FormattedIssues() const; + + private: + std::shared_ptr policy_source_; + absl_nullable std::unique_ptr policy_; + std::vector issues_; +}; + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSE_RESULT_H_ diff --git a/policy/cel_policy_parser.h b/policy/cel_policy_parser.h new file mode 100644 index 000000000..0a11c9e68 --- /dev/null +++ b/policy/cel_policy_parser.h @@ -0,0 +1,40 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSER_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSER_H_ + +#include "absl/status/status.h" +#include "policy/cel_policy_parse_context.h" + +namespace cel { + +// A policy parser for a given policy format. The type `T` parameter is the +// representation of the input file format, such as `` for YAML. +// +// Parsers are intended to be stateless: all state, including the resulting +// policy and any issues encountered, should be kept in the context passed to +// the `ParsePolicy` method. +template +class CelPolicyParser { + public: + virtual ~CelPolicyParser() = default; + + // Parses the input and populates a CelPolicy in the context. + virtual absl::Status ParsePolicy(CelPolicyParseContext& ctx) const = 0; +}; + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSER_H_ diff --git a/policy/cel_policy_test.cc b/policy/cel_policy_test.cc new file mode 100644 index 000000000..dc9d6762d --- /dev/null +++ b/policy/cel_policy_test.cc @@ -0,0 +1,245 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/cel_policy.h" + +#include +#include +#include +#include + +#include "absl/strings/str_replace.h" +#include "common/source.h" +#include "internal/testing.h" + +namespace cel { +namespace { + +using testing::Field; +using testing::Optional; +using testing::SizeIs; + +TEST(CelPolicyBuilderTest, Build) { + CelPolicyElementId next_id = 1; + ASSERT_OK_AND_ASSIGN(SourcePtr source, NewSource("CEL\n policy\n source")); + std::shared_ptr policy_source = + std::make_shared(std::move(source)); + CelPolicy policy(policy_source); + policy.set_name(ValueString(next_id++, "test_policy")); + policy.set_description(ValueString(next_id++, "test_description")); + policy.set_display_name(ValueString(next_id++, "test_display_name")); + ValueString import1_name = ValueString(next_id++, "test_import1"); + policy.mutable_imports().push_back(Import(next_id++, import1_name)); + ValueString import2_name = ValueString(next_id++, "test_import2"); + policy.mutable_imports().push_back(Import(next_id++, import2_name)); + + Rule& rule = policy.mutable_rule(); + rule.set_id(next_id++); + rule.set_rule_id(ValueString(next_id++, "test_rule_id")); + rule.set_description(ValueString(next_id++, "test_rule_description")); + + Variable variable; + variable.set_name(ValueString(next_id++, "test_variable")); + variable.set_expression(ValueString(next_id++, "test_expression")); + variable.set_description(ValueString(next_id++, "test_variable_description")); + variable.set_display_name( + ValueString(next_id++, "test_variable_display_name")); + + Match match1; + match1.set_id(next_id++); + match1.set_condition(ValueString(next_id++, "test_condition")); + CelPolicyElementId output_id = next_id++; + CelPolicyElementId explanation_id = next_id++; + match1.set_result( + OutputBlock(ValueString(output_id, "test_result"), + ValueString(explanation_id, "test_explanation"))); + + Match match2; + match2.set_id(next_id++); + match2.set_condition(ValueString(next_id++, "test_condition2")); + + auto sub_rule = std::make_unique(); + sub_rule->set_id(next_id++); + sub_rule->set_rule_id(ValueString(next_id++, "sub_rule_id")); + sub_rule->set_description(ValueString(next_id++, "sub_rule_description")); + Match sub_rule_match; + sub_rule_match.set_id(next_id++); + sub_rule_match.set_condition(ValueString(next_id++, "sub_rule_condition")); + sub_rule_match.set_result( + OutputBlock(ValueString(next_id++, "sub_rule_result"), std::nullopt)); + sub_rule->mutable_matches().push_back(sub_rule_match); + + match2.set_result(std::move(sub_rule)); + + rule.mutable_variables().push_back(variable); + rule.mutable_matches().push_back(match1); + rule.mutable_matches().push_back(match2); + + EXPECT_EQ(policy.name().value(), "test_policy"); + ASSERT_TRUE(policy.description().has_value()); + EXPECT_EQ(policy.description()->value(), "test_description"); + ASSERT_TRUE(policy.display_name().has_value()); + EXPECT_EQ(policy.display_name()->value(), "test_display_name"); + + ASSERT_THAT(policy.imports(), SizeIs(2)); + + EXPECT_EQ(policy.imports()[0].name().value(), "test_import1"); + EXPECT_EQ(policy.imports()[1].name().value(), "test_import2"); + ASSERT_TRUE(policy.rule().rule_id().has_value()); + EXPECT_EQ(policy.rule().rule_id()->value(), "test_rule_id"); + ASSERT_TRUE(policy.rule().description().has_value()); + EXPECT_EQ(policy.rule().description()->value(), "test_rule_description"); + + ASSERT_THAT(policy.rule().variables(), SizeIs(1)); + + EXPECT_EQ(policy.rule().variables()[0].name().value(), "test_variable"); + EXPECT_EQ(policy.rule().variables()[0].expression().value(), + "test_expression"); + ASSERT_TRUE(policy.rule().variables()[0].description().has_value()); + EXPECT_EQ(policy.rule().variables()[0].description()->value(), + "test_variable_description"); + ASSERT_TRUE(policy.rule().variables()[0].display_name().has_value()); + EXPECT_EQ(policy.rule().variables()[0].display_name()->value(), + "test_variable_display_name"); + + ASSERT_THAT(policy.rule().matches(), SizeIs(2)); + + EXPECT_EQ(policy.rule().matches()[0].condition().value().value(), + "test_condition"); + ASSERT_TRUE(policy.rule().matches()[0].has_output_block()); + EXPECT_EQ(policy.rule().matches()[0].output_block().output().value(), + "test_result"); + ASSERT_TRUE( + policy.rule().matches()[0].output_block().explanation().has_value()); + EXPECT_EQ(policy.rule().matches()[0].output_block().explanation()->value(), + "test_explanation"); + + EXPECT_EQ(policy.rule().matches()[1].condition().value().value(), + "test_condition2"); + ASSERT_TRUE(policy.rule().matches()[1].has_rule()); + ASSERT_TRUE(policy.rule().matches()[1].rule().rule_id().has_value()); + EXPECT_EQ(policy.rule().matches()[1].rule().rule_id()->value(), + "sub_rule_id"); + ASSERT_TRUE(policy.rule().matches()[1].rule().description().has_value()); + EXPECT_EQ(policy.rule().matches()[1].rule().description()->value(), + "sub_rule_description"); + ASSERT_THAT(policy.rule().matches()[1].rule().matches(), SizeIs(1)); + EXPECT_EQ(policy.rule() + .matches()[1] + .rule() + .matches()[0] + .condition() + .value() + .value(), + "sub_rule_condition"); + + std::string actual = policy.DebugString(); + EXPECT_EQ(actual, absl::StrReplaceAll(R"(CelPolicy{ + =========================================================== + CEL + policy + source + =========================================================== + name: #1> "test_policy" + description: #2> "test_description" + display_name: #3> "test_display_name" + imports: + #5> name: #4> "test_import1" + #7> name: #6> "test_import2" + #8> rule: { + rule_id: #9> "test_rule_id" + description: #10> "test_rule_description" + variable: { + name: #11> "test_variable" + expression: #12> "test_expression" + description: #13> "test_variable_description" + display_name: #14> "test_variable_display_name" + } + #15> match: { + condition: #16> "test_condition" + result: { + output: #17> "test_result" + explanation: #18> "test_explanation" + } + } + #19> match: { + condition: #20> "test_condition2" + result: + #21> rule: { + rule_id: #22> "sub_rule_id" + description: #23> "sub_rule_description" + #24> match: { + condition: #25> "sub_rule_condition" + result: { + output: #26> "sub_rule_result" + } + } + } + } + } + })", + {{"\n ", "\n"}})); +} + +TEST(CelPolicySourceTest, Build) { + std::string source = + "name: test_policy\n imports:\n - name: test_import\n"; + + ASSERT_OK_AND_ASSIGN(SourcePtr source_ptr, NewSource(source)); + CelPolicySource policy_source(std::move(source_ptr)); + policy_source.NoteSourcePosition(1, source.find("test_policy")); + policy_source.NoteSourcePosition(2, source.find("test_import")); + + EXPECT_THAT(policy_source.GetSourcePosition(1), Optional(6)); + EXPECT_THAT(policy_source.GetSourceLocation(1), + Optional(AllOf(Field(&SourceLocation::line, 1), + Field(&SourceLocation::column, 6)))); + EXPECT_THAT(policy_source.GetSourcePosition(2), Optional(44)); + EXPECT_THAT(policy_source.GetSourceLocation(2), + Optional(AllOf(Field(&SourceLocation::line, 3), + Field(&SourceLocation::column, 13)))); + EXPECT_EQ(policy_source.GetSourcePosition(3), std::nullopt); + EXPECT_EQ(policy_source.GetSourceLocation(3), std::nullopt); + EXPECT_EQ( + policy_source.DebugString(), + "name: #1> test_policy\n imports:\n - name: #2> test_import\n"); +} + +TEST(CelPolicyBuilderTest, AggregateRule) { + Rule rule; + EXPECT_FALSE(rule.is_aggregate()); + rule.set_id(1); + rule.set_rule_id(ValueString(2, "agg_rule")); + rule.set_is_aggregate(true); + EXPECT_TRUE(rule.is_aggregate()); + + Match match; + match.set_id(3); + match.set_result(OutputBlock(ValueString(4, "test_result"), std::nullopt)); + rule.mutable_matches().push_back(match); + + EXPECT_EQ(rule.DebugString(), absl::StrReplaceAll(R"(#1> rule: { + rule_id: #2> "agg_rule" + is_aggregate: true + #3> match: { + result: { + output: #4> "test_result" + } + } + })", + {{"\n ", "\n"}})); +} + +} // namespace +} // namespace cel diff --git a/policy/cel_policy_validation_result.cc b/policy/cel_policy_validation_result.cc new file mode 100644 index 000000000..e257f064c --- /dev/null +++ b/policy/cel_policy_validation_result.cc @@ -0,0 +1,32 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/cel_policy_validation_result.h" + +#include + +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "policy/cel_policy_parse_result.h" + +namespace cel { + +std::string CelPolicyValidationResult::FormatIssues() const { + return absl::StrJoin( + issues_, "\n", [this](std::string* out, const CelPolicyIssue& issue) { + absl::StrAppend(out, issue.ToDisplayString(source_.get())); + }); +} + +} // namespace cel diff --git a/policy/cel_policy_validation_result.h b/policy/cel_policy_validation_result.h new file mode 100644 index 000000000..a47a2c66a --- /dev/null +++ b/policy/cel_policy_validation_result.h @@ -0,0 +1,85 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_VALIDATION_RESULT_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_VALIDATION_RESULT_H_ + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "common/decl.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" + +namespace cel { + +// CelPolicyValidationResult holds the result of policy compilation. +// +// Policy compilation/validation errors are captured in issues. +class CelPolicyValidationResult { + public: + CelPolicyValidationResult( + std::unique_ptr ast, std::vector issues, + std::shared_ptr source = nullptr) + : ast_(std::move(ast)), + issues_(std::move(issues)), + source_(std::move(source)) {} + + explicit CelPolicyValidationResult( + std::vector issues, + std::shared_ptr source = nullptr) + : ast_(nullptr), issues_(std::move(issues)), source_(std::move(source)) {} + + // Returns true if validation succeeded and an AST is present. + bool IsValid() const { return ast_ != nullptr; } + + // Returns the AST if validation was successful. + const Ast* absl_nullable GetAst() const { return ast_.get(); } + + // Moves out and returns the AST. + absl::StatusOr> ReleaseAst() { + if (ast_ == nullptr) { + return absl::FailedPreconditionError( + "CelPolicyValidationResult is empty. Check for CelPolicyIssues."); + } + return std::move(ast_); + } + + // Returns the list of issues encountered during compilation. + absl::Span GetIssues() const { return issues_; } + + // Returns the contained policy source, if any. + const CelPolicySource* absl_nullable GetSource() const { + return source_.get(); + } + + // Returns a formatted error string of the compiled issues. + std::string FormatIssues() const; + + private: + absl_nullable std::unique_ptr ast_; + std::vector issues_; + std::shared_ptr source_; +}; + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_VALIDATION_RESULT_H_ diff --git a/policy/compiler.cc b/policy/compiler.cc new file mode 100644 index 000000000..4a37a04d7 --- /dev/null +++ b/policy/compiler.cc @@ -0,0 +1,1260 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/compiler.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/base/nullability.h" +#include "absl/cleanup/cleanup.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "checker/type_check_issue.h" +#include "checker/type_checker.h" +#include "checker/validation_result.h" +#include "common/ast.h" +#include "common/ast_rewrite.h" +#include "common/constant.h" +#include "common/container.h" +#include "common/decl.h" +#include "common/expr.h" +#include "common/format_type_name.h" +#include "common/navigable_ast.h" +#include "common/source.h" +#include "common/type.h" +#include "common/type_kind.h" +#include "compiler/compiler.h" +#include "internal/status_macros.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" +#include "policy/cel_policy_validation_result.h" +#include "policy/internal/issue_reporter.h" +#include "policy/internal/optimizer_expr_factory.h" +#include "google/protobuf/arena.h" + +namespace cel { +namespace { + +constexpr absl::string_view kCelBlock = "cel.@block"; + +enum class RuleSemantics { + kFirstMatch, + kAggregate, + + kNotForUseWithExhaustiveSwitchStatements, +}; + +template +void AbslStringify(Sink& s, RuleSemantics semantics) { + switch (semantics) { + case RuleSemantics::kFirstMatch: + s.Append("first_match"); + return; + case RuleSemantics::kAggregate: + s.Append("aggregate"); + return; + default: + s.Append(""); + return; + } +} + +struct EmbeddedAst { + CelPolicyElementId id; + std::unique_ptr ast; +}; + +struct CompiledVariable { + std::string ident; + EmbeddedAst ast; +}; + +struct CompiledOutputBlock { + EmbeddedAst output_ast; + cel::Type result_type; + std::optional explanation_ast; +}; + +struct CompiledRule; + +struct CompiledMatch { + using Production = + std::variant absl_nonnull, + CompiledOutputBlock>; + + CelPolicyElementId id; + std::optional condition; + Production production; +}; + +struct CompiledRule { + CelPolicyElementId id; + std::vector variables; + std::vector matches; + // Not set if cannot be determined. + std::optional result_type; + RuleSemantics semantics = RuleSemantics::kFirstMatch; +}; + +std::optional GetOutputType( + const CompiledMatch::Production& production) { + return std::visit( + [](const auto& production) -> std::optional { + if constexpr (std::is_same_v, + CompiledOutputBlock>) { + return production.result_type; + } else if constexpr (std::is_same_v, + std::unique_ptr>) { + return production->result_type; + } + return std::nullopt; + }, + production); +} + +// Internal representation of the compiled policy elements. +// +// This is used for checking the component expression before composing into the +// final AST based on the provided rule semantics. +class IntermediateCompiledPolicy { + public: + CompiledRule& mutable_root_rule() { return root_rule_; } + + const CompiledRule& root_rule() const { return root_rule_; } + + void set_name(absl::string_view name) { name_ = name; } + absl::string_view name() const { return name_; } + void set_display_name(absl::string_view display_name) { + display_name_ = display_name; + } + absl::string_view display_name() const { return display_name_; } + void set_description(absl::string_view description) { + description_ = description; + } + absl::string_view description() const { return description_; } + + void set_policy_source(const CelPolicySource* absl_nullable src) { + policy_source_ = src; + } + const CelPolicySource* absl_nullable policy_source() const { + return policy_source_; + } + + private: + std::string name_; + std::string display_name_; + std::string description_; + const CelPolicySource* absl_nullable policy_source_ = nullptr; + + CompiledRule root_rule_; +}; + +CelPolicyIssue::Severity MapSeverity(cel::TypeCheckIssue::Severity severity) { + switch (severity) { + case cel::TypeCheckIssue::Severity::kError: + return CelPolicyIssue::Severity::kError; + case cel::TypeCheckIssue::Severity::kWarning: + return CelPolicyIssue::Severity::kWarning; + case cel::TypeCheckIssue::Severity::kDeprecated: + return CelPolicyIssue::Severity::kDeprecated; + default: + return CelPolicyIssue::Severity::kError; + } +} + +bool IsWrapperOf(cel::TypeKind wrapper_kind, cel::TypeKind primitive_kind) { + switch (wrapper_kind) { + case cel::TypeKind::kBoolWrapper: + return primitive_kind == cel::TypeKind::kBool; + case cel::TypeKind::kIntWrapper: + return primitive_kind == cel::TypeKind::kInt; + case cel::TypeKind::kUintWrapper: + return primitive_kind == cel::TypeKind::kUint; + case cel::TypeKind::kDoubleWrapper: + return primitive_kind == cel::TypeKind::kDouble; + case cel::TypeKind::kStringWrapper: + return primitive_kind == cel::TypeKind::kString; + case cel::TypeKind::kBytesWrapper: + return primitive_kind == cel::TypeKind::kBytes; + default: + return false; + } +} + +cel::Type FilterSpecialTypes(cel::Type type) { + if (type.IsTypeParam()) { + // Free type param should not appear in the output type, but if it does, + // force it to dyn. + return DynType(); + } + if (type.IsEnum()) { + return IntType{}; + } + if (type.IsError()) { + return DynType(); + } + if (type.IsType()) { + // drop parameters so all type types are compatible. + return TypeType{}; + } + return type; +} + +// Returns true if `from` is assignable to `to`. +// +// Slightly adjusted from the standard routine to cover some edge cases around +// null and wrappers. +// +// TODO(b/522391716): try to standardize assignability checks. +bool OutputTypeIsAssignable(cel::Type from, cel::Type to) { + from = FilterSpecialTypes(from); + to = FilterSpecialTypes(to); + + // Any and dyn are assignable to/from everything. + if (from.kind() == cel::TypeKind::kAny || + from.kind() == cel::TypeKind::kDyn || to.kind() == cel::TypeKind::kAny || + to.kind() == cel::TypeKind::kDyn) { + return true; + } + + // Wrappers auto-unwrap. + if (IsWrapperOf(from.kind(), to.kind()) || + IsWrapperOf(to.kind(), from.kind())) { + return true; + } + + // Null is assignable to anything that is message-like. + if (from.kind() == cel::TypeKind::kNull) { + switch (to.kind()) { + case cel::TypeKind::kNull: + case cel::TypeKind::kStruct: + case cel::TypeKind::kOpaque: + case cel::TypeKind::kTimestamp: + case cel::TypeKind::kDuration: + case cel::TypeKind::kBytesWrapper: + case cel::TypeKind::kBoolWrapper: + case cel::TypeKind::kIntWrapper: + case cel::TypeKind::kUintWrapper: + case cel::TypeKind::kDoubleWrapper: + case cel::TypeKind::kStringWrapper: + return true; + default: + return false; + } + } + + if (from.kind() != to.kind()) { + return false; + } + + if (from.name() != to.name()) { + return false; + } + + if (from.GetParameters().size() != to.GetParameters().size()) { + return false; + } + + for (int i = 0; i < from.GetParameters().size(); ++i) { + if (!OutputTypeIsAssignable(from.GetParameters()[i], + to.GetParameters()[i])) { + return false; + } + } + + return true; +} + +bool OutputTypeIsCompatible(cel::Type from, cel::Type to) { + // We don't handle widening like in a self-contained CEL expression, but + // permit some cases where one type is more specific than the other. + return OutputTypeIsAssignable(from, to) || OutputTypeIsAssignable(to, from); +} + +bool HasErrors(const policy_internal::IssueReporter& issues) { + for (const auto& issue : issues.issues()) { + if (issue.severity() == CelPolicyIssue::Severity::kError) { + return true; + } + } + return false; +} + +void AdaptTypeCheckIssues(CelPolicyElementId id, const ValidationResult& r, + const CelPolicySource* absl_nullable policy_source, + policy_internal::IssueReporter& issues) { + const Source* source = r.GetSource(); + if (source == nullptr && policy_source != nullptr) { + source = policy_source->content(); + } + + for (const auto& iss : r.GetIssues()) { + std::optional offset; + if (source != nullptr) { + offset = source->GetPosition(iss.location()); + } + if (offset.has_value()) { + issues.ReportOffsetIssue(id, *offset, MapSeverity(iss.severity()), + iss.message()); + continue; + } + issues.ReportIssue(id, MapSeverity(iss.severity()), iss.message()); + } +} + +// Note on lifetime safety: +// +// The output policy will contain references to types that are owned by the +// arena member of this class. This is safe as long as the policy compiler lives +// as long as the output policies. +class PolicyCompiler { + public: + explicit PolicyCompiler(google::protobuf::Arena* absl_nonnull arena, + policy_internal::IssueReporter* issues, + std::unique_ptr base_compiler) + : arena_(arena), + issues_(*issues), + base_compiler_(std::move(base_compiler)) {} + + absl::string_view GetSourceDescription() const { + if (src_ == nullptr) { + return ""; + } + return src_->content()->description(); + } + + absl::StatusOr CompileExpression(CelPolicyElementId id, + absl::string_view val, + const Compiler* env) { + std::unique_ptr source; + if (src_ != nullptr && src_->content() != nullptr) { + std::optional range; + range = src_->GetSourceRange(id); + bool use_subrange = !(src_->IsQuoted(id).value_or(true)); + if (range.has_value() && use_subrange) { + source = std::make_unique(*src_->content(), *range); + } + } + + if (source == nullptr) { + // For quoted strings, the source should be generated from the interpreted + // YAML value. + CEL_ASSIGN_OR_RETURN( + source, cel::NewSource(val, std::string(GetSourceDescription()))); + } + auto result = env->Compile(*source, arena_); + if (!result.ok()) { + return result; + } + result->SetSource(std::move(source)); + return result; + } + + void AdaptTypeCheckIssues(CelPolicyElementId id, const ValidationResult& r) { + cel::AdaptTypeCheckIssues(id, r, src_, issues_); + } + + absl::StatusOr CompileOutputBlock( + const cel::OutputBlock& output_block, const Compiler* env) { + CompiledOutputBlock output; + CEL_ASSIGN_OR_RETURN(auto output_validation, + CompileExpression(output_block.output().id(), + output_block.output().value(), env)); + AdaptTypeCheckIssues(output_block.output().id(), output_validation); + + cel::Type result_type = DynType(); + if (output_validation.IsValid()) { + CEL_ASSIGN_OR_RETURN(auto ast, output_validation.ReleaseAst()); + auto root_expr_id = ast->root_expr().id(); + output.output_ast = + EmbeddedAst{output_block.output().id(), std::move(ast)}; + if (auto it = output_validation.GetResolvedTypeMap().find(root_expr_id); + it != output_validation.GetResolvedTypeMap().end()) { + result_type = it->second; + } + } + if (output_block.explanation().has_value()) { + CEL_ASSIGN_OR_RETURN( + auto explanation_validation, + CompileExpression(output_block.explanation()->id(), + output_block.explanation()->value(), env)); + AdaptTypeCheckIssues(output_block.explanation()->id(), + explanation_validation); + if (explanation_validation.IsValid()) { + CEL_ASSIGN_OR_RETURN(auto ast, explanation_validation.ReleaseAst()); + if (ast->GetReturnType().primitive() != PrimitiveType::kString) { + issues_.ReportError(output_block.explanation()->id(), + "explanation must evaluate to string"); + } else { + output.explanation_ast = + EmbeddedAst{output_block.explanation()->id(), std::move(ast)}; + } + } + } + output.result_type = result_type; + return output; + } + + absl::Status CompileMatch(const Match& match, const Compiler* env, + CompiledRule* out) { + CompiledMatch c_match; + c_match.id = match.id(); + if (match.condition().has_value()) { + CEL_ASSIGN_OR_RETURN(auto validation, + CompileExpression(match.condition()->id(), + match.condition()->value(), env)); + AdaptTypeCheckIssues(match.condition()->id(), validation); + if (validation.IsValid()) { + CEL_ASSIGN_OR_RETURN(auto ast, validation.ReleaseAst()); + if (ast->GetReturnType().primitive() != PrimitiveType::kBool) { + issues_.ReportError(match.condition()->id(), + "condition must evaluate to bool"); + } + c_match.condition = + EmbeddedAst{match.condition()->id(), std::move(ast)}; + } + } + + if (match.has_output_block()) { + CEL_ASSIGN_OR_RETURN(c_match.production, + CompileOutputBlock(match.output_block(), env)); + } else if (match.has_rule()) { + auto rule = std::make_unique(); + CEL_RETURN_IF_ERROR(CompileRule(match.rule(), env, rule.get())); + c_match.production = std::move(rule); + } else { + issues_.ReportError(match.id(), "match must specify an output or rule"); + } + out->matches.push_back(std::move(c_match)); + return absl::OkStatus(); + } + + absl::Status CompileRule(const Rule& rule, const cel::Compiler* env, + CompiledRule* out) { + out->id = rule.id(); + out->semantics = rule.is_aggregate() ? RuleSemantics::kAggregate + : RuleSemantics::kFirstMatch; + std::unique_ptr buf; + + absl::flat_hash_set seen_variables; + for (const auto& variable : rule.variables()) { + std::string name(variable.name().value()); + if (!seen_variables.insert(name).second) { + issues_.ReportError( + variable.expression().id(), + absl::StrCat("overlapping identifier for name 'variables.", name, + "'")); + continue; + } + std::string ident = absl::StrCat("variables.", name); + CEL_ASSIGN_OR_RETURN( + auto validation, + CompileExpression(variable.expression().id(), + variable.expression().value(), env)); + AdaptTypeCheckIssues(variable.expression().id(), validation); + if (!validation.IsValid()) { + continue; + } + CEL_ASSIGN_OR_RETURN(auto ast, validation.ReleaseAst()); + cel::Type result_type = DynType(); + + if (auto it = validation.GetResolvedTypeMap().find(ast->root_expr().id()); + it != validation.GetResolvedTypeMap().end()) { + result_type = it->second; + } + out->variables.push_back(CompiledVariable{ + ident, + EmbeddedAst{variable.expression().id(), std::move(ast)}, + }); + auto next = env->ToBuilder(); + auto status = next->GetCheckerBuilder().AddOrReplaceVariable( + MakeVariableDecl(ident, result_type)); + if (!status.ok()) { + issues_.ReportError(variable.expression().id(), status.message()); + continue; + } + CEL_ASSIGN_OR_RETURN(buf, next->Build()); + env = buf.get(); + } + + std::optional overall_type; + for (const auto& match : rule.matches()) { + CEL_RETURN_IF_ERROR(CompileMatch(match, env, out)); + if (!overall_type.has_value()) { + overall_type = GetOutputType(out->matches.back().production); + continue; + } + + if (std::optional match_type = + GetOutputType(out->matches.back().production); + match_type.has_value()) { + if (!OutputTypeIsCompatible(*match_type, *overall_type)) { + issues_.ReportError( + match.id(), + absl::StrCat("incompatible output types: block has output type ", + FormatTypeName(*match_type), + ", but previous outputs have type ", + FormatTypeName(*overall_type))); + } + } + } + out->result_type = overall_type; + if (overall_type.has_value() && + out->semantics == RuleSemantics::kAggregate) { + // Unlike first-match implicit optionalization (i.e. the user policy can't + // interact with the optional layer injected by the composer), aggregate + // rules are explicitly a list output and can intermingle with other rules + // producing compatible lists (aggregate or not). + out->result_type = cel::ListType(arena_, *overall_type); + } + return absl::OkStatus(); + } + + absl::Status CompilePolicy(const CelPolicy& policy, + IntermediateCompiledPolicy* out) { + src_ = policy.source(); + out->set_policy_source(src_); + out->set_name(policy.name().value()); + out->set_display_name( + policy.display_name().value_or(ValueString{}).value()); + out->set_description(policy.description().value_or(ValueString{}).value()); + + return CompileRule(policy.rule(), base_compiler_.get(), + &out->mutable_root_rule()); + } + + private: + google::protobuf::Arena* absl_nonnull arena_; + const CelPolicySource* absl_nullable src_; + policy_internal::IssueReporter& issues_; + std::unique_ptr base_compiler_; +}; + +bool IsExhaustive(const CompiledRule& rule); + +class PolicyComposer { + public: + PolicyComposer(google::protobuf::Arena* absl_nonnull arena, + const IntermediateCompiledPolicy& icp, + const Compiler& compiler, + policy_internal::IssueReporter& issues) + : arena_(arena), issues_(issues), icp_(icp), compiler_(compiler) {} + + absl::Status Compose(); + + bool success() const { return ast_ != nullptr; } + + std::unique_ptr ReleaseAst() { return std::move(ast_); } + + private: + SourcePosition GetAstOffset(CelPolicyElementId id) const { + if (icp_.policy_source() == nullptr) { + return 0; + } + if (auto range = icp_.policy_source()->GetSourceRange(id); + range.has_value()) { + return range->begin; + } + if (auto pos = icp_.policy_source()->GetSourcePosition(id); + pos.has_value()) { + return *pos; + } + return 0; + } + + using VariableScope = absl::flat_hash_map; + + std::optional ResolvePolicyVariable(absl::string_view reference); + + absl::flat_hash_map ResolveBlockIndexes(const Ast& ast); + + bool CheckMatchStructure(const CompiledRule& rule, + bool has_aggregate_ancestor = false); + + // Returns true if already optional wrapped. + absl::StatusOr ComposeRule(const CompiledRule& rule, Expr& init, + Expr& insertion_expr); + absl::StatusOr ComposeFirstMatchRule(const CompiledRule& rule, + Expr& init, Expr& insertion_expr); + absl::Status ComposeAggregateRule(const CompiledRule& rule, Expr& init, + Expr& insertion_expr); + + // returns true if already optional wrapped. + absl::StatusOr ComposeProduction( + const CompiledRule& rule, const CompiledMatch::Production& production, + Expr& init, Expr& insertion_expr); + + void MapVariables(Ast& ast); + + void ComposeRuleVariables(const CompiledRule& rule, Expr& init, + Expr& insertion_expr); + + google::protobuf::Arena* absl_nonnull arena_; + policy_internal::IssueReporter& issues_; + OptimizerExprFactory factory_; + const IntermediateCompiledPolicy& icp_; + const Compiler& compiler_; + std::vector scopes_; + bool optionalize_ = false; + std::unique_ptr ast_; +}; + +absl::Status PolicyComposer::Compose() { + if (icp_.policy_source() != nullptr) { + ABSL_DCHECK(icp_.policy_source()->content() != nullptr); + absl::Span line_offsets = + icp_.policy_source()->content()->line_offsets(); + factory_.mutable_ast().mutable_source_info().set_line_offsets( + std::vector(line_offsets.begin(), line_offsets.end())); + } + + factory_.mutable_ast().mutable_root_expr() = factory_.NewCall( + "cel.@block", factory_.NewList(), factory_.NewUnspecified()); + auto& block_init_list = factory_.mutable_ast() + .mutable_root_expr() + .mutable_call_expr() + .mutable_args()[0]; + auto& insertion_expr = factory_.mutable_ast() + .mutable_root_expr() + .mutable_call_expr() + .mutable_args()[1]; + optionalize_ = !IsExhaustive(icp_.root_rule()); + if (!CheckMatchStructure(icp_.root_rule())) { + return absl::OkStatus(); + } + CEL_ASSIGN_OR_RETURN( + bool optional_wrapped, + ComposeRule(icp_.root_rule(), block_init_list, insertion_expr)); + + if (optional_wrapped != optionalize_) { + return absl::InternalError( + "composition failed to handle non-exhaustive rules"); + } + + CEL_ASSIGN_OR_RETURN( + cel::ValidationResult result, + compiler_.GetTypeChecker().Check(factory_.ast(), arena_)); + if (!result.IsValid()) { + AdaptTypeCheckIssues(icp_.root_rule().id, result, icp_.policy_source(), + issues_); + return absl::OkStatus(); + } + + CEL_ASSIGN_OR_RETURN(ast_, result.ReleaseAst()); + + return absl::OkStatus(); +} + +bool IsTriviallyTrueCondition(const CompiledMatch& match) { + if (!match.condition.has_value() || match.condition->ast == nullptr) { + return true; + } + const cel::Expr& expr = match.condition->ast->root_expr(); + if (expr.has_const_expr()) { + const cel::Constant& const_expr = expr.const_expr(); + if (const_expr.has_bool_value() && const_expr.bool_value()) { + return true; + } + } + return false; +} + +bool IsLiterallyFalseCondition(const CompiledMatch& match) { + if (!match.condition.has_value() || match.condition->ast == nullptr) { + return false; + } + const cel::Expr& expr = match.condition->ast->root_expr(); + if (expr.has_const_expr()) { + const cel::Constant& const_expr = expr.const_expr(); + if (const_expr.has_bool_value() && !const_expr.bool_value()) { + return true; + } + } + return false; +} + +bool IsExhaustive(const CompiledRule& rule); + +bool IsExhaustive(const CompiledMatch& match) { + if (std::holds_alternative(match.production)) { + return true; + } + + const auto* nested_rule_ptr = + std::get_if>(&match.production); + ABSL_DCHECK(nested_rule_ptr != nullptr); + const CompiledRule& nested_rule = **nested_rule_ptr; + return IsExhaustive(nested_rule); +} + +bool IsExhaustive(const CompiledRule& rule) { + if (rule.semantics == RuleSemantics::kAggregate) { + return true; + } + if (rule.matches.empty()) { + // Validation should fail, but generalization would be false. + return false; + } + bool has_default = false; + for (const auto& match : rule.matches) { + if (IsTriviallyTrueCondition(match) && IsExhaustive(match)) { + // If this isn't the last match in the rule, it should get flagged + // during validation since it means there are trivially unreachable + // matches. + has_default = true; + } + if (!IsTriviallyTrueCondition(match) && !IsExhaustive(match)) { + // There is a nested rule that might return an optional.none(). + return false; + } + } + // Otherwise, everything in this branch is exhaustive so we can defer + // wrapping. + return has_default; +} + +bool PolicyComposer::CheckMatchStructure(const CompiledRule& rule, + bool has_aggregate_ancestor) { + bool valid = true; + if (has_aggregate_ancestor && rule.semantics == RuleSemantics::kAggregate) { + issues_.ReportError(rule.id, "nested aggregate rules are not allowed"); + valid = false; + } + + if (rule.matches.empty()) { + issues_.ReportError(rule.id, "rule does not specify match conditions"); + return false; + } + + bool seen_trivially_true = false; + + for (const auto& match : rule.matches) { + if (seen_trivially_true && rule.semantics == RuleSemantics::kFirstMatch) { + if (std::holds_alternative(match.production)) { + issues_.ReportError(match.id, "match creates unreachable outputs"); + } else if (std::holds_alternative>( + match.production)) { + issues_.ReportError(match.id, "rule creates unreachable outputs"); + } + valid = false; + } + + if (IsTriviallyTrueCondition(match) && IsExhaustive(match)) { + seen_trivially_true = true; + } + + if (IsLiterallyFalseCondition(match)) { + issues_.ReportError(match.id, "condition is always false"); + valid = false; + } + + if (auto* nested_rule = + std::get_if>(&match.production); + nested_rule != nullptr) { + ABSL_DCHECK(*nested_rule != nullptr); + const bool next_has_aggregate_ancestor = + has_aggregate_ancestor || rule.semantics == RuleSemantics::kAggregate; + if (!CheckMatchStructure(**nested_rule, next_has_aggregate_ancestor)) { + valid = false; + } + } + } + + return valid; +} + +std::optional PolicyComposer::ResolvePolicyVariable( + absl::string_view reference) { + for (auto scope_iter = scopes_.rbegin(); scope_iter != scopes_.rend(); + ++scope_iter) { + if (auto it = scope_iter->find(reference); it != scope_iter->end()) { + return it->second; + } + } + return std::nullopt; +} + +class IndexRewrite : public AstRewriterBase { + public: + explicit IndexRewrite(absl::flat_hash_map expr_id_to_index, + OptimizerExprFactory& factory) + : expr_id_to_index_(std::move(expr_id_to_index)), factory_(factory) {} + + bool PreVisitRewrite(Expr& e) override { + if (auto it = expr_id_to_index_.find(e.id()); + it != expr_id_to_index_.end()) { + e.mutable_ident_expr().set_name(absl::StrCat("@index", it->second)); + // Keep any metadata for the rewritten ident. This still replaces + // any references in the macro call map. + factory_.RecordReplacement(e.id(), e, /*keep_metadata=*/true); + return true; + } + return false; + } + + private: + absl::flat_hash_map expr_id_to_index_; + OptimizerExprFactory& factory_; +}; + +absl::StatusOr PolicyComposer::ComposeRule(const CompiledRule& rule, + Expr& init, + Expr& insertion_expr) { + if (rule.semantics == RuleSemantics::kAggregate) { + CEL_RETURN_IF_ERROR(ComposeAggregateRule(rule, init, insertion_expr)); + return false; + } + ABSL_DCHECK(rule.semantics == RuleSemantics::kFirstMatch); + return ComposeFirstMatchRule(rule, init, insertion_expr); +} + +absl::StatusOr PolicyComposer::ComposeFirstMatchRule( + const CompiledRule& rule, Expr& init, Expr& insertion_expr) { + ABSL_DCHECK(rule.semantics == RuleSemantics::kFirstMatch); + scopes_.emplace_back(); + auto pop_scope = absl::MakeCleanup([this]() { scopes_.pop_back(); }); + ComposeRuleVariables(rule, init, insertion_expr); + Expr* insertion_point = &insertion_expr; + const bool has_default = IsTriviallyTrueCondition(rule.matches.back()); + const bool needs_wrap = !IsExhaustive(rule); + size_t end = rule.matches.size() - (has_default ? 1 : 0); + for (size_t i = 0; i < end; i++) { + const auto& match = rule.matches[i]; + if (IsTriviallyTrueCondition(match) && IsExhaustive(match)) { + return absl::InternalError("detected unreachable match after validation"); + } + + Expr production; + CEL_ASSIGN_OR_RETURN( + bool is_wrapped, + ComposeProduction(rule, match.production, init, production)); + if (needs_wrap && !is_wrapped) { + production = factory_.NewCall("optional.of", std::move(production)); + } + + if (!IsTriviallyTrueCondition(match)) { + Ast condition = *match.condition->ast; + MapVariables(condition); + factory_.StartCopyContext(); + auto copy = factory_.Copy(condition.root_expr()); + auto source_info = factory_.RemapSourceInfo( + condition.source_info(), GetAstOffset(match.condition->id)); + factory_.MergeSourceInfo(source_info); + *insertion_point = factory_.NewCall("_?_:_", std::move(copy)); + insertion_point->mutable_call_expr().mutable_args().push_back( + std::move(production)); + ABSL_DCHECK(!(!needs_wrap && is_wrapped)) + << "unexpected wrapping in exhaustive policy."; + insertion_point = &insertion_point->mutable_call_expr().add_args(); + continue; + } + + if (!is_wrapped) { + return absl::InternalError( + "composition failed. expected optional wrapped rule but got a plain " + "value"); + } + auto fn = needs_wrap ? "or" : "orValue"; + *insertion_point = factory_.NewMemberCall(fn, std::move(production)); + insertion_point = &insertion_point->mutable_call_expr().add_args(); + } + + if (has_default) { + const auto& match = rule.matches.back(); + Expr production; + CEL_ASSIGN_OR_RETURN( + bool is_wrapped, + ComposeProduction(rule, match.production, init, production)); + if (needs_wrap && !is_wrapped) { + production = factory_.NewCall("optional.of", std::move(production)); + } + *insertion_point = std::move(production); + ABSL_DCHECK(!(!needs_wrap && is_wrapped)) + << "unexpected wrapping in exhaustive policy."; + + return needs_wrap; + } + + // Otherwise, we fell through a non-exhaustive rule. + *insertion_point = factory_.NewCall("optional.none"); + return true; +} + +absl::Status PolicyComposer::ComposeAggregateRule(const CompiledRule& rule, + Expr& init, + Expr& insertion_expr) { + ABSL_DCHECK(rule.semantics == RuleSemantics::kAggregate); + scopes_.emplace_back(); + auto pop_scope = absl::MakeCleanup([this]() { scopes_.pop_back(); }); + ComposeRuleVariables(rule, init, insertion_expr); + + std::vector elements; + elements.reserve(rule.matches.size()); + + for (const auto& match : rule.matches) { + Expr production; + CEL_ASSIGN_OR_RETURN( + bool is_wrapped, + ComposeProduction(rule, match.production, init, production)); + + if (IsTriviallyTrueCondition(match)) { + elements.push_back( + factory_.NewListElement(std::move(production), is_wrapped)); + continue; + } + + if (!is_wrapped) { + production = factory_.NewCall("optional.of", std::move(production)); + } + + Ast condition = *match.condition->ast; + MapVariables(condition); + factory_.StartCopyContext(); + Expr cond_copy = factory_.Copy(condition.root_expr()); + SourceInfo source_info = factory_.RemapSourceInfo( + condition.source_info(), GetAstOffset(match.condition->id)); + factory_.MergeSourceInfo(source_info); + + Expr ternary = + factory_.NewCall("_?_:_", std::move(cond_copy), std::move(production), + factory_.NewCall("optional.none")); + elements.push_back( + factory_.NewListElement(std::move(ternary), /*optional=*/true)); + } + + insertion_expr = factory_.NewList(std::move(elements)); + return absl::OkStatus(); +} + +absl::StatusOr PolicyComposer::ComposeProduction( + const CompiledRule& rule, const CompiledMatch::Production& production, + Expr& init, Expr& insertion_expr) { + if (auto* nested_rule = + std::get_if>(&production); + nested_rule != nullptr) { + return ComposeRule(**nested_rule, init, insertion_expr); + } + auto* output = std::get_if(&production); + if (output == nullptr) { + return absl::InternalError("unexpected rule production type"); + } + const EmbeddedAst& output_ast = output->output_ast; + Ast ast = *output_ast.ast; + MapVariables(ast); + factory_.StartCopyContext(); + Expr to_insert = factory_.Copy(ast.root_expr()); + auto source_info = + factory_.RemapSourceInfo(ast.source_info(), GetAstOffset(output_ast.id)); + factory_.MergeSourceInfo(source_info); + insertion_expr = std::move(to_insert); + + return false; +} + +absl::flat_hash_map PolicyComposer::ResolveBlockIndexes( + const Ast& ast) { + absl::flat_hash_map out; + for (auto it = ast.reference_map().begin(); it != ast.reference_map().end(); + it++) { + const Reference& ref = it->second; + if (!it->second.overload_id().empty()) { + continue; + } + if (!absl::StartsWith(ref.name(), "variable")) { + continue; + } + if (auto index = ResolvePolicyVariable(ref.name()); index.has_value()) { + out[it->first] = *index; + } + } + return out; +} + +void PolicyComposer::MapVariables(Ast& ast) { + absl::flat_hash_map edit_map = ResolveBlockIndexes(ast); + OptimizerExprFactory ast_factory(std::move(ast)); + IndexRewrite rewriter(std::move(edit_map), ast_factory); + AstRewrite(ast_factory.mutable_ast().mutable_root_expr(), rewriter); + ast = std::move(ast_factory.mutable_ast()); +} + +void PolicyComposer::ComposeRuleVariables(const CompiledRule& rule, Expr& init, + Expr& insertion_expr) { + for (const auto& variable : rule.variables) { + Ast ast = *variable.ast.ast; + MapVariables(ast); + factory_.StartCopyContext(); + auto insertion = factory_.Copy(ast.root_expr()); + auto info = factory_.RemapSourceInfo(ast.source_info(), + GetAstOffset(variable.ast.id)); + factory_.MergeSourceInfo(info); + ABSL_DCHECK(init.has_list_expr()); + int index = init.mutable_list_expr().elements().size(); + init.mutable_list_expr().mutable_elements().push_back( + factory_.NewListElement(std::move(insertion))); + scopes_.back()[variable.ident] = index; + } +} + +bool HasComprehensionParent(const NavigableAstNode& node) { + const NavigableAstNode* curr = &node; + while (curr != nullptr) { + if (curr->node_kind() == NodeKind::kComprehension) { + return true; + } + curr = curr->parent(); + } + return false; +} + +// Unnester implementation. +class Unnester { + public: + Unnester(google::protobuf::Arena* absl_nonnull arena, const TypeChecker& type_checker, + Ast ast, const CompilePolicyOptions& options, + policy_internal::IssueReporter& issues) + : arena_(arena), + type_checker_(type_checker), + factory_(std::move(ast)), + options_(options), + height_(options_.unnesting_height_limit), + issues_(issues) {} + + // Run the unnesting. + // The class cannot be reused after this is called. + absl::StatusOr Unnest() { + if (height_ > 0) { + CEL_RETURN_IF_ERROR(Slice()); + } + CEL_RETURN_IF_ERROR(Cleanup()); + + CEL_ASSIGN_OR_RETURN( + ValidationResult result, + type_checker_.Check(std::move(factory_.mutable_ast()), arena_)); + ABSL_DCHECK(result.IsValid()); + CEL_ASSIGN_OR_RETURN(std::unique_ptr checked_ast, result.ReleaseAst()); + return std::move(*checked_ast); + } + + private: + // The core unnest routine. + absl::Status Slice(); + // Fixup the AST post-unnesting. + absl::Status Cleanup(); + + void ReportErrorAtId(int64_t id, absl::string_view message); + + google::protobuf::Arena* absl_nonnull arena_; + const TypeChecker& type_checker_; + OptimizerExprFactory factory_; + const CompilePolicyOptions& options_; + int height_; + policy_internal::IssueReporter& issues_; +}; + +class UnnestRewriter : public AstRewriterBase { + public: + explicit UnnestRewriter(OptimizerExprFactory& f, Expr& block_list_expr, + absl::Span cuts) + : factory_(f), cuts_(cuts), block_list_expr_(block_list_expr) {} + + bool PostVisitRewrite(Expr& expr) override { + using std::swap; + // Post order so we always see children before parents. + // No need to copy metadata since we're only moving exprs or minting + // new ones. + if (absl::c_contains(cuts_, expr.id())) { + size_t idx = block_list_expr_.list_expr().elements().size(); + Expr value = factory_.NewIdent(absl::StrCat("@index", idx)); + factory_.RecordReplacement(expr.id(), value, /*keep_metadata=*/true); + swap(value, expr); + block_list_expr_.mutable_list_expr().mutable_elements().push_back( + factory_.NewListElement(std::move(value))); + return true; + } + return false; + } + + private: + OptimizerExprFactory& factory_; + absl::Span cuts_; + Expr& block_list_expr_; +}; + +absl::Status Unnester::Slice() { + Expr& root = factory_.mutable_ast().mutable_root_expr(); + if (root.call_expr().function() != kCelBlock || + root.call_expr().args().size() != 2 || + !root.call_expr().args()[0].has_list_expr()) { + return absl::InternalError("malformed AST detected during unnesting"); + } + // Two passes, we identify the slice points (bottom up), then cut + // and paste the leaves into the block list. + NavigableAst nav_ast = NavigableAst::Build(factory_.ast().root_expr()); + + ABSL_DCHECK(nav_ast.IdsAreUnique()); + bool can_cut = true; + std::vector cuts; + for (const NavigableAstNode& node : nav_ast.Root().DescendantsPostorder()) { + // Subsequent cuts will be height_ + 1 in the block indices. Within the + // error margin we specified. + if (node.height() % height_ == 0) { + if (HasComprehensionParent(node)) { + ReportErrorAtId( + node.expr()->id(), + absl::StrCat( + "cannot unnest AST due to comprehension. cannot accommodate " + "height limit of ", + height_)); + can_cut = false; + continue; + } + if (&node == &nav_ast.Root()) { + // If evenly divisible by height, don't cut since it will net a taller + // AST. + continue; + } + cuts.push_back(node.expr()->id()); + } + } + + if (!can_cut || cuts.empty()) { + return absl::OkStatus(); + } + + Expr& block_list_expr = root.mutable_call_expr().mutable_args()[0]; + Expr& insertion_expr = root.mutable_call_expr().mutable_args()[1]; + + UnnestRewriter rewriter(factory_, block_list_expr, cuts); + AstRewrite(insertion_expr, rewriter); + + return absl::OkStatus(); +} + +absl::Status Unnester::Cleanup() { + using std::swap; + + const auto& ast = factory_.ast(); + if (ast.root_expr().call_expr().function() != kCelBlock || + ast.root_expr().call_expr().args().size() != 2 || + !ast.root_expr().call_expr().args()[0].has_list_expr()) { + return absl::InternalError("malformed AST detected during unnesting"); + } + if (ast.root_expr().call_expr().args()[0].list_expr().elements().empty()) { + const ExprId block_id = ast.root_expr().id(); + const ExprId list_id = ast.root_expr().call_expr().args()[0].id(); + + Expr value = std::move(factory_.mutable_ast() + .mutable_root_expr() + .mutable_call_expr() + .mutable_args()[1]); + factory_.mutable_ast().mutable_root_expr() = std::move(value); + + factory_.PruneMetadata(block_id); + factory_.PruneMetadata(list_id); + } + + return absl::OkStatus(); +} + +void Unnester::ReportErrorAtId(int64_t id, absl::string_view message) { + int32_t position = 0; + auto it = factory_.ast().source_info().positions().find(id); + if (it != factory_.ast().source_info().positions().end()) { + position = it->second; + } + issues_.ReportError(-1, position, message); +} +} // namespace + +// Compiles a CEL policy using the provided CEL compiler as a base environment. +absl::StatusOr CompilePolicy( + const Compiler& compiler, const CelPolicy& policy, + const CompilePolicyOptions& options) { + google::protobuf::Arena* arena = options.arena; + std::optional local_arena; + if (arena == nullptr) { + local_arena.emplace(); + arena = &(*local_arena); + } + policy_internal::IssueReporter issues; + if (options.unnesting_height_limit != 0 && + options.unnesting_height_limit < 2) { + return absl::InvalidArgumentError( + "unnesting_height_limit must be at least 2"); + } + auto builder = compiler.ToBuilder(); + ExpressionContainer cont; + for (const auto& import : policy.imports()) { + auto status = cont.AddAbbreviation(import.name().value()); + if (!status.ok()) { + issues.ReportError( + import.name().id(), + absl::StrCat("'", import.name().value(), "': ", status.message())); + } + } + + builder->GetCheckerBuilder().SetExpressionContainer(cont); + CEL_ASSIGN_OR_RETURN(auto base_compiler, builder->Build()); + + PolicyCompiler policy_compiler(arena, &issues, std::move(base_compiler)); + + IntermediateCompiledPolicy icp; + CEL_RETURN_IF_ERROR(policy_compiler.CompilePolicy(policy, &icp)); + + if (HasErrors(issues)) { + return CelPolicyValidationResult(issues.ReleaseIssues(), + policy.source_ptr()); + } + + CEL_ASSIGN_OR_RETURN(base_compiler, builder->Build()); + PolicyComposer composer(arena, icp, *base_compiler, issues); + CEL_RETURN_IF_ERROR(composer.Compose()); + if (!composer.success()) { + return CelPolicyValidationResult(issues.ReleaseIssues(), + policy.source_ptr()); + } + + auto ast = composer.ReleaseAst(); + Unnester unnester(arena, base_compiler->GetTypeChecker(), std::move(*ast), + options, issues); + CEL_ASSIGN_OR_RETURN(Ast unnested_ast, unnester.Unnest()); + + if (HasErrors(issues)) { + return CelPolicyValidationResult(issues.ReleaseIssues(), + policy.source_ptr()); + } + + CelPolicyValidationResult result = CelPolicyValidationResult( + std::make_unique(std::move(unnested_ast)), {}, policy.source_ptr()); + return result; +} + +} // namespace cel diff --git a/policy/compiler.h b/policy/compiler.h new file mode 100644 index 000000000..6f6db99b8 --- /dev/null +++ b/policy/compiler.h @@ -0,0 +1,53 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_COMPILER_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_COMPILER_H_ + +#include "absl/status/statusor.h" +#include "compiler/compiler.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_validation_result.h" +#include "google/protobuf/arena.h" + +namespace cel { + +struct CompilePolicyOptions { + // If greater than 0, the compiler will attempt to unnest rule branches + // at the specified height. The overall height of the final AST may exceed + // this by a small, fixed margin. + // + // To avoid slicing comprehensions, subexpressions within comprehensions + // are not eligible for unnesting. If the height limit cannot be accommodated, + // an error with code InvalidArgument is returned. + // + // If the AST is converted to proto, even relatively low levels of nesting + // can cause problems in serialization/deserialization. This does not apply + // if the AST is used directly by the runtime. + int unnesting_height_limit = 0; + + google::protobuf::Arena* arena = nullptr; +}; + +// Compiles a CEL policy using the provided CEL compiler as a base environment. +// +// TODO(b/506179116): Implementation in progress. Functionally complete, +// but errors are not consistent with other implementations. +absl::StatusOr CompilePolicy( + const Compiler& compiler, const CelPolicy& policy, + const CompilePolicyOptions& options = {}); + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_COMPILER_H_ diff --git a/policy/compiler_test.cc b/policy/compiler_test.cc new file mode 100644 index 000000000..f99cce5e9 --- /dev/null +++ b/policy/compiler_test.cc @@ -0,0 +1,1722 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/compiler.h" + +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "common/ast.h" +#include "common/ast_proto.h" +#include "common/decl.h" +#include "common/expr.h" +#include "common/navigable_ast.h" +#include "common/source.h" +#include "common/type.h" +#include "common/types/message_type.h" +#include "common/value.h" +#include "common/value_testing.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/optional.h" +#include "compiler/standard_library.h" +#include "extensions/bindings_ext.h" +#include "internal/runfiles.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "internal/testing_message_factory.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" +#include "policy/cel_policy_validation_result.h" +#include "policy/yaml_policy_parser.h" +#include "runtime/activation.h" +#include "runtime/optional_types.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" +#include "tools/cel_unparser.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" + +namespace cel { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::StatusIs; +using ::cel::test::IntValueIs; +using ::cel::test::OptionalValueIs; +using ::cel::test::OptionalValueIsEmpty; +using ::cel::test::StringValueIs; +using ::cel::test::ValueMatcher; + +constexpr absl::string_view kTestPolicyFilePath = +"_main/policy/testdata/cel_policy.yaml"; + +absl::StatusOr> BuildTestCompiler() { + CompilerOptions opts; + opts.adapt_parser_errors = true; + opts.parser_options.enable_optional_syntax = true; + opts.parser_options.add_macro_calls = true; + CEL_ASSIGN_OR_RETURN( + auto builder, + NewCompilerBuilder(internal::GetSharedTestingDescriptorPool(), opts)); + + CEL_RETURN_IF_ERROR(builder->AddLibrary(cel::StandardCompilerLibrary())); + CEL_RETURN_IF_ERROR(builder->AddLibrary(cel::OptionalCompilerLibrary())); + CEL_RETURN_IF_ERROR( + builder->AddLibrary(cel::extensions::BindingsCompilerLibrary())); + + CEL_RETURN_IF_ERROR(builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("x", IntType()))); + CEL_RETURN_IF_ERROR(builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("y", IntType()))); + + const google::protobuf::Descriptor* descriptor = + cel::internal::GetSharedTestingDescriptorPool()->FindMessageTypeByName( + "cel.expr.conformance.proto3.TestAllTypes"); + if (descriptor == nullptr) { + return absl::InternalError("Failed to find TestAllTypes descriptor"); + } + CEL_RETURN_IF_ERROR(builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("spec", cel::MessageType(descriptor)))); + + return builder->Build(); +} + +absl::StatusOr> ParsePolicyFromYaml( + absl::string_view yaml_content) { + CEL_ASSIGN_OR_RETURN(auto source, cel::NewSource(yaml_content, "test.yaml")); + + std::shared_ptr policy_source = + std::make_shared(std::move(source)); + CEL_ASSIGN_OR_RETURN(auto parse_result, + cel::ParseYamlCelPolicy(policy_source)); + + if (!parse_result.IsValid()) { + return absl::InvalidArgumentError("Invalid policy YAML structure"); + } + return parse_result.ReleasePolicy(); +} + +[[maybe_unused]] +const Ast* GetAstPointer(const std::unique_ptr& ast) { + return ast.get(); +} + +MATCHER(MetadataMapValid, negation ? "has stale metadata map entries" + : "has valid metadata maps") { + const Ast* ast = GetAstPointer(arg); + if (ast == nullptr) { + *result_listener << "AST is null"; + return false; + } + NavigableAst nav_ast = cel::NavigableAst::Build(ast->root_expr()); + absl::flat_hash_set valid_ids; + for (const NavigableAstNode& node : nav_ast.Root().DescendantsPostorder()) { + valid_ids.insert(node.expr()->id()); + } + absl::flat_hash_set valid_position_ids = valid_ids; + for (const auto& [_, macro_expr] : ast->source_info().macro_calls()) { + NavigableAst macro_nav = cel::NavigableAst::Build(macro_expr); + for (const NavigableAstNode& node : + macro_nav.Root().DescendantsPostorder()) { + valid_position_ids.insert(node.expr()->id()); + } + } + + bool is_valid = true; + for (const auto& [id, _] : ast->type_map()) { + if (!valid_ids.contains(id)) { + *result_listener << "stale type_map entry for id " << id << "; "; + is_valid = false; + } + } + for (const auto& [id, _] : ast->reference_map()) { + if (!valid_ids.contains(id)) { + *result_listener << "stale reference_map entry for id " << id << "; "; + is_valid = false; + } + } + for (const auto& [id, _] : ast->source_info().positions()) { + if (!valid_position_ids.contains(id)) { + *result_listener << "stale source_info position for id " << id << "; "; + is_valid = false; + } + } + for (const auto& [id, _] : ast->source_info().macro_calls()) { + if (!valid_ids.contains(id)) { + *result_listener << "stale source_info macro_call for id " << id << "; "; + is_valid = false; + } + } + return is_valid; +} + +TEST(CompilerTest, SmokeTest) { + std::string contents; + std::string test_file = + cel::internal::ResolveRunfilesPath(kTestPolicyFilePath); + auto read_status = cel::internal::GetFileContents(test_file, &contents); + ASSERT_THAT(read_status, IsOk()); + + auto source_or = cel::NewSource(contents, "cel_policy.yaml"); + ASSERT_THAT(source_or.status(), IsOk()); + auto source = *std::move(source_or); + + std::shared_ptr policy_source = + std::make_shared(std::move(source)); + auto parse_result_or = cel::ParseYamlCelPolicy(policy_source); + ASSERT_THAT(parse_result_or.status(), IsOk()); + auto parse_result = *std::move(parse_result_or); + + ASSERT_TRUE(parse_result.IsValid()); + const CelPolicy* policy = parse_result.GetPolicy(); + + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()); +} + +TEST(CompilerTest, VariableOutOfScopeReportsError) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: variables.non_existent == 10 + output: '"error"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("undeclared reference")); +} + +TEST(CompilerTest, ConditionNotBoolReportsError) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: 10 + output: '"error"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("condition must evaluate to bool")); +} + +TEST(CompilerTest, InvalidOutputExpressionReportsError) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: true + output: undeclared_var +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("undeclared reference")); +} + +TEST(CompilerTest, DisplayErrorFormattingForInvalidExpression) { + absl::string_view yaml = R"yaml(name: cel_policy +rule: + match: + - condition: x > 0 + output: undeclared_var +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT( + result.FormatIssues(), + testing::HasSubstr( + R"err(ERROR: test.yaml:5:15: undeclared reference to 'undeclared_var' (in container '') + | output: undeclared_var + | ..............^)err")); +} + +TEST(CompilerTest, DisplayErrorFormattingForMultilineBlockLiteral) { + absl::string_view yaml = R"yaml(name: cel_policy +rule: + match: + - condition: x > 0 + output: | + undeclared_var +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT( + result.FormatIssues(), + testing::HasSubstr( + R"err(ERROR: test.yaml:6:9: undeclared reference to 'undeclared_var' (in container '') + | undeclared_var + | ........^)err")); +} + +TEST(CompilerTest, DisplayErrorFormattingForDoubleQuotedExpression) { + absl::string_view yaml = R"yaml(name: cel_policy +rule: + match: + - condition: x > 0 + output: "undeclared_var" +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT( + result.FormatIssues(), + testing::HasSubstr( + R"err(ERROR: test.yaml:5:16: undeclared reference to 'undeclared_var' (in container '') + | output: "undeclared_var" + | ...............^)err")); +} + +TEST(CompilerTest, ComposedAstPositionsAreRelativeToMainYaml) { + absl::string_view yaml = R"yaml(name: cel_policy +rule: + variables: + - name: v1 + expression: 10 + match: + - condition: variables.v1 > 5 + output: "variables.v1 + 20" +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()); + const cel::Ast* ast = result.GetAst(); + ASSERT_NE(ast, nullptr); + const cel::Source* source = result.GetSource()->content(); + ASSERT_NE(source, nullptr); + + auto nav_ast = cel::NavigableAst::Build(ast->root_expr()); + const cel::NavigableAstNode* var_node = nullptr; + const cel::NavigableAstNode* cond_node = nullptr; + const cel::NavigableAstNode* out_node = nullptr; + + for (const cel::NavigableAstNode& node : + nav_ast.Root().DescendantsPostorder()) { + if (node.expr()->has_const_expr() && + node.expr()->const_expr().has_int_value() && + node.expr()->const_expr().int_value() == 10) { + var_node = &node; + } else if (node.expr()->has_call_expr() && + node.expr()->call_expr().function() == "_>_") { + cond_node = &node; + } else if (node.expr()->has_call_expr() && + node.expr()->call_expr().function() == "_+_") { + out_node = &node; + } + } + + ASSERT_NE(var_node, nullptr); + ASSERT_NE(cond_node, nullptr); + ASSERT_NE(out_node, nullptr); + + auto var_pos = ast->source_info().positions().find(var_node->expr()->id()); + ASSERT_NE(var_pos, ast->source_info().positions().end()); + auto var_loc = source->GetLocation(var_pos->second); + ASSERT_TRUE(var_loc.has_value()); + EXPECT_EQ(ast->ComputeSourceLocation(var_node->expr()->id()), *var_loc); + EXPECT_THAT( + source->DisplayErrorLocation(*var_loc), + testing::HasSubstr(" expression: 10\n | ..................^")); + + auto cond_pos = ast->source_info().positions().find(cond_node->expr()->id()); + ASSERT_NE(cond_pos, ast->source_info().positions().end()); + auto cond_loc = source->GetLocation(cond_pos->second); + ASSERT_TRUE(cond_loc.has_value()); + EXPECT_EQ(ast->ComputeSourceLocation(cond_node->expr()->id()), *cond_loc); + EXPECT_THAT(source->DisplayErrorLocation(*cond_loc), + testing::HasSubstr(" - condition: variables.v1 > 5\n | " + "..............................^")); + + auto out_pos = ast->source_info().positions().find(out_node->expr()->id()); + ASSERT_NE(out_pos, ast->source_info().positions().end()); + auto out_loc = source->GetLocation(out_pos->second); + ASSERT_TRUE(out_loc.has_value()); + EXPECT_EQ(ast->ComputeSourceLocation(out_node->expr()->id()), *out_loc); + EXPECT_THAT(source->DisplayErrorLocation(*out_loc), + testing::HasSubstr(" output: \"variables.v1 + 20\"\n | " + "............................^")); +} + +TEST(CompilerTest, UnparseComposedAstWithMacros) { + absl::string_view yaml = R"yaml(name: macro_policy +rule: + variables: + - name: var_inner + expression: "[1, 2].all(i, i > 0)" + - name: var_outer + expression: "[3, 4].exists(j, j > 0 && variables.var_inner)" + match: + - condition: "[5, 6].all(k, k > 0)" + output: "[1].map(m, m > 0 && variables.var_outer)" +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()) << result.FormatIssues(); + const cel::Ast* ast = result.GetAst(); + ASSERT_NE(ast, nullptr); + + cel::expr::ParsedExpr parsed_expr; + ASSERT_THAT(cel::AstToParsedExpr(*ast, &parsed_expr), IsOk()); + ASSERT_OK_AND_ASSIGN(std::string unparsed, + google::api::expr::Unparse(parsed_expr)); + + EXPECT_EQ( + unparsed, + "cel.@block([[1, 2].all(i, i > 0), [3, 4].exists(j, j > 0 && @index0)], " + "[5, 6].all(k, k > 0) ? optional.of([1].map(m, m > 0 && @index1)) : " + "optional.none())"); +} + +TEST(CompilerTest, RewrittenVariablesUpdatedInMacroCallsAndReferenceMap) { + absl::string_view yaml = R"yaml(name: macro_policy +rule: + variables: + - name: var_inner + expression: "[1, 2].all(i, i > 0)" + - name: var_outer + expression: "[3, 4].exists(j, j > 0 && variables.var_inner)" + match: + - condition: "[5, 6].all(k, k > 0 && variables.var_inner)" + output: "[1].map(m, m > 0 && variables.var_outer)" +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()) << result.FormatIssues(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, result.ReleaseAst()); + + EXPECT_THAT(ast->source_info().macro_calls(), testing::SizeIs(4)); + EXPECT_THAT(ast, MetadataMapValid()); + + absl::flat_hash_set macro_index_vars; + for (const auto& [macro_id, macro_expr] : ast->source_info().macro_calls()) { + NavigableAst macro_nav = cel::NavigableAst::Build(macro_expr); + for (const NavigableAstNode& node : + macro_nav.Root().DescendantsPostorder()) { + const Expr* expr = node.expr(); + if (expr->has_select_expr()) { + EXPECT_NE(expr->select_expr().field(), "var_inner"); + EXPECT_NE(expr->select_expr().field(), "var_outer"); + } + if (expr->has_ident_expr()) { + absl::string_view name = expr->ident_expr().name(); + EXPECT_FALSE(absl::StartsWith(name, "variables")); + if (absl::StartsWith(name, "@index")) { + macro_index_vars.insert(std::string(name)); + auto ref_it = ast->reference_map().find(expr->id()); + ASSERT_NE(ref_it, ast->reference_map().end()); + EXPECT_EQ(ref_it->second.name(), name); + } + } + } + } + EXPECT_THAT(macro_index_vars, + testing::UnorderedElementsAre("@index0", "@index1")); + + absl::flat_hash_set root_index_vars; + NavigableAst root_nav = cel::NavigableAst::Build(ast->root_expr()); + for (const NavigableAstNode& node : root_nav.Root().DescendantsPostorder()) { + const Expr* expr = node.expr(); + if (expr->has_ident_expr() && + absl::StartsWith(expr->ident_expr().name(), "@index")) { + absl::string_view name = expr->ident_expr().name(); + root_index_vars.insert(std::string(name)); + auto ref_it = ast->reference_map().find(expr->id()); + ASSERT_NE(ref_it, ast->reference_map().end()); + EXPECT_EQ(ref_it->second.name(), name); + EXPECT_TRUE(ast->type_map().contains(expr->id())); + } + } + EXPECT_THAT(root_index_vars, + testing::UnorderedElementsAre("@index0", "@index1")); + + for (const auto& [id, ref] : ast->reference_map()) { + EXPECT_FALSE(absl::StartsWith(ref.name(), "variables")); + } + + cel::expr::ParsedExpr parsed_expr; + ASSERT_THAT(cel::AstToParsedExpr(*ast, &parsed_expr), IsOk()); + ASSERT_OK_AND_ASSIGN(std::string unparsed, + google::api::expr::Unparse(parsed_expr)); + EXPECT_EQ( + unparsed, + "cel.@block([[1, 2].all(i, i > 0), [3, 4].exists(j, j > 0 && @index0)], " + "[5, 6].all(k, k > 0 && @index0) ? " + "optional.of([1].map(m, m > 0 && @index1)) : optional.none())"); +} + +TEST(CompilerTest, UnreachableMatchAfterTriviallyTrueCondition) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: true + output: '"first"' + - condition: true + output: '"second"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("match creates unreachable outputs")); +} + +TEST(CompilerTest, UnreachableMatchAfterUnconditionalExhaustiveSubRule) { + absl::string_view yaml = R"yaml( +name: dead_branch +rule: + match: + - rule: + match: + - output: 1 + - output: 2 +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("match creates unreachable outputs")); +} + +TEST(CompilerTest, UnreachableRuleAfterTriviallyTrueCondition) { + absl::string_view yaml = R"yaml( +name: dead_rule_branch +rule: + match: + - condition: true + output: 1 + - rule: + match: + - output: 2 +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("rule creates unreachable outputs")); +} + +TEST(CompilerTest, UnreachableMatchAfterUnconditionalAggregateSubRule) { + absl::string_view yaml = R"yaml( +name: dead_branch_after_aggregate +rule: + match: + - rule: + aggregate: + - condition: x > 0 + output: 1 + - output: "[2]" +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("match creates unreachable outputs")); +} + +TEST(CompilerTest, LiterallyFalseConditionReportsError) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: false + output: dyn(1) + - condition: x > 5 + output: 2 + - output: '"default"' +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.GetIssues(), testing::SizeIs(1)); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("condition is always false")); +} + +TEST(CompilerTest, AggregateRuleLiterallyFalseConditionReportsError) { + absl::string_view yaml = R"yaml( +name: aggregate_false_condition_policy +rule: + aggregate: + - condition: false + output: '"unreachable"' + - output: '"default"' +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("condition is always false")); +} + +TEST(CompilerTest, RuleWithoutMatchesReportsError) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("rule does not specify match conditions")); +} + +TEST(CompilerTest, ExhaustivePolicyCompiles) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + variables: + - name: test_var + expression: 10 + match: + - condition: variables.test_var > 15 + output: '"greater than 15"' + - condition: variables.test_var > 5 + output: '"greater than 5"' + - output: '"default"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()); + EXPECT_TRUE(result.GetAst()->is_checked()); +} + +TEST(CompilerTest, NonExhaustivePolicyCompiles) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + variables: + - name: test_var + expression: 10 + match: + - condition: variables.test_var > 5 + output: '"greater than 5"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()); +} + +TEST(CompilerTest, PolicyReferencesEnvInput) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: spec.single_int32 > 10 + output: '"greater than 10"' + - output: '"default"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()); + EXPECT_TRUE(result.GetAst()->is_checked()); +} + +struct EvaluationTestCase { + std::string name; + std::string yaml_policy; + struct Input { + int64_t x; + int64_t y; + } input; + ValueMatcher expected_result_matcher; +}; + +class PolicyEvaluationTest : public testing::TestWithParam { +}; + +TEST_P(PolicyEvaluationTest, Evaluate) { + const auto& test_case = GetParam(); + + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(test_case.yaml_policy)); + ASSERT_OK_AND_ASSIGN(auto validation_result, + CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + // Set up runtime + cel::RuntimeOptions opts; + opts.enable_qualified_type_identifiers = true; + ASSERT_OK_AND_ASSIGN( + cel::RuntimeBuilder rt_builder, + cel::CreateStandardRuntimeBuilder( + cel::internal::GetSharedTestingDescriptorPool(), opts)); + ASSERT_THAT(cel::extensions::EnableOptionalTypes(rt_builder), IsOk()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + std::move(rt_builder).Build()); + + ASSERT_OK_AND_ASSIGN(auto program, runtime->CreateProgram(std::move(ast))); + + // Set up activation + cel::Activation activation; + activation.InsertOrAssignValue("x", cel::IntValue(test_case.input.x)); + activation.InsertOrAssignValue("y", cel::IntValue(test_case.input.y)); + + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(cel::Value result, + program->Evaluate(&arena, activation)); + + EXPECT_THAT(result, test_case.expected_result_matcher); +} + +constexpr absl::string_view kEvalPolicyYaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: x > 10 && y > 10 + output: '"both greater than 10"' + - condition: x > 10 + output: '"x greater than 10"' + - condition: y > 10 + output: '"y greater than 10"' + - output: '"default"' +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + PolicyEvaluationTest, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "BothGreaterThan10", + .yaml_policy = std::string(kEvalPolicyYaml), + .input = {.x = 15, .y = 15}, + .expected_result_matcher = StringValueIs("both greater than 10"), + }, + EvaluationTestCase{ + .name = "XGreaterThan10", + .yaml_policy = std::string(kEvalPolicyYaml), + .input = {.x = 15, .y = 5}, + .expected_result_matcher = StringValueIs("x greater than 10"), + }, + EvaluationTestCase{ + .name = "YGreaterThan10", + .yaml_policy = std::string(kEvalPolicyYaml), + .input = {.x = 5, .y = 15}, + .expected_result_matcher = StringValueIs("y greater than 10"), + }, + EvaluationTestCase{ + .name = "Default", + .yaml_policy = std::string(kEvalPolicyYaml), + .input = {.x = 5, .y = 5}, + .expected_result_matcher = StringValueIs("default"), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +constexpr absl::string_view kNonExhaustivePolicyYaml = R"yaml( +name: nested_rule4 +rule: + match: + - condition: x > 0 + rule: + match: + - condition: x < 3 + output: 1 + - condition: x < 5 + output: 2 + - condition: x < 0 + rule: + match: + - condition: x > -2 + output: 3 + - condition: x > -4 + output: 4 + - output: 5 +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + NonExhaustivePolicyEvaluation, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "XEquals0_FallthroughTopLevel", + .yaml_policy = std::string(kNonExhaustivePolicyYaml), + .input = {.x = 0, .y = 0}, + .expected_result_matcher = OptionalValueIsEmpty(), + }, + EvaluationTestCase{ + .name = "XEquals2_MatchesFirstNested", + .yaml_policy = std::string(kNonExhaustivePolicyYaml), + .input = {.x = 2, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(1)), + }, + EvaluationTestCase{ + .name = "XEquals6_FallthroughNested", + .yaml_policy = std::string(kNonExhaustivePolicyYaml), + .input = {.x = 6, .y = 0}, + .expected_result_matcher = OptionalValueIsEmpty(), + }, + EvaluationTestCase{ + .name = "XEqualsMinus1_MatchesMinus2", + .yaml_policy = std::string(kNonExhaustivePolicyYaml), + .input = {.x = -1, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(3)), + }, + EvaluationTestCase{ + .name = "XEqualsMinus3_MatchesMinus4", + .yaml_policy = std::string(kNonExhaustivePolicyYaml), + .input = {.x = -3, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(4)), + }, + EvaluationTestCase{ + .name = "XEqualsMinus5_MatchesDefault", + .yaml_policy = std::string(kNonExhaustivePolicyYaml), + .input = {.x = -5, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(5)), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +constexpr absl::string_view kNestedVariablePolicyYaml = R"yaml( +name: nested_rule4 +rule: + variables: + - name: i + expression: "1" + - name: j + expression: "2" + match: + - condition: x > 0 + rule: + variables: + - name: k + expression: "3" + match: + - output: "variables.i + variables.j + variables.k" + - condition: x < 0 + rule: + variables: + - name: j + expression: "5" + - name: k + expression: "4" + match: + - output: "variables.i + variables.j + variables.k" + - output: "variables.i + variables.j" +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + NestedVariablePolicyEvaluation, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "XGreaterThan0", + .yaml_policy = std::string(kNestedVariablePolicyYaml), + .input = {.x = 1, .y = 0}, + .expected_result_matcher = IntValueIs(6), + }, + EvaluationTestCase{ + .name = "XLessThan0", + .yaml_policy = std::string(kNestedVariablePolicyYaml), + .input = {.x = -1, .y = 0}, + .expected_result_matcher = IntValueIs(10), + }, + EvaluationTestCase{ + .name = "XEquals0", + .yaml_policy = std::string(kNestedVariablePolicyYaml), + .input = {.x = 0, .y = 0}, + .expected_result_matcher = IntValueIs(3), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +constexpr absl::string_view + kOptionalChainingUnconditionalSubRuleOptionalParentYaml = R"yaml( +name: optional_chaining +rule: + match: + - rule: + id: r2 + match: + - condition: x > 0 + output: 1 + - output: 2 + condition: x < 0 +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + OptionalChainingUnconditionalSubRuleOptionalParent, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "XEquals1", + .yaml_policy = std::string( + kOptionalChainingUnconditionalSubRuleOptionalParentYaml), + .input = {.x = 1, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(1)), + }, + EvaluationTestCase{ + .name = "XEqualsMinus1", + .yaml_policy = std::string( + kOptionalChainingUnconditionalSubRuleOptionalParentYaml), + .input = {.x = -1, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(2)), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +constexpr absl::string_view kOptionalChainingUnconditionalSubRuleYaml = R"yaml( +name: optional_chaining +rule: + id: r1 + match: + - rule: + id: r2 + match: + - condition: x > 0 + output: 1 + - output: 2 +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + OptionalChainingUnconditionalSubRule, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "XEquals1", + .yaml_policy = + std::string(kOptionalChainingUnconditionalSubRuleYaml), + .input = {.x = 1, .y = 0}, + .expected_result_matcher = IntValueIs(1), + }, + EvaluationTestCase{ + .name = "XEqualsMinus1", + .yaml_policy = + std::string(kOptionalChainingUnconditionalSubRuleYaml), + .input = {.x = -1, .y = 0}, + .expected_result_matcher = IntValueIs(2), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +constexpr absl::string_view kOptionalChainingUnconditionalComplexYaml = R"yaml( +name: optional_chaining +rule: + match: + - condition: x > 0 + rule: + match: + - rule: + match: + - condition: x == 1 + output: 1 + - output: 2 + - rule: + match: + - condition: x == -1 + output: 3 + - condition: x == -2 + output: 4 +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + OptionalChainingUnconditionalComplex, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "XEquals1", + .yaml_policy = + std::string(kOptionalChainingUnconditionalComplexYaml), + .input = {.x = 1, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(1)), + }, + EvaluationTestCase{ + .name = "XEqualsMinus1", + .yaml_policy = + std::string(kOptionalChainingUnconditionalComplexYaml), + .input = {.x = -1, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(3)), + }, + EvaluationTestCase{ + .name = "XEqualsMinus2", + .yaml_policy = + std::string(kOptionalChainingUnconditionalComplexYaml), + .input = {.x = -2, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(4)), + }, + EvaluationTestCase{ + .name = "XEqualsMinus3", + .yaml_policy = + std::string(kOptionalChainingUnconditionalComplexYaml), + .input = {.x = -3, .y = 0}, + .expected_result_matcher = OptionalValueIsEmpty(), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +constexpr absl::string_view kUnconditionalExhaustiveSubRuleAsLastMatchYaml = + R"yaml( +name: exhaustive_unconditional_subrule +rule: + match: + - condition: x > 0 + output: 1 + - rule: + match: + - condition: y > 0 + output: 2 + - output: 3 +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + UnconditionalExhaustiveSubRuleAsLastMatch, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "XEquals1", + .yaml_policy = + std::string(kUnconditionalExhaustiveSubRuleAsLastMatchYaml), + .input = {.x = 1, .y = 0}, + .expected_result_matcher = IntValueIs(1), + }, + EvaluationTestCase{ + .name = "XEquals0_YEquals1", + .yaml_policy = + std::string(kUnconditionalExhaustiveSubRuleAsLastMatchYaml), + .input = {.x = 0, .y = 1}, + .expected_result_matcher = IntValueIs(2), + }, + EvaluationTestCase{ + .name = "XEquals0_YEquals0", + .yaml_policy = + std::string(kUnconditionalExhaustiveSubRuleAsLastMatchYaml), + .input = {.x = 0, .y = 0}, + .expected_result_matcher = IntValueIs(3), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +constexpr absl::string_view kUnconditionalNonExhaustiveSubRuleAsLastMatchYaml = + R"yaml( +name: non_exhaustive_unconditional_subrule +rule: + match: + - condition: x > 0 + output: 1 + - rule: + match: + - condition: y > 0 + output: 2 +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + UnconditionalNonExhaustiveSubRuleAsLastMatch, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "XEquals1", + .yaml_policy = + std::string(kUnconditionalNonExhaustiveSubRuleAsLastMatchYaml), + .input = {.x = 1, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(1)), + }, + EvaluationTestCase{ + .name = "XEquals0_YEquals1", + .yaml_policy = + std::string(kUnconditionalNonExhaustiveSubRuleAsLastMatchYaml), + .input = {.x = 0, .y = 1}, + .expected_result_matcher = OptionalValueIs(IntValueIs(2)), + }, + EvaluationTestCase{ + .name = "XEquals0_YEquals0", + .yaml_policy = + std::string(kUnconditionalNonExhaustiveSubRuleAsLastMatchYaml), + .input = {.x = 0, .y = 0}, + .expected_result_matcher = OptionalValueIsEmpty(), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +TEST(CompilerTest, ImportsAndAbbreviations) { + absl::string_view yaml = R"yaml( +name: imports_test +imports: + - name: cel.expr.conformance.proto3.TestAllTypes +rule: + match: + - condition: 'spec == TestAllTypes{single_int32: 10}' + output: '"matched"' + - output: '"default"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + auto ast_or = CompilePolicy(*compiler, *policy); + ASSERT_THAT(ast_or, IsOk()); +} + +TEST(CompilerTest, MatchWithoutProductionReportsError) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: true +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("match must specify an output or rule")); +} + +int GetAstHeight(const cel::Ast& ast) { + auto nav_ast = cel::NavigableAst::Build(ast.root_expr()); + return nav_ast.Root().height(); +} + +TEST(CompilerTest, UnnestHeightValidation) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: true + output: '"ok"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + + CompilePolicyOptions options; + options.unnesting_height_limit = 1; + auto status_or = CompilePolicy(*compiler, *policy, options); + EXPECT_THAT(status_or.status(), + StatusIs(absl::StatusCode::kInvalidArgument, + testing::HasSubstr( + "unnesting_height_limit must be at least 2"))); + + options.unnesting_height_limit = 2; + EXPECT_THAT(CompilePolicy(*compiler, *policy, options), IsOk()); +} + +constexpr absl::string_view kDeepPolicyYaml = R"yaml( +name: deep_policy +rule: + match: + - condition: x > 0 + rule: + match: + - condition: x > 1 + rule: + match: + - condition: x > 2 + rule: + match: + - condition: x > 3 + rule: + match: + - condition: x > 4 + rule: + match: + - condition: x > 5 + output: 6 + - output: 5 + - output: 4 + - output: 3 + - output: 2 + - output: 1 + - output: 0 +)yaml"; + +TEST(CompilerTest, UnnestHeightReduction) { + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(kDeepPolicyYaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + + // Compile without unnesting + CompilePolicyOptions options_no_unnest; + options_no_unnest.unnesting_height_limit = 0; + ASSERT_OK_AND_ASSIGN(auto result_no_unnest, + CompilePolicy(*compiler, *policy, options_no_unnest)); + ASSERT_TRUE(result_no_unnest.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast_no_unnest, result_no_unnest.ReleaseAst()); + int height_no_unnest = GetAstHeight(*ast_no_unnest); + + CompilePolicyOptions options_unnest; + options_unnest.unnesting_height_limit = 2; + ASSERT_OK_AND_ASSIGN(auto result_unnest, + CompilePolicy(*compiler, *policy, options_unnest)); + ASSERT_TRUE(result_unnest.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast_unnest, result_unnest.ReleaseAst()); + int height_unnest = GetAstHeight(*ast_unnest); + + EXPECT_EQ(height_no_unnest, 8); + EXPECT_EQ(height_unnest, 5); + EXPECT_LT(height_unnest, height_no_unnest); +} + +TEST(CompilerTest, UnnestComprehensionFailure) { + absl::string_view yaml = R"yaml( +name: comprehension_policy +rule: + match: + - condition: x > 0 + rule: + match: + - condition: "[1, 2].all(i, i > x)" + output: 1 + - output: 2 + - output: 0 +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + + CompilePolicyOptions options; + options.unnesting_height_limit = 2; + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy, options)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("cannot unnest AST due to comprehension")); +} + +struct UnnestEvaluationTestCase { + std::string name; + int64_t x; + ValueMatcher expected; +}; + +class UnnestedDeepPolicyEvaluationTest + : public testing::TestWithParam {}; + +TEST_P(UnnestedDeepPolicyEvaluationTest, Evaluate) { + const auto& tc = GetParam(); + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(kDeepPolicyYaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + + CompilePolicyOptions options; + options.unnesting_height_limit = 2; + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy, options)); + ASSERT_TRUE(result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, result.ReleaseAst()); + + // Set up runtime + cel::RuntimeOptions opts; + opts.enable_qualified_type_identifiers = true; + ASSERT_OK_AND_ASSIGN( + cel::RuntimeBuilder rt_builder, + cel::CreateStandardRuntimeBuilder( + cel::internal::GetSharedTestingDescriptorPool(), opts)); + ASSERT_THAT(cel::extensions::EnableOptionalTypes(rt_builder), IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + std::move(rt_builder).Build()); + + ASSERT_OK_AND_ASSIGN(auto program, runtime->CreateProgram(std::move(ast))); + + cel::Activation activation; + activation.InsertOrAssignValue("x", cel::IntValue(tc.x)); + google::protobuf::Arena arena; + + ASSERT_OK_AND_ASSIGN(cel::Value res, program->Evaluate(&arena, activation)); + + EXPECT_THAT(res, tc.expected); +} + +INSTANTIATE_TEST_SUITE_P( + UnnestedDeepPolicyEvaluation, UnnestedDeepPolicyEvaluationTest, + testing::Values(UnnestEvaluationTestCase{"XEquals6", 6, IntValueIs(6)}, + UnnestEvaluationTestCase{"XEquals5", 5, IntValueIs(5)}, + UnnestEvaluationTestCase{"XEquals4", 4, IntValueIs(4)}, + UnnestEvaluationTestCase{"XEquals3", 3, IntValueIs(3)}, + UnnestEvaluationTestCase{"XEquals2", 2, IntValueIs(2)}, + UnnestEvaluationTestCase{"XEquals1", 1, IntValueIs(1)}, + UnnestEvaluationTestCase{"XEquals0", 0, IntValueIs(0)}, + UnnestEvaluationTestCase{"XEqualsMinus1", -1, + IntValueIs(0)}), + [](const testing::TestParamInfo< + UnnestedDeepPolicyEvaluationTest::ParamType>& info) { + return info.param.name; + }); + +TEST(CompilerTest, UnnestCleanupRunsWhenDisabled) { + // A policy without variables and without nesting. + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: true + output: '"ok"' +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + + CompilePolicyOptions options; + options.unnesting_height_limit = 0; // Disabled + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy, options)); + ASSERT_TRUE(result.IsValid()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, result.ReleaseAst()); + + // If cleanup ran, it should have optimized away the trivial `cel.@block`. + // So the root expression should NOT be a call to `cel.@block`. + // It should be just the constant `"ok"`. + NavigableAst nav_ast = cel::NavigableAst::Build(ast->root_expr()); + EXPECT_FALSE(nav_ast.Root().expr()->has_call_expr() && + nav_ast.Root().expr()->call_expr().function() == "cel.@block"); + EXPECT_TRUE(nav_ast.Root().expr()->has_const_expr()); + + // Stale metadata for cel.@block and empty list must be pruned. + for (const auto& [id, ref] : ast->reference_map()) { + EXPECT_NE(ref.name(), "cel.@block"); + } + + EXPECT_THAT(ast, MetadataMapValid()); +} + +TEST(CompilerTest, UnnestCleanupRemovesStaleMetadata) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: x > 0 + output: 1 + - output: 2 +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, result.ReleaseAst()); + + NavigableAst nav_ast = cel::NavigableAst::Build(ast->root_expr()); + EXPECT_FALSE(nav_ast.Root().expr()->has_call_expr() && + nav_ast.Root().expr()->call_expr().function() == "cel.@block"); + + for (const auto& [id, ref] : ast->reference_map()) { + EXPECT_NE(ref.name(), "cel.@block"); + } + + EXPECT_THAT(ast, MetadataMapValid()); +} + +TEST(CompilerTest, UnnestPreservesVariablesBlock) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + variables: + - name: my_var + expression: 10 + match: + - condition: x > variables.my_var + output: 1 + - output: 2 +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + + CompilePolicyOptions options; + options.unnesting_height_limit = 0; + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy, options)); + ASSERT_TRUE(result.IsValid()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, result.ReleaseAst()); + + NavigableAst nav_ast = cel::NavigableAst::Build(ast->root_expr()); + EXPECT_TRUE(nav_ast.Root().expr()->has_call_expr() && + nav_ast.Root().expr()->call_expr().function() == "cel.@block"); + + EXPECT_THAT(ast, MetadataMapValid()); +} + +TEST(CompilerTest, UnnestRequiredLongConditionalChain) { + absl::string_view yaml = R"yaml( +name: conditional_chain_policy +rule: + match: + - condition: x == 1 + output: 10 + - condition: x == 2 + output: 20 + - condition: x == 3 + output: 30 + - condition: x == 4 + output: 40 + - condition: x == 5 + output: 50 + - condition: x == 6 + output: 60 + - condition: x == 7 + output: 70 + - output: 0 +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + + CompilePolicyOptions options; + options.unnesting_height_limit = 2; + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy, options)); + ASSERT_TRUE(result.IsValid()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, result.ReleaseAst()); + + // Root should be cel.@block because unnesting introduced @index variables. + NavigableAst nav_ast = cel::NavigableAst::Build(ast->root_expr()); + EXPECT_TRUE(nav_ast.Root().expr()->has_call_expr() && + nav_ast.Root().expr()->call_expr().function() == "cel.@block"); + + bool found_index_var = false; + for (const NavigableAstNode& node : nav_ast.Root().DescendantsPostorder()) { + if (node.expr()->has_ident_expr() && + absl::StartsWith(node.expr()->ident_expr().name(), "@index")) { + found_index_var = true; + EXPECT_TRUE(ast->type_map().contains(node.expr()->id())); + EXPECT_TRUE(ast->reference_map().contains(node.expr()->id())); + break; + } + } + EXPECT_TRUE(found_index_var); + + EXPECT_THAT(ast, MetadataMapValid()); + + // Also verify that the compiled unnested AST evaluates correctly. + cel::RuntimeOptions opts; + opts.enable_qualified_type_identifiers = true; + ASSERT_OK_AND_ASSIGN( + cel::RuntimeBuilder rt_builder, + cel::CreateStandardRuntimeBuilder( + cel::internal::GetSharedTestingDescriptorPool(), opts)); + ASSERT_THAT(cel::extensions::EnableOptionalTypes(rt_builder), + absl_testing::IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + std::move(rt_builder).Build()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + google::protobuf::Arena arena; + for (int64_t test_x : {1, 3, 5, 7, 99}) { + cel::Activation activation; + activation.InsertOrAssignValue("x", cel::IntValue(test_x)); + ASSERT_OK_AND_ASSIGN(cel::Value res, program->Evaluate(&arena, activation)); + int64_t expected = (test_x >= 1 && test_x <= 7) ? test_x * 10 : 0; + EXPECT_THAT(res, IntValueIs(expected)); + } +} + +TEST(CompilerTest, CompositionTypeCheckFailureReportsError) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: x > 10 + output: dyn(1) + - condition: x > 5 + output: 2 + - output: '"three"' +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("no matching overload for '_?_:_'")); +} + +TEST(CompilerTest, AggregateRuleComposedAstStructure) { + absl::string_view yaml = R"yaml( +name: aggregate_policy +rule: + variables: + - name: v1 + expression: "'v'" + aggregate: + - condition: x > 10 + output: '"first"' + - output: variables.v1 + - rule: + match: + - condition: y > 5 + output: '"nested_opt"' +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()) << result.FormatIssues(); + const cel::Ast* ast = result.GetAst(); + ASSERT_NE(ast, nullptr); + + cel::expr::ParsedExpr parsed_expr; + ASSERT_THAT(cel::AstToParsedExpr(*ast, &parsed_expr), IsOk()); + ASSERT_OK_AND_ASSIGN(std::string unparsed, + google::api::expr::Unparse(parsed_expr)); + + EXPECT_EQ(unparsed, + "cel.@block([\"v\"], [?(x > 10) ? optional.of(\"first\") : " + "optional.none(), @index0, ?(y > 5) ? optional.of(\"nested_opt\") " + ": optional.none()])"); +} + +TEST(CompilerTest, AggregateRuleConditionVariablesAndMetadata) { + absl::string_view yaml = R"yaml(name: aggregate_condition_metadata_policy +rule: + variables: + - name: v1 + expression: 10 + aggregate: + - condition: "[1, 2].all(i, i > variables.v1)" + output: "[3, 4].map(j, j + variables.v1)" +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()) << result.FormatIssues(); + const cel::Source* source = result.GetSource()->content(); + ASSERT_NE(source, nullptr); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, result.ReleaseAst()); + + EXPECT_THAT(ast, MetadataMapValid()); + EXPECT_THAT(ast->source_info().macro_calls(), testing::SizeIs(2)); + + cel::expr::ParsedExpr parsed_expr; + ASSERT_THAT(cel::AstToParsedExpr(*ast, &parsed_expr), IsOk()); + ASSERT_OK_AND_ASSIGN(std::string unparsed, + google::api::expr::Unparse(parsed_expr)); + EXPECT_EQ(unparsed, + "cel.@block([10], [?[1, 2].all(i, i > @index0) ? " + "optional.of([3, 4].map(j, j + @index0)) : optional.none()])"); + + NavigableAst nav_ast = cel::NavigableAst::Build(ast->root_expr()); + const NavigableAstNode* cond_gt_node = nullptr; + for (const NavigableAstNode& node : nav_ast.Root().DescendantsPostorder()) { + if (node.expr()->has_call_expr() && + node.expr()->call_expr().function() == "_>_") { + cond_gt_node = &node; + break; + } + } + ASSERT_NE(cond_gt_node, nullptr); + std::optional cond_loc = + ast->ComputeSourceLocation(cond_gt_node->expr()->id()); + ASSERT_TRUE(cond_loc.has_value()); + EXPECT_THAT(source->DisplayErrorLocation(*cond_loc), + testing::HasSubstr( + " - condition: \"[1, 2].all(i, i > variables.v1)\"\n | " + "..................................^")); +} + +TEST(CompilerTest, NestedAggregateUnderAggregateReportsError) { + absl::string_view yaml = R"yaml( +name: nested_aggregate_policy +rule: + aggregate: + - condition: x > 10 + rule: + aggregate: + - output: '"first"' +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("nested aggregate rules are not allowed")); +} + +TEST(CompilerTest, NestedAggregateUnderMatchUnderAggregateReportsError) { + absl::string_view yaml = R"yaml( +name: nested_aggregate_mixed_policy +rule: + aggregate: + - condition: x > 10 + rule: + match: + - condition: y > 5 + rule: + aggregate: + - output: '"first"' +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("nested aggregate rules are not allowed")); +} + +TEST(CompilerTest, AggregateRuleAllowsMultipleUnconditionalMatches) { + absl::string_view yaml = R"yaml( +name: aggregate_unconditional_policy +rule: + aggregate: + - output: '"first"' + - condition: true + output: '"second"' + - condition: x > 0 + output: '"third"' +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()) << result.FormatIssues(); +} + +TEST(CompilerTest, AggregateRuleIncompatibleOutputsReportsError) { + absl::string_view yaml = R"yaml( +name: aggregate_incompatible_policy +rule: + aggregate: + - condition: x > 10 + output: '"first"' + - condition: y > 5 + output: 42 +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult result, + CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT( + result.FormatIssues(), + testing::HasSubstr("incompatible output types: block has output type " + "int, but previous outputs have type string")); +} + +TEST(CompilerTest, AggregatePolicyEvaluates) { + absl::string_view yaml = R"yaml( +name: aggregate_eval_policy +rule: + variables: + - name: base + expression: 100 + aggregate: + - condition: x > 0 + output: variables.base + x + - output: variables.base + - rule: + match: + - condition: y > 0 + output: variables.base + y +)yaml"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr policy, + ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(CelPolicyValidationResult validation_result, + CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(validation_result.IsValid()) << validation_result.FormatIssues(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + validation_result.ReleaseAst()); + + cel::RuntimeOptions opts; + opts.enable_qualified_type_identifiers = true; + ASSERT_OK_AND_ASSIGN( + cel::RuntimeBuilder rt_builder, + cel::CreateStandardRuntimeBuilder( + cel::internal::GetSharedTestingDescriptorPool(), opts)); + ASSERT_THAT(cel::extensions::EnableOptionalTypes(rt_builder), IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + std::move(rt_builder).Build()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + google::protobuf::Arena arena; + cel::Activation activation; + activation.InsertOrAssignValue("x", cel::IntValue(5)); + activation.InsertOrAssignValue("y", cel::IntValue(20)); + + ASSERT_OK_AND_ASSIGN(cel::Value result, + program->Evaluate(&arena, activation)); + EXPECT_THAT(result, cel::test::ListValueIs(cel::test::ListValueElements( + testing::ElementsAre(IntValueIs(105), IntValueIs(100), + IntValueIs(120)), + cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena))); +} + +} // namespace +} // namespace cel diff --git a/policy/internal/BUILD b/policy/internal/BUILD new file mode 100644 index 000000000..bd4f7a4f3 --- /dev/null +++ b/policy/internal/BUILD @@ -0,0 +1,90 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_test.bzl", "cc_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "issue_reporter", + srcs = ["issue_reporter.cc"], + hdrs = ["issue_reporter.h"], + deps = [ + "//common:source", + "//policy:cel_policy", + "//policy:cel_policy_parser", + "@com_google_absl//absl/strings", + ], +) + +cc_library( + name = "optimizer_expr_factory", + srcs = ["optimizer_expr_factory.cc"], + hdrs = ["optimizer_expr_factory.h"], + deps = [ + "//common:ast", + "//common:ast_rewrite", + "//common:ast_traverse", + "//common:ast_visitor_base", + "//common:constant", + "//common:expr", + "//common:expr_factory", + "//common:source", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/functional:any_invocable", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:optional", + ], +) + +cc_test( + name = "optimizer_expr_factory_test", + srcs = ["optimizer_expr_factory_test.cc"], + deps = [ + ":optimizer_expr_factory", + "//common:ast", + "//common:ast_proto", + "//common:ast_rewrite", + "//common:decl", + "//common:expr", + "//common:expr_factory", + "//common:expr_printer", + "//common:source", + "//common:type", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", + "//internal:status_macros", + "//internal:testing", + "//internal:testing_descriptor_pool", + "//tools:cel_unparser", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + ], +) + +cc_library( + name = "yaml_string_element_scanner", + srcs = ["yaml_string_element_scanner.cc"], + hdrs = ["yaml_string_element_scanner.h"], + deps = [ + "//common:source", + "//internal:utf8", + "@com_google_absl//absl/strings:string_view", + ], +) + +cc_test( + name = "yaml_string_element_scanner_test", + srcs = ["yaml_string_element_scanner_test.cc"], + deps = [ + ":yaml_string_element_scanner", + "//common:source", + "//internal:testing", + "@com_google_absl//absl/strings:string_view", + ], +) diff --git a/policy/internal/issue_reporter.cc b/policy/internal/issue_reporter.cc new file mode 100644 index 000000000..944e687d6 --- /dev/null +++ b/policy/internal/issue_reporter.cc @@ -0,0 +1,45 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/internal/issue_reporter.h" + +#include "absl/strings/string_view.h" +#include "common/source.h" +#include "policy/cel_policy.h" + +namespace cel::policy_internal { + +void IssueReporter::ReportIssue(CelPolicyElementId element, Severity severity, + absl::string_view message) { + issues_.push_back({element, severity, message}); +} + +void IssueReporter::ReportOffsetIssue(CelPolicyElementId element, + cel::SourcePosition relative_position, + Severity severity, + absl::string_view message) { + issues_.push_back({element, relative_position, severity, message}); +} + +void IssueReporter::ReportError(CelPolicyElementId element, + absl::string_view message) { + ReportIssue(element, Severity::kError, message); +} + +void IssueReporter::ReportError(CelPolicyElementId element, SourcePosition pos, + absl::string_view message) { + ReportOffsetIssue(element, pos, Severity::kError, message); +} + +} // namespace cel::policy_internal diff --git a/policy/internal/issue_reporter.h b/policy/internal/issue_reporter.h new file mode 100644 index 000000000..3f88806ef --- /dev/null +++ b/policy/internal/issue_reporter.h @@ -0,0 +1,57 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_ISSUE_REPORTER_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_ISSUE_REPORTER_H_ + +#include + +#include "absl/strings/string_view.h" +#include "common/source.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" + +namespace cel::policy_internal { + +class IssueReporter { + private: + using Severity = CelPolicyIssue::Severity; + + public: + void ReportIssue(CelPolicyElementId element, Severity severity, + absl::string_view message); + + void ReportOffsetIssue(CelPolicyElementId element, + cel::SourcePosition relative_position, + Severity severity, absl::string_view message); + + void ReportError(CelPolicyElementId element, absl::string_view message); + void ReportError(CelPolicyElementId element, SourcePosition relative_pos, + absl::string_view message); + + std::vector ReleaseIssues() { + using std::swap; + std::vector out; + swap(out, issues_); + return out; + } + const std::vector& issues() const { return issues_; } + + private: + std::vector issues_; +}; + +} // namespace cel::policy_internal + +#endif // THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_ISSUE_REPORTER_H_ diff --git a/policy/internal/optimizer_expr_factory.cc b/policy/internal/optimizer_expr_factory.cc new file mode 100644 index 000000000..84bf711db --- /dev/null +++ b/policy/internal/optimizer_expr_factory.cc @@ -0,0 +1,378 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/internal/optimizer_expr_factory.h" + +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/container/flat_hash_map.h" +#include "absl/functional/any_invocable.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "common/ast.h" +#include "common/ast_rewrite.h" +#include "common/ast_traverse.h" +#include "common/ast_visitor_base.h" +#include "common/constant.h" +#include "common/expr.h" +#include "common/expr_factory.h" +#include "common/source.h" + +namespace cel { + +namespace { + +class MaxIdVisitor final : public AstVisitorBase { + public: + ExprId max_id() const { return max_id_; } + + void PreVisitExpr(const Expr& expr) override { + max_id_ = std::max(max_id_, expr.id()); + } + + void PostVisitExpr(const Expr&) override {} + + void PostVisitStruct(const Expr&, const StructExpr& struct_expr) override { + for (const auto& field : struct_expr.fields()) { + max_id_ = std::max(max_id_, field.id()); + } + } + + void PostVisitMap(const Expr&, const MapExpr& map_expr) override { + for (const auto& entry : map_expr.entries()) { + max_id_ = std::max(max_id_, entry.id()); + } + } + + private: + ExprId max_id_ = 0; +}; + +ExprId GetMaxId(const Expr& expr) { + MaxIdVisitor visitor; + AstTraverse(expr, visitor); + return visitor.max_id(); +} + +ExprId GetMaxId(const Ast& ast) { + ExprId max_id = GetMaxId(ast.root_expr()); + for (const auto& [id, _] : ast.source_info().positions()) { + max_id = std::max(max_id, id); + } + for (const auto& [id, expr] : ast.source_info().macro_calls()) { + max_id = std::max(max_id, id); + max_id = std::max(max_id, GetMaxId(expr)); + } + return max_id; +} + +// Replaces nested macros in a macro_calls expr with reference nodes. +// +// The macro_calls map is used for retaining the original structure of the +// parsed expression before macro expansion. When a macro appears inside another +// macro, the parser will replace the inner macro expr node with an unspecified +// expr with the inner macro's ID in the macro_calls map to save space. +class MakeMacroCallRewrite final : public AstRewriterBase { + public: + explicit MakeMacroCallRewrite(const SourceInfo& source_info) + : source_info_(source_info) {} + + bool PreVisitRewrite(Expr& expr) override { + if (source_info_.macro_calls().find(expr.id()) != + source_info_.macro_calls().end()) { + ExprId id = expr.id(); + expr.mutable_kind() = UnspecifiedExpr(); + expr.set_id(id); + return true; + } + return false; + } + + private: + const SourceInfo& source_info_; +}; + +// Updates macro_calls map entries to reflect a replaced expression in the +// main AST. +class ReplaceMacroCallRewrite final : public AstRewriterBase { + public: + ReplaceMacroCallRewrite(ExprId old_id, const Expr& replacement, + const SourceInfo& source_info) + : old_id_(old_id), replacement_(replacement), source_info_(source_info) {} + + bool PreVisitRewrite(Expr& expr) override { + if (expr.id() == old_id_) { + expr = macro_replacement(); + return true; + } + return false; + } + + Expr macro_replacement() { + if (!macro_replacement_) { + macro_replacement_.emplace(replacement_); + MakeMacroCallRewrite hole_creator(source_info_); + AstRewrite(*macro_replacement_, hole_creator); + } + return *macro_replacement_; + } + + private: + ExprId old_id_; + const Expr& replacement_; + absl::optional macro_replacement_; + const SourceInfo& source_info_; +}; + +void ReplaceSubExpr(Expr& expr, ExprId old_id, const Expr& replacement, + const SourceInfo& source_info) { + ReplaceMacroCallRewrite rewriter(old_id, replacement, source_info); + AstRewrite(expr, rewriter); +} + +class IdRewriter : public AstRewriterBase { + using CopyIdFn = absl::AnyInvocable; + + public: + explicit IdRewriter(CopyIdFn copy_id) : copy_id_(std::move(copy_id)) {} + + // No structure changes just ids. + bool PreVisitRewrite(Expr& expr) override { + expr.set_id(copy_id_(expr.id())); + if (expr.has_struct_expr()) { + for (auto& field : expr.mutable_struct_expr().mutable_fields()) { + field.set_id(copy_id_(field.id())); + } + } else if (expr.has_map_expr()) { + for (auto& entry : expr.mutable_map_expr().mutable_entries()) { + entry.set_id(copy_id_(entry.id())); + } + } + return false; + } + + private: + CopyIdFn copy_id_; +}; + +} // namespace + +OptimizerExprFactory::OptimizerExprFactory(Ast basis) + : ast_(std::move(basis)), next_id_(GetMaxId(ast_) + 1) {} + +OptimizerExprFactory::OptimizerExprFactory() : next_id_(1) {} + +Expr OptimizerExprFactory::Copy(const Expr& expr) { + Expr copied = expr; + IdRewriter rewriter([this](ExprId id) { return CopyId(id); }); + AstRewrite(copied, rewriter); + return copied; +} + +ListExprElement OptimizerExprFactory::Copy(const ListExprElement& element) { + return NewListElement(Copy(element.expr()), element.optional()); +} + +StructExprField OptimizerExprFactory::Copy(const StructExprField& field) { + auto field_id = CopyId(field.id()); + auto field_value = Copy(field.value()); + return NewStructField(field_id, field.name(), std::move(field_value), + field.optional()); +} + +MapExprEntry OptimizerExprFactory::Copy(const MapExprEntry& entry) { + auto entry_id = CopyId(entry.id()); + auto entry_key = Copy(entry.key()); + auto entry_value = Copy(entry.value()); + return NewMapEntry(entry_id, std::move(entry_key), std::move(entry_value), + entry.optional()); +} + +ExprId OptimizerExprFactory::NextId() { return next_id_++; } + +ExprId OptimizerExprFactory::CopyId(ExprId id) { + if (id == 0) { + return 0; + } + auto it = renumbers_.find(id); + if (it != renumbers_.end()) { + return it->second; + } + ExprId new_id = NextId(); + renumbers_[id] = new_id; + return new_id; +} + +SourceInfo OptimizerExprFactory::RemapSourceInfo(const SourceInfo& info, + SourcePosition offset) { + SourceInfo out; + + for (const auto& [old_id, macro_expr] : info.macro_calls()) { + if (auto it = renumbers_.find(old_id); it != renumbers_.end()) { + ExprId new_id = it->second; + out.mutable_macro_calls()[new_id] = Copy(macro_expr); + } + } + + for (const auto& [old_id, new_id] : renumbers_) { + if (auto it = info.positions().find(old_id); it != info.positions().end()) { + out.mutable_positions()[new_id] = it->second + offset; + } + } + + return out; +} + +void OptimizerExprFactory::MergeSourceInfo(const SourceInfo& info) { + auto& target_info = ast_.mutable_source_info(); + + for (const auto& [id, pos] : info.positions()) { + auto [it, inserted] = target_info.mutable_positions().insert({id, pos}); + if (!inserted) { + issues_.push_back(Issue{id, "conflicting ID in positions merge"}); + } + } + + for (const auto& [id, expr] : info.macro_calls()) { + auto [it, inserted] = target_info.mutable_macro_calls().insert({id, expr}); + if (!inserted) { + issues_.push_back(Issue{id, "conflicting ID in macro calls merge"}); + } + } + + // TODO(b/506179116): need to add some check that we aren't + // introducing incompatible tags. Not possible in the policy compiler right + // now. + for (const auto& ext : info.extensions()) { + auto& target_exts = target_info.mutable_extensions(); + if (!absl::c_linear_search(target_exts, ext)) { + target_exts.push_back(ext); + } + } +} + +void OptimizerExprFactory::RecordReplacement(ExprId id, const Expr& replacement, + bool keep_metadata) { + auto& source_info = ast_.mutable_source_info(); + if (!keep_metadata) { + PruneMetadata(id); + } + + for (auto& [macro_id, macro_expr] : source_info.mutable_macro_calls()) { + ReplaceSubExpr(macro_expr, id, replacement, source_info); + } +} + +void OptimizerExprFactory::PruneMetadata(ExprId id) { + SourceInfo& source_info = ast_.mutable_source_info(); + source_info.mutable_positions().erase(id); + source_info.mutable_macro_calls().erase(id); +} + +Expr OptimizerExprFactory::ReportError(absl::string_view message) { + ExprId id = NextId(); + issues_.push_back(Issue{id, std::string(message)}); + return NewUnspecified(id); +} + +Expr OptimizerExprFactory::ReportErrorAt(const Expr& expr, + absl::string_view message) { + issues_.push_back(Issue{expr.id(), std::string(message)}); + return NewUnspecified(NextId()); +} + +Expr OptimizerExprFactory::ReportErrorAtCopy(const Expr& expr, + absl::string_view message) { + issues_.push_back(Issue{CopyId(expr.id()), std::string(message)}); + return NewUnspecified(NextId()); +} + +Expr OptimizerExprFactory::NewUnspecified() { return NewUnspecified(NextId()); } + +Expr OptimizerExprFactory::NewNullConst() { return NewNullConst(NextId()); } + +Expr OptimizerExprFactory::NewBoolConst(bool value) { + return NewBoolConst(NextId(), value); +} + +Expr OptimizerExprFactory::NewIntConst(int64_t value) { + return NewIntConst(NextId(), value); +} + +Expr OptimizerExprFactory::NewUintConst(uint64_t value) { + return NewUintConst(NextId(), value); +} + +Expr OptimizerExprFactory::NewDoubleConst(double value) { + return NewDoubleConst(NextId(), value); +} + +Expr OptimizerExprFactory::NewBytesConst(std::string value) { + return NewBytesConst(NextId(), std::move(value)); +} + +Expr OptimizerExprFactory::NewBytesConst(absl::string_view value) { + return NewBytesConst(NextId(), value); +} + +Expr OptimizerExprFactory::NewBytesConst(const char* value) { + return NewBytesConst(NextId(), value); +} + +Expr OptimizerExprFactory::NewStringConst(std::string value) { + return NewStringConst(NextId(), std::move(value)); +} + +Expr OptimizerExprFactory::NewStringConst(absl::string_view value) { + return NewStringConst(NextId(), value); +} + +Expr OptimizerExprFactory::NewStringConst(const char* value) { + return NewStringConst(NextId(), value); +} + +absl::flat_hash_map OptimizerExprFactory::ConsumeRenumbers() { + using std::swap; + absl::flat_hash_map out; + swap(out, renumbers_); + return out; +} + +void OptimizerExprFactory::StartCopyContext() { renumbers_.clear(); } + +const std::vector& OptimizerExprFactory::issues() + const { + return issues_; +} + +const Ast& OptimizerExprFactory::ast() const { return ast_; } + +Ast& OptimizerExprFactory::mutable_ast() { return ast_; } + +absl::string_view OptimizerExprFactory::AccuVarName() { + return ExprFactory::AccuVarName(); +} + +Expr OptimizerExprFactory::NewAccuIdent() { return NewAccuIdent(NextId()); } + +ExprId OptimizerExprFactory::CopyId(const Expr& expr) { + return CopyId(expr.id()); +} + +} // namespace cel diff --git a/policy/internal/optimizer_expr_factory.h b/policy/internal/optimizer_expr_factory.h new file mode 100644 index 000000000..984350f6e --- /dev/null +++ b/policy/internal/optimizer_expr_factory.h @@ -0,0 +1,422 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_OPTIMIZER_EXPR_FACTORY_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_OPTIMIZER_EXPR_FACTORY_H_ + +#include +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" +#include "absl/strings/string_view.h" +#include "common/ast.h" +#include "common/expr.h" +#include "common/expr_factory.h" +#include "common/source.h" + +namespace cel { + +class ParserMacroExprFactory; +class TestOptimizerExprFactory; + +// `OptimizerExprFactory` is a specialization of `ExprFactory` used for AST +// optimization. It provides utilities for correcting metadata for modified +// ASTs. +class OptimizerExprFactory : protected ExprFactory { + public: + struct Issue { + ExprId location = 0; + std::string message; + }; + + explicit OptimizerExprFactory(Ast basis); + OptimizerExprFactory(); + + protected: + using ExprFactory::IsArrayLike; + using ExprFactory::IsExprLike; + using ExprFactory::IsStringLike; + + template + struct IsRValue + : std::bool_constant< + std::disjunction_v, std::is_same>> {}; + + public: + // Consume the current set of renumberings. + absl::flat_hash_map ConsumeRenumbers(); + + // Starts a new copy context. The current set of renumberings are cleared. + void StartCopyContext(); + + const std::vector& issues() const; + + // Record that a node in the working AST was replaced. This is used to correct + // metadata referencing the old ID. + void RecordReplacement(ExprId id, const Expr& replacement, + bool keep_metadata = false); + + // Prunes metadata for the given ID from source info. + void PruneMetadata(ExprId id); + + // Makes a copy of source metadata that is remapped to new expr Ids using + // current renumberings. This is suitable for merging into the main source + // info. + SourceInfo RemapSourceInfo(const SourceInfo& info, SourcePosition offset = 0); + + // Merge a remapped SourceInfo into the current one. + void MergeSourceInfo(const SourceInfo& info); + + const Ast& ast() const; + Ast& mutable_ast(); + + absl::string_view AccuVarName(); + + ABSL_MUST_USE_RESULT Expr Copy(const Expr& expr); + + ABSL_MUST_USE_RESULT ListExprElement Copy(const ListExprElement& element); + + ABSL_MUST_USE_RESULT StructExprField Copy(const StructExprField& field); + + ABSL_MUST_USE_RESULT MapExprEntry Copy(const MapExprEntry& entry); + + ABSL_MUST_USE_RESULT Expr NewUnspecified(); + + ABSL_MUST_USE_RESULT Expr NewNullConst(); + + ABSL_MUST_USE_RESULT Expr NewBoolConst(bool value); + + ABSL_MUST_USE_RESULT Expr NewIntConst(int64_t value); + + ABSL_MUST_USE_RESULT Expr NewUintConst(uint64_t value); + + ABSL_MUST_USE_RESULT Expr NewDoubleConst(double value); + + ABSL_MUST_USE_RESULT Expr NewBytesConst(std::string value); + + ABSL_MUST_USE_RESULT Expr NewBytesConst(absl::string_view value); + + ABSL_MUST_USE_RESULT Expr NewBytesConst(const char* absl_nullable value); + + ABSL_MUST_USE_RESULT Expr NewStringConst(std::string value); + + ABSL_MUST_USE_RESULT Expr NewStringConst(absl::string_view value); + + ABSL_MUST_USE_RESULT Expr NewStringConst(const char* absl_nullable value); + + template ::value>> + ABSL_MUST_USE_RESULT Expr NewIdent(Name name); + + ABSL_MUST_USE_RESULT Expr NewAccuIdent(); + + template ::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT Expr NewSelect(Operand operand, Field field); + + template ::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT Expr NewPresenceTest(Operand operand, Field field); + + template < + typename Function, typename... Args, + typename = std::enable_if_t::value>, + typename = std::enable_if_t...>>> + ABSL_MUST_USE_RESULT Expr NewCall(Function function, Args&&... args); + + template ::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT Expr NewCall(Function function, Args args); + + template < + typename Function, typename Target, typename... Args, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t...>>> + ABSL_MUST_USE_RESULT Expr NewMemberCall(Function function, Target target, + Args&&... args); + + template ::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT Expr NewMemberCall(Function function, Target target, + Args args); + + using ExprFactory::NewListElement; + + template ...>>> + ABSL_MUST_USE_RESULT Expr NewList(Elements&&... elements); + + template ::value>> + ABSL_MUST_USE_RESULT Expr NewList(Elements elements); + + template ::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT StructExprField NewStructField(Name name, Value value, + bool optional = false); + + template ::value>, + typename = std::enable_if_t< + std::conjunction_v...>>> + ABSL_MUST_USE_RESULT Expr NewStruct(Name name, Fields&&... fields); + + template < + typename Name, typename Fields, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT Expr NewStruct(Name name, Fields fields); + + template ::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT MapExprEntry NewMapEntry(Key key, Value value, + bool optional = false); + + template ...>>> + ABSL_MUST_USE_RESULT Expr NewMap(Entries&&... entries); + + template ::value>> + ABSL_MUST_USE_RESULT Expr NewMap(Entries entries); + + template ::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT Expr NewComprehension(IterVar iter_var, + IterRange iter_range, + AccuVar accu_var, + AccuInit accu_init, + LoopCondition loop_condition, + LoopStep loop_step, Result result); + + template ::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT Expr NewComprehension( + IterVar iter_var, IterVar2 iter_var2, IterRange iter_range, + AccuVar accu_var, AccuInit accu_init, LoopCondition loop_condition, + LoopStep loop_step, Result result); + + ABSL_MUST_USE_RESULT Expr ReportError(absl::string_view message); + + // Reports an error at the id in the optimized AST. + ABSL_MUST_USE_RESULT Expr ReportErrorAt(const Expr& expr, + absl::string_view message); + // Reports an error at the mapped id of the copy of expr in the optimized AST. + ABSL_MUST_USE_RESULT Expr ReportErrorAtCopy(const Expr& expr, + absl::string_view message); + + protected: + ABSL_MUST_USE_RESULT ExprId NextId(); + + ABSL_MUST_USE_RESULT ExprId CopyId(ExprId id); + + ABSL_MUST_USE_RESULT ExprId CopyId(const Expr& expr); + + using ExprFactory::AccuVarName; + using ExprFactory::NewAccuIdent; + using ExprFactory::NewBoolConst; + using ExprFactory::NewBytesConst; + using ExprFactory::NewCall; + using ExprFactory::NewComprehension; + using ExprFactory::NewConst; + using ExprFactory::NewDoubleConst; + using ExprFactory::NewIdent; + using ExprFactory::NewIntConst; + using ExprFactory::NewList; + using ExprFactory::NewMap; + using ExprFactory::NewMapEntry; + using ExprFactory::NewMemberCall; + using ExprFactory::NewNullConst; + using ExprFactory::NewPresenceTest; + using ExprFactory::NewSelect; + using ExprFactory::NewStringConst; + using ExprFactory::NewStruct; + using ExprFactory::NewStructField; + using ExprFactory::NewUintConst; + using ExprFactory::NewUnspecified; + + private: + Ast ast_; + absl::flat_hash_map renumbers_; + std::vector issues_; + + ExprId next_id_ = 1; +}; + +// Implementation details. + +template +Expr OptimizerExprFactory::NewIdent(Name name) { + return NewIdent(NextId(), std::move(name)); +} + +template +Expr OptimizerExprFactory::NewSelect(Operand operand, Field field) { + return NewSelect(NextId(), std::move(operand), std::move(field)); +} + +template +Expr OptimizerExprFactory::NewPresenceTest(Operand operand, Field field) { + return NewPresenceTest(NextId(), std::move(operand), std::move(field)); +} + +template +Expr OptimizerExprFactory::NewCall(Function function, Args&&... args) { + std::vector array; + array.reserve(sizeof...(Args)); + (array.push_back(std::forward(args)), ...); + return NewCall(NextId(), std::move(function), std::move(array)); +} + +template +Expr OptimizerExprFactory::NewCall(Function function, Args args) { + return NewCall(NextId(), std::move(function), std::move(args)); +} + +template +Expr OptimizerExprFactory::NewMemberCall(Function function, Target target, + Args&&... args) { + std::vector array; + array.reserve(sizeof...(Args)); + (array.push_back(std::forward(args)), ...); + return NewMemberCall(NextId(), std::move(function), std::move(target), + std::move(array)); +} + +template +Expr OptimizerExprFactory::NewMemberCall(Function function, Target target, + Args args) { + return NewMemberCall(NextId(), std::move(function), std::move(target), + std::move(args)); +} + +template +Expr OptimizerExprFactory::NewList(Elements&&... elements) { + std::vector array; + array.reserve(sizeof...(Elements)); + (array.push_back(std::forward(elements)), ...); + return NewList(NextId(), std::move(array)); +} + +template +Expr OptimizerExprFactory::NewList(Elements elements) { + return NewList(NextId(), std::move(elements)); +} + +template +StructExprField OptimizerExprFactory::NewStructField(Name name, Value value, + bool optional) { + return NewStructField(NextId(), std::move(name), std::move(value), optional); +} + +template +Expr OptimizerExprFactory::NewStruct(Name name, Fields&&... fields) { + std::vector array; + array.reserve(sizeof...(Fields)); + (array.push_back(std::forward(fields)), ...); + return NewStruct(NextId(), std::move(name), std::move(array)); +} + +template +Expr OptimizerExprFactory::NewStruct(Name name, Fields fields) { + return NewStruct(NextId(), std::move(name), std::move(fields)); +} + +template +MapExprEntry OptimizerExprFactory::NewMapEntry(Key key, Value value, + bool optional) { + return NewMapEntry(NextId(), std::move(key), std::move(value), optional); +} + +template +Expr OptimizerExprFactory::NewMap(Entries&&... entries) { + std::vector array; + array.reserve(sizeof...(Entries)); + (array.push_back(std::forward(entries)), ...); + return NewMap(NextId(), std::move(array)); +} + +template +Expr OptimizerExprFactory::NewMap(Entries entries) { + return NewMap(NextId(), std::move(entries)); +} + +template +Expr OptimizerExprFactory::NewComprehension(IterVar iter_var, + IterRange iter_range, + AccuVar accu_var, + AccuInit accu_init, + LoopCondition loop_condition, + LoopStep loop_step, Result result) { + return NewComprehension(NextId(), std::move(iter_var), std::move(iter_range), + std::move(accu_var), std::move(accu_init), + std::move(loop_condition), std::move(loop_step), + std::move(result)); +} + +template +Expr OptimizerExprFactory::NewComprehension( + IterVar iter_var, IterVar2 iter_var2, IterRange iter_range, + AccuVar accu_var, AccuInit accu_init, LoopCondition loop_condition, + LoopStep loop_step, Result result) { + return NewComprehension(NextId(), std::move(iter_var), std::move(iter_var2), + std::move(iter_range), std::move(accu_var), + std::move(accu_init), std::move(loop_condition), + std::move(loop_step), std::move(result)); +} + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_OPTIMIZER_EXPR_FACTORY_H_ diff --git a/policy/internal/optimizer_expr_factory_test.cc b/policy/internal/optimizer_expr_factory_test.cc new file mode 100644 index 000000000..f240bb308 --- /dev/null +++ b/policy/internal/optimizer_expr_factory_test.cc @@ -0,0 +1,592 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/internal/optimizer_expr_factory.h" + +#include +#include +#include +#include + +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "common/ast_proto.h" +#include "common/ast_rewrite.h" +#include "common/decl.h" +#include "common/expr.h" +#include "common/expr_factory.h" +#include "common/expr_printer.h" +#include "common/source.h" +#include "common/type.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "tools/cel_unparser.h" + +namespace cel { + +using ::testing::SizeIs; + +// Expose protected members of OptimizerExprFactory for use in tests +// +// These allow setting explicit IDs which is not safe for the optimizing +// factory. +class TestOptimizerExprFactory final : public OptimizerExprFactory { + public: + using OptimizerExprFactory::OptimizerExprFactory; + + using OptimizerExprFactory::NewBoolConst; + using OptimizerExprFactory::NewCall; + using OptimizerExprFactory::NewComprehension; + using OptimizerExprFactory::NewIdent; + using OptimizerExprFactory::NewList; + using OptimizerExprFactory::NewListElement; + using OptimizerExprFactory::NewMap; + using OptimizerExprFactory::NewMapEntry; + using OptimizerExprFactory::NewMemberCall; + using OptimizerExprFactory::NewSelect; + using OptimizerExprFactory::NewStruct; + using OptimizerExprFactory::NewStructField; + using OptimizerExprFactory::NewUnspecified; + using OptimizerExprFactory::NextId; +}; + +namespace { + +class ReplaceExprRewriter final : public AstRewriterBase { + public: + ReplaceExprRewriter(ExprId old_id, const Expr& replacement) + : old_id_(old_id), replacement_(replacement) {} + + bool PreVisitRewrite(Expr& expr) override { + if (expr.id() == old_id_) { + expr = replacement_; + return true; + } + return false; + } + + private: + ExprId old_id_; + const Expr& replacement_; +}; + +void ReplaceExprInTree(Expr& expr, ExprId old_id, const Expr& replacement) { + ReplaceExprRewriter rewriter(old_id, replacement); + AstRewrite(expr, rewriter); +} + +absl::StatusOr> CreateTestCompiler() { + CompilerOptions opts; + opts.parser_options.add_macro_calls = true; + CEL_ASSIGN_OR_RETURN( + auto builder, cel::NewCompilerBuilder( + cel::internal::GetSharedTestingDescriptorPool(), opts)); + CEL_RETURN_IF_ERROR(builder->AddLibrary(cel::StandardCompilerLibrary())); + CEL_RETURN_IF_ERROR(builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("to_replace", cel::DynType()))); + return builder->Build(); +} + +TEST(OptimizerExprFactory, CopyUnspecified) { + TestOptimizerExprFactory factory{Ast()}; + EXPECT_EQ(factory.Copy(factory.NewUnspecified()), factory.NewUnspecified(2)); +} + +TEST(OptimizerExprFactory, CopyIdent) { + TestOptimizerExprFactory factory{Ast()}; + EXPECT_EQ(factory.Copy(factory.NewIdent("foo")), factory.NewIdent(2, "foo")); +} + +TEST(OptimizerExprFactory, CopyConst) { + TestOptimizerExprFactory factory{Ast()}; + EXPECT_EQ(factory.Copy(factory.NewBoolConst(true)), + factory.NewBoolConst(2, true)); +} + +TEST(OptimizerExprFactory, CopySelect) { + TestOptimizerExprFactory factory{Ast()}; + EXPECT_EQ(factory.Copy(factory.NewSelect(factory.NewIdent("foo"), "bar")), + factory.NewSelect(3, factory.NewIdent(4, "foo"), "bar")); +} + +TEST(OptimizerExprFactory, CopyCall) { + TestOptimizerExprFactory factory{Ast()}; + std::vector copied_args; + copied_args.reserve(1); + copied_args.push_back(factory.NewIdent(6, "baz")); + EXPECT_EQ(factory.Copy(factory.NewMemberCall("bar", factory.NewIdent("foo"), + factory.NewIdent("baz"))), + factory.NewMemberCall(4, "bar", factory.NewIdent(5, "foo"), + absl::MakeSpan(copied_args))); +} + +TEST(OptimizerExprFactory, CopyList) { + TestOptimizerExprFactory factory{Ast()}; + std::vector copied_elements; + copied_elements.reserve(1); + copied_elements.push_back(factory.NewListElement(factory.NewIdent(4, "foo"))); + EXPECT_EQ(factory.Copy(factory.NewList( + factory.NewListElement(factory.NewIdent("foo")))), + factory.NewList(3, absl::MakeSpan(copied_elements))); +} + +TEST(OptimizerExprFactory, CopyStruct) { + TestOptimizerExprFactory factory{Ast()}; + std::vector copied_fields; + copied_fields.reserve(1); + copied_fields.push_back( + factory.NewStructField(5, "bar", factory.NewIdent(6, "baz"))); + EXPECT_EQ(factory.Copy(factory.NewStruct( + "foo", factory.NewStructField("bar", factory.NewIdent("baz")))), + factory.NewStruct(4, "foo", absl::MakeSpan(copied_fields))); +} + +TEST(OptimizerExprFactory, CopyMap) { + TestOptimizerExprFactory factory{Ast()}; + std::vector copied_entries; + copied_entries.reserve(1); + copied_entries.push_back(factory.NewMapEntry(6, factory.NewIdent(7, "bar"), + factory.NewIdent(8, "baz"))); + EXPECT_EQ(factory.Copy(factory.NewMap(factory.NewMapEntry( + factory.NewIdent("bar"), factory.NewIdent("baz")))), + factory.NewMap(5, absl::MakeSpan(copied_entries))); +} + +TEST(OptimizerExprFactory, CopyComprehension) { + TestOptimizerExprFactory factory{Ast()}; + EXPECT_EQ( + factory.Copy(factory.NewComprehension( + "foo", factory.NewList(), "bar", factory.NewBoolConst(true), + factory.NewIdent("baz"), factory.NewIdent("foo"), + factory.NewIdent("bar"))), + factory.NewComprehension( + 7, "foo", factory.NewList(8, std::vector()), "bar", + factory.NewBoolConst(9, true), factory.NewIdent(10, "baz"), + factory.NewIdent(11, "foo"), factory.NewIdent(12, "bar"))); +} + +TEST(OptimizerExprFactory, RemapSourceInfo) { + TestOptimizerExprFactory factory{Ast()}; + Expr orig = factory.NewIdent("foo"); // allocates ID 1 + Expr copied = factory.Copy(orig); // copies ID 1 to mapped ID 2 + + SourceInfo info; + info.mutable_positions()[1] = 42; // old ID 1 has position 42 + + SourceInfo remapped = factory.RemapSourceInfo(info, 10); + + // remapped should have ID 2 mapped to position 42 + 10 = 52 + auto it = remapped.positions().find(2); + ASSERT_NE(it, remapped.positions().end()); + EXPECT_EQ(it->second, 52); +} + +TEST(OptimizerExprFactory, RemapSourceInfoWithMacroCalls) { + TestOptimizerExprFactory factory{Ast()}; + Expr orig = factory.NewIdent("foo"); // allocates ID 1 + Expr copied = factory.Copy(orig); // copies ID 1 to mapped ID 2 + + SourceInfo info; + // old ID 1 has macro call with ID 3 + info.mutable_macro_calls()[1] = factory.NewIdent("bar"); + + SourceInfo remapped = factory.RemapSourceInfo(info, 10); + + // remapped should have ID 2 mapped to the copied macro call + // since "bar" has ID 3, Copy(bar) should map ID 3 to ID 4 + + auto it = remapped.macro_calls().find(2); + ASSERT_NE(it, remapped.macro_calls().end()); + + // The macro call should be an Ident with new ID 4 + EXPECT_EQ(it->second.id(), 4); + EXPECT_TRUE(it->second.has_ident_expr()); + EXPECT_EQ(it->second.ident_expr().name(), "bar"); +} + +TEST(OptimizerExprFactory, ReportError) { + TestOptimizerExprFactory factory{Ast()}; + Expr err_expr = factory.ReportError("something went wrong"); + + // err_expr should be unspecified with ID 1 + EXPECT_EQ(err_expr.id(), 1); + EXPECT_EQ(err_expr.kind_case(), ExprKindCase::kUnspecifiedExpr); + + // issues_ should have 1 entry with ID 1 and correct message + ASSERT_EQ(factory.issues().size(), 1); + EXPECT_EQ(factory.issues()[0].location, 1); + EXPECT_EQ(factory.issues()[0].message, "something went wrong"); +} + +TEST(OptimizerExprFactory, ReportErrorAt) { + TestOptimizerExprFactory factory{Ast()}; + Expr orig = factory.NewIdent("foo"); // allocates ID 1 + Expr copied = factory.Copy(orig); // copies ID 1 to mapped ID 2 + + Expr err_expr = factory.ReportErrorAtCopy(orig, "error on foo"); + + // err_expr should be unspecified with ID 3 (NextId) + EXPECT_EQ(err_expr.id(), 3); + EXPECT_EQ(err_expr.kind_case(), ExprKindCase::kUnspecifiedExpr); + + // issues_ should have 1 entry with mapped ID 2 and correct message + ASSERT_EQ(factory.issues().size(), 1); + EXPECT_EQ(factory.issues()[0].location, 2); + EXPECT_EQ(factory.issues()[0].message, "error on foo"); +} + +TEST(OptimizerExprFactory, MergeSourceInfo) { + // Create a base AST with some source info + SourceInfo base_info; + base_info.set_syntax_version("cel1"); + base_info.set_location("test.cel"); + base_info.mutable_positions()[1] = 10; + + Ast base_ast(Expr(), std::move(base_info)); + + TestOptimizerExprFactory factory{std::move(base_ast)}; + + // Create a new source info to merge + SourceInfo new_info; + new_info.mutable_positions()[2] = 20; + + factory.MergeSourceInfo(new_info); + + // The merged source info should have both positions + const auto& merged_info = factory.ast().source_info(); + EXPECT_EQ(merged_info.syntax_version(), "cel1"); + EXPECT_EQ(merged_info.location(), "test.cel"); + + auto it1 = merged_info.positions().find(1); + ASSERT_NE(it1, merged_info.positions().end()); + EXPECT_EQ(it1->second, 10); + + auto it2 = merged_info.positions().find(2); + ASSERT_NE(it2, merged_info.positions().end()); + EXPECT_EQ(it2->second, 20); +} + +TEST(OptimizerExprFactory, MergeSourceInfoConflict) { + SourceInfo base_info; + base_info.mutable_positions()[1] = 10; + + Ast base_ast(Expr(), std::move(base_info)); + TestOptimizerExprFactory factory{std::move(base_ast)}; + + SourceInfo new_info; + new_info.mutable_positions()[1] = 20; // conflicting ID 1 + + factory.MergeSourceInfo(new_info); + + // Should report an error for the conflict + ASSERT_EQ(factory.issues().size(), 1); + EXPECT_EQ(factory.issues()[0].location, 1); + EXPECT_EQ(factory.issues()[0].message, "conflicting ID in positions merge"); +} + +TEST(OptimizerExprFactory, RecordReplacement) { + SourceInfo base_info; + base_info.mutable_positions()[1] = 10; + base_info.mutable_positions()[2] = 20; + + TestOptimizerExprFactory factory{Ast()}; + + // macro_calls[1] maps ID 1 to macro call "bar(foo)" (where "foo" has ID 1) + base_info.mutable_macro_calls()[1] = + factory.NewCall("bar", factory.NewIdent(1, "foo")); + + // macro_calls[2] maps ID 2 to macro call "baz(foo)" (where "foo" has ID 1) + base_info.mutable_macro_calls()[2] = + factory.NewCall("baz", factory.NewIdent(1, "foo")); + + Ast base_ast(Expr(), std::move(base_info)); + TestOptimizerExprFactory optimizer{std::move(base_ast)}; + + // Record the replacement of ID 1 by a new Ident "replacement" with ID 3 + optimizer.RecordReplacement(1, factory.NewIdent(3, "replacement")); + + const auto& result_info = optimizer.ast().source_info(); + + // 1. ID 1 should be erased from positions + EXPECT_EQ(result_info.positions().find(1), result_info.positions().end()); + EXPECT_NE(result_info.positions().find(2), result_info.positions().end()); + + // 2. ID 1 should be erased from macro_calls keys + EXPECT_EQ(result_info.macro_calls().find(1), result_info.macro_calls().end()); + + // 3. macro_calls[2] should still exist, but its argument referencing ID 1 + // should be replaced with the Ident "replacement" with ID 3 inline + auto it = result_info.macro_calls().find(2); + ASSERT_NE(it, result_info.macro_calls().end()); + + const Expr& macro_expr = it->second; + ASSERT_TRUE(macro_expr.has_call_expr()); + ASSERT_EQ(macro_expr.call_expr().args().size(), 1); + + const Expr& arg = macro_expr.call_expr().args()[0]; + EXPECT_EQ(arg.id(), 3); + EXPECT_TRUE(arg.has_ident_expr()); + EXPECT_EQ(arg.ident_expr().name(), "replacement"); +} + +class IdAdorner : public cel::ExpressionAdorner { + public: + std::string Adorn(const cel::Expr& e) const override { + return absl::StrCat("#", e.id()); + } + + std::string AdornStructField(const cel::StructExprField& e) const override { + return absl::StrCat("#", e.id()); + } + + std::string AdornMapEntry(const cel::MapExprEntry& e) const override { + return absl::StrCat("#", e.id()); + } +}; + +TEST(OptimizerExprFactory, UnparseCopiedMacroCall) { + // Arrange: create an template expression and one to inline. + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + CreateTestCompiler()); + + ASSERT_OK_AND_ASSIGN(auto basis_result, + compiler->Compile("[1].map(x, x + to_replace)")); + ASSERT_TRUE(basis_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto basis_ast, basis_result.ReleaseAst()); + + ASSERT_OK_AND_ASSIGN(auto copy_result, + compiler->Compile("[1].filter(x, x > 2).size()")); + ASSERT_TRUE(copy_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto copy_ast, copy_result.ReleaseAst()); + + // Locate the "to_replace" IdentExpr node in reference_map + ExprId to_replace_id = 0; + for (const auto& [id, ref] : basis_ast->reference_map()) { + if (ref.name() == "to_replace") { + to_replace_id = id; + break; + } + } + ASSERT_NE(to_replace_id, 0); + + // Act: implement the optimization. + TestOptimizerExprFactory factory{std::move(*basis_ast)}; + Expr copied_expr = factory.Copy(copy_ast->root_expr()); + SourceInfo remapped_info = factory.RemapSourceInfo(copy_ast->source_info()); + factory.MergeSourceInfo(remapped_info); + + ReplaceExprInTree(factory.mutable_ast().mutable_root_expr(), to_replace_id, + copied_expr); + factory.RecordReplacement(to_replace_id, copied_expr); + + // Test AST structure. + EXPECT_THAT(cel::ExprPrinter(IdAdorner()).Print(factory.ast().root_expr()), + ::testing::StrEq(R"(__comprehension__( + // Variable + x, + // Target + [ + 1#2 + ]#1, + // Accumulator + @result, + // Init + []#8, + // LoopCondition + true#9, + // LoopStep + _+_( + @result#10, + [ + _+_( + x#5, + __comprehension__( + // Variable + x, + // Target + [ + 1#18 + ]#17, + // Accumulator + @result, + // Init + []#19, + // LoopCondition + true#20, + // LoopStep + _?_:_( + _>_( + x#23, + 2#24 + )#22, + _+_( + @result#26, + [ + x#28 + ]#27 + )#25, + @result#29 + )#21, + // Result + @result#30)#16.size()#15 + )#6 + ]#11 + )#12, + // Result + @result#13)#14)")); + + // Check that the structure is compatible with unparser. + cel::expr::ParsedExpr optimized_parsed; + auto status = AstToParsedExpr(factory.ast(), &optimized_parsed); + ASSERT_THAT(status, absl_testing::IsOk()); + ASSERT_OK_AND_ASSIGN(std::string unparsed, + google::api::expr::Unparse(optimized_parsed)); + + EXPECT_EQ(unparsed, "[1].map(x, x + [1].filter(x, x > 2).size())"); + + const CallExpr& call_expr = factory.mutable_ast() + .mutable_source_info() + .mutable_macro_calls()[14] + .mutable_call_expr(); + ASSERT_THAT(call_expr.args(), SizeIs(2)); + ASSERT_THAT(call_expr.args()[1].call_expr().args(), SizeIs(2)); + EXPECT_EQ(call_expr.args()[1].call_expr().args()[1].id(), 15); + + EXPECT_EQ(call_expr.args()[1].call_expr().args()[1].call_expr().target().id(), + 16); + EXPECT_EQ(call_expr.args()[1] + .call_expr() + .args()[1] + .call_expr() + .target() + .kind_case(), + ExprKindCase::kUnspecifiedExpr); +} + +TEST(OptimizerExprFactory, CopyMultipleAstsWithConsumeRenumbers) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + CreateTestCompiler()); + + ASSERT_OK_AND_ASSIGN(auto ast1_result, compiler->Compile("[1]")); + ASSERT_TRUE(ast1_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast1, ast1_result.ReleaseAst()); + + ASSERT_OK_AND_ASSIGN(auto ast2_result, compiler->Compile("2")); + ASSERT_TRUE(ast2_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast2, ast2_result.ReleaseAst()); + + TestOptimizerExprFactory factory{Ast()}; + + Expr copied1 = factory.Copy(ast1->root_expr()); + auto renumbers1 = factory.ConsumeRenumbers(); + + Expr copied2 = factory.Copy(ast2->root_expr()); + auto renumbers2 = factory.ConsumeRenumbers(); + + EXPECT_EQ(renumbers1.size(), 2); + EXPECT_EQ(renumbers2.size(), 1); + + EXPECT_NE(copied1.id(), copied2.id()); + EXPECT_GT(copied2.id(), copied1.id()); +} + +TEST(OptimizerExprFactory, MaxIdVisitorExprKinds) { + ASSERT_OK_AND_ASSIGN(auto compiler, CreateTestCompiler()); + + // Expression that covers all the kinds. + ASSERT_OK_AND_ASSIGN(auto source, NewSource(R"cel( + Struct{field : 1} || + {'key' : 'value'} || [1].exists(x, x) || foo(bar))cel")); + ASSERT_OK_AND_ASSIGN(auto ast, compiler->GetParser().Parse(*source)); + + TestOptimizerExprFactory factory{std::move(*ast)}; + + EXPECT_EQ(factory.NextId(), 26); +} + +TEST(OptimizerExprFactory, CopyListElement) { + TestOptimizerExprFactory factory{Ast()}; + ListExprElement orig = factory.NewListElement(factory.NewIdent("foo")); + ListExprElement copied = factory.Copy(orig); + EXPECT_EQ(copied.expr(), factory.NewIdent(2, "foo")); +} + +TEST(OptimizerExprFactory, CopyStructField) { + TestOptimizerExprFactory factory{Ast()}; + StructExprField orig = factory.NewStructField("bar", factory.NewIdent("baz")); + StructExprField copied = factory.Copy(orig); + EXPECT_EQ(copied.id(), 3); + EXPECT_EQ(copied.name(), "bar"); + EXPECT_EQ(copied.value(), factory.NewIdent(4, "baz")); +} + +TEST(OptimizerExprFactory, CopyMapEntry) { + TestOptimizerExprFactory factory{Ast()}; + MapExprEntry orig = + factory.NewMapEntry(factory.NewIdent("bar"), factory.NewIdent("baz")); + MapExprEntry copied = factory.Copy(orig); + EXPECT_EQ(copied.id(), 4); + EXPECT_EQ(copied.key(), factory.NewIdent(5, "bar")); + EXPECT_EQ(copied.value(), factory.NewIdent(6, "baz")); +} + +TEST(OptimizerExprFactory, MergeSourceInfoMacroConflict) { + SourceInfo base_info; + base_info.mutable_macro_calls()[1] = Expr(); + + Ast base_ast(Expr(), std::move(base_info)); + TestOptimizerExprFactory factory{std::move(base_ast)}; + + SourceInfo new_info; + new_info.mutable_macro_calls()[1] = Expr(); + + factory.MergeSourceInfo(new_info); + + ASSERT_EQ(factory.issues().size(), 1); + EXPECT_EQ(factory.issues()[0].location, 1); + EXPECT_EQ(factory.issues()[0].message, "conflicting ID in macro calls merge"); +} + +TEST(OptimizerExprFactory, PruneMetadataSingle) { + SourceInfo base_info; + base_info.mutable_positions()[1] = 10; + base_info.mutable_positions()[2] = 20; + base_info.mutable_macro_calls()[1] = Expr(); + base_info.mutable_macro_calls()[2] = Expr(); + + Ast base_ast(Expr(), std::move(base_info)); + TestOptimizerExprFactory factory{std::move(base_ast)}; + + factory.PruneMetadata(1); + + EXPECT_EQ(factory.ast().source_info().positions().find(1), + factory.ast().source_info().positions().end()); + EXPECT_NE(factory.ast().source_info().positions().find(2), + factory.ast().source_info().positions().end()); + EXPECT_EQ(factory.ast().source_info().macro_calls().find(1), + factory.ast().source_info().macro_calls().end()); + EXPECT_NE(factory.ast().source_info().macro_calls().find(2), + factory.ast().source_info().macro_calls().end()); +} + +} // namespace +} // namespace cel diff --git a/policy/internal/yaml_string_element_scanner.cc b/policy/internal/yaml_string_element_scanner.cc new file mode 100644 index 000000000..06bd8e899 --- /dev/null +++ b/policy/internal/yaml_string_element_scanner.cc @@ -0,0 +1,127 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/internal/yaml_string_element_scanner.h" + +#include + +#include "absl/strings/string_view.h" +#include "common/source.h" +#include "internal/utf8.h" + +namespace cel::policy_internal { +namespace { + +SourceRange ScanDoubleQuotedExpression(const SourceContentView& view, + SourcePosition pos) { + SourcePosition start = pos + 1; + SourcePosition cur = start; + while (cur < view.size()) { + char32_t ch = view.at(cur); + if (ch == '\\') { + cur += 2; + continue; + } + if (ch == '"') { + break; + } + cur++; + } + return SourceRange{start, cur}; +} + +SourceRange ScanSingleQuotedExpression(const SourceContentView& view, + SourcePosition pos) { + SourcePosition start = pos + 1; + SourcePosition cur = start; + while (cur < view.size()) { + char32_t ch = view.at(cur); + if (ch == '\'') { + if (cur + 1 < view.size() && view.at(cur + 1) == '\'') { + cur += 2; + continue; + } + break; + } + cur++; + } + return SourceRange{start, cur}; +} + +SourceRange ScanPlainOrBlockExpression(const SourceContentView& view, + SourcePosition pos, + absl::string_view val) { + if (val.empty()) { + return SourceRange{pos, pos}; + } + + char32_t first_char = view.at(pos); + SourcePosition start = pos; + if (first_char == '|' || first_char == '>') { + // Skip block header line + while (start < view.size() && view.at(start) != '\n') { + start++; + } + if (start < view.size() && view.at(start) == '\n') { + start++; + } + while (start < view.size() && + (view.at(start) == ' ' || view.at(start) == '\t')) { + start++; + } + } + + SourcePosition cur = start; + absl::string_view remaining = val; + while (!remaining.empty()) { + auto [code_point, code_units] = cel::internal::Utf8Decode(remaining); + remaining.remove_prefix(code_units); + if (code_point == ' ' || code_point == '\t' || code_point == '\n') continue; + while (cur < view.size() && view.at(cur) != code_point) { + cur++; + } + if (cur < view.size()) cur++; + } + return SourceRange{start, cur}; +} + +} // namespace + +// Scans the YAML string element starting at `pos` in `view` with the parsed +// string value `val`. +// +// Returns a `YamlStringElement` reporting the starting position, optional +// source range, and whether it was quoted. +// +// TODO(b/525495513): Implement support for correctly aligning the source YAML +// to the decoded string value (the component CEL expression). +YamlStringElement ScanYamlStringElement(const SourceContentView& view, + SourcePosition pos, + absl::string_view val) { + if (pos < 0 || pos >= view.size()) { + return YamlStringElement{pos, std::nullopt, false}; + } + + char32_t first_char = view.at(pos); + if (first_char == '"') { + return YamlStringElement{pos, ScanDoubleQuotedExpression(view, pos), true}; + } + if (first_char == '\'') { + return YamlStringElement{pos, ScanSingleQuotedExpression(view, pos), true}; + } + return YamlStringElement{pos, ScanPlainOrBlockExpression(view, pos, val), + false}; +} + +} // namespace cel::policy_internal diff --git a/policy/internal/yaml_string_element_scanner.h b/policy/internal/yaml_string_element_scanner.h new file mode 100644 index 000000000..f4f7536b9 --- /dev/null +++ b/policy/internal/yaml_string_element_scanner.h @@ -0,0 +1,40 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_YAML_STRING_ELEMENT_SCANNER_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_YAML_STRING_ELEMENT_SCANNER_H_ + +#include + +#include "absl/strings/string_view.h" +#include "common/source.h" + +namespace cel::policy_internal { + +struct YamlStringElement { + SourcePosition starting_position = -1; + std::optional source_range; + bool quoted = false; +}; + +// Scans a YAML scalar string element directly from the SourceContentView +// (behaving as an array of char32_t unicode codepoints) starting at `pos`, +// matching against the decoded value `val`. +YamlStringElement ScanYamlStringElement(const SourceContentView& view, + SourcePosition pos, + absl::string_view val); + +} // namespace cel::policy_internal + +#endif // THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_YAML_STRING_ELEMENT_SCANNER_H_ diff --git a/policy/internal/yaml_string_element_scanner_test.cc b/policy/internal/yaml_string_element_scanner_test.cc new file mode 100644 index 000000000..94cab4ba8 --- /dev/null +++ b/policy/internal/yaml_string_element_scanner_test.cc @@ -0,0 +1,105 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/internal/yaml_string_element_scanner.h" + +#include "absl/strings/string_view.h" +#include "common/source.h" +#include "internal/testing.h" + +namespace cel::policy_internal { +namespace { + +using ::testing::Eq; + +TEST(YamlStringElementScannerTest, QuotedScalars) { + ASSERT_OK_AND_ASSIGN(auto source, NewSource("expression: \"a + b\"")); + YamlStringElement element = + ScanYamlStringElement(source->content(), 12, "a + b"); + EXPECT_THAT(element.starting_position, Eq(12)); + EXPECT_THAT(element.quoted, Eq(true)); + ASSERT_THAT(element.source_range.has_value(), Eq(true)); + EXPECT_THAT(element.source_range->begin, Eq(13)); + EXPECT_THAT(element.source_range->end, Eq(18)); + + ASSERT_OK_AND_ASSIGN(auto source2, NewSource("expression: 'a + b'")); + YamlStringElement element2 = + ScanYamlStringElement(source2->content(), 12, "a + b"); + EXPECT_THAT(element2.starting_position, Eq(12)); + EXPECT_THAT(element2.quoted, Eq(true)); + ASSERT_THAT(element2.source_range.has_value(), Eq(true)); + EXPECT_THAT(element2.source_range->begin, Eq(13)); + EXPECT_THAT(element2.source_range->end, Eq(18)); + + ASSERT_OK_AND_ASSIGN(auto source3, NewSource("expression: \"a + \\n b\"")); + YamlStringElement element3 = + ScanYamlStringElement(source3->content(), 12, "a + \n b"); + EXPECT_THAT(element3.starting_position, Eq(12)); + EXPECT_THAT(element3.quoted, Eq(true)); + ASSERT_THAT(element3.source_range.has_value(), Eq(true)); + EXPECT_THAT(element3.source_range->begin, Eq(13)); + EXPECT_THAT(element3.source_range->end, Eq(21)); +} + +TEST(YamlStringElementScannerTest, PlainScalars) { + ASSERT_OK_AND_ASSIGN(auto source, NewSource("expression: a + b")); + YamlStringElement element = + ScanYamlStringElement(source->content(), 12, "a + b"); + EXPECT_THAT(element.starting_position, Eq(12)); + EXPECT_THAT(element.quoted, Eq(false)); + ASSERT_THAT(element.source_range.has_value(), Eq(true)); + EXPECT_THAT(element.source_range->begin, Eq(12)); + EXPECT_THAT(element.source_range->end, Eq(17)); + + ASSERT_OK_AND_ASSIGN(auto source2, NewSource("expression: a +\n b")); + YamlStringElement element2 = + ScanYamlStringElement(source2->content(), 12, "a + b"); + EXPECT_THAT(element2.starting_position, Eq(12)); + EXPECT_THAT(element2.quoted, Eq(false)); + ASSERT_THAT(element2.source_range.has_value(), Eq(true)); + EXPECT_THAT(element2.source_range->begin, Eq(12)); + EXPECT_THAT(element2.source_range->end, Eq(19)); +} + +TEST(YamlStringElementScannerTest, BlockScalars) { + ASSERT_OK_AND_ASSIGN(auto source, NewSource("expression: |\n a + b\n")); + YamlStringElement element = + ScanYamlStringElement(source->content(), 12, "a + b\n"); + EXPECT_THAT(element.starting_position, Eq(12)); + EXPECT_THAT(element.quoted, Eq(false)); + ASSERT_THAT(element.source_range.has_value(), Eq(true)); + EXPECT_THAT(element.source_range->begin, Eq(16)); + EXPECT_THAT(element.source_range->end, Eq(21)); + + ASSERT_OK_AND_ASSIGN(auto source2, NewSource("expression: >2-\n a + b\n")); + YamlStringElement element2 = + ScanYamlStringElement(source2->content(), 12, "a + b"); + EXPECT_THAT(element2.starting_position, Eq(12)); + EXPECT_THAT(element2.quoted, Eq(false)); + ASSERT_THAT(element2.source_range.has_value(), Eq(true)); + EXPECT_THAT(element2.source_range->begin, Eq(20)); + EXPECT_THAT(element2.source_range->end, Eq(25)); +} + +TEST(YamlStringElementScannerTest, InvalidPosition) { + ASSERT_OK_AND_ASSIGN(auto source, NewSource("expression: a + b")); + YamlStringElement element = + ScanYamlStringElement(source->content(), 100, "a + b"); + EXPECT_THAT(element.starting_position, Eq(100)); + EXPECT_THAT(element.quoted, Eq(false)); + EXPECT_THAT(element.source_range.has_value(), Eq(false)); +} + +} // namespace +} // namespace cel::policy_internal diff --git a/policy/test_custom_yaml_policy_parser.cc b/policy/test_custom_yaml_policy_parser.cc new file mode 100644 index 000000000..faced6952 --- /dev/null +++ b/policy/test_custom_yaml_policy_parser.cc @@ -0,0 +1,188 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "internal/status_macros.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_context.h" +#include "policy/cel_policy_parser.h" +#include "policy/yaml_policy_parser.h" +#include "yaml-cpp/node/node.h" +#include "yaml-cpp/yaml.h" // IWYU pragma: keep + +namespace cel::internal { + +// TestCustomYamlPolicyParser is used to support unit tests for custom tags +// and custom policy structures. It demonstrates the versatility of the +// cel::YamlPolicyParser framework API by implementing custom tag and block +// parsing without needing to modify the core parser. +class TestCustomYamlPolicyParser : public cel::YamlPolicyParser { + absl::StatusOr ParsePolicyTag(CelPolicyParseContext& ctx, + const ValueString& tag_name, + const YAML::Node& node) const override { + if (tag_name.value() == "name" || tag_name.value() == "description" || + tag_name.value() == "imports") { + return cel::YamlPolicyParser::ParsePolicyTag(ctx, tag_name, node); + } + if (tag_name.value() == "purpose") { + std::optional purpose = + GetValueString(ctx, node, "Policy purpose is not a string"); + if (purpose.has_value()) { + ctx.policy().mutable_metadata()["purpose"] = *purpose; + } + return true; + } + if (tag_name.value() == "version") { + std::optional version = + GetValueString(ctx, node, "Policy version is not a string"); + if (!version.has_value()) { + return true; + } + int version_int; + if (!absl::SimpleAtoi(version->value(), &version_int)) { + ctx.ReportError(version->id(), + absl::StrCat("Policy version is not an integer: ", + version->value())); + return true; + } + ctx.policy().mutable_metadata()["version"] = version_int; + return true; + } + + if (tag_name.value() == "conditions") { + if (!node.IsSequence()) { + ctx.ReportError(tag_name.id(), "Policy 'conditions' is not a sequence"); + return true; + } + for (const YAML::Node& condition : node) { + // Track the number of existing matches before parsing. When ParseMatch + // evaluates an 'else' block, it recursively triggers parsing and adds + // internal inner matches directly to the rule's match vector. + // Inserting the outer match at begin() + size_before ensures that the + // primary outer 'if' condition is always evaluated before its nested + // 'else' fallbacks. + // + // Example: + // if: x > 0 + // then: "positive" + // else: "negative" + // + // The inner "negative" match is parsed and appended to rule.matches() + // by the inner recursive call, before the outer "x > 0" match finishes. + // Inserting at size_before places the "x > 0" match ahead of the inner + // one. + size_t size_before = ctx.policy().rule().matches().size(); + CEL_ASSIGN_OR_RETURN(Match match, + cel::YamlPolicyParser::ParseMatch( + ctx, condition, ctx.policy().mutable_rule())); + ctx.policy().mutable_rule().mutable_matches().insert( + ctx.policy().mutable_rule().mutable_matches().begin() + size_before, + std::move(match)); + } + + return true; + } + return false; + } + + absl::Status ParseThenBlock(CelPolicyParseContext& ctx, + const YAML::Node& value_node, + Match& match) const { + if (value_node.IsScalar()) { + std::optional val = GetValueString( + ctx, value_node, "Policy condition 'then' is not a string"); + if (val.has_value()) { + OutputBlock output; + output.set_output(*val); + match.set_result(output); + } + } else if (value_node.IsMap()) { + auto nested_rule = std::make_unique(); + CEL_ASSIGN_OR_RETURN( + Match nested_match, + cel::YamlPolicyParser::ParseMatch(ctx, value_node, *nested_rule)); + nested_rule->mutable_matches().insert( + nested_rule->mutable_matches().begin(), std::move(nested_match)); + match.set_result(std::move(nested_rule)); + } else { + ctx.ReportError(CollectMetadata(ctx, value_node), + "Bad syntax in 'if/then' block"); + } + return absl::OkStatus(); + } + + absl::Status ParseElseBlock(CelPolicyParseContext& ctx, + const YAML::Node& value_node, Rule& rule) const { + if (value_node.IsScalar()) { + std::optional val = GetValueString( + ctx, value_node, "Policy condition 'else' is not a string"); + if (val.has_value()) { + Match else_match; + else_match.set_id(CollectMetadata(ctx, value_node)); + OutputBlock output; + output.set_output(*val); + else_match.set_result(output); + rule.mutable_matches().push_back(std::move(else_match)); + } + } else if (value_node.IsMap()) { + size_t size_before = rule.matches().size(); + CEL_ASSIGN_OR_RETURN(Match match, cel::YamlPolicyParser::ParseMatch( + ctx, value_node, rule)); + rule.mutable_matches().insert( + rule.mutable_matches().begin() + size_before, std::move(match)); + } else { + ctx.ReportError(CollectMetadata(ctx, value_node), + "Bad syntax in 'if/then' block"); + } + return absl::OkStatus(); + } + + absl::StatusOr ParseMatchTag(CelPolicyParseContext& ctx, + const ValueString& tag_name, + const YAML::Node& node, Match& match, + Rule& rule) const override { + if (tag_name.value() == "if") { + std::optional condition = + GetValueString(ctx, node, "Policy 'if' condition is not a string"); + if (condition.has_value()) { + match.set_condition(*condition); + } + return true; + } + if (tag_name.value() == "then") { + CEL_RETURN_IF_ERROR(ParseThenBlock(ctx, node, match)); + return true; + } + if (tag_name.value() == "else") { + CEL_RETURN_IF_ERROR(ParseElseBlock(ctx, node, rule)); + return true; + } + return false; + } +}; + +const CelPolicyParser& GetTestCustomYamlPolicyParser() { + static const auto* const parser = new TestCustomYamlPolicyParser(); + return *parser; +} + +} // namespace cel::internal diff --git a/policy/test_util.cc b/policy/test_util.cc new file mode 100644 index 000000000..a567e3862 --- /dev/null +++ b/policy/test_util.cc @@ -0,0 +1,274 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +#include "policy/test_util.h" + +#include +#include +#include +#include + +#include "cel/expr/eval.pb.h" +#include "cel/expr/value.pb.h" +#include "google/protobuf/struct.pb.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "internal/status_macros.h" +#include "yaml-cpp/yaml.h" + +namespace cel::test { + +namespace { + +absl::Status YamlToExprValue(const YAML::Node& node, + cel::expr::Value* proto) { + if (node.IsNull()) { + proto->set_null_value(google::protobuf::NULL_VALUE); + return absl::OkStatus(); + } + if (node.IsScalar()) { + // Try bool + try { + proto->set_bool_value(node.as()); + return absl::OkStatus(); + } catch (...) { + } + // Try int64 + try { + int64_t val; + if (YAML::convert::decode(node, val)) { + proto->set_int64_value(val); + return absl::OkStatus(); + } + } catch (...) { + } + // Try double + try { + double val; + if (YAML::convert::decode(node, val)) { + proto->set_double_value(val); + return absl::OkStatus(); + } + } catch (...) { + } + // Fallback to string + proto->set_string_value(node.as()); + return absl::OkStatus(); + } + if (node.IsSequence()) { + auto* list = proto->mutable_list_value(); + for (const auto& elem : node) { + CEL_RETURN_IF_ERROR(YamlToExprValue(elem, list->add_values())); + } + return absl::OkStatus(); + } + if (node.IsMap()) { + auto* map_val = proto->mutable_map_value(); + for (auto it = node.begin(); it != node.end(); ++it) { + auto* entry = map_val->add_entries(); + CEL_RETURN_IF_ERROR(YamlToExprValue(it->first, entry->mutable_key())); + CEL_RETURN_IF_ERROR(YamlToExprValue(it->second, entry->mutable_value())); + } + return absl::OkStatus(); + } + return absl::InvalidArgumentError("Unknown YAML node type"); +} + +absl::Status ParseInputValue( + const YAML::Node& node, + cel::expr::conformance::test::InputValue* input_val) { + if (node.IsMap() && node["expr"].IsDefined()) { + input_val->set_expr(node["expr"].as()); + return absl::OkStatus(); + } + if (node.IsMap() && node["value"].IsDefined()) { + return YamlToExprValue(node["value"], input_val->mutable_value()); + } + return YamlToExprValue(node, input_val->mutable_value()); +} + +absl::Status ParseTestOutput(const YAML::Node& node, + cel::expr::conformance::test::TestOutput* output) { + if (!node.IsDefined()) { + return absl::InvalidArgumentError("Missing output node"); + } + if (node.IsMap()) { + if (node["expr"].IsDefined()) { + output->set_result_expr(node["expr"].as()); + return absl::OkStatus(); + } + if (node["value"].IsDefined()) { + return YamlToExprValue(node["value"], output->mutable_result_value()); + } + if (node["error"].IsDefined()) { + auto* eval_error = output->mutable_eval_error(); + eval_error->add_errors()->set_message(node["error"].as()); + return absl::OkStatus(); + } + if (node["error_set"].IsDefined()) { + auto* eval_error = output->mutable_eval_error(); + for (const auto& err : node["error_set"]) { + eval_error->add_errors()->set_message(err.as()); + } + return absl::OkStatus(); + } + if (node["unknown"].IsDefined()) { + auto* unknown = output->mutable_unknown(); + for (const auto& expr_id_node : node["unknown"]) { + unknown->add_exprs(expr_id_node.as()); + } + return absl::OkStatus(); + } + } + return YamlToExprValue(node, output->mutable_result_value()); +} + +absl::Status ParseTestCaseNode( + const YAML::Node& test_node, + cel::expr::conformance::test::TestCase* test_case) { + if (test_node.IsNull()) { + return absl::OkStatus(); + } + if (!test_node.IsMap()) { + return absl::InvalidArgumentError("Test case must be a YAML mapping"); + } + if (test_node["name"].IsDefined()) { + test_case->set_name(test_node["name"].as()); + } + if (test_node["description"].IsDefined()) { + test_case->set_description(test_node["description"].as()); + } + if (test_node["expr"].IsDefined()) { + test_case->set_expr(test_node["expr"].as()); + } + if (test_node["context_expr"].IsDefined()) { + test_case->mutable_input_context()->set_context_expr( + test_node["context_expr"].as()); + } else if (test_node["input_context"].IsDefined() && + test_node["input_context"].IsMap() && + test_node["input_context"]["context_expr"].IsDefined()) { + test_case->mutable_input_context()->set_context_expr( + test_node["input_context"]["context_expr"].as()); + } + if (test_node["disable_check"].IsDefined()) { + test_case->set_disable_check(test_node["disable_check"].as()); + } + + YAML::Node input_node = test_node["input"]; + if (input_node.IsDefined() && input_node.IsMap()) { + auto* input_map = test_case->mutable_input(); + for (auto it = input_node.begin(); it != input_node.end(); ++it) { + std::string var_name = it->first.as(); + cel::expr::conformance::test::InputValue input_val; + CEL_RETURN_IF_ERROR(ParseInputValue(it->second, &input_val)); + (*input_map)[var_name] = std::move(input_val); + } + } + + YAML::Node output_node = test_node["output"]; + if (output_node.IsDefined()) { + CEL_RETURN_IF_ERROR( + ParseTestOutput(output_node, test_case->mutable_output())); + } + return absl::OkStatus(); +} + +absl::StatusOr ParseTestCaseYamlImpl( + absl::string_view yaml_content) { + if (yaml_content.empty()) { + return cel::expr::conformance::test::TestCase(); + } + YAML::Node test_node; + try { + test_node = YAML::Load(std::string(yaml_content)); + } catch (const std::exception& e) { + return absl::InvalidArgumentError( + absl::StrCat("Failed to parse YAML: ", e.what())); + } + cel::expr::conformance::test::TestCase test_case; + CEL_RETURN_IF_ERROR(ParseTestCaseNode(test_node, &test_case)); + return test_case; +} + +absl::StatusOr +ParsePolicyTestSuiteYamlImpl(absl::string_view yaml_content) { + YAML::Node tests_node; + try { + tests_node = YAML::Load(std::string(yaml_content)); + } catch (const std::exception& e) { + return absl::InvalidArgumentError( + absl::StrCat("Failed to parse YAML: ", e.what())); + } + + cel::expr::conformance::test::TestSuite test_suite; + if (tests_node["description"].IsDefined()) { + test_suite.set_description(tests_node["description"].as()); + } + + YAML::Node sections = tests_node["sections"]; + if (!sections.IsDefined()) { + sections = tests_node["section"]; // support singular format + } + if (!sections.IsDefined()) { + return absl::InvalidArgumentError( + "Missing 'sections' or 'section' in tests YAML"); + } + + for (const auto& section_node : sections) { + auto* section = test_suite.add_sections(); + if (section_node["name"].IsDefined()) { + section->set_name(section_node["name"].as()); + } + if (section_node["description"].IsDefined()) { + section->set_description(section_node["description"].as()); + } + + YAML::Node tests = section_node["tests"]; + if (!tests.IsDefined()) { + tests = section_node["test"]; // support singular format + } + if (!tests.IsDefined()) { + continue; + } + + for (const auto& test_node : tests) { + auto* test_case = section->add_tests(); + CEL_RETURN_IF_ERROR(ParseTestCaseNode(test_node, test_case)); + } + } + + return test_suite; +} + +} // namespace + +absl::StatusOr +ParsePolicyTestSuiteYaml(absl::string_view yaml_content) { + try { + return ParsePolicyTestSuiteYamlImpl(yaml_content); + } catch (...) { + return absl::InvalidArgumentError("Failed to parse YAML"); + } +} + +absl::StatusOr ParseTestCaseYaml( + absl::string_view yaml_content) { + try { + return ParseTestCaseYamlImpl(yaml_content); + } catch (const std::exception& e) { + return absl::InvalidArgumentError( + absl::StrCat("Failed to parse YAML: ", e.what())); + } catch (...) { + return absl::InvalidArgumentError("Failed to parse YAML"); + } +} + +} // namespace cel::test diff --git a/policy/test_util.h b/policy/test_util.h new file mode 100644 index 000000000..c917a2581 --- /dev/null +++ b/policy/test_util.h @@ -0,0 +1,38 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_TEST_UTIL_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_TEST_UTIL_H_ + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "cel/expr/conformance/test/suite.pb.h" + +namespace cel::test { + +// Parses a YAML content representing a policy test suite (tests.yaml) +// and adapts it to the cel.expr.conformance.test.TestSuite protobuf message. +// +// TODO(uncreated-issue/92): Move to the testrunner library. +absl::StatusOr +ParsePolicyTestSuiteYaml(absl::string_view yaml_content); + +// Parses a YAML string representing an individual test case and adapts it to +// the cel.expr.conformance.test.TestCase protobuf message. +absl::StatusOr ParseTestCaseYaml( + absl::string_view yaml_content); + +} // namespace cel::test + +#endif // THIRD_PARTY_CEL_CPP_POLICY_TEST_UTIL_H_ diff --git a/policy/testdata/BUILD b/policy/testdata/BUILD new file mode 100644 index 000000000..10a26fa0b --- /dev/null +++ b/policy/testdata/BUILD @@ -0,0 +1,19 @@ +package( + default_testonly = True, + default_visibility = ["//visibility:public"], +) + +filegroup( + name = "policy_testdata", + srcs = glob([ + "*.yaml", + "*.baseline", + ]), +) + +exports_files( + srcs = glob([ + "*.yaml", + "*.baseline", + ]), +) diff --git a/policy/testdata/cel_policy.yaml b/policy/testdata/cel_policy.yaml new file mode 100644 index 000000000..010ad8855 --- /dev/null +++ b/policy/testdata/cel_policy.yaml @@ -0,0 +1,42 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Environment: +# spec: TestAllTypes +name: cel_policy +description: A test policy for CEL +display_name: Cel Policy +imports: +- name: cel.expr.conformance.proto3.TestAllTypes +- name: cel.expr.conformance.proto3.TestAllTypes.NestedEnum +rule: + id: test_rule + description: test rule description + variables: + - name: test_var + expression: > + TestAllTypes{single_int64: 10}.single_int64 + match: + - condition: > + spec.single_int32 > TestAllTypes{single_int64: 10}.single_int64 + output: | + "invalid spec, got single_int32=" + string(spec.single_int32) + ", wanted <= 10" + explanation: | + "invalid spec, spec is greater than 10" + - condition: > + spec.standalone_enum == NestedEnum.BAR + output: | + "invalid spec, reference to BAR is not allowed" + - condition: spec.single_int64 == variables.test_var + output: '"invalid spec: exactly matches test_var"' + explanation: '"the spec cannot have single_int64 set to a known bad value"' \ No newline at end of file diff --git a/policy/testdata/cel_policy_parser.baseline b/policy/testdata/cel_policy_parser.baseline new file mode 100644 index 000000000..7a6678bfe --- /dev/null +++ b/policy/testdata/cel_policy_parser.baseline @@ -0,0 +1,89 @@ +POLICY SOURCE: cel_policy.yaml +-------------------------------------------------------------------- +PARSED POLICY: +CelPolicy{ + =========================================================== + # Copyright 2026 Google LLC + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # https://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + # Environment: + # spec: TestAllTypes + #0> name: #1> cel_policy + #2> description: #3> A test policy for CEL + #4> display_name: #5> Cel Policy + #6> imports: + - #7> name: #8> cel.expr.conformance.proto3.TestAllTypes + - #9> name: #10> cel.expr.conformance.proto3.TestAllTypes.NestedEnum + #11> rule: + #13> #12> id: #14> test_rule + #15> description: #16> test rule description + #17> variables: + - #18> name: #19> test_var + #20> expression: #21> > + TestAllTypes{single_int64: 10}.single_int64 + #22> match: + - #24> #23> condition: #25> > + spec.single_int32 > TestAllTypes{single_int64: 10}.single_int64 + #26> output: #27> | + "invalid spec, got single_int32=" + string(spec.single_int32) + ", wanted <= 10" + #28> explanation: #29> | + "invalid spec, spec is greater than 10" + - #31> #30> condition: #32> > + spec.standalone_enum == NestedEnum.BAR + #33> output: #34> | + "invalid spec, reference to BAR is not allowed" + - #36> #35> condition: #37> spec.single_int64 == variables.test_var + #38> output: #39> '"invalid spec: exactly matches test_var"' + #40> explanation: #41> '"the spec cannot have single_int64 set to a known bad value"' + =========================================================== + name: #1> "cel_policy" + description: #3> "A test policy for CEL" + display_name: #5> "Cel Policy" + imports: + #7> name: #8> "cel.expr.conformance.proto3.TestAllTypes" + #9> name: #10> "cel.expr.conformance.proto3.TestAllTypes.NestedEnum" + #12> rule: { + rule_id: #14> "test_rule" + description: #16> "test rule description" + variable: { + name: #19> "test_var" + expression: #21> "TestAllTypes{single_int64: 10}.single_int64 + " + } + #23> match: { + condition: #25> "spec.single_int32 > TestAllTypes{single_int64: 10}.single_int64 + " + result: { + output: #27> ""invalid spec, got single_int32=" + string(spec.single_int32) + ", wanted <= 10" + " + explanation: #29> ""invalid spec, spec is greater than 10" + " + } + } + #30> match: { + condition: #32> "spec.standalone_enum == NestedEnum.BAR + " + result: { + output: #34> ""invalid spec, reference to BAR is not allowed" + " + } + } + #35> match: { + condition: #37> "spec.single_int64 == variables.test_var" + result: { + output: #39> ""invalid spec: exactly matches test_var"" + explanation: #41> ""the spec cannot have single_int64 set to a known bad value"" + } + } + } +} diff --git a/policy/testdata/custom_policy_format.yaml b/policy/testdata/custom_policy_format.yaml new file mode 100644 index 000000000..a67356906 --- /dev/null +++ b/policy/testdata/custom_policy_format.yaml @@ -0,0 +1,29 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: cel_policy_custom_tags +description: A custom policy format +imports: +- name: cel.expr.conformance.proto3.TestAllTypes +purpose: test +version: 42 +conditions: +- if: spec.single_string == "none" + then: "'zero'" + else: + if: spec.single_string == "integer" + then: + if: spec.single_int32 > 0 + then: "'positive integer'" + else: "'negative integer'" + else: "'not an integer'" diff --git a/policy/testdata/custom_policy_format_parser.baseline b/policy/testdata/custom_policy_format_parser.baseline new file mode 100644 index 000000000..d5b1a2235 --- /dev/null +++ b/policy/testdata/custom_policy_format_parser.baseline @@ -0,0 +1,75 @@ +POLICY SOURCE: custom_policy_format.yaml +-------------------------------------------------------------------- +PARSED POLICY: +CelPolicy{ + =========================================================== + # Copyright 2026 Google LLC + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # https://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #0> name: #1> cel_policy_custom_tags + #2> description: #3> A custom policy format + #4> imports: + - #5> name: #6> cel.expr.conformance.proto3.TestAllTypes + #7> purpose: #8> test + #9> version: #10> 42 + #11> conditions: + - #13> #12> if: #14> spec.single_string == "none" + #15> then: #16> "'zero'" + #17> else: + #19> #18> if: #20> spec.single_string == "integer" + #21> then: + #23> #22> if: #24> spec.single_int32 > 0 + #25> then: #26> "'positive integer'" + #27> else: #29> #28> "'negative integer'" + #30> else: #32> #31> "'not an integer'" + + =========================================================== + name: #1> "cel_policy_custom_tags" + description: #3> "A custom policy format" + metadata: { + purpose: #8> "test" + version: 42 + } + imports: + #5> name: #6> "cel.expr.conformance.proto3.TestAllTypes" + rule: { + #12> match: { + condition: #14> "spec.single_string == "none"" + result: { + output: #16> "'zero'" + } + } + #18> match: { + condition: #20> "spec.single_string == "integer"" + result: + rule: { + #22> match: { + condition: #24> "spec.single_int32 > 0" + result: { + output: #26> "'positive integer'" + } + } + #29> match: { + result: { + output: #28> "'negative integer'" + } + } + } + } + #32> match: { + result: { + output: #31> "'not an integer'" + } + } + } +} diff --git a/policy/testdata/custom_policy_format_with_errors.yaml b/policy/testdata/custom_policy_format_with_errors.yaml new file mode 100644 index 000000000..594747c60 --- /dev/null +++ b/policy/testdata/custom_policy_format_with_errors.yaml @@ -0,0 +1,33 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: cel_policy_custom_tags +description: A custom policy format +imports: +- name: cel.expr.conformance.proto3.TestAllTypes +purpose: + - testing +version: new +conditions: +- if: + spec.single_string: "none" + then: "'zero'" + else: "'not zero'" +- if: spec.single_string == "number" + then: + if: spec.single_int32 > 0 + then: "'positive integer'" + else: + - ignore +- else: "'negative integer'" + diff --git a/policy/testdata/custom_policy_format_with_errors_parser.baseline b/policy/testdata/custom_policy_format_with_errors_parser.baseline new file mode 100644 index 000000000..978d27bda --- /dev/null +++ b/policy/testdata/custom_policy_format_with_errors_parser.baseline @@ -0,0 +1,16 @@ +POLICY SOURCE: custom_policy_format_with_errors.yaml +-------------------------------------------------------------------- +-------------------------------------------------------------------- +PARSER ISSUES: +ERROR: custom_policy_format_with_errors.yaml:19:3: Policy purpose is not a string + | - testing + | ..^ +ERROR: custom_policy_format_with_errors.yaml:20:10: Policy version is not an integer: new + | version: new + | .........^ +ERROR: custom_policy_format_with_errors.yaml:23:5: Policy 'if' condition is not a string + | spec.single_string: "none" + | ....^ +ERROR: custom_policy_format_with_errors.yaml:31:7: Bad syntax in 'if/then' block + | - ignore + | ......^ diff --git a/policy/testdata/nested_rule.yaml b/policy/testdata/nested_rule.yaml new file mode 100644 index 000000000..2b07faa64 --- /dev/null +++ b/policy/testdata/nested_rule.yaml @@ -0,0 +1,37 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: nested_rule +rule: + variables: + - name: "permitted_regions" + expression: "['us', 'uk', 'es']" + match: + - rule: + id: "banned regions" + description: > + determine whether the resource origin is in the banned + list. If the region is also in the permitted list, the + ban has no effect. + variables: + - name: "banned_regions" + expression: "{'us': false, 'ru': false, 'ir': false}" + match: + - condition: | + resource.origin in variables.banned_regions && + !(resource.origin in variables.permitted_regions) + output: "{'banned': true}" + - condition: resource.origin in variables.permitted_regions + output: "{'banned': false}" + - output: "{'banned': true}" + explanation: "'resource is in the banned region ' + resource.origin" \ No newline at end of file diff --git a/policy/testdata/nested_rule_parser.baseline b/policy/testdata/nested_rule_parser.baseline new file mode 100644 index 000000000..128f81bda --- /dev/null +++ b/policy/testdata/nested_rule_parser.baseline @@ -0,0 +1,84 @@ +POLICY SOURCE: nested_rule.yaml +-------------------------------------------------------------------- +PARSED POLICY: +CelPolicy{ + =========================================================== + # Copyright 2024 Google LLC + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # https://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #0> name: #1> nested_rule + #2> rule: + #4> #3> variables: + - #5> name: #6> "permitted_regions" + #7> expression: #8> "['us', 'uk', 'es']" + #9> match: + - #11> #10> rule: + #13> #12> id: #14> "banned regions" + #15> description: #16> > + determine whether the resource origin is in the banned + list. If the region is also in the permitted list, the + ban has no effect. + #17> variables: + - #18> name: #19> "banned_regions" + #20> expression: #21> "{'us': false, 'ru': false, 'ir': false}" + #22> match: + - #24> #23> condition: #25> | + resource.origin in variables.banned_regions && + !(resource.origin in variables.permitted_regions) + #26> output: #27> "{'banned': true}" + - #29> #28> condition: #30> resource.origin in variables.permitted_regions + #31> output: #32> "{'banned': false}" + - #34> #33> output: #35> "{'banned': true}" + #36> explanation: #37> "'resource is in the banned region ' + resource.origin" + =========================================================== + name: #1> "nested_rule" + description: "nested_rule.yaml" + #3> rule: { + variable: { + name: #6> "permitted_regions" + expression: #8> "['us', 'uk', 'es']" + } + #10> match: { + result: + #12> rule: { + rule_id: #14> "banned regions" + description: #16> "determine whether the resource origin is in the banned list. If the region is also in the permitted list, the ban has no effect. + " + variable: { + name: #19> "banned_regions" + expression: #21> "{'us': false, 'ru': false, 'ir': false}" + } + #23> match: { + condition: #25> "resource.origin in variables.banned_regions && + !(resource.origin in variables.permitted_regions) + " + result: { + output: #27> "{'banned': true}" + } + } + } + } + #28> match: { + condition: #30> "resource.origin in variables.permitted_regions" + result: { + output: #32> "{'banned': false}" + } + } + #33> match: { + result: { + output: #35> "{'banned': true}" + explanation: #37> "'resource is in the banned region ' + resource.origin" + } + } + } +} diff --git a/policy/yaml_policy_parser.cc b/policy/yaml_policy_parser.cc new file mode 100644 index 000000000..8b44250ea --- /dev/null +++ b/policy/yaml_policy_parser.cc @@ -0,0 +1,449 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/yaml_policy_parser.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "common/source.h" +#include "internal/status_macros.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_context.h" +#include "policy/cel_policy_parse_result.h" +#include "policy/cel_policy_parser.h" +#include "policy/internal/yaml_string_element_scanner.h" +#include "yaml-cpp/exceptions.h" +#include "yaml-cpp/mark.h" +#include "yaml-cpp/node/node.h" +#include "yaml-cpp/node/parse.h" +#include "yaml-cpp/null.h" +#include "yaml-cpp/yaml.h" // IWYU pragma: keep + +namespace cel { +namespace { + +SourcePosition GetMarkCodepointPosition(const CelPolicyParseContext& ctx, + const YAML::Mark& mark) { + if (mark.is_null() || mark.pos < 0 || + ctx.policy_source().content() == nullptr) { + return -1; + } + const Source& source = *ctx.policy_source().content(); + return source.GetPositionFromByteOffset(mark.pos).value_or( + source.content().size()); +} + +} // namespace + +CelPolicyElementId YamlPolicyParser::CollectMetadata( + CelPolicyParseContext& ctx, const YAML::Node& node) const { + CelPolicyElementId element_id = ctx.next_element_id(); + if (!node.Mark().is_null()) { + ctx.policy_source().NoteSourcePosition( + element_id, GetMarkCodepointPosition(ctx, node.Mark())); + } + return element_id; +} + +std::optional YamlPolicyParser::GetValueString( + CelPolicyParseContext& ctx, const YAML::Node& node, + std::string_view error_message) const { + if (!node.IsDefined()) { + // This should never happen since the YAML syntax has already been checked. + return std::nullopt; + } + + CelPolicyElementId id = CollectMetadata(ctx, node); + if (!node.IsScalar()) { + ctx.ReportError(id, error_message); + return std::nullopt; + } + + if (!node.Mark().is_null() && ctx.policy_source().content() != nullptr) { + SourcePosition codepoint_pos = GetMarkCodepointPosition(ctx, node.Mark()); + policy_internal::YamlStringElement element = + policy_internal::ScanYamlStringElement( + ctx.policy_source().content()->content(), codepoint_pos, + node.as()); + + ctx.policy_source().NoteSourcePosition(id, element.starting_position); + ctx.policy_source().NoteSourceRange(id, element.source_range, + element.quoted); + } + + try { + return ValueString(id, node.as()); + } catch (YAML::Exception& e) { + // This should never happen since we already checked that the node is a + // scalar and all scalars can be converted to strings. + return std::nullopt; + } +} + +absl::Status YamlPolicyParser::ParsePolicy(CelPolicyParseContext& ctx) const { + const Source* source = ctx.policy_source().content(); + if (source == nullptr) { + return absl::OkStatus(); + } + + ctx.policy().set_description(ValueString(-1, source->description())); + YAML::Node node; + try { + node = YAML::Load(source->content().ToString()); + } catch (YAML::Exception& e) { + if (!e.mark.is_null()) { + ctx.policy_source().NoteSourcePosition( + 0, GetMarkCodepointPosition(ctx, e.mark)); + } + ctx.ReportError(0, "Invalid CEL policy YAML syntax"); + return absl::OkStatus(); + } + + if (!node.IsMap()) { + ctx.ReportError(CollectMetadata(ctx, node), "Policy is not a map"); + return absl::OkStatus(); + } + + for (auto it = node.begin(); it != node.end(); ++it) { + const YAML::Node key_node = it->first; + const YAML::Node value_node = it->second; + std::optional key = + GetValueString(ctx, key_node, "Policy tag is not a string"); + if (!key.has_value()) { + continue; + } + CEL_ASSIGN_OR_RETURN(bool handled, ParsePolicyTag(ctx, *key, value_node)); + if (!handled) { + ctx.ReportError( + key->id(), + absl::StrCat("Unrecognized top-level policy tag: ", key->value())); + } + } + + return absl::OkStatus(); +} + +absl::StatusOr YamlPolicyParser::ParsePolicyTag( + CelPolicyParseContext& ctx, const ValueString& tag_name, + const YAML::Node& node) const { + if (tag_name.value() == "imports") { + CEL_RETURN_IF_ERROR(ParseImports(ctx, node)); + return true; + } + if (tag_name.value() == "name") { + std::optional name = + GetValueString(ctx, node, "Policy 'name' is not a string"); + if (name.has_value()) { + ctx.policy().set_name(*name); + } + return true; + } + if (tag_name.value() == "description") { + std::optional description = + GetValueString(ctx, node, "Policy 'description' is not a string"); + if (description.has_value()) { + ctx.policy().set_description(*description); + } + return true; + } + if (tag_name.value() == "display_name") { + std::optional display_name = + GetValueString(ctx, node, "Policy 'display_name' is not a string"); + if (display_name.has_value()) { + ctx.policy().set_display_name(*display_name); + } + return true; + } + if (tag_name.value() == "rule") { + CEL_RETURN_IF_ERROR(ParseRule(ctx, node, ctx.policy().mutable_rule())); + return true; + } + return false; +} + +absl::Status YamlPolicyParser::ParseImports(CelPolicyParseContext& ctx, + const YAML::Node& node) const { + if (!node.IsSequence()) { + ctx.ReportError(CollectMetadata(ctx, node), + "Policy 'imports' is not a sequence"); + return absl::OkStatus(); + } + + for (const YAML::Node& import : node) { + CelPolicyElementId import_id = CollectMetadata(ctx, import); + if (!import.IsMap()) { + ctx.ReportError(import_id, "Import is not a map"); + continue; + } + const YAML::Node& name_node = import["name"]; + if (!name_node.IsDefined()) { + ctx.ReportError(import_id, "No 'name' tag in import"); + continue; + } + std::optional import_name = + GetValueString(ctx, name_node, "Import name is not a string"); + if (import_name.has_value()) { + ctx.policy().mutable_imports().push_back(Import(import_id, *import_name)); + } + } + return absl::OkStatus(); +} + +absl::Status YamlPolicyParser::ParseRule(CelPolicyParseContext& ctx, + const YAML::Node& node, + Rule& rule) const { + if (!node.IsMap()) { + ctx.ReportError(CollectMetadata(ctx, node), "Policy 'rule' is not a map"); + return absl::OkStatus(); + } + rule.set_id(CollectMetadata(ctx, node)); + + for (auto it = node.begin(); it != node.end(); ++it) { + const YAML::Node key_node = it->first; + const YAML::Node value_node = it->second; + std::optional key = + GetValueString(ctx, key_node, "Policy rule tag is not a string"); + if (!key.has_value()) { + continue; + } + CEL_ASSIGN_OR_RETURN(bool handled, + ParseRuleTag(ctx, *key, value_node, rule)); + if (!handled) { + ctx.ReportError(key->id(), absl::StrCat("Unrecognized policy rule tag: ", + key->value())); + } + } + return absl::OkStatus(); +} + +absl::StatusOr YamlPolicyParser::ParseRuleTag(CelPolicyParseContext& ctx, + const ValueString& tag_name, + const YAML::Node& node, + Rule& rule) const { + if (tag_name.value() == "id") { + std::optional rule_id = + GetValueString(ctx, node, "Policy rule 'id' is not a string"); + if (rule_id.has_value()) { + rule.set_rule_id(*rule_id); + } + return true; + } + if (tag_name.value() == "description") { + std::optional description = + GetValueString(ctx, node, "Policy rule 'description' is not a string"); + if (description.has_value()) { + rule.set_description(*description); + } + return true; + } + if (tag_name.value() == "variables") { + if (!node.IsSequence()) { + ctx.ReportError(CollectMetadata(ctx, node), + "Policy rule 'variables' is not a sequence"); + return true; + } + for (const YAML::Node& variable_node : node) { + CEL_ASSIGN_OR_RETURN(Variable variable, + ParseVariable(ctx, variable_node, rule)); + rule.mutable_variables().push_back(std::move(variable)); + } + return true; + } + if (tag_name.value() == "match" || tag_name.value() == "aggregate") { + const bool is_aggregate = (tag_name.value() == "aggregate"); + if (!rule.matches().empty() || (rule.is_aggregate() && !is_aggregate)) { + ctx.ReportError( + tag_name.id(), + "Only one of 'match' or 'aggregate' may be set in a rule"); + return true; + } + rule.set_is_aggregate(is_aggregate); + if (!node.IsSequence()) { + ctx.ReportError(CollectMetadata(ctx, node), + absl::StrCat("Policy rule '", tag_name.value(), + "' is not a sequence")); + return true; + } + for (const YAML::Node& match_node : node) { + CEL_ASSIGN_OR_RETURN(Match match, ParseMatch(ctx, match_node, rule)); + rule.mutable_matches().push_back(std::move(match)); + } + return true; + } + return false; +} + +absl::StatusOr YamlPolicyParser::ParseVariable( + CelPolicyParseContext& ctx, const YAML::Node& node, Rule& rule) const { + Variable variable; + if (!node.IsMap()) { + ctx.ReportError(CollectMetadata(ctx, node), + "Policy rule 'variable' is not a map"); + return variable; + } + for (auto it = node.begin(); it != node.end(); ++it) { + const YAML::Node key_node = it->first; + const YAML::Node value_node = it->second; + std::optional key = + GetValueString(ctx, key_node, "Policy variable tag is not a string"); + if (!key.has_value()) { + continue; + } + CEL_ASSIGN_OR_RETURN(bool handled, + ParseVariableTag(ctx, *key, value_node, variable)); + if (!handled) { + ctx.ReportError( + key->id(), + absl::StrCat("Unrecognized policy variable tag: ", key->value())); + } + } + return variable; +} + +absl::StatusOr YamlPolicyParser::ParseVariableTag( + CelPolicyParseContext& ctx, const ValueString& tag_name, + const YAML::Node& node, Variable& variable) const { + if (tag_name.value() == "name") { + std::optional name = + GetValueString(ctx, node, "Policy variable 'name' is not a string"); + if (name.has_value()) { + variable.set_name(*name); + } + return true; + } + if (tag_name.value() == "expression") { + std::optional expression = GetValueString( + ctx, node, "Policy variable 'expression' is not a string"); + if (expression.has_value()) { + variable.set_expression(*expression); + } + return true; + } + return false; +} + +absl::StatusOr YamlPolicyParser::ParseMatch(CelPolicyParseContext& ctx, + const YAML::Node& node, + Rule& rule) const { + Match match; + match.set_id(CollectMetadata(ctx, node)); + if (!node.IsMap()) { + ctx.ReportError(match.id(), "Policy rule 'match' is not a map"); + return match; + } + for (auto it = node.begin(); it != node.end(); ++it) { + const YAML::Node key_node = it->first; + const YAML::Node value_node = it->second; + std::optional key = + GetValueString(ctx, key_node, "Policy match tag is not a string"); + if (!key.has_value()) { + continue; + } + CEL_ASSIGN_OR_RETURN(bool handled, + ParseMatchTag(ctx, *key, value_node, match, rule)); + if (!handled) { + ctx.ReportError(key->id(), absl::StrCat("Unrecognized policy match tag: ", + key->value())); + } + } + + if (match.has_output_block()) { + if (match.output_block().output().value().empty() && + match.output_block().explanation().has_value()) { + ctx.ReportError(match.id(), "Match specifies explanation but no output"); + } + } + + return match; +} + +absl::StatusOr YamlPolicyParser::ParseMatchTag( + CelPolicyParseContext& ctx, const ValueString& tag_name, + const YAML::Node& node, Match& match, Rule& rule) const { + if (tag_name.value() == "condition") { + std::optional condition = + GetValueString(ctx, node, "Policy match 'condition' is not a string"); + if (condition.has_value()) { + match.set_condition(*condition); + } + return true; + } + if (tag_name.value() == "explanation") { + std::optional explanation = + GetValueString(ctx, node, "Policy match 'explanation' is not a string"); + if (explanation.has_value()) { + if (match.has_rule()) { + ctx.ReportError( + tag_name.id(), + "Cannot specify explanation when a nested rule is present"); + } else { + match.mutable_output_block().set_explanation(*explanation); + } + } + return true; + } + if (tag_name.value() == "output") { + std::optional output = + GetValueString(ctx, node, "Policy match 'output' is not a string"); + if (output.has_value()) { + if (match.has_rule()) { + ctx.ReportError(tag_name.id(), + "Cannot specify output when a nested rule is present"); + } else { + match.mutable_output_block().set_output(*output); + } + } + return true; + } + if (tag_name.value() == "rule") { + if (match.has_output_block()) { + ctx.ReportError(tag_name.id(), + "Cannot specify nested rule when output/explanation is " + "present"); + } + auto nested_rule = std::make_unique(); + CEL_RETURN_IF_ERROR(ParseRule(ctx, node, *nested_rule)); + match.set_result(std::move(nested_rule)); + return true; + } + return false; +} + +const CelPolicyParser& GetDefaultYamlPolicyParser() { + static const auto* const parser = new YamlPolicyParser(); + return *parser; +} + +absl::StatusOr ParseYamlCelPolicy( + std::shared_ptr policy_source) { + return ParseYamlCelPolicy(std::move(policy_source), + GetDefaultYamlPolicyParser()); +} + +absl::StatusOr ParseYamlCelPolicy( + std::shared_ptr policy_source, + const CelPolicyParser& parser) { + CelPolicyParseContext ctx(std::move(policy_source)); + CEL_RETURN_IF_ERROR(parser.ParsePolicy(ctx)); + return ctx.GetResult(); +} + +} // namespace cel diff --git a/policy/yaml_policy_parser.h b/policy/yaml_policy_parser.h new file mode 100644 index 000000000..469209333 --- /dev/null +++ b/policy/yaml_policy_parser.h @@ -0,0 +1,135 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_YAML_POLICY_PARSER_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_YAML_POLICY_PARSER_H_ + +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_context.h" +#include "policy/cel_policy_parse_result.h" +#include "policy/cel_policy_parser.h" +#include "yaml-cpp/node/node.h" + +namespace cel { + +// A parser for YAML-based CEL policies. +// +// To support additional or alternative YAML elements, subclass +// `YamlPolicyParser` and override specific parsing methods, `Parse*` +class YamlPolicyParser : public CelPolicyParser { + public: + std::optional GetValueString( + CelPolicyParseContext& ctx, const YAML::Node& node, + std::string_view error_message) const; + + absl::Status ParsePolicy(CelPolicyParseContext& ctx) const override; + + protected: + // Collects metadata (e.g. source position) for the given YAML node, stores it + // in the context, and returns an ID that can be used to refer to it. + virtual CelPolicyElementId CollectMetadata(CelPolicyParseContext& ctx, + const YAML::Node& node) const; + + // Parses a top-level tag in the policy YAML. + // Returns true if the tag was handled. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::StatusOr ParsePolicyTag(CelPolicyParseContext& ctx, + const ValueString& tag_name, + const YAML::Node& node) const; + + // Parses the imports section of the policy YAML. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::Status ParseImports(CelPolicyParseContext& ctx, + const YAML::Node& node) const; + + // Parses a rule element of the policy YAML, which may be the top-level rule + // or a sub-rule of a match. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::Status ParseRule(CelPolicyParseContext& ctx, + const YAML::Node& node, Rule& rule) const; + + // Parses a tag in a policy YAML rule. + // Returns true if the tag was handled. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::StatusOr ParseRuleTag(CelPolicyParseContext& ctx, + const ValueString& tag_name, + const YAML::Node& node, + Rule& rule) const; + + // Parses a variable element of the policy YAML. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::StatusOr ParseVariable(CelPolicyParseContext& ctx, + const YAML::Node& node, + Rule& rule) const; + + // Parses a tag in a policy YAML variable. + // Returns true if the tag was handled. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::StatusOr ParseVariableTag(CelPolicyParseContext& ctx, + const ValueString& tag_name, + const YAML::Node& node, + Variable& variable) const; + + // Parses a match element of the policy YAML. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::StatusOr ParseMatch(CelPolicyParseContext& ctx, + const YAML::Node& node, + Rule& rule) const; + + // Parses a tag in a policy YAML match. + // Returns true if the tag was handled. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::StatusOr ParseMatchTag(CelPolicyParseContext& ctx, + const ValueString& tag_name, + const YAML::Node& node, + Match& match, Rule& rule) const; +}; + +// Returns a default implementation of YamlPolicyParser. +const CelPolicyParser& GetDefaultYamlPolicyParser(); + +absl::StatusOr ParseYamlCelPolicy( + std::shared_ptr policy_source, + const CelPolicyParser& parser); + +// YAML CelPolicy parser that uses the default format as implemented by +// `YamlPolicyParser`. +absl::StatusOr ParseYamlCelPolicy( + std::shared_ptr policy_source); + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_YAML_POLICY_PARSER_H_ diff --git a/policy/yaml_policy_parser_test.cc b/policy/yaml_policy_parser_test.cc new file mode 100644 index 000000000..6986c5472 --- /dev/null +++ b/policy/yaml_policy_parser_test.cc @@ -0,0 +1,399 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/yaml_policy_parser.h" + +#include +#include +#include +#include +#include + +#include "absl/log/absl_log.h" +#include "absl/status/status_matchers.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "common/source.h" +#include "internal/runfiles.h" +#include "internal/testing.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" +#include "policy/cel_policy_parser.h" +#include "yaml-cpp/node/node.h" + +namespace cel { + +namespace internal { +const CelPolicyParser& GetTestCustomYamlPolicyParser(); +} // namespace internal + +namespace { + +using ::absl_testing::IsOk; +using ::testing::HasSubstr; +using ::testing::IsNull; + +constexpr absl::string_view kTestPolicyFilePath = +"_main/policy/testdata/"; + +constexpr absl::string_view kBaselineSeparator = + "--------------------------------------------------------------------\n"; + +struct YamlPolicyParserTestCase { + std::string policy_source_file; + std::string baseline_file; + const cel::CelPolicyParser& (*parser_factory)(); +}; + +using YamlPolicyParserTest = testing::TestWithParam; + +TEST_P(YamlPolicyParserTest, Parse) { + std::string contents; + std::string test_file = cel::internal::ResolveRunfilesPath( + absl::StrCat(kTestPolicyFilePath, GetParam().policy_source_file)); + ASSERT_THAT(cel::internal::GetFileContents(test_file, &contents), IsOk()); + + std::string baseline; + std::string baseline_file = cel::internal::ResolveRunfilesPath( + absl::StrCat(kTestPolicyFilePath, GetParam().baseline_file)); + ASSERT_THAT(cel::internal::GetFileContents(baseline_file, &baseline), IsOk()); + baseline = absl::StripAsciiWhitespace(baseline); + + std::ostringstream out; + out << "POLICY SOURCE: " << GetParam().policy_source_file << "\n"; + + ASSERT_OK_AND_ASSIGN(cel::SourcePtr source, + cel::NewSource(contents, GetParam().policy_source_file)); + std::shared_ptr policy_source = + std::make_shared(std::move(source)); + + ASSERT_OK_AND_ASSIGN( + CelPolicyParseResult parse_result, + cel::ParseYamlCelPolicy(policy_source, GetParam().parser_factory())); + + out << kBaselineSeparator; + if (parse_result.IsValid()) { + out << "PARSED POLICY:\n"; + out << parse_result.GetPolicy()->DebugString(); + } else { + ASSERT_THAT(parse_result.GetPolicy(), IsNull()); + out << kBaselineSeparator; + out << "PARSER ISSUES:\n"; + for (const auto& issue : parse_result.GetIssues()) { + out << issue.ToDisplayString(*policy_source) << "\n"; + } + } + + std::string actual(absl::StripAsciiWhitespace(out.str())); + if (actual != baseline) { + // Log the actual result to make it easier to copy/paste into the baseline + // file when updating the tests. + ABSL_LOG(INFO) << "Actual:\n" << actual; + EXPECT_EQ(actual, baseline); + } +} + +INSTANTIATE_TEST_SUITE_P( + Formats, YamlPolicyParserTest, + testing::ValuesIn({ + YamlPolicyParserTestCase{ + .policy_source_file = "cel_policy.yaml", + .baseline_file = "cel_policy_parser.baseline", + .parser_factory = GetDefaultYamlPolicyParser, + }, + YamlPolicyParserTestCase{ + .policy_source_file = "nested_rule.yaml", + .baseline_file = "nested_rule_parser.baseline", + .parser_factory = GetDefaultYamlPolicyParser, + }, + YamlPolicyParserTestCase{ + .policy_source_file = "custom_policy_format.yaml", + .baseline_file = "custom_policy_format_parser.baseline", + .parser_factory = internal::GetTestCustomYamlPolicyParser, + }, + YamlPolicyParserTestCase{ + .policy_source_file = "custom_policy_format_with_errors.yaml", + .baseline_file = "custom_policy_format_with_errors_parser.baseline", + .parser_factory = internal::GetTestCustomYamlPolicyParser, + }, + })); + +struct ParseTestCase { + std::string yaml; + std::string expected_error; +}; + +using YamlPolicyParseErrorTest = testing::TestWithParam; + +TEST_P(YamlPolicyParseErrorTest, YamlSyntaxError) { + const ParseTestCase& param = GetParam(); + ASSERT_OK_AND_ASSIGN(cel::SourcePtr source, + cel::NewSource(param.yaml, "test")); + std::shared_ptr policy_source = + std::make_shared(std::move(source)); + ASSERT_OK_AND_ASSIGN(CelPolicyParseResult parse_result, + cel::ParseYamlCelPolicy(policy_source)); + EXPECT_THAT(parse_result.FormattedIssues(), HasSubstr(param.expected_error)); +} + +std::vector GetParseTestCases() { + return { + ParseTestCase{ + .yaml = "name: \"unclosed", + .expected_error = "1:16: Invalid CEL policy YAML syntax\n" + " | name: \"unclosed\n" + " | ...............^", + }, + ParseTestCase{ + .yaml = R"yaml( ? [ John, Doe ]: age: 30 )yaml", + .expected_error = "1:22: Invalid CEL policy YAML syntax\n" + " | ? [ John, Doe ]: age: 30 \n" + " | .....................^", + }, + ParseTestCase{ + .yaml = R"yaml( invalid yaml )yaml", + .expected_error = "1:2: Policy is not a map\n" + " | invalid yaml \n" + " | .^", + }, + ParseTestCase{ + .yaml = R"yaml( + ? [1, 2, 3] + : "Prime numbers sequence" + )yaml", + .expected_error = "2:23: Policy tag is not a string\n" + " | ? [1, 2, 3]\n" + " | ......................^", + }, + ParseTestCase{ + .yaml = R"yaml( + imports: N/A + )yaml", + .expected_error = "2:28: Policy 'imports' is not a sequence\n" + " | imports: N/A\n" + " | ...........................^", + }, + ParseTestCase{ + .yaml = R"yaml( + imports: + - cel.expr.conformance + )yaml", + .expected_error = "3:21: Import is not a map\n" + " | - cel.expr.conformance\n" + " | ....................^", + }, + ParseTestCase{ + .yaml = R"yaml( + imports: + - name: + - cel.expr.conformance + )yaml", + .expected_error = "4:21: Import name is not a string\n" + " | - cel.expr.conformance\n" + " | ....................^", + }, + ParseTestCase{ + .yaml = R"yaml( + # Comment with multi-byte char: € + imports: + - name: + - cel.expr.conformance + )yaml", + .expected_error = "5:21: Import name is not a string\n" + " | - cel.expr.conformance\n" + " | ....................^", + }, + ParseTestCase{ + .yaml = R"yaml( + rule: do something + )yaml", + .expected_error = "2:25: Policy 'rule' is not a map\n" + " | rule: do something\n" + " | ........................^", + }, + ParseTestCase{ + .yaml = R"yaml( + rule: + id: + - 22 + )yaml", + .expected_error = "4:21: Policy rule 'id' is not a string\n" + " | - 22\n" + " | ....................^", + }, + ParseTestCase{ + .yaml = R"yaml( + rule: + variables: + no vars + )yaml", + .expected_error = "4:23: Policy rule 'variables' is not a sequence\n" + " | no vars\n" + " | ......................^", + }, + ParseTestCase{ + .yaml = R"yaml( + rule: + variables: + - name: + foo: bar + )yaml", + .expected_error = "5:25: Policy variable 'name' is not a string\n" + " | foo: bar\n" + " | ........................^", + }, + ParseTestCase{ + .yaml = R"yaml( + rule: + variables: + - name: test_var + expression: + - 22 + )yaml", + .expected_error = + "6:23: Policy variable 'expression' is not a string\n" + " | - 22\n" + " | ......................^", + }, + ParseTestCase{ + .yaml = R"yaml( + rule: + variables: + - name: '\u0041\u00a9\u20ac\U0001f680' + - '\u0041\u00a9\u20ac\U0001f680': name + )yaml", + .expected_error = + "5:23: Unrecognized policy variable tag: " + "\\u0041\\u00a9\\u20ac\\U0001f680\n" + " | - '\\u0041\\u00a9\\u20ac\\U0001f680': " + "name\n" + " | ......................^", + }, + ParseTestCase{ + .yaml = R"yaml( + name: test + rule: + match: + - output: 'true' + aggregate: + - output: 'true' + )yaml", + .expected_error = + "6:21: Only one of 'match' or 'aggregate' may be set in a rule\n" + " | aggregate:\n" + " | ....................^", + }, + ParseTestCase{ + .yaml = R"yaml( + name: test + rule: + aggregate: + - output: 'true' + match: + - output: 'true' + )yaml", + .expected_error = + "6:21: Only one of 'match' or 'aggregate' may be set in a rule\n" + " | match:\n" + " | ....................^", + }, + ParseTestCase{ + .yaml = R"yaml( + rule: + aggregate: illegal + )yaml", + .expected_error = "3:32: Policy rule 'aggregate' is not a sequence\n" + " | aggregate: illegal\n" + " | ...............................^", + }, + }; +} + +INSTANTIATE_TEST_SUITE_P(YamlPolicyParseErrorTest, YamlPolicyParseErrorTest, + ::testing::ValuesIn(GetParseTestCases())); + +TEST(YamlPolicyParserTest, ParseAggregateRule) { + absl::string_view yaml = R"yaml( +name: aggregate_policy +rule: + id: top_agg + aggregate: + - condition: 'x > 0' + output: '"positive"' + - condition: 'true' + rule: + match: + - condition: 'y > 0' + rule: + aggregate: + - output: '"nested_agg"' +)yaml"; + ASSERT_OK_AND_ASSIGN(cel::SourcePtr source, + cel::NewSource(yaml, "aggregate.yaml")); + std::shared_ptr policy_source = + std::make_shared(std::move(source)); + ASSERT_OK_AND_ASSIGN(CelPolicyParseResult parse_result, + cel::ParseYamlCelPolicy(policy_source)); + ASSERT_TRUE(parse_result.IsValid()) << parse_result.FormattedIssues(); + const CelPolicy* policy = parse_result.GetPolicy(); + ASSERT_NE(policy, nullptr); + EXPECT_TRUE(policy->rule().is_aggregate()); + ASSERT_EQ(policy->rule().matches().size(), 2); + EXPECT_TRUE(policy->rule().matches()[0].has_output_block()); + ASSERT_TRUE(policy->rule().matches()[1].has_rule()); + const Rule& sub_rule = policy->rule().matches()[1].rule(); + EXPECT_FALSE(sub_rule.is_aggregate()); + ASSERT_EQ(sub_rule.matches().size(), 1); + ASSERT_TRUE(sub_rule.matches()[0].has_rule()); + const Rule& nested_agg_rule = sub_rule.matches()[0].rule(); + EXPECT_TRUE(nested_agg_rule.is_aggregate()); + ASSERT_EQ(nested_agg_rule.matches().size(), 1); + EXPECT_EQ(nested_agg_rule.matches()[0].output_block().output().value(), + "\"nested_agg\""); +} + +TEST(YamlPolicyParserTest, OffsetIssueFormatting) { + // TODO(b/506179116): will need to copy the go implementation in extracting + // the source string from the YAML document instead of the interpreted string + // value to fix up error locations in folded and block literals. + std::string contents; + std::string test_file = cel::internal::ResolveRunfilesPath( + absl::StrCat(kTestPolicyFilePath, "cel_policy.yaml")); + ASSERT_THAT(cel::internal::GetFileContents(test_file, &contents), IsOk()); + + ASSERT_OK_AND_ASSIGN(cel::SourcePtr source, + cel::NewSource(contents, "cel_policy.yaml")); + std::shared_ptr policy_source = + std::make_shared(std::move(source)); + ASSERT_OK_AND_ASSIGN(CelPolicyParseResult parse_result, + cel::ParseYamlCelPolicy(policy_source)); + + ASSERT_TRUE(parse_result.IsValid()); + const CelPolicy* policy = parse_result.GetPolicy(); + + CelPolicyElementId name_id = policy->name().id(); + + CelPolicyIssue issue(name_id, 4, CelPolicyIssue::Severity::kError, + "Test error"); + + std::string formatted = issue.ToDisplayString(*policy_source); + + EXPECT_THAT(formatted, HasSubstr("ERROR: cel_policy.yaml:16:11: Test error")); + EXPECT_THAT(formatted, HasSubstr(" | name: cel_policy")); + EXPECT_THAT(formatted, HasSubstr(" | ..........^")); +} + +} // namespace +} // namespace cel diff --git a/runtime/BUILD b/runtime/BUILD index 776a8223d..00753a616 100644 --- a/runtime/BUILD +++ b/runtime/BUILD @@ -27,15 +27,14 @@ cc_library( hdrs = ["activation_interface.h"], deps = [ ":function_overload_reference", + "//base:attribute_matcher", "//base:attributes", "//common:value", "//internal:status_macros", - "//runtime/internal:attribute_matcher", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", "@com_google_protobuf//:protobuf", ], ) @@ -68,16 +67,17 @@ cc_library( ":activation_interface", ":function", ":function_overload_reference", + "//base:attribute_matcher", "//base:attributes", "//common:function_descriptor", "//common:value", "//internal:status_macros", - "//runtime/internal:attribute_matcher", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/functional:any_invocable", "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", @@ -110,6 +110,40 @@ cc_test( ], ) +cc_library( + name = "bind_proto_to_activation", + srcs = ["bind_proto_to_activation.cc"], + hdrs = ["bind_proto_to_activation.h"], + deps = [ + ":activation", + "//common:value", + "//internal:status_macros", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "bind_proto_to_activation_test", + srcs = ["bind_proto_to_activation_test.cc"], + deps = [ + ":activation", + ":bind_proto_to_activation", + "//common:value", + "//common:value_testing", + "//internal:testing", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", + "@com_google_protobuf//:protobuf", + "@com_google_protobuf//:wrappers_cc_proto", + ], +) + cc_library( name = "register_function_helper", hdrs = ["register_function_helper.h"], @@ -281,7 +315,6 @@ cc_test( "//parser", "//parser:macro_registry", "//parser:standard_macros", - "//runtime/internal:runtime_impl", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", @@ -293,6 +326,67 @@ cc_test( ], ) +cc_library( + name = "optimized_runtime_builder_factory", + srcs = ["optimized_runtime_builder_factory.cc"], + hdrs = ["optimized_runtime_builder_factory.h"], + deps = [ + ":constant_folding", + ":reference_resolver", + ":regex_precompilation", + ":runtime_builder", + ":runtime_options", + ":standard_runtime_builder_factory", + "//extensions:select_optimization", + "//extensions:set_membership_optimization", + "//internal:noop_delete", + "//internal:status_macros", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status:statusor", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "optimized_runtime_builder_factory_test", + srcs = ["optimized_runtime_builder_factory_test.cc"], + deps = [ + ":activation", + ":optimized_runtime_builder_factory", + ":runtime", + ":runtime_builder", + ":runtime_options", + "//checker:type_checker_builder", + "//checker:validation_result", + "//common:ast", + "//common:decl", + "//common:type", + "//common:value", + "//common:value_testing", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", + "//extensions:bindings_ext", + "//extensions:comprehensions_v2", + "//extensions:comprehensions_v2_functions", + "//extensions:sets_functions", + "//extensions/protobuf:enum_adapter", + "//internal:noop_delete", + "//internal:status_macros", + "//internal:testing", + "//internal:testing_descriptor_pool", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", + "@com_google_protobuf//:protobuf", + ], +) + cc_library( name = "standard_functions", srcs = ["standard_functions.cc"], @@ -539,11 +633,11 @@ cc_library( srcs = ["optional_types.cc"], hdrs = ["optional_types.h"], deps = [ + ":function", ":function_registry", ":runtime_builder", ":runtime_options", "//base:function_adapter", - "//common:casting", "//common:type", "//common:value", "//internal:casts", @@ -583,7 +677,6 @@ cc_test( "//internal:testing_descriptor_pool", "//parser", "//parser:options", - "//runtime/internal:runtime_impl", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", @@ -615,6 +708,7 @@ cc_test( ":activation", ":constant_folding", ":function_adapter", + ":optional_types", ":reference_resolver", ":regex_precompilation", ":runtime", diff --git a/runtime/activation.cc b/runtime/activation.cc index e999f7a02..2ce3eb364 100644 --- a/runtime/activation.cc +++ b/runtime/activation.cc @@ -102,14 +102,14 @@ std::vector Activation::FindFunctionOverloads( bool Activation::InsertOrAssignValue(absl::string_view name, Value value) { return values_ - .insert_or_assign(name, ValueEntry{std::move(value), absl::nullopt}) + .insert_or_assign(name, ValueEntry{std::move(value), std::nullopt}) .second; } bool Activation::InsertOrAssignValueProvider(absl::string_view name, ValueProvider provider) { return values_ - .insert_or_assign(name, ValueEntry{absl::nullopt, std::move(provider)}) + .insert_or_assign(name, ValueEntry{std::nullopt, std::move(provider)}) .second; } diff --git a/runtime/activation.h b/runtime/activation.h index 8c4fb4073..e05a0c295 100644 --- a/runtime/activation.h +++ b/runtime/activation.h @@ -23,18 +23,20 @@ #include "absl/base/nullability.h" #include "absl/container/flat_hash_map.h" #include "absl/functional/any_invocable.h" +#include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" #include "absl/types/optional.h" #include "absl/types/span.h" #include "base/attribute.h" +#include "base/attribute_matcher.h" #include "common/function_descriptor.h" #include "common/value.h" +#include "internal/status_macros.h" #include "runtime/activation_interface.h" #include "runtime/function.h" #include "runtime/function_overload_reference.h" -#include "runtime/internal/attribute_matcher.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -75,16 +77,6 @@ class Activation final : public ActivationInterface { std::vector FindFunctionOverloads( absl::string_view name) const override; - absl::Span GetUnknownAttributes() - const override { - return unknown_patterns_; - } - - absl::Span GetMissingAttributes() - const override { - return missing_patterns_; - } - // Bind a value to a named variable. // // Returns false if the entry for name was overwritten. @@ -97,20 +89,64 @@ class Activation final : public ActivationInterface { bool InsertOrAssignValueProvider(absl::string_view name, ValueProvider provider); - void AddUnknownPattern(cel::AttributePattern pattern) { - unknown_patterns_.push_back(std::move(pattern)); + absl::Status AddUnknownPattern(const cel::AttributePattern& pattern) { + // This function used to return void, thus they could not fail. We use + // upsert because it just works most of the time, especially if the caller + // provided overlapped attribute patterns. Ideally we would use insert. + return unknown_attributes_.UpsertAttribute(pattern); + } + + absl::Status SetUnknownPatterns( + absl::Span patterns) { + AttributeMatcher matcher; + for (const auto& pattern : patterns) { + // This function used to return void, thus they could not fail. We use + // upsert because it just works most of the time, especially if the caller + // provided overlapped attribute patterns. Ideally we would use insert. + CEL_RETURN_IF_ERROR(matcher.UpsertAttribute(pattern)); + } + unknown_attributes_ = std::move(matcher); + return absl::OkStatus(); } - void SetUnknownPatterns(std::vector patterns) { - unknown_patterns_ = std::move(patterns); + absl::Status AddMissingPattern(const cel::AttributePattern& pattern) { + // This function used to return void, thus they could not fail. We use + // upsert because it just works most of the time, especially if the caller + // provided overlapped attribute patterns. Ideally we would use insert. + return missing_attributes_.UpsertAttribute(pattern); } - void AddMissingPattern(cel::AttributePattern pattern) { - missing_patterns_.push_back(std::move(pattern)); + absl::Status SetMissingPatterns( + absl::Span patterns) { + AttributeMatcher matcher; + for (const auto& pattern : patterns) { + // This function used to return void, thus they could not fail. We use + // upsert because it just works most of the time, especially if the caller + // provided overlapped attribute patterns. Ideally we would use insert. + CEL_RETURN_IF_ERROR(matcher.UpsertAttribute(pattern)); + } + missing_attributes_ = std::move(matcher); + return absl::OkStatus(); } - void SetMissingPatterns(std::vector patterns) { - missing_patterns_ = std::move(patterns); + absl::Status AddKnownPattern(const cel::AttributePattern& pattern) { + // This function used to return void, thus they could not fail. We use + // upsert because it just works most of the time, especially if the caller + // provided overlapped attribute patterns. Ideally we would use insert. + return known_attributes_.UpsertAttribute(pattern); + } + + absl::Status SetKnownPatterns( + absl::Span patterns) { + AttributeMatcher matcher; + for (const auto& pattern : patterns) { + // This function used to return void, thus they could not fail. We use + // upsert because it just works most of the time, especially if the caller + // provided overlapped attribute patterns. Ideally we would use insert. + CEL_RETURN_IF_ERROR(matcher.UpsertAttribute(pattern)); + } + known_attributes_ = std::move(matcher); + return absl::OkStatus(); } // Returns true if the function was inserted (no other registered function has @@ -118,6 +154,18 @@ class Activation final : public ActivationInterface { bool InsertFunction(const cel::FunctionDescriptor& descriptor, std::unique_ptr impl); + const AttributeMatcher& GetUnknownAttributeMatcher() const final { + return unknown_attributes_; + } + + const AttributeMatcher& GetKnownAttributeMatcher() const final { + return known_attributes_; + } + + const AttributeMatcher& GetMissingAttributeMatcher() const final { + return missing_attributes_; + } + private: struct ValueEntry { // If provider is present, then access must be synchronized to maintain @@ -131,29 +179,13 @@ class Activation final : public ActivationInterface { std::unique_ptr implementation; }; - friend class runtime_internal::ActivationAttributeMatcherAccess; - - void SetAttributeMatcher(const runtime_internal::AttributeMatcher* matcher) { - attribute_matcher_ = matcher; - } - - void SetAttributeMatcher( - std::unique_ptr matcher) { - owned_attribute_matcher_ = std::move(matcher); - attribute_matcher_ = owned_attribute_matcher_.get(); - } - - const runtime_internal::AttributeMatcher* absl_nullable GetAttributeMatcher() - const override { - return attribute_matcher_; - } - friend void swap(Activation& a, Activation& b) { using std::swap; swap(a.values_, b.values_); swap(a.functions_, b.functions_); - swap(a.unknown_patterns_, b.unknown_patterns_); - swap(a.missing_patterns_, b.missing_patterns_); + swap(a.unknown_attributes_, b.unknown_attributes_); + swap(a.known_attributes_, b.known_attributes_); + swap(a.missing_attributes_, b.missing_attributes_); } // Internal getter for provided values. @@ -169,12 +201,9 @@ class Activation final : public ActivationInterface { mutable absl::Mutex mutex_; mutable absl::flat_hash_map values_; - std::vector unknown_patterns_; - std::vector missing_patterns_; - - const runtime_internal::AttributeMatcher* attribute_matcher_ = nullptr; - std::unique_ptr - owned_attribute_matcher_; + AttributeMatcher unknown_attributes_; + AttributeMatcher known_attributes_; + AttributeMatcher missing_attributes_; absl::flat_hash_map> functions_; }; diff --git a/runtime/activation_interface.h b/runtime/activation_interface.h index c589468de..7c7ae069d 100644 --- a/runtime/activation_interface.h +++ b/runtime/activation_interface.h @@ -21,12 +21,11 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "absl/types/span.h" #include "base/attribute.h" +#include "base/attribute_matcher.h" #include "common/value.h" #include "internal/status_macros.h" #include "runtime/function_overload_reference.h" -#include "runtime/internal/attribute_matcher.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -77,30 +76,37 @@ class ActivationInterface { // If an attribute (select path) encountered during evaluation matches any of // the patterns, the value will be treated as unknown and propagated in an // unknown set. + std::vector GetUnknownAttributes() const { + return GetUnknownAttributeMatcher().GetAttributes(); + } + + // Return a list of known attribute patterns. // - // The returned span must remain valid for the duration of any evaluation - // using this this activation. - virtual absl::Span GetUnknownAttributes() - const = 0; + // If an attribute (select path) encountered during evaluation matches any of + // the patterns, the value will be treated as known. + std::vector GetKnownAttributes() const { + return GetKnownAttributeMatcher().GetAttributes(); + } // Return a list of missing attribute patterns. // // If an attribute (select path) encountered during evaluation matches any of // the patterns, the value will be treated as missing and propagated as an // error. - // - // The returned span must remain valid for the duration of any evaluation - // using this activation. - virtual absl::Span GetMissingAttributes() - const = 0; - - private: - friend class runtime_internal::ActivationAttributeMatcherAccess; - - // Returns the attribute matcher for this activation. - virtual const runtime_internal::AttributeMatcher* absl_nullable - GetAttributeMatcher() const { - return nullptr; + std::vector GetMissingAttributes() const { + return GetMissingAttributeMatcher().GetAttributes(); + } + + virtual const AttributeMatcher& GetUnknownAttributeMatcher() const { + return EmptyAttributeMatcher(); + } + + virtual const cel::AttributeMatcher& GetKnownAttributeMatcher() const { + return EmptyAttributeMatcher(); + } + + virtual const AttributeMatcher& GetMissingAttributeMatcher() const { + return EmptyAttributeMatcher(); } }; diff --git a/runtime/activation_test.cc b/runtime/activation_test.cc index e6a74f027..780258464 100644 --- a/runtime/activation_test.cc +++ b/runtime/activation_test.cc @@ -39,6 +39,7 @@ namespace cel { namespace { +using ::absl_testing::IsOk; using ::absl_testing::IsOkAndHolds; using ::absl_testing::StatusIs; using testing::ElementsAre; @@ -79,7 +80,7 @@ TEST_F(ActivationTest, ValueNotFound) { EXPECT_THAT(activation.FindVariable("var1", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ActivationTest, InsertValue) { @@ -122,11 +123,11 @@ TEST_F(ActivationTest, InsertProviderForwardsNotFound) { "var1", [](absl::string_view name, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - google::protobuf::Arena* absl_nonnull) { return absl::nullopt; })); + google::protobuf::Arena* absl_nonnull) { return std::nullopt; })); EXPECT_THAT(activation.FindVariable("var1", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ActivationTest, InsertProviderForwardsStatus) { @@ -212,11 +213,13 @@ TEST_F(ActivationTest, ValuesAndProvidersShareNamespace) { TEST_F(ActivationTest, SetUnknownAttributes) { Activation activation; - activation.SetUnknownPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); + ASSERT_THAT( + activation.SetUnknownPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); EXPECT_THAT( activation.GetUnknownAttributes(), @@ -229,12 +232,14 @@ TEST_F(ActivationTest, SetUnknownAttributes) { TEST_F(ActivationTest, ClearUnknownAttributes) { Activation activation; - activation.SetUnknownPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); - activation.SetUnknownPatterns({}); + ASSERT_THAT( + activation.SetUnknownPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); + ASSERT_THAT(activation.SetUnknownPatterns({}), IsOk()); EXPECT_THAT(activation.GetUnknownAttributes(), IsEmpty()); } @@ -242,11 +247,13 @@ TEST_F(ActivationTest, ClearUnknownAttributes) { TEST_F(ActivationTest, SetMissingAttributes) { Activation activation; - activation.SetMissingPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); + ASSERT_THAT( + activation.SetMissingPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); EXPECT_THAT( activation.GetMissingAttributes(), @@ -259,12 +266,14 @@ TEST_F(ActivationTest, SetMissingAttributes) { TEST_F(ActivationTest, ClearMissingAttributes) { Activation activation; - activation.SetMissingPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); - activation.SetMissingPatterns({}); + ASSERT_THAT( + activation.SetMissingPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); + ASSERT_THAT(activation.SetMissingPatterns({}), IsOk()); EXPECT_THAT(activation.GetMissingAttributes(), IsEmpty()); } @@ -326,17 +335,21 @@ TEST_F(ActivationTest, MoveAssignment) { "val_provided", [](absl::string_view name, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, google::protobuf::Arena* absl_nonnull) - -> absl::StatusOr> { return IntValue(42); })); - moved_from.SetUnknownPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); - moved_from.SetMissingPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); + -> absl::StatusOr> { return IntValue(42); })); + ASSERT_THAT( + moved_from.SetUnknownPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); + ASSERT_THAT( + moved_from.SetMissingPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); Activation moved_to; moved_to = std::move(moved_from); @@ -355,10 +368,10 @@ TEST_F(ActivationTest, MoveAssignment) { // NOLINTBEGIN(bugprone-use-after-move) EXPECT_THAT(moved_from.FindVariable("val", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(moved_from.FindVariable("val_provided", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(moved_from.FindFunctionOverloads("Fn"), SizeIs(0)); EXPECT_THAT(moved_from.GetUnknownAttributes(), SizeIs(0)); EXPECT_THAT(moved_from.GetMissingAttributes(), SizeIs(0)); @@ -377,17 +390,21 @@ TEST_F(ActivationTest, MoveCtor) { "val_provided", [](absl::string_view name, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, google::protobuf::Arena* absl_nonnull) - -> absl::StatusOr> { return IntValue(42); })); - moved_from.SetUnknownPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); - moved_from.SetMissingPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); + -> absl::StatusOr> { return IntValue(42); })); + ASSERT_THAT( + moved_from.SetUnknownPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); + ASSERT_THAT( + moved_from.SetMissingPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); Activation moved_to = std::move(moved_from); @@ -405,10 +422,10 @@ TEST_F(ActivationTest, MoveCtor) { // NOLINTBEGIN(bugprone-use-after-move) EXPECT_THAT(moved_from.FindVariable("val", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(moved_from.FindVariable("val_provided", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(moved_from.FindFunctionOverloads("Fn"), SizeIs(0)); EXPECT_THAT(moved_from.GetUnknownAttributes(), SizeIs(0)); EXPECT_THAT(moved_from.GetMissingAttributes(), SizeIs(0)); diff --git a/runtime/bind_proto_to_activation.cc b/runtime/bind_proto_to_activation.cc new file mode 100644 index 000000000..011feea7c --- /dev/null +++ b/runtime/bind_proto_to_activation.cc @@ -0,0 +1,92 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "runtime/bind_proto_to_activation.h" + +#include + +#include "absl/base/nullability.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "common/value.h" +#include "internal/status_macros.h" +#include "runtime/activation.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" + +namespace cel::runtime_internal { + +namespace { + +using ::google::protobuf::Descriptor; + +absl::StatusOr ShouldBindField( + const google::protobuf::FieldDescriptor* field_desc, const StructValue& struct_value, + BindProtoUnsetFieldBehavior unset_field_behavior) { + if (unset_field_behavior == BindProtoUnsetFieldBehavior::kBindDefaultValue || + field_desc->is_repeated()) { + return true; + } + return struct_value.HasFieldByNumber(field_desc->number()); +} + +absl::StatusOr GetFieldValue( + const google::protobuf::FieldDescriptor* field_desc, const StructValue& struct_value, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) { + // Special case unset any. + if (field_desc->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE && + field_desc->message_type()->well_known_type() == + Descriptor::WELLKNOWNTYPE_ANY) { + CEL_ASSIGN_OR_RETURN(bool present, + struct_value.HasFieldByNumber(field_desc->number())); + if (!present) { + return NullValue(); + } + } + + return struct_value.GetFieldByNumber(field_desc->number(), descriptor_pool, + message_factory, arena); +} + +} // namespace + +absl::Status BindProtoToActivation( + const Descriptor& descriptor, const StructValue& struct_value, + BindProtoUnsetFieldBehavior unset_field_behavior, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { + for (int i = 0; i < descriptor.field_count(); i++) { + const google::protobuf::FieldDescriptor* field_desc = descriptor.field(i); + CEL_ASSIGN_OR_RETURN( + bool should_bind, + ShouldBindField(field_desc, struct_value, unset_field_behavior)); + if (!should_bind) { + continue; + } + + CEL_ASSIGN_OR_RETURN( + Value field, GetFieldValue(field_desc, struct_value, descriptor_pool, + message_factory, arena)); + + activation->InsertOrAssignValue(field_desc->name(), std::move(field)); + } + + return absl::OkStatus(); +} + +} // namespace cel::runtime_internal diff --git a/runtime/bind_proto_to_activation.h b/runtime/bind_proto_to_activation.h new file mode 100644 index 000000000..03476b386 --- /dev/null +++ b/runtime/bind_proto_to_activation.h @@ -0,0 +1,174 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_RUNTIME_BIND_PROTO_TO_ACTIVATION_H_ +#define THIRD_PARTY_CEL_CPP_RUNTIME_BIND_PROTO_TO_ACTIVATION_H_ + +#include + +#include "absl/base/nullability.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "common/value.h" +#include "runtime/activation.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" + +namespace cel { + +// Option for handling unset fields on the context proto. +enum class BindProtoUnsetFieldBehavior { + // Bind the message defined default or zero value. + kBindDefaultValue, + // Skip binding unset fields, no value is bound for the corresponding + // variable. + kSkip +}; + +namespace runtime_internal { + +// Implements binding provided the context message has already +// been adapted to a suitable struct value. +absl::Status BindProtoToActivation( + const google::protobuf::Descriptor& descriptor, const StructValue& struct_value, + BindProtoUnsetFieldBehavior unset_field_behavior, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation); + +template +absl::Status BindProtoToActivationImpl( + const T& context, BindProtoUnsetFieldBehavior unset_field_behavior, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { + static_assert(std::is_base_of_v); + + Value parent; + if constexpr (kBorrow) { + parent = Value::WrapMessageUnsafe(&context, descriptor_pool, + message_factory, arena); + } else { + parent = + Value::FromMessage(context, descriptor_pool, message_factory, arena); + } + + if (!parent.IsStruct()) { + return absl::InvalidArgumentError( + absl::StrCat("context is a well-known type: ", context.GetTypeName())); + } + StructValue struct_value = parent.GetStruct(); + + const google::protobuf::Descriptor* descriptor = context.GetDescriptor(); + ABSL_DCHECK(descriptor != nullptr); + if (descriptor == nullptr) { + // Generally not possible, but don't crash in case of a misbehaving + // implementation in normal builds. + return absl::InvalidArgumentError( + absl::StrCat("context missing descriptor: ", context.GetTypeName())); + } + + return BindProtoToActivation(*descriptor, struct_value, unset_field_behavior, + descriptor_pool, message_factory, arena, + activation); +} + +} // namespace runtime_internal + +// Utility method, that takes a protobuf Message and interprets it as a +// namespace, binding its fields to Activation. This is often referred to as a +// context message. +// +// Field names and values become respective names and values of parameters +// bound to the Activation object. +// Example: +// Assume we have a protobuf message of type: +// message Person { +// int age = 1; +// string name = 2; +// } +// +// The sample code snippet will look as follows: +// +// Person person; +// person.set_name("John Doe"); +// person.age(42); +// +// CEL_RETURN_IF_ERROR(BindProtoToActivation(person, value_factory, +// activation)); +// +// After this snippet, activation will have two parameters bound: +// "name", with string value of "John Doe" +// "age", with int value of 42. +// +// The default behavior for unset fields is to skip them. E.g. if the name field +// is not set on the Person message, it will not be bound in to the activation. +// BindProtoUnsetFieldBehavior::kBindDefault, will bind the cc proto api default +// for the field (either an explicit default value or a type specific default). +// +// For repeated fields, an unset field is bound as an empty list. +template +absl::Status BindProtoToActivation( + const T& context, BindProtoUnsetFieldBehavior unset_field_behavior, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { + return runtime_internal::BindProtoToActivationImpl( + context, unset_field_behavior, descriptor_pool, message_factory, arena, + activation); +} + +template +absl::Status BindProtoToActivation( + const T& context, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { + return BindProtoToActivation(context, BindProtoUnsetFieldBehavior::kSkip, + descriptor_pool, message_factory, arena, + activation); +} + +// Like `BindProtoToActivation`, but uses `Value::WrapMessageUnsafe` to borrow +// from `context` rather than copying fields to `arena`. +// +// Requires the caller to keep the context message valid as long as the +// activation or any derived value. +template +absl::Status BindProtoViewToActivation( + const T& context, BindProtoUnsetFieldBehavior unset_field_behavior, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { + return runtime_internal::BindProtoToActivationImpl( + context, unset_field_behavior, descriptor_pool, message_factory, arena, + activation); +} + +template +absl::Status BindProtoViewToActivation( + const T& context, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { + return BindProtoViewToActivation(context, BindProtoUnsetFieldBehavior::kSkip, + descriptor_pool, message_factory, arena, + activation); +} + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_RUNTIME_BIND_PROTO_TO_ACTIVATION_H_ diff --git a/runtime/bind_proto_to_activation_test.cc b/runtime/bind_proto_to_activation_test.cc new file mode 100644 index 000000000..e28fbef1b --- /dev/null +++ b/runtime/bind_proto_to_activation_test.cc @@ -0,0 +1,278 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "runtime/bind_proto_to_activation.h" + +#include + +#include "google/protobuf/wrappers.pb.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "common/value.h" +#include "common/value_testing.h" +#include "internal/testing.h" +#include "runtime/activation.h" +#include "cel/expr/conformance/proto2/test_all_types.pb.h" +#include "google/protobuf/arena.h" + +namespace cel { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::IsOkAndHolds; +using ::absl_testing::StatusIs; +using ::cel::expr::conformance::proto2::TestAllTypes; +using ::cel::test::IntValueIs; +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::Optional; + +using BindProtoToActivationTest = common_internal::ValueTest<>; + +TEST_F(BindProtoToActivationTest, BindProtoToActivation) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("single_int64", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(123)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationWktUnsupported) { + google::protobuf::Int64Value int64_value; + int64_value.set_value(123); + Activation activation; + + EXPECT_THAT(BindProtoToActivation(int64_value, descriptor_pool(), + message_factory(), arena(), &activation), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("google.protobuf.Int64Value"))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationSkip) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("single_int32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT(activation.FindVariable("single_sint32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationDefault) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT( + BindProtoToActivation( + test_all_types, BindProtoUnsetFieldBehavior::kBindDefaultValue, + descriptor_pool(), message_factory(), arena(), &activation), + IsOk()); + + // from test_all_types.proto + // optional int32 single_int32 = 1 [default = -32]; + EXPECT_THAT(activation.FindVariable("single_int32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(-32)))); + EXPECT_THAT(activation.FindVariable("single_sint32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(0)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationDefaultAny) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT( + BindProtoToActivation( + test_all_types, BindProtoUnsetFieldBehavior::kBindDefaultValue, + descriptor_pool(), message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("single_any", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(test::IsNullValue()))); +} + +MATCHER_P(IsListValueOfSize, size, "") { + const Value& v = arg; + + auto value = v.AsList(); + if (!value) { + return false; + } + auto s = value->Size(); + return s.ok() && *s == size; +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationRepeated) { + TestAllTypes test_all_types; + test_all_types.add_repeated_int64(123); + test_all_types.add_repeated_int64(456); + test_all_types.add_repeated_int64(789); + + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("repeated_int64", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IsListValueOfSize(3)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationRepeatedEmpty) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("repeated_int32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IsListValueOfSize(0)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationRepeatedComplex) { + TestAllTypes test_all_types; + auto* nested = test_all_types.add_repeated_nested_message(); + nested->set_bb(123); + nested = test_all_types.add_repeated_nested_message(); + nested->set_bb(456); + nested = test_all_types.add_repeated_nested_message(); + nested->set_bb(789); + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT( + activation.FindVariable("repeated_nested_message", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IsListValueOfSize(3)))); +} + +MATCHER_P(IsMapValueOfSize, size, "") { + const Value& v = arg; + + auto value = v.AsMap(); + if (!value) { + return false; + } + auto s = value->Size(); + return s.ok() && *s == size; +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationMap) { + TestAllTypes test_all_types; + (*test_all_types.mutable_map_int64_int64())[1] = 2; + (*test_all_types.mutable_map_int64_int64())[2] = 4; + + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("map_int64_int64", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IsMapValueOfSize(2)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationMapEmpty) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("map_int32_int32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IsMapValueOfSize(0)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationMapComplex) { + TestAllTypes test_all_types; + TestAllTypes::NestedMessage value; + value.set_bb(42); + (*test_all_types.mutable_map_int64_message())[1] = value; + (*test_all_types.mutable_map_int64_message())[2] = value; + + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("map_int64_message", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IsMapValueOfSize(2)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoViewToActivation) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT( + BindProtoViewToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("single_int64", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(123)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoViewToActivationDefault) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT( + BindProtoViewToActivation( + test_all_types, BindProtoUnsetFieldBehavior::kBindDefaultValue, + descriptor_pool(), message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("single_int32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(-32)))); + EXPECT_THAT(activation.FindVariable("single_sint32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(0)))); +} + +} // namespace +} // namespace cel diff --git a/runtime/constant_folding_test.cc b/runtime/constant_folding_test.cc index c59d5602a..0770039f3 100644 --- a/runtime/constant_folding_test.cc +++ b/runtime/constant_folding_test.cc @@ -87,9 +87,11 @@ TEST_P(ConstantFoldingExtTest, Runner) { const StringValue&>:: RegisterGlobalOverload( "prepend", - [](const StringValue& value, const StringValue& prefix) { - return StringValue( - absl::StrCat(prefix.ToString(), value.ToString())); + [](const StringValue& value, const StringValue& prefix, + const Function::InvokeContext& context) { + return StringValue::From( + absl::StrCat(prefix.ToString(), value.ToString()), + context.arena()); }, builder.function_registry()); ASSERT_THAT(status, IsOk()); @@ -150,9 +152,11 @@ TEST(ConstantFoldingExtTest, LazyFunctionNotFolded) { BinaryFunctionAdapter, const StringValue&, const StringValue&>; auto fn = FunctionAdapter::WrapFunction( - [&call_count](const StringValue& value, const StringValue& prefix) { + [&call_count](const StringValue& value, const StringValue& prefix, + const Function::InvokeContext& context) { call_count++; - return StringValue(absl::StrCat(prefix.ToString(), value.ToString())); + return StringValue::From( + absl::StrCat(prefix.ToString(), value.ToString()), context.arena()); }); FunctionDescriptor descriptor = FunctionAdapter::CreateDescriptor( "lazy_prepend", /*receiver_style=*/false); @@ -189,19 +193,21 @@ TEST(ConstantFoldingExtTest, ContextualFunctionNotFolded) { internal::GetTestingDescriptorPool(), options)); int call_count = 0; - auto status = BinaryFunctionAdapter< - absl::StatusOr, const StringValue&, - const StringValue&>::Register("contextual_prepend", - /*receiver_style=*/false, - [&call_count](const StringValue& value, - const StringValue& prefix) { - call_count++; - return StringValue(absl::StrCat( - prefix.ToString(), value.ToString())); - }, - builder.function_registry(), - {/*.is_strict=*/true, - /*is_contextual=*/true}); + auto status = BinaryFunctionAdapter, const StringValue&, + const StringValue&>:: + Register( + "contextual_prepend", + /*receiver_style=*/false, + [&call_count](const StringValue& value, const StringValue& prefix, + const Function::InvokeContext& context) { + call_count++; + return StringValue::From( + absl::StrCat(prefix.ToString(), value.ToString()), + context.arena()); + }, + builder.function_registry(), + {/*.is_strict=*/true, + /*is_contextual=*/true}); ASSERT_THAT(status, IsOk()); ASSERT_THAT(EnableConstantFolding(builder), IsOk()); diff --git a/runtime/embedder_context_test.cc b/runtime/embedder_context_test.cc index d8cbbb736..4de80f5d3 100644 --- a/runtime/embedder_context_test.cc +++ b/runtime/embedder_context_test.cc @@ -29,11 +29,11 @@ TEST(EmbedderContextTest, From) { struct TestScope {}; EmbedderContext context = EmbedderContext::From(int64_t{42}); EXPECT_THAT((context.Get()), Optional(42)); - EXPECT_EQ((context.Get()), absl::nullopt); + EXPECT_EQ((context.Get()), std::nullopt); EmbedderContext context2 = EmbedderContext::From(uint64_t{42}); EXPECT_THAT((context2.Get()), Optional(42)); - EXPECT_EQ((context2.Get()), absl::nullopt); + EXPECT_EQ((context2.Get()), std::nullopt); // Side effect, but checking that we keep a dense range. EXPECT_EQ(cel::TypeIdInSet::Size(), 2); @@ -47,7 +47,7 @@ TEST(EmbedderContextTest, FromOutOfLine) { EXPECT_THAT((context.Get()), Optional(42)); EXPECT_THAT((context.Get()), Optional(43)); EXPECT_THAT((context.Get()), Optional(44)); - EXPECT_EQ((context.Get()), absl::nullopt); + EXPECT_EQ((context.Get()), std::nullopt); // Note: Referencing a type not intended to be stored will still reserve a // slot in the TypeIdInSet. @@ -76,7 +76,7 @@ TEST(EmbedderContextTest, FromPtrs) { TEST(EmbedderContextTest, FromDefaultScope) { EmbedderContext context = EmbedderContext::From(int64_t{42}); EXPECT_THAT((context.Get()), Optional(42)); - EXPECT_EQ((context.Get()), absl::nullopt); + EXPECT_EQ((context.Get()), std::nullopt); } // These death assertions are only enabled when compiled in debug mode. diff --git a/runtime/function_adapter_test.cc b/runtime/function_adapter_test.cc index 910020fdf..5895df87e 100644 --- a/runtime/function_adapter_test.cc +++ b/runtime/function_adapter_test.cc @@ -162,13 +162,14 @@ TEST_F(FunctionAdapterTest, UnaryFunctionAdapterWrapFunctionDuration) { TEST_F(FunctionAdapterTest, UnaryFunctionAdapterWrapFunctionString) { using FunctionAdapter = UnaryFunctionAdapter; - std::unique_ptr wrapped = - FunctionAdapter::WrapFunction([](const StringValue& x) -> StringValue { - return StringValue("pre_" + x.ToString()); + std::unique_ptr wrapped = FunctionAdapter::WrapFunction( + [](const StringValue& x, + const Function::InvokeContext& context) -> StringValue { + return StringValue::From("pre_" + x.ToString(), context.arena()); }); std::vector args; - args.emplace_back() = StringValue("string"); + args.emplace_back() = StringValue::WrapUnsafe("string"); ASSERT_OK_AND_ASSIGN(auto result, wrapped->Invoke(args, test_invoke_context())); @@ -178,13 +179,14 @@ TEST_F(FunctionAdapterTest, UnaryFunctionAdapterWrapFunctionString) { TEST_F(FunctionAdapterTest, UnaryFunctionAdapterWrapFunctionBytes) { using FunctionAdapter = UnaryFunctionAdapter; - std::unique_ptr wrapped = - FunctionAdapter::WrapFunction([](const BytesValue& x) -> BytesValue { - return BytesValue("pre_" + x.ToString()); + std::unique_ptr wrapped = FunctionAdapter::WrapFunction( + [](const BytesValue& x, + const Function::InvokeContext& context) -> BytesValue { + return BytesValue::From("pre_" + x.ToString(), context.arena()); }); std::vector args; - args.emplace_back() = BytesValue("bytes"); + args.emplace_back() = BytesValue::WrapUnsafe("bytes"); ASSERT_OK_AND_ASSIGN(auto result, wrapped->Invoke(args, test_invoke_context())); @@ -208,8 +210,9 @@ TEST_F(FunctionAdapterTest, UnaryFunctionAdapterWrapFunctionAny) { TEST_F(FunctionAdapterTest, UnaryFunctionAdapterWrapFunctionReturnError) { using FunctionAdapter = UnaryFunctionAdapter; std::unique_ptr wrapped = - FunctionAdapter::WrapFunction([](uint64_t x) -> Value { - return ErrorValue(absl::InvalidArgumentError("test_error")); + FunctionAdapter::WrapFunction([this](uint64_t x) -> Value { + return ErrorValue::From(absl::InvalidArgumentError("test_error"), + arena()); }); std::vector args{UintValue(44)}; @@ -480,14 +483,15 @@ TEST_F(FunctionAdapterTest, BinaryFunctionAdapterWrapFunctionString) { BinaryFunctionAdapter, const StringValue&, const StringValue&>; std::unique_ptr wrapped = FunctionAdapter::WrapFunction( - [](const StringValue& x, - const StringValue& y) -> absl::StatusOr { - return StringValue(x.ToString() + y.ToString()); + [](const StringValue& x, const StringValue& y, + const Function::InvokeContext& context) + -> absl::StatusOr { + return StringValue::From(x.ToString() + y.ToString(), context.arena()); }); std::vector args; - args.emplace_back() = StringValue("abc"); - args.emplace_back() = StringValue("def"); + args.emplace_back() = StringValue::WrapUnsafe("abc"); + args.emplace_back() = StringValue::WrapUnsafe("def"); ASSERT_OK_AND_ASSIGN(auto result, wrapped->Invoke(args, test_invoke_context())); @@ -501,14 +505,14 @@ TEST_F(FunctionAdapterTest, BinaryFunctionAdapterWrapFunctionBytes) { BinaryFunctionAdapter, const BytesValue&, const BytesValue&>; std::unique_ptr wrapped = FunctionAdapter::WrapFunction( - [](const BytesValue& x, - const BytesValue& y) -> absl::StatusOr { - return BytesValue(x.ToString() + y.ToString()); + [](const BytesValue& x, const BytesValue& y, + const Function::InvokeContext& context) -> absl::StatusOr { + return BytesValue::From(x.ToString() + y.ToString(), context.arena()); }); std::vector args; - args.emplace_back() = BytesValue("abc"); - args.emplace_back() = BytesValue("def"); + args.emplace_back() = BytesValue::WrapUnsafe("abc"); + args.emplace_back() = BytesValue::WrapUnsafe("def"); ASSERT_OK_AND_ASSIGN(auto result, wrapped->Invoke(args, test_invoke_context())); @@ -536,8 +540,9 @@ TEST_F(FunctionAdapterTest, BinaryFunctionAdapterWrapFunctionAny) { TEST_F(FunctionAdapterTest, BinaryFunctionAdapterWrapFunctionReturnError) { using FunctionAdapter = BinaryFunctionAdapter; std::unique_ptr wrapped = - FunctionAdapter::WrapFunction([](int64_t x, uint64_t y) -> Value { - return ErrorValue(absl::InvalidArgumentError("test_error")); + FunctionAdapter::WrapFunction([this](int64_t x, uint64_t y) -> Value { + return ErrorValue::From(absl::InvalidArgumentError("test_error"), + arena()); }); std::vector args{IntValue(44), UintValue(44)}; @@ -712,7 +717,7 @@ TEST_F(FunctionAdapterTest, NaryFunctionAdapterCreateDescriptor0Args) { TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction0Args) { std::unique_ptr fn = NullaryFunctionAdapter>::WrapFunction( - []() { return StringValue("abc"); }); + []() { return StringValue::WrapUnsafe("abc"); }); ASSERT_OK_AND_ASSIGN(auto result, fn->Invoke({}, descriptor_pool(), message_factory(), arena())); @@ -734,16 +739,18 @@ TEST_F(FunctionAdapterTest, NaryFunctionAdapterCreateDescriptor3Args) { TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction3Args) { std::unique_ptr fn = NaryFunctionAdapter< - absl::StatusOr, int64_t, bool, - const StringValue&>::WrapFunction([](int64_t int_val, bool bool_val, - const StringValue& string_val) - -> absl::StatusOr { - return StringValue(absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), - "_", string_val.ToString())); - }); + absl::StatusOr, int64_t, bool, const StringValue&>:: + WrapFunction( + [](int64_t int_val, bool bool_val, const StringValue& string_val, + const Function::InvokeContext& context) -> absl::StatusOr { + return StringValue::From( + absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), "_", + string_val.ToString()), + context.arena()); + }); std::vector args{IntValue(42), BoolValue(false)}; - args.emplace_back() = StringValue("abcd"); + args.emplace_back() = StringValue::WrapUnsafe("abcd"); ASSERT_OK_AND_ASSIGN(auto result, fn->Invoke(args, descriptor_pool(), message_factory(), arena())); ASSERT_TRUE(result->Is()); @@ -752,13 +759,15 @@ TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction3Args) { TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction3ArgsBadArgType) { std::unique_ptr fn = NaryFunctionAdapter< - absl::StatusOr, int64_t, bool, - const StringValue&>::WrapFunction([](int64_t int_val, bool bool_val, - const StringValue& string_val) - -> absl::StatusOr { - return StringValue(absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), - "_", string_val.ToString())); - }); + absl::StatusOr, int64_t, bool, const StringValue&>:: + WrapFunction( + [](int64_t int_val, bool bool_val, const StringValue& string_val, + const Function::InvokeContext& context) -> absl::StatusOr { + return StringValue::From( + absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), "_", + string_val.ToString()), + context.arena()); + }); std::vector args{IntValue(42), BoolValue(false)}; args.emplace_back() = TimestampValue(absl::UnixEpoch()); @@ -769,13 +778,15 @@ TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction3ArgsBadArgType) { TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction3ArgsBadArgCount) { std::unique_ptr fn = NaryFunctionAdapter< - absl::StatusOr, int64_t, bool, - const StringValue&>::WrapFunction([](int64_t int_val, bool bool_val, - const StringValue& string_val) - -> absl::StatusOr { - return StringValue(absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), - "_", string_val.ToString())); - }); + absl::StatusOr, int64_t, bool, const StringValue&>:: + WrapFunction( + [](int64_t int_val, bool bool_val, const StringValue& string_val, + const Function::InvokeContext& context) -> absl::StatusOr { + return StringValue::From( + absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), "_", + string_val.ToString()), + context.arena()); + }); std::vector args{IntValue(42), BoolValue(false)}; EXPECT_THAT(fn->Invoke(args, test_invoke_context()), @@ -802,15 +813,17 @@ TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction5Args) { absl::StatusOr, int64_t, bool, const StringValue&, int64_t, int64_t>::WrapFunction([](int64_t int_val, bool bool_val, const StringValue& string_val, - int64_t extra_arg, - int64_t extra_arg2) -> absl::StatusOr { - return StringValue(absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), - "_", string_val.ToString(), "_", extra_arg, - "_", extra_arg2)); + int64_t extra_arg, int64_t extra_arg2, + const Function::InvokeContext& context) + -> absl::StatusOr { + return StringValue::From( + absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), "_", + string_val.ToString(), "_", extra_arg, "_", extra_arg2), + context.arena()); }); std::vector args{IntValue(42), BoolValue(false)}; - args.emplace_back() = StringValue("abcd"); + args.emplace_back() = StringValue::WrapUnsafe("abcd"); args.push_back(IntValue(123)); args.push_back(IntValue(456)); ASSERT_OK_AND_ASSIGN(auto result, fn->Invoke(args, descriptor_pool(), @@ -824,12 +837,15 @@ TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction5ArgsBadArgType) { absl::StatusOr, int64_t, bool, const StringValue&, int64_t, int64_t>::WrapFunction([](int64_t int_val, bool bool_val, const StringValue& string_val, - int64_t extra_arg, - int64_t extra_arg2) -> absl::StatusOr { + int64_t extra_arg, int64_t extra_arg2, + const Function::InvokeContext& context) + -> absl::StatusOr { static_cast(extra_arg); static_cast(extra_arg2); - return StringValue(absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), - "_", string_val.ToString())); + return StringValue::From( + absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), "_", + string_val.ToString()), + context.arena()); }); std::vector args{IntValue(42), BoolValue(false)}; @@ -846,12 +862,15 @@ TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction5ArgsBadArgCount) { absl::StatusOr, int64_t, bool, const StringValue&, int64_t, int64_t>::WrapFunction([](int64_t int_val, bool bool_val, const StringValue& string_val, - int64_t extra_arg, - int64_t extra_arg2) -> absl::StatusOr { + int64_t extra_arg, int64_t extra_arg2, + const Function::InvokeContext& context) + -> absl::StatusOr { static_cast(extra_arg); static_cast(extra_arg2); - return StringValue(absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), - "_", string_val.ToString())); + return StringValue::From( + absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), "_", + string_val.ToString()), + context.arena()); }); std::vector args{IntValue(42), BoolValue(false)}; diff --git a/runtime/function_registry.cc b/runtime/function_registry.cc index ac1e53eb5..b1f08e00a 100644 --- a/runtime/function_registry.cc +++ b/runtime/function_registry.cc @@ -44,14 +44,13 @@ class ActivationFunctionProviderImpl public: ActivationFunctionProviderImpl() = default; - absl::StatusOr> GetFunction( + absl::StatusOr> GetFunction( const cel::FunctionDescriptor& descriptor, const cel::ActivationInterface& activation) const override { std::vector overloads = activation.FindFunctionOverloads(descriptor.name()); - absl::optional matching_overload = - absl::nullopt; + std::optional matching_overload = std::nullopt; for (const auto& overload : overloads) { if (overload.descriptor.ShapeMatches(descriptor)) { diff --git a/runtime/function_registry_test.cc b/runtime/function_registry_test.cc index af7f5bc06..40445f43a 100644 --- a/runtime/function_registry_test.cc +++ b/runtime/function_registry_test.cc @@ -120,11 +120,11 @@ TEST(FunctionRegistryTest, DefaultLazyProviderNoOverloadFound) { ASSERT_THAT(providers, SizeIs(1)); const FunctionProvider& provider = providers[0].provider; ASSERT_OK_AND_ASSIGN( - absl::optional func, + std::optional func, provider.GetFunction({"LazyFunc", false, {cel::Kind::kInt64}}, activation)); - EXPECT_EQ(func, absl::nullopt); + EXPECT_EQ(func, std::nullopt); } TEST(FunctionRegistryTest, DefaultLazyProviderReturnsImpl) { @@ -146,7 +146,7 @@ TEST(FunctionRegistryTest, DefaultLazyProviderReturnsImpl) { ASSERT_THAT(providers, SizeIs(1)); const FunctionProvider& provider = providers[0].provider; ASSERT_OK_AND_ASSIGN( - absl::optional func, + std::optional func, provider.GetFunction( FunctionDescriptor("LazyFunction", false, {Kind::kInt}), activation)); diff --git a/runtime/internal/BUILD b/runtime/internal/BUILD index 1223ff6d1..42a885958 100644 --- a/runtime/internal/BUILD +++ b/runtime/internal/BUILD @@ -62,11 +62,7 @@ cc_library( "//common:native_type", "//common:value", "//eval/compiler:flat_expr_builder", - "//eval/eval:attribute_trail", - "//eval/eval:comprehension_slots", - "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", - "//internal:casts", "//internal:status_macros", "//internal:well_known_types", "//runtime", @@ -87,15 +83,15 @@ cc_library( srcs = ["convert_constant.cc"], hdrs = ["convert_constant.h"], deps = [ - "//common:allocator", - "//common:ast", "//common:constant", "//common:value", "//eval/internal:errors", + "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/time", "@com_google_absl//absl/types:variant", + "@com_google_protobuf//:protobuf", ], ) @@ -138,7 +134,6 @@ cc_library( "function_adapter.h", ], deps = [ - "//common:casting", "//common:kind", "//common:value", "@com_google_absl//absl/status", @@ -159,6 +154,7 @@ cc_test( "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/time", + "@com_google_protobuf//:protobuf", ], ) @@ -174,16 +170,40 @@ cc_library( "//internal:testing_message_factory", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:die_if_null", "@com_google_protobuf//:protobuf", ], ) cc_library( name = "legacy_runtime_type_provider", + srcs = ["legacy_runtime_type_provider.cc"], hdrs = ["legacy_runtime_type_provider.h"], deps = [ - "//eval/public/structs:protobuf_descriptor_type_provider", + ":runtime_type_provider", + "//common:legacy_value", + "//common:type", + "//common:value", + "//internal:status_macros", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "legacy_runtime_type_provider_test", + srcs = ["legacy_runtime_type_provider_test.cc"], + deps = [ + ":legacy_runtime_type_provider", + ":runtime_type_provider", + "//common:type", + "//common:value", + "//internal:testing", + "//internal:testing_descriptor_pool", + "//internal:testing_message_factory", + "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", "@com_google_protobuf//:protobuf", ], ) @@ -193,9 +213,9 @@ cc_library( srcs = ["runtime_type_provider.cc"], hdrs = ["runtime_type_provider.h"], deps = [ + "//common:descriptor_pool_type_introspector", "//common:type", "//common:value", - "//internal:status_macros", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", @@ -206,21 +226,3 @@ cc_library( "@com_google_protobuf//:protobuf", ], ) - -cc_library( - name = "attribute_matcher", - hdrs = ["attribute_matcher.h"], - deps = ["//base:attributes"], -) - -cc_library( - name = "activation_attribute_matcher_access", - srcs = ["activation_attribute_matcher_access.cc"], - hdrs = ["activation_attribute_matcher_access.h"], - deps = [ - ":attribute_matcher", - "//eval/public:activation", - "//runtime:activation", - "@com_google_absl//absl/base:nullability", - ], -) diff --git a/runtime/internal/activation_attribute_matcher_access.cc b/runtime/internal/activation_attribute_matcher_access.cc deleted file mode 100644 index 7d358ba23..000000000 --- a/runtime/internal/activation_attribute_matcher_access.cc +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "runtime/internal/activation_attribute_matcher_access.h" - -#include -#include - -#include "absl/base/nullability.h" -#include "eval/public/activation.h" -#include "runtime/activation.h" -#include "runtime/internal/attribute_matcher.h" - -namespace cel::runtime_internal { - -void ActivationAttributeMatcherAccess::SetAttributeMatcher( - google::api::expr::runtime::Activation& activation, - const AttributeMatcher* matcher) { - activation.SetAttributeMatcher(matcher); -} - -void ActivationAttributeMatcherAccess::SetAttributeMatcher( - google::api::expr::runtime::Activation& activation, - std::unique_ptr matcher) { - activation.SetAttributeMatcher(std::move(matcher)); -} - -const AttributeMatcher* absl_nullable -ActivationAttributeMatcherAccess::GetAttributeMatcher( - const google::api::expr::runtime::BaseActivation& activation) { - return activation.GetAttributeMatcher(); -} - -void ActivationAttributeMatcherAccess::SetAttributeMatcher( - Activation& activation, const AttributeMatcher* matcher) { - activation.SetAttributeMatcher(matcher); -} - -void ActivationAttributeMatcherAccess::SetAttributeMatcher( - Activation& activation, std::unique_ptr matcher) { - activation.SetAttributeMatcher(std::move(matcher)); -} - -const AttributeMatcher* absl_nullable -ActivationAttributeMatcherAccess::GetAttributeMatcher( - const ActivationInterface& activation) { - return activation.GetAttributeMatcher(); -} - -} // namespace cel::runtime_internal diff --git a/runtime/internal/activation_attribute_matcher_access.h b/runtime/internal/activation_attribute_matcher_access.h deleted file mode 100644 index 2741be692..000000000 --- a/runtime/internal/activation_attribute_matcher_access.h +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_ACTIVATION_MATCHER_ACCESS_H_ -#define THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_ACTIVATION_MATCHER_ACCESS_H_ - -#include - -#include "absl/base/nullability.h" -#include "runtime/internal/attribute_matcher.h" - -namespace google::api::expr::runtime { -class Activation; -class BaseActivation; -} // namespace google::api::expr::runtime - -namespace cel { -class Activation; -class ActivationInterface; -} // namespace cel - -namespace cel::runtime_internal { - -class ActivationAttributeMatcherAccess { - public: - static void SetAttributeMatcher( - google::api::expr::runtime::Activation& activation, - const AttributeMatcher* matcher); - - static void SetAttributeMatcher( - google::api::expr::runtime::Activation& activation, - std::unique_ptr matcher); - - static const AttributeMatcher* absl_nullable GetAttributeMatcher( - const google::api::expr::runtime::BaseActivation& activation); - - static void SetAttributeMatcher(Activation& activation, - const AttributeMatcher* matcher); - - static void SetAttributeMatcher( - Activation& activation, std::unique_ptr matcher); - - static const AttributeMatcher* absl_nullable GetAttributeMatcher( - const ActivationInterface& activation); -}; - -} // namespace cel::runtime_internal - -#endif // THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_ACTIVATION_MATCHER_ACCESS_H_ diff --git a/runtime/internal/attribute_matcher.h b/runtime/internal/attribute_matcher.h deleted file mode 100644 index a168b714c..000000000 --- a/runtime/internal/attribute_matcher.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_ATTRIBUTE_MATCHER_H_ -#define THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_ATTRIBUTE_MATCHER_H_ - -#include "base/attribute.h" - -namespace cel::runtime_internal { - -// Interface for matching unknown and missing attributes against the -// observed attribute trail at runtime. -class AttributeMatcher { - public: - using MatchResult = cel::AttributePattern::MatchType; - - virtual ~AttributeMatcher() = default; - - // Checks whether the attribute trail matches any unknown patterns. - // Used to identify and collect referenced unknowns in an UnknownValue. - virtual MatchResult CheckForUnknown(const Attribute& attr - [[maybe_unused]]) const { - return MatchResult::NONE; - }; - - // Checks whether the attribute trail matches any missing patterns. - // Used to identify missing attributes, and report an error if referenced - // directly. - virtual MatchResult CheckForMissing(const Attribute& attr - [[maybe_unused]]) const { - return MatchResult::NONE; - }; -}; - -} // namespace cel::runtime_internal - -#endif // THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_ATTRIBUTE_MATCHER_H_ diff --git a/runtime/internal/convert_constant.cc b/runtime/internal/convert_constant.cc index a9effd229..edeedb4f8 100644 --- a/runtime/internal/convert_constant.cc +++ b/runtime/internal/convert_constant.cc @@ -17,23 +17,24 @@ #include #include +#include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/time/time.h" #include "absl/types/variant.h" -#include "common/allocator.h" #include "common/constant.h" #include "common/value.h" #include "eval/internal/errors.h" +#include "google/protobuf/arena.h" namespace cel::runtime_internal { namespace { using ::cel::Constant; struct ConvertVisitor { - Allocator<> allocator; + google::protobuf::Arena* const arena; - absl::StatusOr operator()(absl::monostate) { + absl::StatusOr operator()(std::monostate) { return absl::InvalidArgumentError("unspecified constant"); } absl::StatusOr operator()(std::nullptr_t) { return NullValue(); } @@ -48,14 +49,14 @@ struct ConvertVisitor { return DoubleValue(value); } absl::StatusOr operator()(const cel::StringConstant& value) { - return StringValue(allocator, value); + return StringValue::From(value, arena); } absl::StatusOr operator()(const cel::BytesConstant& value) { - return BytesValue(allocator, value); + return BytesValue::From(value, arena); } absl::StatusOr operator()(const absl::Duration duration) { if (duration >= kDurationHigh || duration <= kDurationLow) { - return ErrorValue(*DurationOverflowError()); + return ErrorValue::WrapUnsafe(DurationOverflowError()); } return UnsafeDurationValue(duration); } @@ -71,8 +72,8 @@ struct ConvertVisitor { // // A status maybe returned if value creation fails. absl::StatusOr ConvertConstant(const Constant& constant, - Allocator<> allocator) { - return absl::visit(ConvertVisitor{allocator}, constant.constant_kind()); + google::protobuf::Arena* absl_nonnull arena) { + return absl::visit(ConvertVisitor{arena}, constant.constant_kind()); } } // namespace cel::runtime_internal diff --git a/runtime/internal/convert_constant.h b/runtime/internal/convert_constant.h index f1ac0c850..e891ca45d 100644 --- a/runtime/internal/convert_constant.h +++ b/runtime/internal/convert_constant.h @@ -14,10 +14,11 @@ #ifndef THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_CONVERT_CONSTANT_H_ #define THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_CONVERT_CONSTANT_H_ +#include "absl/base/nullability.h" #include "absl/status/statusor.h" -#include "common/allocator.h" -#include "common/ast.h" +#include "common/constant.h" #include "common/value.h" +#include "google/protobuf/arena.h" namespace cel::runtime_internal { @@ -32,7 +33,7 @@ namespace cel::runtime_internal { // A status may still be returned if value creation fails according to // value_factory's policy. absl::StatusOr ConvertConstant(const Constant& constant, - Allocator<> allocator); + google::protobuf::Arena* absl_nonnull arena); } // namespace cel::runtime_internal diff --git a/runtime/internal/function_adapter.h b/runtime/internal/function_adapter.h index 9b191e577..3a0ed1beb 100644 --- a/runtime/internal/function_adapter.h +++ b/runtime/internal/function_adapter.h @@ -24,7 +24,6 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/time/time.h" -#include "common/casting.h" #include "common/kind.h" #include "common/value.h" @@ -162,24 +161,24 @@ struct ValueToAdaptedVisitor { template absl::Status operator()(T* out) const { - if (!InstanceOf>(input)) { + if (!input.Is>()) { return absl::InvalidArgumentError( absl::StrCat("expected ", ValueKindToString(T::kKind), " value")); } - *out = Cast>(input); + *out = input.Get>(); return absl::OkStatus(); } template absl::Status operator()(T** out) const { - if (!InstanceOf>(input)) { + if (!input.Is>()) { return absl::InvalidArgumentError( absl::StrCat("expected ", ValueKindToString(T::kKind), " value")); } static_assert(std::is_lvalue_reference_v< - decltype(Cast>(input))>, + decltype(input.Get>())>, "expected l-value reference return type for Cast."); - *out = &Cast>(input); + *out = &input.Get>(); return absl::OkStatus(); } @@ -200,13 +199,13 @@ struct AdaptedToValueVisitor { absl::StatusOr operator()(absl::Time in) { // Type matching may have already occurred. It's too late to change up the // type and return an error. - return TimestampValue(in); + return UnsafeTimestampValue(in); } absl::StatusOr operator()(absl::Duration in) { // Type matching may have already occurred. It's too late to change up the // type and return an error. - return DurationValue(in); + return UnsafeDurationValue(in); } absl::StatusOr operator()(Value in) { return in; } diff --git a/runtime/internal/function_adapter_test.cc b/runtime/internal/function_adapter_test.cc index 643f08090..fed91528e 100644 --- a/runtime/internal/function_adapter_test.cc +++ b/runtime/internal/function_adapter_test.cc @@ -23,6 +23,7 @@ #include "common/kind.h" #include "common/value.h" #include "internal/testing.h" +#include "google/protobuf/arena.h" namespace cel::runtime_internal { namespace { @@ -179,7 +180,7 @@ TEST_F(ValueToAdaptedVisitorTest, DurationWrongKind) { } TEST_F(ValueToAdaptedVisitorTest, String) { - Value v = cel::StringValue("string"); + Value v = cel::StringValue::WrapUnsafe("string"); StringValue out; ASSERT_THAT(ValueToAdaptedVisitor{v}(&out), IsOk()); @@ -197,7 +198,7 @@ TEST_F(ValueToAdaptedVisitorTest, StringWrongKind) { } TEST_F(ValueToAdaptedVisitorTest, Bytes) { - Value v = cel::BytesValue("bytes"); + Value v = cel::BytesValue::WrapUnsafe("bytes"); BytesValue out; ASSERT_THAT(ValueToAdaptedVisitor{v}(&out), IsOk()); @@ -272,7 +273,7 @@ TEST_F(AdaptedToValueVisitorTest, Duration) { } TEST_F(AdaptedToValueVisitorTest, String) { - StringValue value = cel::StringValue("str"); + StringValue value = cel::StringValue::WrapUnsafe("str"); ASSERT_OK_AND_ASSIGN(auto result, AdaptedToValueVisitor{}(value)); @@ -281,7 +282,7 @@ TEST_F(AdaptedToValueVisitorTest, String) { } TEST_F(AdaptedToValueVisitorTest, Bytes) { - BytesValue value = cel::BytesValue("bytes"); + BytesValue value = cel::BytesValue::WrapUnsafe("bytes"); ASSERT_OK_AND_ASSIGN(auto result, AdaptedToValueVisitor{}(value)); @@ -306,7 +307,9 @@ TEST_F(AdaptedToValueVisitorTest, StatusOrError) { } TEST_F(AdaptedToValueVisitorTest, Any) { - auto handle = cel::ErrorValue(absl::InternalError("test_error")); + google::protobuf::Arena arena; + auto handle = + cel::ErrorValue::From(absl::InternalError("test_error"), &arena); ASSERT_OK_AND_ASSIGN(auto result, AdaptedToValueVisitor{}(handle)); diff --git a/runtime/internal/legacy_runtime_type_provider.cc b/runtime/internal/legacy_runtime_type_provider.cc new file mode 100644 index 000000000..54113113c --- /dev/null +++ b/runtime/internal/legacy_runtime_type_provider.cc @@ -0,0 +1,87 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "runtime/internal/legacy_runtime_type_provider.h" + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "common/legacy_value.h" +#include "common/value.h" +#include "common/values/value_builder.h" +#include "internal/status_macros.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/message.h" + +namespace cel::runtime_internal { + +namespace { + +class LegacyValueBuilder final : public cel::ValueBuilder { + public: + LegacyValueBuilder(google::protobuf::Arena* absl_nonnull arena, + cel::ValueBuilderPtr builder) + : arena_(arena), builder_(std::move(builder)) {} + + absl::StatusOr> SetFieldByName( + absl::string_view name, cel::Value value) override { + return builder_->SetFieldByName(name, std::move(value)); + } + + absl::StatusOr> SetFieldByNumber( + int64_t number, cel::Value value) override { + return builder_->SetFieldByNumber(number, std::move(value)); + } + + absl::StatusOr Build() && override { + CEL_ASSIGN_OR_RETURN(auto value, std::move(*builder_).Build(), + _.With(cel::ErrorValueReturn(arena_))); + if (value.Is()) { + // Make the value behave like a legacy message. Minimizes further + // legacy/modern conversions (e.g. on return and when accessing fields). + CEL_ASSIGN_OR_RETURN(auto legacy_value, LegacyValue(arena_, value), + _.With(cel::ErrorValueReturn(arena_))); + CEL_ASSIGN_OR_RETURN(auto result, ModernValue(arena_, legacy_value), + _.With(cel::ErrorValueReturn(arena_))); + return result; + } + return value; + } + + private: + google::protobuf::Arena* const arena_; + cel::ValueBuilderPtr builder_; +}; + +} // namespace + +absl::StatusOr +LegacyRuntimeTypeProvider::NewValueBuilder( + absl::string_view name, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) const { + auto builder = common_internal::NewValueBuilder(arena, descriptor_pool_, + message_factory, name); + if (builder == nullptr) { + return nullptr; + } + return std::make_unique(arena, std::move(builder)); +} + +} // namespace cel::runtime_internal diff --git a/runtime/internal/legacy_runtime_type_provider.h b/runtime/internal/legacy_runtime_type_provider.h index 503a79b46..2d289f3ed 100644 --- a/runtime/internal/legacy_runtime_type_provider.h +++ b/runtime/internal/legacy_runtime_type_provider.h @@ -15,21 +15,52 @@ #ifndef THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_LEGACY_RUNTIME_TYPE_PROVIDER_H_ #define THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_LEGACY_RUNTIME_TYPE_PROVIDER_H_ +#include + #include "absl/base/nullability.h" -#include "eval/public/structs/protobuf_descriptor_type_provider.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "common/type.h" +#include "common/type_reflector.h" +#include "common/value.h" +#include "runtime/internal/runtime_type_provider.h" +#include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" namespace cel::runtime_internal { -class LegacyRuntimeTypeProvider final - : public google::api::expr::runtime::ProtobufDescriptorProvider { +// LegacyRuntimeTypeProvider is a TypeReflector that uses a RuntimeTypeProvider +// internally to provide types with the google::api::expr::runtime::CelValue +// APIs. It prefers to create wrapped legacy values but otherwise proxies to +// the standard RuntimeTypeProvider. +class LegacyRuntimeTypeProvider final : public TypeReflector { public: LegacyRuntimeTypeProvider( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nullable message_factory) - : google::api::expr::runtime::ProtobufDescriptorProvider( - descriptor_pool, message_factory) {} + const RuntimeTypeProvider* absl_nonnull runtime_type_provider) + : descriptor_pool_(descriptor_pool), + runtime_type_provider_(runtime_type_provider) {} + + absl::StatusOr NewValueBuilder( + absl::string_view name, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) const override; + + protected: + absl::StatusOr> FindTypeImpl( + absl::string_view name) const override { + return runtime_type_provider_->FindType(name); + } + + absl::StatusOr> FindStructTypeFieldByNameImpl( + absl::string_view type, absl::string_view name) const override { + return runtime_type_provider_->FindStructTypeFieldByName(type, name); + } + + private: + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool_; + const RuntimeTypeProvider* absl_nonnull runtime_type_provider_; }; } // namespace cel::runtime_internal diff --git a/runtime/internal/legacy_runtime_type_provider_test.cc b/runtime/internal/legacy_runtime_type_provider_test.cc new file mode 100644 index 000000000..bd3caddc4 --- /dev/null +++ b/runtime/internal/legacy_runtime_type_provider_test.cc @@ -0,0 +1,148 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "runtime/internal/legacy_runtime_type_provider.h" + +#include +#include + +#include "common/type.h" +#include "common/value.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "internal/testing_message_factory.h" +#include "runtime/internal/runtime_type_provider.h" +#include "cel/expr/conformance/proto3/test_all_types.pb.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" + +namespace cel::runtime_internal { +namespace { + +using ::cel::expr::conformance::proto3::TestAllTypes; + +TEST(LegacyRuntimeTypeProviderTest, FindType) { + RuntimeTypeProvider type_provider(cel::internal::GetTestingDescriptorPool()); + LegacyRuntimeTypeProvider provider(cel::internal::GetTestingDescriptorPool(), + &type_provider); + ASSERT_OK_AND_ASSIGN(std::optional wrapper_type, + provider.FindType("google.protobuf.Int64Value")); + ASSERT_TRUE(wrapper_type.has_value()); + EXPECT_TRUE(wrapper_type->Is()); + EXPECT_EQ(wrapper_type->name(), "google.protobuf.Int64Value"); + + ASSERT_OK_AND_ASSIGN( + std::optional msg_type, + provider.FindType("cel.expr.conformance.proto3.TestAllTypes")); + ASSERT_TRUE(msg_type.has_value()); + EXPECT_TRUE(msg_type->Is()); + EXPECT_EQ(msg_type->name(), "cel.expr.conformance.proto3.TestAllTypes"); +} + +TEST(LegacyRuntimeTypeProviderTest, FindTypeNotFound) { + RuntimeTypeProvider type_provider(google::protobuf::DescriptorPool::generated_pool()); + LegacyRuntimeTypeProvider provider(google::protobuf::DescriptorPool::generated_pool(), + &type_provider); + ASSERT_OK_AND_ASSIGN(std::optional type, + provider.FindType("UnknownType")); + EXPECT_FALSE(type.has_value()); +} + +TEST(LegacyRuntimeTypeProviderTest, FindStructTypeFieldByName) { + RuntimeTypeProvider type_provider(google::protobuf::DescriptorPool::generated_pool()); + LegacyRuntimeTypeProvider provider(google::protobuf::DescriptorPool::generated_pool(), + &type_provider); + ASSERT_OK_AND_ASSIGN(std::optional field, + provider.FindStructTypeFieldByName( + "google.protobuf.Int64Value", "value")); + ASSERT_TRUE(field.has_value()); + EXPECT_EQ(field->name(), "value"); + EXPECT_EQ(field->number(), 1); + EXPECT_EQ(field->GetType(), IntType()); +} + +TEST(LegacyRuntimeTypeProviderTest, FindStructTypeFieldByNameNotFound) { + RuntimeTypeProvider type_provider(google::protobuf::DescriptorPool::generated_pool()); + LegacyRuntimeTypeProvider provider(google::protobuf::DescriptorPool::generated_pool(), + &type_provider); + ASSERT_OK_AND_ASSIGN(std::optional field, + provider.FindStructTypeFieldByName( + "google.protobuf.Int64Value", "unknown_field")); + EXPECT_FALSE(field.has_value()); + + ASSERT_OK_AND_ASSIGN( + std::optional field2, + provider.FindStructTypeFieldByName("UnknownType", "value")); + EXPECT_FALSE(field2.has_value()); +} + +TEST(LegacyRuntimeTypeProviderTest, NewValueBuilderMessage) { + RuntimeTypeProvider type_provider(cel::internal::GetTestingDescriptorPool()); + LegacyRuntimeTypeProvider provider(cel::internal::GetTestingDescriptorPool(), + &type_provider); + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(auto builder, + provider.NewValueBuilder( + "cel.expr.conformance.proto3.TestAllTypes", + cel::internal::GetTestingMessageFactory(), &arena)); + ASSERT_NE(builder, nullptr); + + ASSERT_OK_AND_ASSIGN(auto field_result, + builder->SetFieldByName("single_int64", IntValue(42))); + EXPECT_FALSE(field_result.has_value()); + + ASSERT_OK_AND_ASSIGN(auto field_result2, + builder->SetFieldByNumber(1, IntValue(100))); + EXPECT_FALSE(field_result2.has_value()); + + ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); + EXPECT_TRUE(value.Is()); +} + +TEST(LegacyRuntimeTypeProviderTest, NewValueBuilderWellKnownType) { + RuntimeTypeProvider type_provider(cel::internal::GetTestingDescriptorPool()); + LegacyRuntimeTypeProvider provider(cel::internal::GetTestingDescriptorPool(), + &type_provider); + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(auto builder, + provider.NewValueBuilder( + "google.protobuf.Int64Value", + cel::internal::GetTestingMessageFactory(), &arena)); + ASSERT_NE(builder, nullptr); + + ASSERT_OK_AND_ASSIGN(auto field_result, + builder->SetFieldByName("value", IntValue(42))); + EXPECT_FALSE(field_result.has_value()); + + ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); + ASSERT_TRUE(value.Is()); + EXPECT_EQ(value.As()->NativeValue(), 42); +} + +TEST(LegacyRuntimeTypeProviderTest, NewValueBuilderNotFound) { + RuntimeTypeProvider type_provider(google::protobuf::DescriptorPool::generated_pool()); + LegacyRuntimeTypeProvider provider(google::protobuf::DescriptorPool::generated_pool(), + &type_provider); + google::protobuf::LinkMessageReflection(); + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN( + auto builder, + provider.NewValueBuilder( + "UnknownType", google::protobuf::MessageFactory::generated_factory(), &arena)); + EXPECT_EQ(builder, nullptr); +} + +} // namespace +} // namespace cel::runtime_internal diff --git a/runtime/internal/runtime_env.h b/runtime/internal/runtime_env.h index cb9d9b93d..efc68cd5f 100644 --- a/runtime/internal/runtime_env.h +++ b/runtime/internal/runtime_env.h @@ -70,7 +70,7 @@ struct RuntimeEnv final { bool IsInitialized() const { return well_known_types.IsInitialized(); } - ABSL_ATTRIBUTE_UNUSED + [[maybe_unused]] const absl_nonnull std::shared_ptr descriptor_pool; diff --git a/runtime/internal/runtime_env_testing.h b/runtime/internal/runtime_env_testing.h index 71b2096cd..8c391784d 100644 --- a/runtime/internal/runtime_env_testing.h +++ b/runtime/internal/runtime_env_testing.h @@ -18,12 +18,43 @@ #include #include "absl/base/nullability.h" +#include "absl/log/absl_check.h" +#include "absl/log/die_if_null.h" +#include "internal/testing_descriptor_pool.h" +#include "internal/testing_message_factory.h" #include "runtime/internal/runtime_env.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" namespace cel::runtime_internal { absl_nonnull std::shared_ptr NewTestingRuntimeEnv(); +template +const google::protobuf::Descriptor* absl_nonnull GetTestingEnvDescriptor() { + const google::protobuf::Descriptor* descriptor = + internal::GetTestingDescriptorPool()->FindMessageTypeByName( + T::descriptor()->full_name()); + ABSL_CHECK(descriptor != nullptr) + << "Could not find CEL test env descriptor for type " + << T::descriptor()->full_name(); + return descriptor; +} + +template +google::protobuf::Message* absl_nonnull MakeTestingEnvDynamicProto( + const T& in, google::protobuf::Arena* absl_nonnull arena) { + const google::protobuf::Descriptor* descriptor = GetTestingEnvDescriptor(); + google::protobuf::Message* out = + ABSL_DIE_IF_NULL( + internal::GetTestingMessageFactory()->GetPrototype(descriptor)) + ->New(arena); + + ABSL_CHECK(out->MergeFromString(in.SerializeAsString())); + return out; +} + } // namespace cel::runtime_internal #endif // THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_RUNTIME_ENV_TESTING_H_ diff --git a/runtime/internal/runtime_impl.cc b/runtime/internal/runtime_impl.cc index 92d097b2c..8d8cda12b 100644 --- a/runtime/internal/runtime_impl.cc +++ b/runtime/internal/runtime_impl.cc @@ -21,13 +21,8 @@ #include "absl/status/statusor.h" #include "base/ast.h" #include "base/type_provider.h" -#include "common/native_type.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/comprehension_slots.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "internal/casts.h" #include "internal/status_macros.h" #include "runtime/activation_interface.h" #include "runtime/runtime.h" @@ -36,12 +31,7 @@ namespace cel::runtime_internal { namespace { -using ::google::api::expr::runtime::AttributeTrail; -using ::google::api::expr::runtime::ComprehensionSlots; -using ::google::api::expr::runtime::DirectExpressionStep; -using ::google::api::expr::runtime::ExecutionFrameBase; using ::google::api::expr::runtime::FlatExpression; -using ::google::api::expr::runtime::WrappedDirectStep; class ProgramImpl final : public TraceableProgram { public: @@ -76,46 +66,6 @@ class ProgramImpl final : public TraceableProgram { FlatExpression impl_; }; -class RecursiveProgramImpl final : public TraceableProgram { - public: - using EvaluationListener = TraceableProgram::EvaluationListener; - RecursiveProgramImpl( - const std::shared_ptr& environment, - FlatExpression impl, const DirectExpressionStep* absl_nonnull root) - : environment_(environment), impl_(std::move(impl)), root_(root) {} - - absl::StatusOr TraceImpl( - const ActivationInterface& activation, - EvaluationListener evaluation_listener, google::protobuf::Arena* absl_nonnull arena, - const EvaluateOptions& options) const override { - ABSL_DCHECK(arena != nullptr); - ComprehensionSlots slots(impl_.comprehension_slots_size()); - ExecutionFrameBase frame(activation, std::move(evaluation_listener), - impl_.options(), GetTypeProvider(), - environment_->descriptor_pool.get(), - options.message_factory != nullptr - ? options.message_factory - : environment_->MutableMessageFactory(), - arena, options.embedder_context, slots); - - Value result; - AttributeTrail attribute; - CEL_RETURN_IF_ERROR(root_->Evaluate(frame, result, attribute)); - - return result; - } - - const TypeProvider& GetTypeProvider() const override { - return environment_->type_registry.GetComposedTypeProvider(); - } - - private: - // Keep the Runtime environment alive while programs reference it. - std::shared_ptr environment_; - FlatExpression impl_; - const DirectExpressionStep* absl_nonnull root_; -}; - } // namespace absl::StatusOr> RuntimeImpl::CreateProgram( @@ -131,29 +81,7 @@ RuntimeImpl::CreateTraceableProgram( CEL_ASSIGN_OR_RETURN(auto flat_expr, expr_builder_.CreateExpressionImpl( std::move(ast), options.issues)); - // Special case if the program is fully recursive. - // - // This implementation avoids unnecessary allocs at evaluation time which - // improves performance notably for small expressions. - if (expr_builder_.options().max_recursion_depth != 0 && - !flat_expr.subexpressions().empty() && - // mainline expression is exactly one recursive step. - flat_expr.subexpressions().front().size() == 1 && - flat_expr.subexpressions().front().front()->GetNativeTypeId() == - NativeTypeId::For()) { - const DirectExpressionStep* root = - internal::down_cast( - flat_expr.subexpressions().front().front().get()) - ->wrapped(); - return std::make_unique(environment_, - std::move(flat_expr), root); - } - return std::make_unique(environment_, std::move(flat_expr)); } -bool TestOnly_IsRecursiveImpl(const Program* program) { - return dynamic_cast(program) != nullptr; -} - } // namespace cel::runtime_internal diff --git a/runtime/internal/runtime_impl.h b/runtime/internal/runtime_impl.h index 7c5d445f9..ed5e5ad47 100644 --- a/runtime/internal/runtime_impl.h +++ b/runtime/internal/runtime_impl.h @@ -115,11 +115,6 @@ class RuntimeImpl : public Runtime { google::api::expr::runtime::FlatExprBuilder expr_builder_; }; -// Exposed for testing to validate program is recursively planned. -// -// Uses dynamic_casts to test. -bool TestOnly_IsRecursiveImpl(const Program* program); - } // namespace cel::runtime_internal #endif // THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_RUNTIME_IMPL_H_ diff --git a/runtime/internal/runtime_type_provider.cc b/runtime/internal/runtime_type_provider.cc index 40f5ff575..5314918bf 100644 --- a/runtime/internal/runtime_type_provider.cc +++ b/runtime/internal/runtime_type_provider.cc @@ -14,6 +14,7 @@ #include "runtime/internal/runtime_type_provider.h" +#include #include #include "absl/base/nullability.h" @@ -28,7 +29,6 @@ #include "common/value.h" #include "common/values/value_builder.h" #include "google/protobuf/arena.h" -#include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" namespace cel::runtime_internal { @@ -48,15 +48,20 @@ absl::StatusOr> RuntimeTypeProvider::FindTypeImpl( if (type.has_value()) { return type; } - const auto* desc = descriptor_pool_->FindMessageTypeByName(name); - if (desc != nullptr) { - return MessageType(desc); + + auto result = descriptor_pool_provider_.FindType(name); + if (!result.ok()) { + return result; + } + + if (result->has_value() && !result->value().IsEnum()) { + return result; } if (const auto it = types_.find(name); it != types_.end()) { return it->second; } - return absl::nullopt; + return std::nullopt; } absl::StatusOr> @@ -66,24 +71,7 @@ RuntimeTypeProvider::FindEnumConstantImpl(absl::string_view type, if (enum_constant.has_value()) { return enum_constant; } - const google::protobuf::EnumDescriptor* enum_desc = - descriptor_pool_->FindEnumTypeByName(type); - if (enum_desc == nullptr) { - return absl::nullopt; - } - - // Note: we don't support strong enum typing at this time so only the fully - // qualified enum values are meaningful, so we don't provide any signal if the - // enum type is found but can't match the value name. - const google::protobuf::EnumValueDescriptor* value_desc = - enum_desc->FindValueByName(value); - if (value_desc == nullptr) { - return absl::nullopt; - } - - return TypeIntrospector::EnumConstant{ - EnumType(enum_desc), enum_desc->full_name(), value_desc->name(), - value_desc->number()}; + return descriptor_pool_provider_.FindEnumConstant(type, value); } absl::StatusOr> @@ -93,18 +81,7 @@ RuntimeTypeProvider::FindStructTypeFieldByNameImpl( if (field.has_value()) { return field; } - const auto* desc = descriptor_pool_->FindMessageTypeByName(type); - if (desc == nullptr) { - return absl::nullopt; - } - const auto* field_desc = desc->FindFieldByName(name); - if (field_desc == nullptr) { - field_desc = descriptor_pool_->FindExtensionByPrintableName(desc, name); - if (field_desc == nullptr) { - return absl::nullopt; - } - } - return MessageTypeField(field_desc); + return descriptor_pool_provider_.FindStructTypeFieldByName(type, name); } absl::StatusOr diff --git a/runtime/internal/runtime_type_provider.h b/runtime/internal/runtime_type_provider.h index 3f418af4d..fc7163f6c 100644 --- a/runtime/internal/runtime_type_provider.h +++ b/runtime/internal/runtime_type_provider.h @@ -21,6 +21,7 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "common/descriptor_pool_type_introspector.h" #include "common/type.h" #include "common/type_reflector.h" #include "common/value.h" @@ -34,7 +35,8 @@ class RuntimeTypeProvider final : public TypeReflector { public: explicit RuntimeTypeProvider( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool) - : descriptor_pool_(descriptor_pool) {} + : descriptor_pool_(descriptor_pool), + descriptor_pool_provider_(descriptor_pool) {} absl::Status RegisterType(const OpaqueType& type); @@ -55,6 +57,7 @@ class RuntimeTypeProvider final : public TypeReflector { private: const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool_; + DescriptorPoolTypeIntrospector descriptor_pool_provider_; absl::flat_hash_map types_; }; diff --git a/runtime/memory_safety_test.cc b/runtime/memory_safety_test.cc index 7e864ecf6..f2903d820 100644 --- a/runtime/memory_safety_test.cc +++ b/runtime/memory_safety_test.cc @@ -45,6 +45,7 @@ #include "runtime/activation.h" #include "runtime/constant_folding.h" #include "runtime/function_adapter.h" +#include "runtime/optional_types.h" #include "runtime/reference_resolver.h" #include "runtime/regex_precompilation.h" #include "runtime/runtime.h" @@ -73,7 +74,7 @@ struct TestCase { std::string name; std::string expression; absl::flat_hash_map> + std::variant> activation; test::ValueMatcher expected_matcher; bool reference_resolver_enabled = false; @@ -174,6 +175,7 @@ absl::StatusOr> ConfigureRuntimeImpl( if (resolve_references) { CEL_RETURN_IF_ERROR(EnableReferenceResolver( runtime_builder, ReferenceResolverEnabled::kAlways)); + CEL_RETURN_IF_ERROR(extensions::EnableOptionalTypes(runtime_builder)); } if (evaluation_options == Options::kFoldConstants) { CEL_RETURN_IF_ERROR(extensions::EnableConstantFolding(runtime_builder)); @@ -315,6 +317,14 @@ INSTANTIATE_TEST_SUITE_P( {{"condition", BoolValue(false)}}, test::StringValueIs("long_right_hand_string_0123456789"), }, + {"optional_of_long_const_string", + "condition ? optional.of('lhs_short') : " + "optional.of('long_right_hand_string_0123456789')", + {{"condition", BoolValue(false)}}, + test::OptionalValueIs( + test::StringValueIs("long_right_hand_string_0123456789")), + // optional.of is a namespaced function. + /*enable_reference_resolver=*/true}, { "computed_string", "(condition ? 'a.b' : 'b.c') + '.d.e.f'", @@ -772,6 +782,50 @@ TEST_P(ViewTypesMemorySafetyTest, UnsafeWrappedMessageDifferentArena) { EXPECT_THAT(result_msg, IsSameInstance(&proto)); } +TEST_P(ViewTypesMemorySafetyTest, UnsafeWrappedMessageDifferentExplicitArena) { + // Arrange: create the runtime and expression. + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + ConfigureRuntimeImpl(false, EvaluationOptions())); + constexpr absl::string_view kProtoValue = R"pb( + child { payload { repeated_int32: [ 1, 2, 3 ] } } + payload { repeated_string: [ "foo", "bar", "baz" ] } + )pb"; + + ASSERT_OK_AND_ASSIGN( + ValidationResult validation, + GetCompiler().Compile( + "condition ? nested_test_all_types : NestedTestAllTypes{}")); + ASSERT_TRUE(validation.IsValid()) << validation.FormatError(); + ASSERT_OK_AND_ASSIGN(auto ast, validation.ReleaseAst()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + // Act: wrap the message and evaluate the expression. + // The unsafe version will alias the input message, so caller must ensure + // the input outlives the use of the `Value` rather than assuming it + // is managed by the evaluation arena. + google::protobuf::Arena arena; + google::protobuf::Arena other_arena; + auto* proto = google::protobuf::Arena::Create(&other_arena); + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(kProtoValue, proto)); + Activation activation; + activation.InsertOrAssignValue("condition", BoolValue(true)); + activation.InsertOrAssignValue( + "nested_test_all_types", + Value::WrapMessageUnsafe(proto, google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), + &arena)); + ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); + + // Assert: the result is an alias of the input message. + ASSERT_TRUE(result.IsParsedMessage()); + const ParsedMessageValue& result_msg = result.GetParsedMessage(); + EXPECT_THAT(result_msg, + test::StructValueIs(ParsedProtoStructEquals(kProtoValue))); + EXPECT_EQ(result_msg->GetArena(), &other_arena); + EXPECT_THAT(result_msg, IsSameInstance(proto)); +} + TEST_P(ViewTypesMemorySafetyTest, UnsafeWrappedMessageFields) { // Arrange: create the runtime and expression. ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, diff --git a/runtime/optimized_runtime_builder_factory.cc b/runtime/optimized_runtime_builder_factory.cc new file mode 100644 index 000000000..d57c90373 --- /dev/null +++ b/runtime/optimized_runtime_builder_factory.cc @@ -0,0 +1,64 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "runtime/optimized_runtime_builder_factory.h" + +#include +#include + +#include "absl/base/nullability.h" +#include "absl/log/absl_check.h" +#include "absl/status/statusor.h" +#include "extensions/select_optimization.h" +#include "extensions/set_membership_optimization.h" +#include "internal/noop_delete.h" +#include "internal/status_macros.h" +#include "runtime/constant_folding.h" +#include "runtime/reference_resolver.h" +#include "runtime/regex_precompilation.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" +#include "google/protobuf/descriptor.h" + +namespace cel { + +absl::StatusOr CreateOptimizedRuntimeBuilder( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + const RuntimeOptions& options) { + ABSL_DCHECK(descriptor_pool != nullptr); + return CreateOptimizedRuntimeBuilder( + std::shared_ptr( + descriptor_pool, + internal::NoopDeleteFor()), + options); +} + +absl::StatusOr CreateOptimizedRuntimeBuilder( + absl_nonnull std::shared_ptr descriptor_pool, + const RuntimeOptions& options) { + ABSL_DCHECK(descriptor_pool != nullptr); + CEL_ASSIGN_OR_RETURN( + RuntimeBuilder builder, + CreateStandardRuntimeBuilder(std::move(descriptor_pool), options)); + CEL_RETURN_IF_ERROR(EnableReferenceResolver( + builder, ReferenceResolverEnabled::kCheckedExpressionOnly)); + CEL_RETURN_IF_ERROR(extensions::EnableConstantFolding(builder)); + CEL_RETURN_IF_ERROR(extensions::EnableRegexPrecompilation(builder)); + CEL_RETURN_IF_ERROR(extensions::EnableSelectOptimization(builder)); + CEL_RETURN_IF_ERROR(extensions::EnableSetMembershipOptimization(builder)); + return builder; +} + +} // namespace cel diff --git a/runtime/optimized_runtime_builder_factory.h b/runtime/optimized_runtime_builder_factory.h new file mode 100644 index 000000000..a74105154 --- /dev/null +++ b/runtime/optimized_runtime_builder_factory.h @@ -0,0 +1,99 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_RUNTIME_OPTIMIZED_RUNTIME_BUILDER_FACTORY_H_ +#define THIRD_PARTY_CEL_CPP_RUNTIME_OPTIMIZED_RUNTIME_BUILDER_FACTORY_H_ + +#include + +#include "absl/base/attributes.h" +#include "absl/base/nullability.h" +#include "absl/status/statusor.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "google/protobuf/descriptor.h" + +namespace cel { + +// Returns `RuntimeOptions` preconfigured with the default optimization flags +// used by `CreateOptimizedRuntimeBuilder`: +// - `enable_comprehension_list_append = true` +// - `enable_comprehension_mutable_map = true` +// - `enable_subexpression_inlining = true` +inline RuntimeOptions OptimizedRuntimeOptions() { + RuntimeOptions options; + options.enable_comprehension_list_append = true; + options.enable_comprehension_mutable_map = true; + options.enable_subexpression_inlining = true; + return options; +} + +// Creates a `RuntimeBuilder` preconfigured with CEL standard definitions and +// extended planner and runtime optimizations enabled. +// +// In addition to the standard functions registered by +// `CreateStandardRuntimeBuilder`, this factory enables: +// - In-place mutable list and map construction for comprehensions +// (`enable_comprehension_list_append` and +// `enable_comprehension_mutable_map` via `OptimizedRuntimeOptions`). +// - Inlining of trivial lazy subexpressions bound via `cel.bind` or +// `cel.@block` (`enable_subexpression_inlining` via +// `OptimizedRuntimeOptions`). +// - Qualified reference resolution for type-checked expressions +// (`EnableReferenceResolver` with `kCheckedExpressionOnly`). +// - Constant folding (`extensions::EnableConstantFolding`). +// - Regular expression precompilation +// (`extensions::EnableRegexPrecompilation`). +// - Batched struct/map/list select path optimization +// (`extensions::EnableSelectOptimization`). +// - Precomputed bitest and hash set optimization for constant integer/enum and +// string set membership checks +// (`extensions::EnableSetMembershipOptimization`). +// +// When passing custom `RuntimeOptions`, initialize from +// `OptimizedRuntimeOptions()` to retain the option-controlled optimizations. +// +// Trade-offs and requirements: +// - **Type-checked ASTs**: These optimizations generally expect a validated, +// type-checked AST (e.g., produced by `cel::Compiler`). Parse-only ASTs will +// skip type-dependent optimizations (such as reference resolution, select +// optimization, and typed list optimizations), and hand-rolled ASTs +// that violate comprehension accumulator invariants are not safe with +// in-place comprehension mutations. +// - **Higher planning cost**: `Runtime::CreateProgram` performs additional AST +// rewrites, type lookups, constant sub-expression evaluation, and regex +// compilation. This increases planning latency and memory usage in exchange +// for faster program evaluation, making it best suited for workflows that +// plan once and evaluate many times. +// - **Larger binary footprint**: Linking this library pulls in the extended +// optimizer passes and specialized evaluation steps. +// - **Tracing fidelity**: Optimizations rewrite the AST and collapse multiple +// AST nodes (such as constant subtrees, select chains, or set membership +// expressions) into single execution steps. As a result, evaluation traces +// (`Runtime::Trace`) will not invoke callbacks for elided intermediate nodes +// and can be difficult to map back to the original input AST. +// +// See `CreateRuntimeBuilder` for a description of the requirements related to +// `descriptor_pool`. +absl::StatusOr CreateOptimizedRuntimeBuilder( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool + ABSL_ATTRIBUTE_LIFETIME_BOUND, + const RuntimeOptions& options = OptimizedRuntimeOptions()); +absl::StatusOr CreateOptimizedRuntimeBuilder( + absl_nonnull std::shared_ptr descriptor_pool, + const RuntimeOptions& options = OptimizedRuntimeOptions()); + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_RUNTIME_OPTIMIZED_RUNTIME_BUILDER_FACTORY_H_ diff --git a/runtime/optimized_runtime_builder_factory_test.cc b/runtime/optimized_runtime_builder_factory_test.cc new file mode 100644 index 000000000..bdec3c542 --- /dev/null +++ b/runtime/optimized_runtime_builder_factory_test.cc @@ -0,0 +1,274 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "runtime/optimized_runtime_builder_factory.h" + +#include +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "checker/type_checker_builder.h" +#include "checker/validation_result.h" +#include "common/ast.h" +#include "common/decl.h" +#include "common/type.h" +#include "common/value.h" +#include "common/value_testing.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" +#include "extensions/bindings_ext.h" +#include "extensions/comprehensions_v2.h" +#include "extensions/comprehensions_v2_functions.h" +#include "extensions/protobuf/enum_adapter.h" +#include "extensions/sets_functions.h" +#include "internal/noop_delete.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "runtime/activation.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "cel/expr/conformance/proto2/test_all_types.pb.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" + +namespace cel { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::StatusIs; +using ::cel::expr::conformance::proto2::TestAllTypes; +using ::cel::test::BoolValueIs; + +absl::StatusOr> CreateTestCompiler() { + CEL_ASSIGN_OR_RETURN( + std::unique_ptr builder, + NewCompilerBuilder(internal::GetTestingDescriptorPool())); + CEL_RETURN_IF_ERROR(builder->AddLibrary(StandardCompilerLibrary())); + CEL_RETURN_IF_ERROR( + builder->AddLibrary(extensions::BindingsCompilerLibrary())); + CEL_RETURN_IF_ERROR(builder->AddLibrary(extensions::SetsCompilerLibrary())); + CEL_RETURN_IF_ERROR( + builder->AddLibrary(extensions::ComprehensionsV2CompilerLibrary())); + + TypeCheckerBuilder& checker = builder->GetCheckerBuilder(); + checker.set_container("cel.expr.conformance.proto2"); + CEL_RETURN_IF_ERROR(checker.AddContextDeclaration( + "cel.expr.conformance.proto2.TestAllTypes")); + CEL_RETURN_IF_ERROR(checker.AddVariable( + MakeVariableDecl("msg", MessageType(TestAllTypes::descriptor())))); + CEL_RETURN_IF_ERROR(checker.AddVariable(MakeVariableDecl("x", IntType()))); + CEL_RETURN_IF_ERROR(checker.AddVariable(MakeVariableDecl("s", StringType()))); + CEL_RETURN_IF_ERROR(checker.AddVariable( + MakeVariableDecl("int_list", ListType(checker.arena(), IntType())))); + CEL_RETURN_IF_ERROR(checker.AddVariable( + MakeVariableDecl("str_list", ListType(checker.arena(), StringType())))); + + return builder->Build(); +} + +absl::StatusOr> CompileTestExpr(absl::string_view expr) { + static const Compiler* const kCompiler = []() { + absl::StatusOr> compiler = CreateTestCompiler(); + ABSL_CHECK_OK(compiler.status()); + return compiler->release(); + }(); + + CEL_ASSIGN_OR_RETURN(ValidationResult result, kCompiler->Compile(expr)); + if (!result.IsValid()) { + return absl::InvalidArgumentError(result.FormatError()); + } + return result.ReleaseAst(); +} + +absl::StatusOr> BuildTestRuntime() { + RuntimeOptions options = OptimizedRuntimeOptions(); + options.container = "cel.expr.conformance.proto2"; + CEL_ASSIGN_OR_RETURN(RuntimeBuilder builder, + CreateOptimizedRuntimeBuilder( + internal::GetTestingDescriptorPool(), options)); + CEL_RETURN_IF_ERROR(extensions::RegisterProtobufEnum( + builder.type_registry(), TestAllTypes::NestedEnum_descriptor())); + CEL_RETURN_IF_ERROR( + extensions::RegisterSetsFunctions(builder.function_registry(), options)); + CEL_RETURN_IF_ERROR(extensions::RegisterComprehensionsV2Functions( + builder.function_registry(), options)); + return std::move(builder).Build(); +} + +TEST(OptimizedRuntimeBuilderFactoryTest, OptimizedRuntimeOptionsDefaults) { + RuntimeOptions options = OptimizedRuntimeOptions(); + EXPECT_TRUE(options.enable_comprehension_list_append); + EXPECT_TRUE(options.enable_comprehension_mutable_map); + EXPECT_TRUE(options.enable_subexpression_inlining); +} + +TEST(OptimizedRuntimeBuilderFactoryTest, SharedPtrDescriptorPoolOverload) { + std::shared_ptr pool( + internal::GetTestingDescriptorPool(), + internal::NoopDeleteFor()); + ASSERT_OK_AND_ASSIGN(RuntimeBuilder builder, + CreateOptimizedRuntimeBuilder(std::move(pool))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + std::move(builder).Build()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, CompileTestExpr("1 + 2 == 3")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + google::protobuf::Arena arena; + Activation activation; + ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); + EXPECT_THAT(result, BoolValueIs(true)); +} + +TEST(OptimizedRuntimeBuilderFactoryTest, RegexPrecompilation) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + BuildTestRuntime()); + + // Valid constant regex succeeds. + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + CompileTestExpr("s.matches('^a+b+$')")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + google::protobuf::Arena arena; + Activation activation; + activation.InsertOrAssignValue("s", StringValue::From("aaabbb", &arena)); + ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); + EXPECT_THAT(result, BoolValueIs(true)); + } + + // Invalid constant regex fails at plan time when regex precompilation is + // enabled. + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + CompileTestExpr("s.matches('[invalid')")); + EXPECT_THAT(runtime->CreateProgram(std::move(ast)), + StatusIs(absl::StatusCode::kInvalidArgument)); + } +} + +struct EvaluateResultTestCase { + std::string name; + std::string expression; + bool expected_result; + + template + friend void AbslStringify(S& sink, const EvaluateResultTestCase& tc) { + sink.Append(tc.name); + } +}; + +class OptimizedRuntimeEvalTest + : public ::testing::TestWithParam {}; + +TEST_P(OptimizedRuntimeEvalTest, EvaluatesCorrectly) { + const EvaluateResultTestCase& test_case = GetParam(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + BuildTestRuntime()); + + TestAllTypes msg; + msg.mutable_single_nested_message()->set_bb(42); + msg.set_standalone_enum(TestAllTypes::BAR); + + google::protobuf::Arena arena; + Activation activation; + activation.InsertOrAssignValue( + "msg", + Value::WrapMessage(&msg, internal::GetTestingDescriptorPool(), + google::protobuf::MessageFactory::generated_factory(), &arena)); + activation.InsertOrAssignValue("x", IntValue(3)); + activation.InsertOrAssignValue("s", StringValue::From("delta", &arena)); + + ListValueBuilderPtr int_list_builder = NewListValueBuilder(&arena); + ASSERT_THAT(int_list_builder->Add(IntValue(1)), IsOk()); + ASSERT_THAT(int_list_builder->Add(IntValue(3)), IsOk()); + activation.InsertOrAssignValue("int_list", + std::move(*int_list_builder).Build()); + + ListValueBuilderPtr str_list_builder = NewListValueBuilder(&arena); + ASSERT_THAT(str_list_builder->Add(StringValue::From("alpha", &arena)), + IsOk()); + ASSERT_THAT(str_list_builder->Add(StringValue::From("delta", &arena)), + IsOk()); + activation.InsertOrAssignValue("str_list", + std::move(*str_list_builder).Build()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + CompileTestExpr(test_case.expression)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); + EXPECT_THAT(result, BoolValueIs(test_case.expected_result)) + << test_case.expression; +} + +INSTANTIATE_TEST_SUITE_P( + ReferenceResolverAndSelectOptimization, OptimizedRuntimeEvalTest, + testing::ValuesIn(std::vector{ + {"nested_message_presence_and_select", + "has(msg.single_nested_message) && msg.single_nested_message.bb == 42", + true}, + {"unset_message_presence", "!has(msg.standalone_message)", true}, + {"qualified_enum_reference", + "msg.standalone_enum == TestAllTypes.NestedEnum.BAR", true}, + })); + +INSTANTIATE_TEST_SUITE_P( + ComprehensionListAndMapOptimizations, OptimizedRuntimeEvalTest, + testing::ValuesIn(std::vector{ + {"filter_and_map_list_append", + "[1, 2, 3, 4].filter(i, i % 2 == 0).map(i, i * 10) == [20, 40]", true}, + {"transform_map_mutable_map", + "{'a': 1, 'b': 2}.transformMap(k, v, v + 10)['b'] == 12", true}, + {"bind_subexpression_inlining", "cel.bind(a, 1 + 2, a + x == 6)", true}, + })); + +INSTANTIATE_TEST_SUITE_P( + ConstantFoldingAndSetMembershipOptimization, OptimizedRuntimeEvalTest, + testing::ValuesIn(std::vector{ + {"int_in_constant_list_true", "x in [1, 2, 3, 4]", true}, + {"int_in_folded_list_true", "x in ([1, 2] + [3, 4])", true}, + {"sets_contains_int_true", "sets.contains([1, 2, 3, 4], int_list)", + true}, + {"sets_intersects_int_true", "sets.intersects(int_list, [3, 4, 5])", + true}, + {"sets_equivalent_int_true", "sets.equivalent(int_list, [3, 1])", true}, + {"string_in_constant_list_true", + "s in ['alpha', 'beta', 'gamma', 'delta']", true}, + {"sets_contains_string_true", + "sets.contains(['alpha', 'beta', 'gamma', 'delta'], str_list)", true}, + {"sets_intersects_string_true", + "sets.intersects(str_list, ['gamma', 'delta', 'epsilon', 'zeta'])", + true}, + {"int_in_constant_list_false", "x in [10, 20, 30]", false}, + {"sets_contains_int_false", "sets.contains(int_list, [1, 2, 3])", + false}, + {"string_in_constant_list_false", + "s in ['one', 'two', 'three', 'four']", false}, + })); + +} // namespace +} // namespace cel diff --git a/runtime/optional_types.cc b/runtime/optional_types.cc index 6678a05ed..91e5b0543 100644 --- a/runtime/optional_types.cc +++ b/runtime/optional_types.cc @@ -27,12 +27,12 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "base/function_adapter.h" -#include "common/casting.h" #include "common/type.h" #include "common/value.h" #include "internal/casts.h" #include "internal/number.h" #include "internal/status_macros.h" +#include "runtime/function.h" #include "runtime/function_registry.h" #include "runtime/internal/errors.h" #include "runtime/internal/runtime_friend_access.h" @@ -66,19 +66,24 @@ Value OptionalOfNonZeroValue( return OptionalOf(value, descriptor_pool, message_factory, arena); } -absl::StatusOr OptionalGetValue(const OpaqueValue& opaque_value) { +absl::StatusOr OptionalGetValue(const OpaqueValue& opaque_value, + const Function::InvokeContext& context) { if (auto optional_value = opaque_value.AsOptional(); optional_value) { return optional_value->Value(); } - return ErrorValue{runtime_internal::CreateNoMatchingOverloadError("value")}; + return ErrorValue::From( + runtime_internal::CreateNoMatchingOverloadError("value"), + context.arena()); } -absl::StatusOr OptionalHasValue(const OpaqueValue& opaque_value) { +absl::StatusOr OptionalHasValue(const OpaqueValue& opaque_value, + const Function::InvokeContext& context) { if (auto optional_value = opaque_value.AsOptional(); optional_value) { return BoolValue{optional_value->HasValue()}; } - return ErrorValue{ - runtime_internal::CreateNoMatchingOverloadError("hasValue")}; + return ErrorValue::From( + runtime_internal::CreateNoMatchingOverloadError("hasValue"), + context.arena()); } absl::StatusOr SelectOptionalFieldStruct( @@ -132,7 +137,8 @@ absl::StatusOr SelectOptionalField( message_factory, arena); } } - return ErrorValue{runtime_internal::CreateNoMatchingOverloadError("_[?_]")}; + return ErrorValue::From( + runtime_internal::CreateNoMatchingOverloadError("_[?_]"), arena); } absl::StatusOr MapOptIndexOptionalValue( @@ -141,7 +147,7 @@ absl::StatusOr MapOptIndexOptionalValue( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) { absl::optional value; - if (auto double_key = cel::As(key); double_key) { + if (auto double_key = key.AsDouble(); double_key) { // Try int/uint. auto number = internal::Number::FromDouble(double_key->NativeValue()); if (number.LosslessConvertibleToInt()) { @@ -210,23 +216,24 @@ absl::StatusOr OptionalOptIndexOptionalValue( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) { - if (auto optional_value = As(opaque_value); optional_value) { + if (auto optional_value = opaque_value.AsOptional(); optional_value) { if (!optional_value->HasValue()) { return OptionalValue::None(); } auto container = optional_value->Value(); - if (auto map_value = cel::As(container); map_value) { + if (auto map_value = container.AsMap(); map_value) { return MapOptIndexOptionalValue(*map_value, key, descriptor_pool, message_factory, arena); } - if (auto list_value = cel::As(container); list_value) { - if (auto int_value = cel::As(key); int_value) { + if (auto list_value = container.AsList(); list_value) { + if (auto int_value = key.AsInt(); int_value) { return ListOptIndexOptionalInt(*list_value, int_value->NativeValue(), descriptor_pool, message_factory, arena); } } } - return ErrorValue{runtime_internal::CreateNoMatchingOverloadError("_[?_]")}; + return ErrorValue::From( + runtime_internal::CreateNoMatchingOverloadError("_[?_]"), arena); } absl::StatusOr ListFirst(const cel::ListValue& list, @@ -281,7 +288,7 @@ absl::StatusOr ListUnwrapOpt( }, descriptor_pool, message_factory, arena); if (!status.ok()) { - return ErrorValue(status); + return ErrorValue::From(status, arena); } return std::move(*builder).Build(); } diff --git a/runtime/optional_types_test.cc b/runtime/optional_types_test.cc index 455e51988..578b6d5bc 100644 --- a/runtime/optional_types_test.cc +++ b/runtime/optional_types_test.cc @@ -37,7 +37,6 @@ #include "parser/parser.h" #include "runtime/activation.h" #include "runtime/function.h" -#include "runtime/internal/runtime_impl.h" #include "runtime/reference_resolver.h" #include "runtime/runtime.h" #include "runtime/runtime_builder.h" @@ -185,41 +184,6 @@ class OptionalTypesTest bool EnableShortCircuiting() { return std::get<1>(GetParam()); } }; -TEST_P(OptionalTypesTest, RecursivePlan) { - RuntimeOptions opts; - opts.enable_qualified_type_identifiers = true; - opts.max_recursion_depth = -1; - opts.short_circuiting = EnableShortCircuiting(); - - const EvaluateResultTestCase& test_case = GetTestCase(); - - ASSERT_OK_AND_ASSIGN( - auto builder, - CreateStandardRuntimeBuilder(internal::GetTestingDescriptorPool(), opts)); - - ASSERT_OK(EnableOptionalTypes(builder)); - ASSERT_OK( - EnableReferenceResolver(builder, ReferenceResolverEnabled::kAlways)); - - ASSERT_OK_AND_ASSIGN(auto runtime, std::move(builder).Build()); - - ASSERT_OK_AND_ASSIGN(ParsedExpr expr, - Parse(test_case.expression, "", - ParserOptions{.enable_optional_syntax = true})); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr program, - ProtobufRuntimeAdapter::CreateProgram(*runtime, expr)); - - EXPECT_TRUE(runtime_internal::TestOnly_IsRecursiveImpl(program.get())); - - google::protobuf::Arena arena; - Activation activation; - - ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); - - EXPECT_THAT(result, test_case.value_matcher) << test_case.expression; -} - TEST_P(OptionalTypesTest, Defaults) { RuntimeOptions opts; opts.enable_qualified_type_identifiers = true; @@ -310,7 +274,7 @@ class UnreachableFunction final : public cel::Function { absl::StatusOr Invoke(absl::Span args, const InvokeContext& context) const override { ++(*count_); - return ErrorValue(absl::CancelledError()); + return ErrorValue::From(absl::CancelledError(), context.arena()); } private: diff --git a/runtime/regex_precompilation_test.cc b/runtime/regex_precompilation_test.cc index 85b47ef45..f31d34455 100644 --- a/runtime/regex_precompilation_test.cc +++ b/runtime/regex_precompilation_test.cc @@ -87,9 +87,11 @@ TEST_P(RegexPrecompilationTest, Basic) { absl::StatusOr, const StringValue&, const StringValue&>>:: RegisterGlobalOverload( "prepend", - [](const StringValue& value, const StringValue& prefix) { - return StringValue( - absl::StrCat(prefix.ToString(), value.ToString())); + [](const StringValue& value, const StringValue& prefix, + const Function::InvokeContext& context) { + return StringValue::From( + absl::StrCat(prefix.ToString(), value.ToString()), + context.arena()); }, builder.function_registry()); ASSERT_THAT(status, IsOk()); @@ -114,7 +116,7 @@ TEST_P(RegexPrecompilationTest, Basic) { google::protobuf::Arena arena; Activation activation; activation.InsertOrAssignValue("string_var", - StringValue(&arena, "string_var")); + StringValue::WrapUnsafe("string_var")); ASSERT_OK_AND_ASSIGN(Value value, program->Evaluate(&arena, activation)); EXPECT_THAT(value, test_case.result_matcher); @@ -131,9 +133,11 @@ TEST_P(RegexPrecompilationTest, WithConstantFolding) { absl::StatusOr, const StringValue&, const StringValue&>>:: RegisterGlobalOverload( "prepend", - [](const StringValue& value, const StringValue& prefix) { - return StringValue( - absl::StrCat(prefix.ToString(), value.ToString())); + [](const StringValue& value, const StringValue& prefix, + const Function::InvokeContext& context) { + return StringValue::From( + absl::StrCat(prefix.ToString(), value.ToString()), + context.arena()); }, builder.function_registry()); ASSERT_THAT(status, IsOk()); @@ -158,7 +162,7 @@ TEST_P(RegexPrecompilationTest, WithConstantFolding) { google::protobuf::Arena arena; Activation activation; activation.InsertOrAssignValue("string_var", - StringValue(&arena, "string_var")); + StringValue::WrapUnsafe("string_var")); ASSERT_OK_AND_ASSIGN(Value value, program->Evaluate(&arena, activation)); EXPECT_THAT(value, test_case.result_matcher); diff --git a/runtime/runtime.h b/runtime/runtime.h index 2db39b0e3..02ad9fd06 100644 --- a/runtime/runtime.h +++ b/runtime/runtime.h @@ -76,11 +76,20 @@ class Program { // Activation manages instances of variables available in the cel expression's // environment. // - // The arena will be used to as necessary to allocate values and must outlive - // the returned value, as must this program. + // Notes on lifetimes: // - // For consistency, users should use the same arena to create values - // in the activation and for Program evaluation. + // The provided arena will be used as necessary to allocate complex values + // and must outlive any returned value. Values created by the program may + // depend on internal state in the runtime. In particular protobuf messages + // may depend on the descriptor pool and message factory managed by the + // runtime or program. + // + // Programs implicitly keep shared state in the runtime object alive so it + // is sufficient to ensure that any cel::Value result is destroyed before the + // cel::Program that created it. + // + // For consistency, users should use the same arena to create values placed in + // the activation for calls to Program::Evaluate. absl::StatusOr Evaluate( google::protobuf::Arena* absl_nonnull arena ABSL_ATTRIBUTE_LIFETIME_BOUND, const ActivationInterface& activation, @@ -121,6 +130,13 @@ class TraceableProgram : public Program { // to an AST expression node. The value provided is the top of the value // stack, corresponding to the result of evaluating the given sub expression. // + // Expression IDs outside of the range [0, INT32_MAX] are not supported and + // will not invoke the listener. While the AST allows any int64, supported + // parser implementations should use a dense range starting at 1. In practice, + // no AST should contain more than ~ 1e9 nodes. + // + // ID 0 should not be considered valid, but is supported for legacy reasons. + // // A returning a non-ok status stops evaluation and forwards the error. using EvaluationListener = absl::AnyInvocable CreateRuntimeBuilder( diff --git a/runtime/runtime_options.h b/runtime/runtime_options.h index 7a61208a0..1c52e9b4c 100644 --- a/runtime/runtime_options.h +++ b/runtime/runtime_options.h @@ -156,12 +156,14 @@ struct RuntimeOptions { // -1 means unbounded. // 0 means disabled (using a heap-based stack machine instead), which is the // default. + ABSL_DEPRECATED("Recursive evaluation has been removed") int max_recursion_depth = 0; // Enable tracing support for recursively planned programs. // // Unlike the stack machine implementation, supporting tracing can affect // performance whether or not tracing is requested for a given evaluation. + ABSL_DEPRECATED("Recursive evaluation has been removed") bool enable_recursive_tracing = false; // Enable fast implementations for some CEL standard functions. @@ -188,6 +190,26 @@ struct RuntimeOptions { // // If disabled, will use the legacy behavior of rounding to 6 decimal places. bool enable_precision_preserving_double_format = true; + + // When enabled, the planner will attempt to use a more performant execution + // path for field access when the type is known at plan time, instead of using + // the generic field access implementation. + // + // The runtime will try to verify that the field access is compatible with the + // actual type at evaluation time, and will fall back to the generic + // implementation if the value is not what was expected. + // + // This is not recommended if the values bound to the activation are typically + // not what the planner expected (e.g. a map that was declared as a proto or + // a different message with matching field names). + bool enable_typed_field_access = true; + + // When enabled, trivial lazy subexpressions (such as constants or + // identifiers) bound via cel.bind or cel.@block will be inlined directly + // at reference sites during planning. + // + // Note: Enabling this option may change observed expression tracing behavior. + bool enable_subexpression_inlining = false; }; // LINT.ThenChange(//depot/google3/eval/public/cel_options.h) diff --git a/runtime/standard/BUILD b/runtime/standard/BUILD index 02a23ef1b..d08ac2116 100644 --- a/runtime/standard/BUILD +++ b/runtime/standard/BUILD @@ -159,6 +159,7 @@ cc_library( "//base:function_adapter", "//common:value", "//internal:status_macros", + "//runtime:function", "//runtime:function_registry", "//runtime:register_function_helper", "//runtime:runtime_options", @@ -276,6 +277,7 @@ cc_library( "//common:value", "//internal:overflow", "//internal:status_macros", + "//runtime:function", "//runtime:function_registry", "//runtime:runtime_options", "@com_google_absl//absl/status", @@ -307,12 +309,14 @@ cc_library( "//common:value", "//internal:overflow", "//internal:status_macros", + "//runtime:function", "//runtime:function_registry", "//runtime:runtime_options", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", + "@com_google_protobuf//:protobuf", ], ) @@ -373,6 +377,7 @@ cc_library( "//common:value", "//internal:re2_options", "//internal:status_macros", + "//runtime:function", "//runtime:function_registry", "//runtime:runtime_options", "@com_google_absl//absl/status", diff --git a/runtime/standard/arithmetic_functions.cc b/runtime/standard/arithmetic_functions.cc index a851ceb39..c77dd1a12 100644 --- a/runtime/standard/arithmetic_functions.cc +++ b/runtime/standard/arithmetic_functions.cc @@ -24,6 +24,7 @@ #include "common/value.h" #include "internal/overflow.h" #include "internal/status_macros.h" +#include "runtime/function.h" #include "runtime/function_registry.h" #include "runtime/runtime_options.h" @@ -32,93 +33,100 @@ namespace { // Template functions providing arithmetic operations template -Value Add(Type v0, Type v1); +Value Add(Type v0, Type v1, const Function::InvokeContext& context); template <> -Value Add(int64_t v0, int64_t v1) { +Value Add(int64_t v0, int64_t v1, + const Function::InvokeContext& context) { auto sum = cel::internal::CheckedAdd(v0, v1); if (!sum.ok()) { - return ErrorValue(sum.status()); + return ErrorValue::From(sum.status(), context.arena()); } return IntValue(*sum); } template <> -Value Add(uint64_t v0, uint64_t v1) { +Value Add(uint64_t v0, uint64_t v1, + const Function::InvokeContext& context) { auto sum = cel::internal::CheckedAdd(v0, v1); if (!sum.ok()) { - return ErrorValue(sum.status()); + return ErrorValue::From(sum.status(), context.arena()); } return UintValue(*sum); } template <> -Value Add(double v0, double v1) { +Value Add(double v0, double v1, const Function::InvokeContext&) { return DoubleValue(v0 + v1); } template -Value Sub(Type v0, Type v1); +Value Sub(Type v0, Type v1, const Function::InvokeContext& context); template <> -Value Sub(int64_t v0, int64_t v1) { +Value Sub(int64_t v0, int64_t v1, + const Function::InvokeContext& context) { auto diff = cel::internal::CheckedSub(v0, v1); if (!diff.ok()) { - return ErrorValue(diff.status()); + return ErrorValue::From(diff.status(), context.arena()); } return IntValue(*diff); } template <> -Value Sub(uint64_t v0, uint64_t v1) { +Value Sub(uint64_t v0, uint64_t v1, + const Function::InvokeContext& context) { auto diff = cel::internal::CheckedSub(v0, v1); if (!diff.ok()) { - return ErrorValue(diff.status()); + return ErrorValue::From(diff.status(), context.arena()); } return UintValue(*diff); } template <> -Value Sub(double v0, double v1) { +Value Sub(double v0, double v1, const Function::InvokeContext&) { return DoubleValue(v0 - v1); } template -Value Mul(Type v0, Type v1); +Value Mul(Type v0, Type v1, const Function::InvokeContext& context); template <> -Value Mul(int64_t v0, int64_t v1) { +Value Mul(int64_t v0, int64_t v1, + const Function::InvokeContext& context) { auto prod = cel::internal::CheckedMul(v0, v1); if (!prod.ok()) { - return ErrorValue(prod.status()); + return ErrorValue::From(prod.status(), context.arena()); } return IntValue(*prod); } template <> -Value Mul(uint64_t v0, uint64_t v1) { +Value Mul(uint64_t v0, uint64_t v1, + const Function::InvokeContext& context) { auto prod = cel::internal::CheckedMul(v0, v1); if (!prod.ok()) { - return ErrorValue(prod.status()); + return ErrorValue::From(prod.status(), context.arena()); } return UintValue(*prod); } template <> -Value Mul(double v0, double v1) { +Value Mul(double v0, double v1, const Function::InvokeContext&) { return DoubleValue(v0 * v1); } template -Value Div(Type v0, Type v1); +Value Div(Type v0, Type v1, const Function::InvokeContext& context); // Division operations for integer types should check for // division by 0 template <> -Value Div(int64_t v0, int64_t v1) { +Value Div(int64_t v0, int64_t v1, + const Function::InvokeContext& context) { auto quot = cel::internal::CheckedDiv(v0, v1); if (!quot.ok()) { - return ErrorValue(quot.status()); + return ErrorValue::From(quot.status(), context.arena()); } return IntValue(*quot); } @@ -126,16 +134,17 @@ Value Div(int64_t v0, int64_t v1) { // Division operations for integer types should check for // division by 0 template <> -Value Div(uint64_t v0, uint64_t v1) { +Value Div(uint64_t v0, uint64_t v1, + const Function::InvokeContext& context) { auto quot = cel::internal::CheckedDiv(v0, v1); if (!quot.ok()) { - return ErrorValue(quot.status()); + return ErrorValue::From(quot.status(), context.arena()); } return UintValue(*quot); } template <> -Value Div(double v0, double v1) { +Value Div(double v0, double v1, const Function::InvokeContext&) { static_assert(std::numeric_limits::is_iec559, "Division by zero for doubles must be supported"); @@ -145,24 +154,26 @@ Value Div(double v0, double v1) { // Modulo operation template -Value Modulo(Type v0, Type v1); +Value Modulo(Type v0, Type v1, const Function::InvokeContext& context); // Modulo operations for integer types should check for // division by 0 template <> -Value Modulo(int64_t v0, int64_t v1) { +Value Modulo(int64_t v0, int64_t v1, + const Function::InvokeContext& context) { auto mod = cel::internal::CheckedMod(v0, v1); if (!mod.ok()) { - return ErrorValue(mod.status()); + return ErrorValue::From(mod.status(), context.arena()); } return IntValue(*mod); } template <> -Value Modulo(uint64_t v0, uint64_t v1) { +Value Modulo(uint64_t v0, uint64_t v1, + const Function::InvokeContext& context) { auto mod = cel::internal::CheckedMod(v0, v1); if (!mod.ok()) { - return ErrorValue(mod.status()); + return ErrorValue::From(mod.status(), context.arena()); } return UintValue(*mod); } @@ -211,17 +222,17 @@ absl::Status RegisterArithmeticFunctions(FunctionRegistry& registry, &Modulo))); // Negation group - CEL_RETURN_IF_ERROR( - registry.Register(UnaryFunctionAdapter::CreateDescriptor( - cel::builtin::kNeg, false), - UnaryFunctionAdapter::WrapFunction( - [](int64_t value) -> Value { - auto inv = cel::internal::CheckedNegation(value); - if (!inv.ok()) { - return ErrorValue(inv.status()); - } - return IntValue(*inv); - }))); + CEL_RETURN_IF_ERROR(registry.Register( + UnaryFunctionAdapter::CreateDescriptor(cel::builtin::kNeg, + false), + UnaryFunctionAdapter::WrapFunction( + [](int64_t value, const Function::InvokeContext& context) -> Value { + auto inv = cel::internal::CheckedNegation(value); + if (!inv.ok()) { + return ErrorValue::From(inv.status(), context.arena()); + } + return IntValue(*inv); + }))); return registry.Register( UnaryFunctionAdapter::CreateDescriptor(cel::builtin::kNeg, diff --git a/runtime/standard/container_membership_functions.cc b/runtime/standard/container_membership_functions.cc index cc0638429..26c115103 100644 --- a/runtime/standard/container_membership_functions.cc +++ b/runtime/standard/container_membership_functions.cc @@ -16,7 +16,6 @@ #include #include -#include #include "absl/base/nullability.h" #include "absl/status/status.h" @@ -50,7 +49,7 @@ bool ValueEquals(const Value& value, T other); template <> bool ValueEquals(const Value& value, bool other) { - if (auto bool_value = As(value); bool_value) { + if (auto bool_value = value.As(); bool_value) { return bool_value->NativeValue() == other; } return false; @@ -58,7 +57,7 @@ bool ValueEquals(const Value& value, bool other) { template <> bool ValueEquals(const Value& value, int64_t other) { - if (auto int_value = As(value); int_value) { + if (auto int_value = value.As(); int_value) { return int_value->NativeValue() == other; } return false; @@ -66,7 +65,7 @@ bool ValueEquals(const Value& value, int64_t other) { template <> bool ValueEquals(const Value& value, uint64_t other) { - if (auto uint_value = As(value); uint_value) { + if (auto uint_value = value.As(); uint_value) { return uint_value->NativeValue() == other; } return false; @@ -74,7 +73,7 @@ bool ValueEquals(const Value& value, uint64_t other) { template <> bool ValueEquals(const Value& value, double other) { - if (auto double_value = As(value); double_value) { + if (auto double_value = value.As(); double_value) { return double_value->NativeValue() == other; } return false; @@ -82,7 +81,7 @@ bool ValueEquals(const Value& value, double other) { template <> bool ValueEquals(const Value& value, const StringValue& other) { - if (auto string_value = As(value); string_value) { + if (auto string_value = value.As(); string_value) { return string_value->Equals(other); } return false; @@ -90,7 +89,7 @@ bool ValueEquals(const Value& value, const StringValue& other) { template <> bool ValueEquals(const Value& value, const BytesValue& other) { - if (auto bytes_value = As(value); bytes_value) { + if (auto bytes_value = value.As(); bytes_value) { return bytes_value->Equals(other); } return false; diff --git a/runtime/standard/equality_functions.cc b/runtime/standard/equality_functions.cc index 6546db16c..03e29223f 100644 --- a/runtime/standard/equality_functions.cc +++ b/runtime/standard/equality_functions.cc @@ -169,7 +169,7 @@ absl::StatusOr> OpaqueEqual( if (auto bool_value = result.AsBool(); bool_value) { return bool_value->NativeValue(); } - return TypeConversionError(result.GetTypeName(), "bool").NativeValue(); + return common_internal::MakeTypeConversionError(result.GetTypeName(), "bool"); } absl::optional NumberFromValue(const Value& value) { @@ -181,7 +181,7 @@ absl::optional NumberFromValue(const Value& value) { return Number::FromDouble(value.GetDouble().NativeValue()); } - return absl::nullopt; + return std::nullopt; } absl::StatusOr> CheckAlternativeNumericType( @@ -192,7 +192,7 @@ absl::StatusOr> CheckAlternativeNumericType( absl::optional number = NumberFromValue(key); if (!number.has_value()) { - return absl::nullopt; + return std::nullopt; } if (!key.IsInt() && number->LosslessConvertibleToInt()) { @@ -215,7 +215,7 @@ absl::StatusOr> CheckAlternativeNumericType( } } - return absl::nullopt; + return std::nullopt; } // Equality for maps. Template parameter provides either heterogeneous or @@ -284,8 +284,8 @@ WrapComparison(Op op, absl::string_view name) { return BoolValue(*result); } - return ErrorValue( - cel::runtime_internal::CreateNoMatchingOverloadError(name)); + return ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(name), arena); }; } @@ -317,8 +317,8 @@ auto ComplexEquality(Op&& op) { CEL_ASSIGN_OR_RETURN(absl::optional result, op(t1, t2, descriptor_pool, message_factory, arena)); if (!result.has_value()) { - return ErrorValue( - cel::runtime_internal::CreateNoMatchingOverloadError(kEqual)); + return ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(kEqual), arena); } return BoolValue(*result); }; @@ -334,8 +334,9 @@ auto ComplexInequality(Op&& op) { CEL_ASSIGN_OR_RETURN(absl::optional result, op(t1, t2, descriptor_pool, message_factory, arena)); if (!result.has_value()) { - return ErrorValue( - cel::runtime_internal::CreateNoMatchingOverloadError(kInequal)); + return ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(kInequal), + arena); } return BoolValue(!*result); }; @@ -440,7 +441,7 @@ absl::StatusOr> HomogenousValueEqual( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) { if (v1.kind() != v2.kind()) { - return absl::nullopt; + return std::nullopt; } static_assert(std::is_lvalue_reference_v, @@ -483,7 +484,7 @@ absl::StatusOr> HomogenousValueEqual( return OpaqueEqual(v1.GetOpaque(), v2.GetOpaque(), descriptor_pool, message_factory, arena); default: - return absl::nullopt; + return std::nullopt; } } @@ -498,8 +499,8 @@ absl::StatusOr EqualOverloadImpl( if (result.has_value()) { return BoolValue(*result); } - return ErrorValue( - cel::runtime_internal::CreateNoMatchingOverloadError(kEqual)); + return ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(kEqual), arena); } absl::StatusOr InequalOverloadImpl( @@ -513,8 +514,8 @@ absl::StatusOr InequalOverloadImpl( if (result.has_value()) { return BoolValue(!*result); } - return ErrorValue( - cel::runtime_internal::CreateNoMatchingOverloadError(kInequal)); + return ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(kInequal), arena); } absl::Status RegisterHeterogeneousEqualityFunctions( @@ -582,7 +583,7 @@ absl::StatusOr> ValueEqualImpl( // map containing an Error. Return no matching overload to propagate an error // instead of a false result. if (v1.IsError() || v1.IsUnknown() || v2.IsError() || v2.IsUnknown()) { - return absl::nullopt; + return std::nullopt; } return false; diff --git a/runtime/standard/logical_functions.cc b/runtime/standard/logical_functions.cc index cd3dd3cb5..6eed8b695 100644 --- a/runtime/standard/logical_functions.cc +++ b/runtime/standard/logical_functions.cc @@ -20,6 +20,7 @@ #include "base/function_adapter.h" #include "common/value.h" #include "internal/status_macros.h" +#include "runtime/function.h" #include "runtime/function_registry.h" #include "runtime/internal/errors.h" #include "runtime/register_function_helper.h" @@ -30,7 +31,8 @@ namespace { using ::cel::runtime_internal::CreateNoMatchingOverloadError; -Value NotStrictlyFalseImpl(const Value& value) { +Value NotStrictlyFalseImpl(const Value& value, + const Function::InvokeContext& context) { if (value.IsBool()) { return value; } @@ -40,7 +42,9 @@ Value NotStrictlyFalseImpl(const Value& value) { } // Should only accept bool unknown or error. - return ErrorValue(CreateNoMatchingOverloadError(builtin::kNotStrictlyFalse)); + return ErrorValue::From( + CreateNoMatchingOverloadError(builtin::kNotStrictlyFalse), + context.arena()); } } // namespace diff --git a/runtime/standard/logical_functions_test.cc b/runtime/standard/logical_functions_test.cc index de50f5312..6e106cb62 100644 --- a/runtime/standard/logical_functions_test.cc +++ b/runtime/standard/logical_functions_test.cc @@ -168,9 +168,7 @@ INSTANTIATE_TEST_SUITE_P( []() -> std::vector { return {BoolValue(false)}; }, IsBool(false)}, TestCase{builtin::kNotStrictlyFalse, - []() -> std::vector { - return {ErrorValue(absl::InternalError("test"))}; - }, + []() -> std::vector { return {ErrorValue()}; }, IsBool(true)}, TestCase{builtin::kNotStrictlyFalse, []() -> std::vector { return {UnknownValue()}; }, diff --git a/runtime/standard/regex_functions.cc b/runtime/standard/regex_functions.cc index 6833f7804..e51ef821f 100644 --- a/runtime/standard/regex_functions.cc +++ b/runtime/standard/regex_functions.cc @@ -20,6 +20,7 @@ #include "common/value.h" #include "internal/re2_options.h" #include "internal/status_macros.h" +#include "runtime/function.h" #include "runtime/function_registry.h" #include "runtime/runtime_options.h" #include "re2/re2.h" @@ -32,10 +33,11 @@ absl::Status RegisterRegexFunctions(FunctionRegistry& registry, if (options.enable_regex) { auto regex_matches = [max_size = options.regex_max_program_size]( const StringValue& target, - const StringValue& regex) -> Value { + const StringValue& regex, + const Function::InvokeContext& context) -> Value { RE2 re2(regex.ToString(), cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, max_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(context.arena())); return BoolValue(RE2::PartialMatch(target.ToString(), re2)); }; diff --git a/runtime/standard/time_functions.cc b/runtime/standard/time_functions.cc index a0ec5377c..d517e6f40 100644 --- a/runtime/standard/time_functions.cc +++ b/runtime/standard/time_functions.cc @@ -30,8 +30,10 @@ #include "common/value.h" #include "internal/overflow.h" #include "internal/status_macros.h" +#include "runtime/function.h" #include "runtime/function_registry.h" #include "runtime/runtime_options.h" +#include "google/protobuf/arena.h" namespace cel { namespace { @@ -73,55 +75,73 @@ absl::Status FindTimeBreakdown(absl::Time timestamp, absl::string_view tz, Value GetTimeBreakdownPart( absl::Time timestamp, absl::string_view tz, const std::function& - extractor_func) { + extractor_func, + google::protobuf::Arena* arena) { absl::TimeZone::CivilInfo breakdown; auto status = FindTimeBreakdown(timestamp, tz, &breakdown); if (!status.ok()) { - return ErrorValue(status); + return ErrorValue::From(status, arena); } return IntValue(extractor_func(breakdown)); } -Value GetFullYear(absl::Time timestamp, absl::string_view tz) { - return GetTimeBreakdownPart(timestamp, tz, - [](const absl::TimeZone::CivilInfo& breakdown) { - return breakdown.cs.year(); - }); +Value GetFullYear(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { + return GetTimeBreakdownPart( + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { + return breakdown.cs.year(); + }, + context.arena()); } -Value GetMonth(absl::Time timestamp, absl::string_view tz) { - return GetTimeBreakdownPart(timestamp, tz, - [](const absl::TimeZone::CivilInfo& breakdown) { - return breakdown.cs.month() - 1; - }); +Value GetMonth(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { + return GetTimeBreakdownPart( + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { + return breakdown.cs.month() - 1; + }, + context.arena()); } -Value GetDayOfYear(absl::Time timestamp, absl::string_view tz) { +Value GetDayOfYear(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { return GetTimeBreakdownPart( - timestamp, tz, [](const absl::TimeZone::CivilInfo& breakdown) { + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { return absl::GetYearDay(absl::CivilDay(breakdown.cs)) - 1; - }); + }, + context.arena()); } -Value GetDayOfMonth(absl::Time timestamp, absl::string_view tz) { - return GetTimeBreakdownPart(timestamp, tz, - [](const absl::TimeZone::CivilInfo& breakdown) { - return breakdown.cs.day() - 1; - }); +Value GetDayOfMonth(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { + return GetTimeBreakdownPart( + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { + return breakdown.cs.day() - 1; + }, + context.arena()); } -Value GetDate(absl::Time timestamp, absl::string_view tz) { - return GetTimeBreakdownPart(timestamp, tz, - [](const absl::TimeZone::CivilInfo& breakdown) { - return breakdown.cs.day(); - }); +Value GetDate(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { + return GetTimeBreakdownPart( + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { + return breakdown.cs.day(); + }, + context.arena()); } -Value GetDayOfWeek(absl::Time timestamp, absl::string_view tz) { +Value GetDayOfWeek(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { return GetTimeBreakdownPart( - timestamp, tz, [](const absl::TimeZone::CivilInfo& breakdown) { + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { absl::Weekday weekday = absl::GetWeekday(breakdown.cs); // get day of week from the date in UTC, zero-based, zero for Sunday, @@ -129,35 +149,48 @@ Value GetDayOfWeek(absl::Time timestamp, absl::string_view tz) { int weekday_num = static_cast(weekday); weekday_num = (weekday_num == 6) ? 0 : weekday_num + 1; return weekday_num; - }); + }, + context.arena()); } -Value GetHours(absl::Time timestamp, absl::string_view tz) { - return GetTimeBreakdownPart(timestamp, tz, - [](const absl::TimeZone::CivilInfo& breakdown) { - return breakdown.cs.hour(); - }); +Value GetHours(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { + return GetTimeBreakdownPart( + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { + return breakdown.cs.hour(); + }, + context.arena()); } -Value GetMinutes(absl::Time timestamp, absl::string_view tz) { - return GetTimeBreakdownPart(timestamp, tz, - [](const absl::TimeZone::CivilInfo& breakdown) { - return breakdown.cs.minute(); - }); +Value GetMinutes(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { + return GetTimeBreakdownPart( + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { + return breakdown.cs.minute(); + }, + context.arena()); } -Value GetSeconds(absl::Time timestamp, absl::string_view tz) { - return GetTimeBreakdownPart(timestamp, tz, - [](const absl::TimeZone::CivilInfo& breakdown) { - return breakdown.cs.second(); - }); +Value GetSeconds(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { + return GetTimeBreakdownPart( + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { + return breakdown.cs.second(); + }, + context.arena()); } -Value GetMilliseconds(absl::Time timestamp, absl::string_view tz) { +Value GetMilliseconds(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { return GetTimeBreakdownPart( - timestamp, tz, [](const absl::TimeZone::CivilInfo& breakdown) { + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { return absl::ToInt64Milliseconds(breakdown.subsecond); - }); + }, + context.arena()); } absl::Status RegisterTimestampFunctions(FunctionRegistry& registry, @@ -166,141 +199,171 @@ absl::Status RegisterTimestampFunctions(FunctionRegistry& registry, BinaryFunctionAdapter:: CreateDescriptor(builtin::kFullYear, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetFullYear(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetFullYear(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor( builtin::kFullYear, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetFullYear(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetFullYear(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kMonth, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetMonth(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetMonth(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor(builtin::kMonth, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetMonth(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetMonth(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kDayOfYear, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetDayOfYear(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetDayOfYear(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor( builtin::kDayOfYear, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetDayOfYear(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetDayOfYear(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kDayOfMonth, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetDayOfMonth(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetDayOfMonth(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor( builtin::kDayOfMonth, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetDayOfMonth(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetDayOfMonth(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kDate, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetDate(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetDate(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor(builtin::kDate, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetDate(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetDate(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kDayOfWeek, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetDayOfWeek(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetDayOfWeek(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor( builtin::kDayOfWeek, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetDayOfWeek(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetDayOfWeek(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kHours, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetHours(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetHours(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor(builtin::kHours, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetHours(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetHours(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kMinutes, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetMinutes(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetMinutes(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor( builtin::kMinutes, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetMinutes(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetMinutes(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kSeconds, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetSeconds(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetSeconds(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor( builtin::kSeconds, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetSeconds(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetSeconds(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kMilliseconds, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetMilliseconds(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetMilliseconds(ts, tz.ToString(), context); }))); return registry.Register( UnaryFunctionAdapter::CreateDescriptor( builtin::kMilliseconds, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetMilliseconds(ts, ""); })); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetMilliseconds(ts, "", context); + })); } absl::Status RegisterCheckedTimeArithmeticFunctions( @@ -310,84 +373,90 @@ absl::Status RegisterCheckedTimeArithmeticFunctions( absl::Duration>::CreateDescriptor(builtin::kAdd, false), BinaryFunctionAdapter, absl::Time, absl::Duration>:: - WrapFunction( - [](absl::Time t1, absl::Duration d2) -> absl::StatusOr { - auto sum = cel::internal::CheckedAdd(t1, d2); - if (!sum.ok()) { - return ErrorValue(sum.status()); - } - return TimestampValue(*sum); - }))); + WrapFunction([](absl::Time t1, absl::Duration d2, + const Function::InvokeContext& context) + -> absl::StatusOr { + auto sum = cel::internal::CheckedAdd(t1, d2); + if (!sum.ok()) { + return ErrorValue::From(sum.status(), context.arena()); + } + return TimestampValue(*sum); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter, absl::Duration, absl::Time>::CreateDescriptor(builtin::kAdd, false), BinaryFunctionAdapter, absl::Duration, absl::Time>:: - WrapFunction( - [](absl::Duration d2, absl::Time t1) -> absl::StatusOr { - auto sum = cel::internal::CheckedAdd(t1, d2); - if (!sum.ok()) { - return ErrorValue(sum.status()); - } - return TimestampValue(*sum); - }))); + WrapFunction([](absl::Duration d2, absl::Time t1, + const Function::InvokeContext& context) + -> absl::StatusOr { + auto sum = cel::internal::CheckedAdd(t1, d2); + if (!sum.ok()) { + return ErrorValue::From(sum.status(), context.arena()); + } + return TimestampValue(*sum); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter, absl::Duration, absl::Duration>::CreateDescriptor(builtin::kAdd, false), - BinaryFunctionAdapter< - absl::StatusOr, absl::Duration, - absl::Duration>::WrapFunction([](absl::Duration d1, absl::Duration d2) - -> absl::StatusOr { - auto sum = cel::internal::CheckedAdd(d1, d2); - if (!sum.ok()) { - return ErrorValue(sum.status()); - } - return DurationValue(*sum); - }))); + BinaryFunctionAdapter, absl::Duration, + absl::Duration>:: + WrapFunction([](absl::Duration d1, absl::Duration d2, + const Function::InvokeContext& context) + -> absl::StatusOr { + auto sum = cel::internal::CheckedAdd(d1, d2); + if (!sum.ok()) { + return ErrorValue::From(sum.status(), context.arena()); + } + return DurationValue(*sum); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter, absl::Time, absl::Duration>:: CreateDescriptor(builtin::kSubtract, false), BinaryFunctionAdapter, absl::Time, absl::Duration>:: - WrapFunction( - [](absl::Time t1, absl::Duration d2) -> absl::StatusOr { - auto diff = cel::internal::CheckedSub(t1, d2); - if (!diff.ok()) { - return ErrorValue(diff.status()); - } - return TimestampValue(*diff); - }))); + WrapFunction([](absl::Time t1, absl::Duration d2, + const Function::InvokeContext& context) + -> absl::StatusOr { + auto diff = cel::internal::CheckedSub(t1, d2); + if (!diff.ok()) { + return ErrorValue::From(diff.status(), context.arena()); + } + return TimestampValue(*diff); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter, absl::Time, absl::Time>::CreateDescriptor(builtin::kSubtract, false), BinaryFunctionAdapter, absl::Time, absl::Time>:: - WrapFunction( - [](absl::Time t1, absl::Time t2) -> absl::StatusOr { - auto diff = cel::internal::CheckedSub(t1, t2); - if (!diff.ok()) { - return ErrorValue(diff.status()); - } - return DurationValue(*diff); - }))); + WrapFunction([](absl::Time t1, absl::Time t2, + const Function::InvokeContext& context) + -> absl::StatusOr { + auto diff = cel::internal::CheckedSub(t1, t2); + if (!diff.ok()) { + return ErrorValue::From(diff.status(), context.arena()); + } + return DurationValue(*diff); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter< absl::StatusOr, absl::Duration, absl::Duration>::CreateDescriptor(builtin::kSubtract, false), - BinaryFunctionAdapter< - absl::StatusOr, absl::Duration, - absl::Duration>::WrapFunction([](absl::Duration d1, absl::Duration d2) - -> absl::StatusOr { - auto diff = cel::internal::CheckedSub(d1, d2); - if (!diff.ok()) { - return ErrorValue(diff.status()); - } - return DurationValue(*diff); - }))); + BinaryFunctionAdapter, absl::Duration, + absl::Duration>:: + WrapFunction([](absl::Duration d1, absl::Duration d2, + const Function::InvokeContext& context) + -> absl::StatusOr { + auto diff = cel::internal::CheckedSub(d1, d2); + if (!diff.ok()) { + return ErrorValue::From(diff.status(), context.arena()); + } + return DurationValue(*diff); + }))); return absl::OkStatus(); } diff --git a/runtime/standard/type_conversion_functions.cc b/runtime/standard/type_conversion_functions.cc index 76e95751b..2c5130aa3 100644 --- a/runtime/standard/type_conversion_functions.cc +++ b/runtime/standard/type_conversion_functions.cc @@ -23,7 +23,6 @@ #include "absl/status/statusor.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" #include "base/builtins.h" @@ -66,8 +65,11 @@ Value FormatDouble(double v, const Function::InvokeContext& context) { std::to_chars_result result = std::to_chars(buf, buf + kBufSize, v, std::chars_format::general); if (result.ec != std::errc()) { - return cel::ErrorValue(absl::InvalidArgumentError(absl::StrCat( - "double format error: ", std::make_error_code(result.ec).message()))); + return cel::ErrorValue::From( + absl::InvalidArgumentError( + absl::StrCat("double format error: ", + std::make_error_code(result.ec).message())), + context.arena()); } absl::string_view out(buf, result.ptr - buf); return StringValue::From(out, arena); @@ -89,7 +91,8 @@ absl::Status RegisterBoolConversionFunctions(FunctionRegistry& registry, // string -> bool return UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kBool, - [](const StringValue& v) -> Value { + [](const StringValue& v, + const Function::InvokeContext& context) -> Value { if ((v == "true") || (v == "True") || (v == "TRUE") || (v == "t") || (v == "1")) { return TrueValue(); @@ -97,8 +100,10 @@ absl::Status RegisterBoolConversionFunctions(FunctionRegistry& registry, (v == "f") || (v == "0")) { return FalseValue(); } else { - return ErrorValue(absl::InvalidArgumentError( - "Type conversion error from 'string' to 'bool'")); + return ErrorValue::From( + absl::InvalidArgumentError( + "Type conversion error from 'string' to 'bool'"), + context.arena()); } }, registry); @@ -116,10 +121,10 @@ absl::Status RegisterIntConversionFunctions(FunctionRegistry& registry, // double -> int status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kInt, - [](double v) -> Value { + [](double v, const Function::InvokeContext& context) -> Value { auto conv = cel::internal::CheckedDoubleToInt64(v); if (!conv.ok()) { - return ErrorValue(conv.status()); + return ErrorValue::From(conv.status(), context.arena()); } return IntValue(*conv); }, @@ -135,11 +140,13 @@ absl::Status RegisterIntConversionFunctions(FunctionRegistry& registry, status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kInt, - [](const StringValue& s) -> Value { + [](const StringValue& s, + const Function::InvokeContext& context) -> Value { int64_t result; if (!absl::SimpleAtoi(s.ToString(), &result)) { - return ErrorValue( - absl::InvalidArgumentError("cannot convert string to int")); + return ErrorValue::From( + absl::InvalidArgumentError("cannot convert string to int"), + context.arena()); } return IntValue(result); }, @@ -155,10 +162,10 @@ absl::Status RegisterIntConversionFunctions(FunctionRegistry& registry, // uint -> int return UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kInt, - [](uint64_t v) -> Value { + [](uint64_t v, const Function::InvokeContext& context) -> Value { auto conv = cel::internal::CheckedUint64ToInt64(v); if (!conv.ok()) { - return ErrorValue(conv.status()); + return ErrorValue::From(conv.status(), context.arena()); } return IntValue(*conv); }, @@ -176,15 +183,17 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kString, - [](const BytesValue& value) -> Value { + [](const BytesValue& value, + const Function::InvokeContext& context) -> Value { auto valid = value.NativeValue([](const auto& value) -> bool { return internal::Utf8IsValid(value); }); if (!valid) { - return ErrorValue( - absl::InvalidArgumentError("malformed UTF-8 bytes")); + return ErrorValue::From( + absl::InvalidArgumentError("malformed UTF-8 bytes"), + context.arena()); } - return StringValue(value.ToString()); + return StringValue(value); }, registry); CEL_RETURN_IF_ERROR(status); @@ -193,7 +202,7 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kString, [](bool value) -> StringValue { - return StringValue(value ? "true" : "false"); + return StringValue::WrapUnsafe(value ? "true" : "false"); }, registry); CEL_RETURN_IF_ERROR(status); @@ -209,8 +218,8 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, // int -> string status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kString, - [](int64_t value) -> StringValue { - return StringValue(absl::StrCat(value)); + [](int64_t value, const Function::InvokeContext& context) -> StringValue { + return StringValue::From(absl::StrCat(value), context.arena()); }, registry); CEL_RETURN_IF_ERROR(status); @@ -225,8 +234,9 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, // uint -> string status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kString, - [](uint64_t value) -> StringValue { - return StringValue(absl::StrCat(value)); + [](uint64_t value, + const Function::InvokeContext& context) -> StringValue { + return StringValue::From(absl::StrCat(value), context.arena()); }, registry); CEL_RETURN_IF_ERROR(status); @@ -234,12 +244,13 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, // duration -> string status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kString, - [](absl::Duration value) -> Value { + [](absl::Duration value, + const Function::InvokeContext& context) -> Value { auto encode = EncodeDurationToJson(value); if (!encode.ok()) { - return ErrorValue(encode.status()); + return ErrorValue::From(encode.status(), context.arena()); } - return StringValue(*encode); + return StringValue::From(*encode, context.arena()); }, registry); CEL_RETURN_IF_ERROR(status); @@ -247,12 +258,12 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, // timestamp -> string return UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kString, - [](absl::Time value) -> Value { + [](absl::Time value, const Function::InvokeContext& context) -> Value { auto encode = EncodeTimestampToJson(value); if (!encode.ok()) { - return ErrorValue(encode.status()); + return ErrorValue::From(encode.status(), context.arena()); } - return StringValue(*encode); + return StringValue::From(*encode, context.arena()); }, registry); } @@ -263,10 +274,10 @@ absl::Status RegisterUintConversionFunctions(FunctionRegistry& registry, absl::Status status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kUint, - [](double v) -> Value { + [](double v, const Function::InvokeContext& context) -> Value { auto conv = cel::internal::CheckedDoubleToUint64(v); if (!conv.ok()) { - return ErrorValue(conv.status()); + return ErrorValue::From(conv.status(), context.arena()); } return UintValue(*conv); }, @@ -276,10 +287,10 @@ absl::Status RegisterUintConversionFunctions(FunctionRegistry& registry, // int -> uint status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kUint, - [](int64_t v) -> Value { + [](int64_t v, const Function::InvokeContext& context) -> Value { auto conv = cel::internal::CheckedInt64ToUint64(v); if (!conv.ok()) { - return ErrorValue(conv.status()); + return ErrorValue::From(conv.status(), context.arena()); } return UintValue(*conv); }, @@ -290,11 +301,13 @@ absl::Status RegisterUintConversionFunctions(FunctionRegistry& registry, status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kUint, - [](const StringValue& s) -> Value { + [](const StringValue& s, + const Function::InvokeContext& context) -> Value { uint64_t result; if (!absl::SimpleAtoi(s.ToString(), &result)) { - return ErrorValue( - absl::InvalidArgumentError("cannot convert string to uint")); + return ErrorValue::From( + absl::InvalidArgumentError("cannot convert string to uint"), + context.arena()); } return UintValue(result); }, @@ -320,8 +333,7 @@ absl::Status RegisterBytesConversionFunctions(FunctionRegistry& registry, return UnaryFunctionAdapter, const StringValue&>:: RegisterGlobalOverload( cel::builtin::kBytes, - [](const StringValue& value) { return BytesValue(value.ToString()); }, - registry); + [](const StringValue& value) { return BytesValue(value); }, registry); } absl::Status RegisterDoubleConversionFunctions(FunctionRegistry& registry, @@ -342,13 +354,15 @@ absl::Status RegisterDoubleConversionFunctions(FunctionRegistry& registry, status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kDouble, - [](const StringValue& s) -> Value { + [](const StringValue& s, + const Function::InvokeContext& context) -> Value { double result; if (absl::SimpleAtod(s.ToString(), &result)) { return DoubleValue(result); } else { - return ErrorValue(absl::InvalidArgumentError( - "cannot convert string to double")); + return ErrorValue::From( + absl::InvalidArgumentError("cannot convert string to double"), + context.arena()); } }, registry); @@ -360,16 +374,18 @@ absl::Status RegisterDoubleConversionFunctions(FunctionRegistry& registry, registry); } -Value CreateDurationFromString(const StringValue& dur_str) { +Value CreateDurationFromString(const StringValue& dur_str, + const Function::InvokeContext& context) { absl::Duration d; if (!absl::ParseDuration(dur_str.ToString(), &d)) { - return ErrorValue( - absl::InvalidArgumentError("String to Duration conversion failed")); + return ErrorValue::From( + absl::InvalidArgumentError("String to Duration conversion failed"), + context.arena()); } auto status = internal::ValidateDuration(d); if (!status.ok()) { - return ErrorValue(std::move(status)); + return ErrorValue::From(std::move(status), context.arena()); } return DurationValue(d); } @@ -388,11 +404,14 @@ absl::Status RegisterTimeConversionFunctions(FunctionRegistry& registry, CEL_RETURN_IF_ERROR( (UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kTimestamp, - [=](int64_t epoch_seconds) -> Value { + [=](int64_t epoch_seconds, + const Function::InvokeContext& context) -> Value { absl::Time ts = absl::FromUnixSeconds(epoch_seconds); if (enable_timestamp_duration_overflow_errors) { if (ts < MinTimestamp() || ts > MaxTimestamp()) { - return ErrorValue(absl::OutOfRangeError("timestamp overflow")); + return ErrorValue::From( + absl::OutOfRangeError("timestamp overflow"), + context.arena()); } } return UnsafeTimestampValue(ts); @@ -403,30 +422,37 @@ absl::Status RegisterTimeConversionFunctions(FunctionRegistry& registry, CEL_RETURN_IF_ERROR( (UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kTimestamp, - [](absl::Time value) -> Value { return TimestampValue(value); }, + [](absl::Time value) -> Value { return UnsafeTimestampValue(value); }, registry))); // duration -> duration CEL_RETURN_IF_ERROR( (UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kDuration, - [](absl::Duration value) -> Value { return DurationValue(value); }, + [](absl::Duration value) -> Value { + return UnsafeDurationValue(value); + }, registry))); // timestamp() conversion from string. return UnaryFunctionAdapter:: RegisterGlobalOverload( cel::builtin::kTimestamp, - [=](const StringValue& time_str) -> Value { + [=](const StringValue& time_str, + const Function::InvokeContext& context) -> Value { absl::Time ts; if (!absl::ParseTime(absl::RFC3339_full, time_str.ToString(), &ts, nullptr)) { - return ErrorValue(absl::InvalidArgumentError( - "String to Timestamp conversion failed")); + return ErrorValue::From( + absl::InvalidArgumentError( + "String to Timestamp conversion failed"), + context.arena()); } if (enable_timestamp_duration_overflow_errors) { if (ts < MinTimestamp() || ts > MaxTimestamp()) { - return ErrorValue(absl::OutOfRangeError("timestamp overflow")); + return ErrorValue::From( + absl::OutOfRangeError("timestamp overflow"), + context.arena()); } } return UnsafeTimestampValue(ts); diff --git a/runtime/standard_runtime_builder_factory_test.cc b/runtime/standard_runtime_builder_factory_test.cc index 029897233..07bc7f1f9 100644 --- a/runtime/standard_runtime_builder_factory_test.cc +++ b/runtime/standard_runtime_builder_factory_test.cc @@ -38,7 +38,6 @@ #include "parser/parser.h" #include "parser/standard_macros.h" #include "runtime/activation.h" -#include "runtime/internal/runtime_impl.h" #include "runtime/runtime.h" #include "runtime/runtime_issue.h" #include "runtime/runtime_options.h" @@ -52,11 +51,9 @@ using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::extensions::ProtobufRuntimeAdapter; using ::cel::test::BoolValueIs; -using ::cel::test::IntValueIs; using ::cel::expr::ParsedExpr; using ::google::api::expr::parser::Parse; using ::testing::ElementsAre; -using ::testing::HasSubstr; using ::testing::TestWithParam; using ::testing::Truly; @@ -78,72 +75,6 @@ absl::StatusOr ParseWithTestMacros(absl::string_view expression) { return Parse(**src, GetMacros()); } -TEST(StandardRuntimeTest, RecursionLimitExceeded) { - RuntimeOptions opts; - opts.max_recursion_depth = 1; - - ASSERT_OK_AND_ASSIGN(auto builder, - CreateStandardRuntimeBuilder( - google::protobuf::DescriptorPool::generated_pool(), opts)); - - ASSERT_OK_AND_ASSIGN(auto runtime, std::move(builder).Build()); - - ASSERT_OK_AND_ASSIGN(ParsedExpr expr, ParseWithTestMacros("1 + 2")); - - EXPECT_THAT(ProtobufRuntimeAdapter::CreateProgram(*runtime, expr), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("Maximum recursion depth of 1 exceeded"))); -} - -TEST(StandardRuntimeTest, RecursionUnderLimit) { - RuntimeOptions opts; - opts.max_recursion_depth = 2; - - ASSERT_OK_AND_ASSIGN(auto builder, - CreateStandardRuntimeBuilder( - google::protobuf::DescriptorPool::generated_pool(), opts)); - - ASSERT_OK_AND_ASSIGN(auto runtime, std::move(builder).Build()); - - ASSERT_OK_AND_ASSIGN(ParsedExpr expr, ParseWithTestMacros("1 + 2")); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr program, - ProtobufRuntimeAdapter::CreateProgram(*runtime, expr)); - - // Whether the implementation is recursive shouldn't affect observable - // behavior, but it does have performance implications (it will skip - // allocating a value stack). - EXPECT_TRUE(runtime_internal::TestOnly_IsRecursiveImpl(program.get())); - - google::protobuf::Arena arena; - Activation activation; - - ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); - EXPECT_THAT(result, IntValueIs(3)); -} - -TEST(StandardRuntimeTest, RecursionLimitTracksLazyExpressions) { - RuntimeOptions opts; - opts.max_recursion_depth = 8; - - ASSERT_OK_AND_ASSIGN(auto builder, - CreateStandardRuntimeBuilder( - google::protobuf::DescriptorPool::generated_pool(), opts)); - - ASSERT_OK_AND_ASSIGN(auto runtime, std::move(builder).Build()); - - ASSERT_OK_AND_ASSIGN(ParsedExpr expr, ParseWithTestMacros(R"cel( - cel.bind(a, 4 + (3 + (2 + 1)), - cel.bind(b, 7 + (6 + (5 + a)), - 9 + (8 + b) - ) - ))cel")); - - EXPECT_THAT(ProtobufRuntimeAdapter::CreateProgram(*runtime, expr), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("Maximum recursion depth of 8 exceeded"))); -} - struct EvaluateResultTestCase { std::string name; std::string expression; @@ -177,8 +108,6 @@ TEST_P(StandardRuntimeTest, Defaults) { ASSERT_OK_AND_ASSIGN(std::unique_ptr program, ProtobufRuntimeAdapter::CreateProgram(*runtime, expr)); - EXPECT_FALSE(runtime_internal::TestOnly_IsRecursiveImpl(program.get())); - google::protobuf::Arena arena; Activation activation; if (test_case.activation_builder != nullptr) { @@ -191,39 +120,6 @@ TEST_P(StandardRuntimeTest, Defaults) { << test_case.expression; } -TEST_P(StandardRuntimeTest, Recursive) { - RuntimeOptions opts; - opts.max_recursion_depth = -1; - const EvaluateResultTestCase& test_case = GetTestCase(); - - ASSERT_OK_AND_ASSIGN(auto builder, - CreateStandardRuntimeBuilder( - google::protobuf::DescriptorPool::generated_pool(), opts)); - - ASSERT_OK_AND_ASSIGN(auto runtime, std::move(builder).Build()); - - ASSERT_OK_AND_ASSIGN(ParsedExpr expr, - ParseWithTestMacros(test_case.expression)); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr program, - ProtobufRuntimeAdapter::CreateProgram(*runtime, expr)); - - // Whether the implementation is recursive shouldn't affect observable - // behavior, but it does have performance implications (it will skip - // allocating a value stack). - EXPECT_TRUE(runtime_internal::TestOnly_IsRecursiveImpl(program.get())); - - google::protobuf::Arena arena; - Activation activation; - if (test_case.activation_builder != nullptr) { - ASSERT_THAT(test_case.activation_builder(activation), IsOk()); - } - - ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); - EXPECT_THAT(result, BoolValueIs(test_case.expected_result)) - << test_case.expression; -} - TEST_P(StandardRuntimeTest, FastBuiltins) { RuntimeOptions opts; opts.enable_fast_builtins = true; @@ -241,8 +137,6 @@ TEST_P(StandardRuntimeTest, FastBuiltins) { ASSERT_OK_AND_ASSIGN(std::unique_ptr program, ProtobufRuntimeAdapter::CreateProgram(*runtime, expr)); - EXPECT_FALSE(runtime_internal::TestOnly_IsRecursiveImpl(program.get())); - google::protobuf::Arena arena; Activation activation; if (test_case.activation_builder != nullptr) { @@ -255,40 +149,6 @@ TEST_P(StandardRuntimeTest, FastBuiltins) { << test_case.expression; } -TEST_P(StandardRuntimeTest, RecursiveFastBuiltins) { - RuntimeOptions opts; - opts.enable_fast_builtins = true; - opts.max_recursion_depth = -1; - const EvaluateResultTestCase& test_case = GetTestCase(); - - ASSERT_OK_AND_ASSIGN(auto builder, - CreateStandardRuntimeBuilder( - google::protobuf::DescriptorPool::generated_pool(), opts)); - - ASSERT_OK_AND_ASSIGN(auto runtime, std::move(builder).Build()); - - ASSERT_OK_AND_ASSIGN(ParsedExpr expr, - ParseWithTestMacros(test_case.expression)); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr program, - ProtobufRuntimeAdapter::CreateProgram(*runtime, expr)); - - // Whether the implementation is recursive shouldn't affect observable - // behavior, but it does have performance implications (it will skip - // allocating a value stack). - EXPECT_TRUE(runtime_internal::TestOnly_IsRecursiveImpl(program.get())); - - google::protobuf::Arena arena; - Activation activation; - if (test_case.activation_builder != nullptr) { - ASSERT_THAT(test_case.activation_builder(activation), IsOk()); - } - - ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); - EXPECT_THAT(result, BoolValueIs(test_case.expected_result)) - << test_case.expression; -} - INSTANTIATE_TEST_SUITE_P( Basic, StandardRuntimeTest, testing::ValuesIn(std::vector{ @@ -668,20 +528,11 @@ TEST(StandardRuntimeTest, RuntimeIssueSupport) { } } -enum class EvalStrategy { kIterative, kRecursive }; - -class StandardRuntimeEvalStrategyTest - : public ::testing::TestWithParam {}; +class StandardRuntimeEvalTest : public ::testing::Test {}; // Check that calls to specialized builtins are validated. -TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinBoolOp) { - EvalStrategy eval_strategy = GetParam(); +TEST_F(StandardRuntimeEvalTest, InvalidBuiltinBoolOp) { RuntimeOptions options; - if (eval_strategy == EvalStrategy::kRecursive) { - options.max_recursion_depth = -1; - } else { - options.max_recursion_depth = 0; - } ASSERT_OK_AND_ASSIGN(auto builder, @@ -699,14 +550,8 @@ TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinBoolOp) { StatusIs(absl::StatusCode::kInvalidArgument)); } -TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinTernaryOp) { - EvalStrategy eval_strategy = GetParam(); +TEST_F(StandardRuntimeEvalTest, InvalidBuiltinTernaryOp) { RuntimeOptions options; - if (eval_strategy == EvalStrategy::kRecursive) { - options.max_recursion_depth = -1; - } else { - options.max_recursion_depth = 0; - } ASSERT_OK_AND_ASSIGN(auto builder, @@ -743,14 +588,8 @@ TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinTernaryOp) { StatusIs(absl::StatusCode::kInvalidArgument)); } -TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinIndex) { - EvalStrategy eval_strategy = GetParam(); +TEST_F(StandardRuntimeEvalTest, InvalidBuiltinIndex) { RuntimeOptions options; - if (eval_strategy == EvalStrategy::kRecursive) { - options.max_recursion_depth = -1; - } else { - options.max_recursion_depth = 0; - } ASSERT_OK_AND_ASSIGN(auto builder, @@ -771,15 +610,8 @@ TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinIndex) { StatusIs(absl::StatusCode::kInvalidArgument)); } -TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinEq) { - EvalStrategy eval_strategy = GetParam(); +TEST_F(StandardRuntimeEvalTest, InvalidBuiltinEq) { RuntimeOptions options; - if (eval_strategy == EvalStrategy::kRecursive) { - options.max_recursion_depth = -1; - } else { - options.max_recursion_depth = 0; - } - ASSERT_OK_AND_ASSIGN(auto builder, CreateStandardRuntimeBuilder( @@ -799,14 +631,8 @@ TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinEq) { StatusIs(absl::StatusCode::kInvalidArgument)); } -TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinIn) { - EvalStrategy eval_strategy = GetParam(); +TEST_F(StandardRuntimeEvalTest, InvalidBuiltinIn) { RuntimeOptions options; - if (eval_strategy == EvalStrategy::kRecursive) { - options.max_recursion_depth = -1; - } else { - options.max_recursion_depth = 0; - } ASSERT_OK_AND_ASSIGN(auto builder, @@ -827,14 +653,8 @@ TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinIn) { StatusIs(absl::StatusCode::kInvalidArgument)); } -TEST_P(StandardRuntimeEvalStrategyTest, PrecisionPreservingDoubleFormat) { - EvalStrategy eval_strategy = GetParam(); +TEST_F(StandardRuntimeEvalTest, PrecisionPreservingDoubleFormat) { RuntimeOptions options; - if (eval_strategy == EvalStrategy::kRecursive) { - options.max_recursion_depth = -1; - } else { - options.max_recursion_depth = 0; - } options.enable_precision_preserving_double_format = true; @@ -861,12 +681,5 @@ TEST_P(StandardRuntimeEvalStrategyTest, PrecisionPreservingDoubleFormat) { } } -INSTANTIATE_TEST_SUITE_P( - StandardRuntimeEvalStrategyTest, StandardRuntimeEvalStrategyTest, - testing::Values(EvalStrategy::kIterative, EvalStrategy::kRecursive), - [](const auto& info) -> std::string { - return info.param == EvalStrategy::kIterative ? "Iterative" : "Recursive"; - }); - } // namespace } // namespace cel diff --git a/runtime/type_registry.cc b/runtime/type_registry.cc index a1e8b0328..58bce8afd 100644 --- a/runtime/type_registry.cc +++ b/runtime/type_registry.cc @@ -35,9 +35,7 @@ TypeRegistry::TypeRegistry( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nullable message_factory) : type_provider_(descriptor_pool), - legacy_type_provider_( - std::make_shared( - descriptor_pool, message_factory)) { + legacy_type_provider_(descriptor_pool, &type_provider_) { RegisterEnum("google.protobuf.NullValue", {{"NULL_VALUE", 0}}); } diff --git a/runtime/type_registry.h b/runtime/type_registry.h index eadd1f1ea..22f923bd6 100644 --- a/runtime/type_registry.h +++ b/runtime/type_registry.h @@ -41,8 +41,8 @@ class TypeRegistry; namespace runtime_internal { const RuntimeTypeProvider& GetRuntimeTypeProvider( const TypeRegistry& type_registry); -const absl_nonnull std::shared_ptr& -GetLegacyRuntimeTypeProvider(const TypeRegistry& type_registry); +const LegacyRuntimeTypeProvider& GetLegacyRuntimeTypeProvider( + const TypeRegistry& type_registry); // Returns a memoized table of fully qualified enum values. // @@ -104,10 +104,9 @@ class TypeRegistry { private: friend const runtime_internal::RuntimeTypeProvider& runtime_internal::GetRuntimeTypeProvider(const TypeRegistry& type_registry); - friend const - absl_nonnull std::shared_ptr& - runtime_internal::GetLegacyRuntimeTypeProvider( - const TypeRegistry& type_registry); + friend const runtime_internal::LegacyRuntimeTypeProvider& + runtime_internal::GetLegacyRuntimeTypeProvider( + const TypeRegistry& type_registry); friend std::shared_ptr> runtime_internal::GetEnumValueTable(const TypeRegistry& type_registry); @@ -116,8 +115,7 @@ class TypeRegistry { GetEnumValueTable() const; runtime_internal::RuntimeTypeProvider type_provider_; - absl_nonnull std::shared_ptr - legacy_type_provider_; + runtime_internal::LegacyRuntimeTypeProvider legacy_type_provider_; absl::flat_hash_map enum_types_; // memoized fully qualified enumerator names. @@ -139,8 +137,8 @@ inline const RuntimeTypeProvider& GetRuntimeTypeProvider( const TypeRegistry& type_registry) { return type_registry.type_provider_; } -inline const absl_nonnull std::shared_ptr& -GetLegacyRuntimeTypeProvider(const TypeRegistry& type_registry) { +inline const LegacyRuntimeTypeProvider& GetLegacyRuntimeTypeProvider( + const TypeRegistry& type_registry) { return type_registry.legacy_type_provider_; } inline std::shared_ptr> diff --git a/testing/testrunner/BUILD b/testing/testrunner/BUILD index b80167487..f01f892f3 100644 --- a/testing/testrunner/BUILD +++ b/testing/testrunner/BUILD @@ -18,6 +18,7 @@ cc_library( "//eval/public:cel_expression", "//runtime", "//runtime:activation", + "//runtime:activation_interface", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/memory", @@ -51,6 +52,7 @@ cc_library( "//internal:testing_no_main", "//runtime", "//runtime:activation", + "//runtime:activation_interface", "@com_google_absl//absl/functional:overload", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", @@ -90,6 +92,7 @@ cc_test( ":cel_test_context", ":coverage_index", ":runner_lib", + "//base:attributes", "//checker:type_checker_builder", "//checker:validation_result", "//common:ast_proto", @@ -107,6 +110,8 @@ cc_test( "//internal:testing_descriptor_pool", "//runtime", "//runtime:activation", + "//runtime:activation_interface", + "//runtime:function_overload_reference", "//runtime:runtime_builder", "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/container:flat_hash_map", @@ -115,6 +120,7 @@ cc_test( "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto", "@com_google_protobuf//:protobuf", diff --git a/testing/testrunner/cel_test_context.h b/testing/testrunner/cel_test_context.h index 0e0f21e28..988c37c66 100644 --- a/testing/testrunner/cel_test_context.h +++ b/testing/testrunner/cel_test_context.h @@ -29,7 +29,7 @@ #include "common/value.h" #include "compiler/compiler.h" #include "eval/public/cel_expression.h" -#include "runtime/activation.h" +#include "runtime/activation_interface.h" #include "runtime/runtime.h" #include "testing/testrunner/cel_expression_source.h" #include "cel/expr/conformance/test/suite.pb.h" @@ -40,9 +40,10 @@ namespace cel::test { // compiled CEL expressions. class CelTestContext { public: - using CelActivationFactoryFn = std::function( - const cel::expr::conformance::test::TestCase& test_case, - google::protobuf::Arena* arena)>; + using CelActivationFactoryFn = + std::function>( + const cel::expr::conformance::test::TestCase& test_case, + google::protobuf::Arena* arena)>; using AssertFn = std::function EvalWithModernBindings( const CheckedExpr& checked_expr, const CelTestContext& context, - const cel::Activation& activation, google::protobuf::Arena* arena) { + const cel::ActivationInterface& activation, google::protobuf::Arena* arena) { CEL_ASSIGN_OR_RETURN(std::unique_ptr program, Plan(checked_expr, context.runtime())); return program->Evaluate(arena, activation); @@ -205,25 +205,37 @@ absl::Status AddTestCaseBindingsToModernActivation( return absl::OkStatus(); } -absl::StatusOr GetActivation(const CelTestContext& context, - const TestCase& test_case, - google::protobuf::Arena* arena) { +absl::StatusOr> GetActivation( + const CelTestContext& context, const TestCase& test_case, + google::protobuf::Arena* arena) { if (context.activation_factory() != nullptr) { return context.activation_factory()(test_case, arena); } - return cel::Activation(); + return std::make_unique(); } -absl::StatusOr CreateModernActivationFromBindings( - const TestCase& test_case, const CelTestContext& context, - google::protobuf::Arena* arena) { - CEL_ASSIGN_OR_RETURN(cel::Activation activation, +absl::StatusOr> +CreateModernActivationFromBindings(const TestCase& test_case, + const CelTestContext& context, + google::protobuf::Arena* arena) { + CEL_ASSIGN_OR_RETURN(std::unique_ptr activation, GetActivation(context, test_case, arena)); - CEL_RETURN_IF_ERROR( - AddCustomBindingsToModernActivation(context, activation, arena)); - CEL_RETURN_IF_ERROR(AddTestCaseBindingsToModernActivation(test_case, context, - activation, arena)); + const bool has_custom_bindings = + !context.custom_bindings().empty() || !test_case.input().empty(); + if (has_custom_bindings) { + auto* cel_activation = dynamic_cast(activation.get()); + if (cel_activation == nullptr) { + return absl::InvalidArgumentError( + "Custom bindings or test case input bindings cannot be combined with " + "a custom cel::ActivationInterface implementation returned by " + "activation_factory."); + } + CEL_RETURN_IF_ERROR( + AddCustomBindingsToModernActivation(context, *cel_activation, arena)); + CEL_RETURN_IF_ERROR(AddTestCaseBindingsToModernActivation( + test_case, context, *cel_activation, arena)); + } return activation; } @@ -362,9 +374,9 @@ absl::StatusOr TestRunner::EvalWithRuntime( const CheckedExpr& checked_expr, const TestCase& test_case, google::protobuf::Arena* arena) { CEL_ASSIGN_OR_RETURN( - cel::Activation activation, + std::unique_ptr activation, CreateModernActivationFromBindings(test_case, *test_context_, arena)); - return EvalWithModernBindings(checked_expr, *test_context_, activation, + return EvalWithModernBindings(checked_expr, *test_context_, *activation, arena); } diff --git a/testing/testrunner/runner_lib_test.cc b/testing/testrunner/runner_lib_test.cc index 804826b6c..c34a1b1b0 100644 --- a/testing/testrunner/runner_lib_test.cc +++ b/testing/testrunner/runner_lib_test.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include "gtest/gtest-spi.h" #include "absl/container/flat_hash_map.h" @@ -25,6 +26,8 @@ #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "base/attribute.h" #include "checker/type_checker_builder.h" #include "checker/validation_result.h" #include "common/ast_proto.h" @@ -41,6 +44,8 @@ #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" #include "runtime/activation.h" +#include "runtime/activation_interface.h" +#include "runtime/function_overload_reference.h" #include "runtime/runtime.h" #include "runtime/runtime_builder.h" #include "runtime/standard_runtime_builder_factory.h" @@ -625,11 +630,11 @@ TEST(TestRunnerStandaloneTest, BasicTestWithActivationFactorySucceeds) { std::unique_ptr context = CelTestContext::CreateFromRuntime(std::move(runtime)); context->SetActivationFactory( - [](const TestCase& test_case, - google::protobuf::Arena* arena) -> absl::StatusOr { - cel::Activation activation; - activation.InsertOrAssignValue("x", cel::IntValue(10)); - activation.InsertOrAssignValue("y", cel::IntValue(5)); + [](const TestCase& test_case, google::protobuf::Arena* arena) + -> absl::StatusOr> { + auto activation = std::make_unique(); + activation->InsertOrAssignValue("x", cel::IntValue(10)); + activation->InsertOrAssignValue("y", cel::IntValue(5)); return activation; }); context->SetExpressionSource( @@ -652,6 +657,90 @@ TEST(TestRunnerStandaloneTest, BasicTestWithActivationFactorySucceeds) { EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); } +namespace { +class TestCustomActivation : public cel::ActivationInterface { + public: + absl::StatusOr FindVariable( + absl::string_view name, const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, google::protobuf::Arena* arena, + cel::Value* result) const override { + if (name == "x") { + *result = cel::IntValue(100); + return true; + } + if (name == "y") { + *result = cel::IntValue(200); + return true; + } + return false; + } + + std::vector FindFunctionOverloads( + absl::string_view name) const override { + return {}; + } +}; +} // namespace + +TEST(TestRunnerStandaloneTest, CustomActivationInterfaceFactorySucceeds) { + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + DefaultCompiler().Compile("x + y")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + std::unique_ptr context = + CelTestContext::CreateFromRuntime(std::move(runtime)); + context->SetActivationFactory( + [](const TestCase& test_case, google::protobuf::Arena* arena) + -> absl::StatusOr> { + return std::make_unique(); + }); + context->SetExpressionSource( + CelExpressionSource::FromCheckedExpr(std::move(checked_expr))); + + TestCase test_case = ParseTextProtoOrDie(R"pb( + output { result_value { int64_value: 300 } } + )pb"); + TestRunner test_runner(std::move(context)); + EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); +} + +TEST(TestRunnerStandaloneTest, + CustomActivationInterfaceWithInputsReturnsError) { + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + DefaultCompiler().Compile("x + y")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + std::unique_ptr context = + CelTestContext::CreateFromRuntime(std::move(runtime)); + context->SetActivationFactory( + [](const TestCase& test_case, google::protobuf::Arena* arena) + -> absl::StatusOr> { + return std::make_unique(); + }); + context->SetExpressionSource( + CelExpressionSource::FromCheckedExpr(std::move(checked_expr))); + + static auto* test_case_ptr = new TestCase(ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { value { int64_value: 4 } } + } + output { result_value { int64_value: 300 } } + )pb")); + static auto* test_runner_ptr = new TestRunner(std::move(context)); + EXPECT_FATAL_FAILURE( + test_runner_ptr->RunTest(*test_case_ptr), + "Custom bindings or test case input bindings cannot be combined"); +} + TEST(TestRunnerStandaloneTest, CustomAssertFnIsUsed) { // Compile the expression. ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, diff --git a/testutil/BUILD b/testutil/BUILD index 292696033..6574db5d5 100644 --- a/testutil/BUILD +++ b/testutil/BUILD @@ -15,62 +15,52 @@ load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_cc//cc:cc_test.bzl", "cc_test") +load("//bazel:cel_cc_embed.bzl", "cel_cc_embed") +load("//bazel:cel_proto_transitive_descriptor_set.bzl", "cel_proto_transitive_descriptor_set") package(default_visibility = ["//visibility:public"]) licenses(["notice"]) cc_library( - name = "expr_printer", - srcs = ["expr_printer.cc"], - hdrs = ["expr_printer.h"], - deps = [ - "//common:ast", - "//common:ast_proto", - "//common:constant", - "//common:expr", - "//internal:strings", - "@com_google_absl//absl/base:no_destructor", - "@com_google_absl//absl/log:absl_log", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", + name = "util", + testonly = True, + hdrs = [ + "util.h", ], + deps = ["//internal:proto_matchers"], ) -cc_test( - name = "expr_printer_test", - srcs = ["expr_printer_test.cc"], +cc_library( + name = "test_macros", + testonly = True, + srcs = ["test_macros.cc"], + hdrs = ["test_macros.h"], deps = [ - ":expr_printer", "//common:expr", - "//internal:testing", - "//parser", - "//parser:options", + "//internal:status_macros", + "//parser:macro", + "//parser:macro_expr_factory", + "//parser:macro_registry", + "//parser:parser_interface", "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", ], ) -cc_library( - name = "util", - testonly = True, - hdrs = [ - "util.h", - ], - deps = ["//internal:proto_matchers"], -) - cc_library( name = "baseline_tests", testonly = True, srcs = ["baseline_tests.cc"], hdrs = ["baseline_tests.h"], deps = [ - ":expr_printer", "//common:ast", "//common:expr", + "//common:expr_printer", "//extensions/protobuf:ast_converters", "@com_google_absl//absl/strings", "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", @@ -92,3 +82,47 @@ proto_library( name = "test_json_names_proto", srcs = ["test_json_names.proto"], ) + +proto_library( + name = "test_external_extensions_proto", + srcs = ["test_external_extensions.proto"], + deps = ["@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_proto"], +) + +cel_proto_transitive_descriptor_set( + name = "test_external_extensions_transitive_descriptor_set", + testonly = True, + deps = [ + ":test_external_extensions_proto", + ], +) + +cel_cc_embed( + name = "test_external_extensions_descriptor_set_embed", + testonly = True, + src = ":test_external_extensions_transitive_descriptor_set", +) + +cc_library( + name = "test_external_extensions_descriptor_set", + testonly = True, + srcs = ["test_external_extensions_descriptor_set.cc"], + hdrs = ["test_external_extensions_descriptor_set.h"], + textual_hdrs = [":test_external_extensions_descriptor_set_embed"], + deps = [ + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/log:absl_check", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "test_external_extensions_descriptor_set_test", + srcs = ["test_external_extensions_descriptor_set_test.cc"], + deps = [ + ":test_external_extensions_descriptor_set", + "//internal:testing", + "@com_google_protobuf//:protobuf", + ], +) diff --git a/testutil/baseline_tests.cc b/testutil/baseline_tests.cc index 8ce43e63d..08db0827a 100644 --- a/testutil/baseline_tests.cc +++ b/testutil/baseline_tests.cc @@ -22,8 +22,8 @@ #include "absl/strings/str_join.h" #include "common/ast.h" #include "common/expr.h" +#include "common/expr_printer.h" #include "extensions/protobuf/ast_converters.h" -#include "testutil/expr_printer.h" namespace cel::test { namespace { diff --git a/testutil/test_external_extensions.proto b/testutil/test_external_extensions.proto new file mode 100644 index 000000000..b78827b67 --- /dev/null +++ b/testutil/test_external_extensions.proto @@ -0,0 +1,42 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Tests for external extensions added in an overlay. + +edition = "2024"; + +package cel.cpp.testutil; + +import "cel/expr/conformance/proto2/test_all_types.proto"; + +// Test message for external extensions. +message TestExternalExtensions { + string string_field = 1; + int64 int64_field = 2; +} + +extend cel.expr.conformance.proto2.TestAllTypes { + TestExternalExtensions test_external_extensions_message = 2001; + string test_external_extensions_string = 2002; + int64 test_external_extensions_int64 = 2003; +} + +// Message scoped extension. +message ScopedTestExternalExtensions { + extend cel.expr.conformance.proto2.TestAllTypes { + TestExternalExtensions scoped_ext_message = 3001; + string scoped_ext_string = 3002; + int64 scoped_ext_int64 = 3003; + } +} diff --git a/testutil/test_external_extensions_descriptor_set.cc b/testutil/test_external_extensions_descriptor_set.cc new file mode 100644 index 000000000..d18633a03 --- /dev/null +++ b/testutil/test_external_extensions_descriptor_set.cc @@ -0,0 +1,62 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "testutil/test_external_extensions_descriptor_set.h" + +#include +#include + +#include "google/protobuf/descriptor.pb.h" +#include "absl/base/attributes.h" +#include "absl/base/no_destructor.h" +#include "absl/base/optimization.h" +#include "absl/log/absl_check.h" + +namespace cel::test { + +namespace { + +ABSL_CONST_INIT const uint8_t kTestExternalExtensionsDescriptorSet[] = { +#include "testutil/test_external_extensions_descriptor_set_embed.inc" +}; + +} // namespace + +const google::protobuf::FileDescriptorSet& GetTestExternalExtensionsFileDescriptorSet() { + static const absl::NoDestructor + file_descriptor_set([]() { + google::protobuf::FileDescriptorSet file_desc_set; + ABSL_CHECK(file_desc_set.ParseFromArray( // Crash OK + kTestExternalExtensionsDescriptorSet, + std::size(kTestExternalExtensionsDescriptorSet))); + return file_desc_set; + }()); + return *file_descriptor_set; +} + +const google::protobuf::FileDescriptorProto& GetTestExternalExtensionsFileDescriptor() { + static const google::protobuf::FileDescriptorProto* const file_desc = []() { + const auto& file_desc_set = GetTestExternalExtensionsFileDescriptorSet(); + for (const auto& file_desc : file_desc_set.file()) { + if (file_desc.name() == + "testutil/test_external_extensions.proto") { + return &file_desc; + } + } + ABSL_UNREACHABLE(); + }(); + return *file_desc; +} + +} // namespace cel::test diff --git a/testutil/test_external_extensions_descriptor_set.h b/testutil/test_external_extensions_descriptor_set.h new file mode 100644 index 000000000..a5302e1bb --- /dev/null +++ b/testutil/test_external_extensions_descriptor_set.h @@ -0,0 +1,28 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_TESTUTIL_TEST_EXTERNAL_EXTENSIONS_DESCRIPTOR_SET_H_ +#define THIRD_PARTY_CEL_CPP_TESTUTIL_TEST_EXTERNAL_EXTENSIONS_DESCRIPTOR_SET_H_ + +#include "google/protobuf/descriptor.pb.h" + +namespace cel::test { + +const google::protobuf::FileDescriptorSet& GetTestExternalExtensionsFileDescriptorSet(); + +const google::protobuf::FileDescriptorProto& GetTestExternalExtensionsFileDescriptor(); + +} // namespace cel::test + +#endif // THIRD_PARTY_CEL_CPP_TESTUTIL_TEST_EXTERNAL_EXTENSIONS_DESCRIPTOR_SET_H_ diff --git a/testutil/test_external_extensions_descriptor_set_test.cc b/testutil/test_external_extensions_descriptor_set_test.cc new file mode 100644 index 000000000..de4b61cba --- /dev/null +++ b/testutil/test_external_extensions_descriptor_set_test.cc @@ -0,0 +1,56 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "testutil/test_external_extensions_descriptor_set.h" + +#include "internal/testing.h" +#include "google/protobuf/descriptor.h" + +namespace cel::test { +namespace { + +using ::testing::NotNull; + +TEST(GetTestExternalExtensionsFileDescriptorSet, BuildsPool) { + const auto& fds = GetTestExternalExtensionsFileDescriptorSet(); + EXPECT_GT(fds.file_size(), 0); + + google::protobuf::DescriptorPool pool; + for (const auto& file : fds.file()) { + EXPECT_THAT(pool.BuildFile(file), NotNull()); + } + + EXPECT_THAT( + pool.FindMessageTypeByName("cel.cpp.testutil.TestExternalExtensions"), + NotNull()); + EXPECT_THAT(pool.FindExtensionByName( + "cel.cpp.testutil.test_external_extensions_message"), + NotNull()); + EXPECT_THAT( + pool.FindExtensionByName( + "cel.cpp.testutil.ScopedTestExternalExtensions.scoped_ext_string"), + NotNull()); +} + +TEST(GetTestExternalExtensionsFileDescriptor, CorrectFileDescriptor) { + const auto& fd = GetTestExternalExtensionsFileDescriptor(); + EXPECT_EQ(fd.name(), + "testutil/test_external_extensions.proto"); + EXPECT_EQ(fd.package(), "cel.cpp.testutil"); + EXPECT_GT(fd.message_type_size(), 0); + EXPECT_GT(fd.extension_size(), 0); +} + +} // namespace +} // namespace cel::test diff --git a/testutil/test_macros.cc b/testutil/test_macros.cc new file mode 100644 index 000000000..30b94280d --- /dev/null +++ b/testutil/test_macros.cc @@ -0,0 +1,187 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "testutil/test_macros.h" + +#include +#include +#include + +#include "absl/base/no_destructor.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/types/span.h" +#include "common/expr.h" +#include "internal/status_macros.h" +#include "parser/macro.h" +#include "parser/macro_expr_factory.h" +#include "parser/macro_registry.h" +#include "parser/parser_interface.h" + +namespace cel::test { + +namespace { + +bool IsCelNamespace(const Expr& target) { + return target.has_ident_expr() && target.ident_expr().name() == "cel"; +} + +std::optional CelBlockMacroExpander(MacroExprFactory& factory, + Expr& target, absl::Span args) { + if (!IsCelNamespace(target)) { + return std::nullopt; + } + Expr& bindings_arg = args[0]; + if (!bindings_arg.has_list_expr()) { + return factory.ReportErrorAt( + bindings_arg, "cel.block requires the first arg to be a list literal"); + } + return factory.NewCall("cel.@block", args); +} + +std::optional CelIndexMacroExpander(MacroExprFactory& factory, + Expr& target, absl::Span args) { + if (!IsCelNamespace(target)) { + return std::nullopt; + } + Expr& index_arg = args[0]; + if (!index_arg.has_const_expr() || !index_arg.const_expr().has_int_value()) { + return factory.ReportErrorAt( + index_arg, "cel.index requires a single non-negative int constant arg"); + } + int64_t index = index_arg.const_expr().int_value(); + if (index < 0) { + return factory.ReportErrorAt( + index_arg, "cel.index requires a single non-negative int constant arg"); + } + return factory.NewIdent(absl::StrCat("@index", index)); +} + +std::optional CelIterVarMacroExpander(MacroExprFactory& factory, + Expr& target, + absl::Span args) { + if (!IsCelNamespace(target)) { + return std::nullopt; + } + Expr& depth_arg = args[0]; + if (!depth_arg.has_const_expr() || !depth_arg.const_expr().has_int_value() || + depth_arg.const_expr().int_value() < 0) { + return factory.ReportErrorAt( + depth_arg, "cel.iterVar requires two non-negative int constant args"); + } + Expr& unique_arg = args[1]; + if (!unique_arg.has_const_expr() || + !unique_arg.const_expr().has_int_value() || + unique_arg.const_expr().int_value() < 0) { + return factory.ReportErrorAt( + unique_arg, "cel.iterVar requires two non-negative int constant args"); + } + return factory.NewIdent( + absl::StrCat("@it:", depth_arg.const_expr().int_value(), ":", + unique_arg.const_expr().int_value())); +} + +std::optional CelAccuVarMacroExpander(MacroExprFactory& factory, + Expr& target, + absl::Span args) { + if (!IsCelNamespace(target)) { + return std::nullopt; + } + Expr& depth_arg = args[0]; + if (!depth_arg.has_const_expr() || !depth_arg.const_expr().has_int_value() || + depth_arg.const_expr().int_value() < 0) { + return factory.ReportErrorAt( + depth_arg, "cel.accuVar requires two non-negative int constant args"); + } + Expr& unique_arg = args[1]; + if (!unique_arg.has_const_expr() || + !unique_arg.const_expr().has_int_value() || + unique_arg.const_expr().int_value() < 0) { + return factory.ReportErrorAt( + unique_arg, "cel.accuVar requires two non-negative int constant args"); + } + return factory.NewIdent( + absl::StrCat("@ac:", depth_arg.const_expr().int_value(), ":", + unique_arg.const_expr().int_value())); +} + +Macro MakeCelBlockMacro() { + auto macro_or_status = Macro::Receiver("block", 2, CelBlockMacroExpander); + ABSL_CHECK_OK(macro_or_status); // Crash OK + return std::move(*macro_or_status); +} + +Macro MakeCelIndexMacro() { + auto macro_or_status = Macro::Receiver("index", 1, CelIndexMacroExpander); + ABSL_CHECK_OK(macro_or_status); // Crash OK + return std::move(*macro_or_status); +} + +Macro MakeCelIterVarMacro() { + auto macro_or_status = Macro::Receiver("iterVar", 2, CelIterVarMacroExpander); + ABSL_CHECK_OK(macro_or_status); // Crash OK + return std::move(*macro_or_status); +} + +Macro MakeCelAccuVarMacro() { + auto macro_or_status = Macro::Receiver("accuVar", 2, CelAccuVarMacroExpander); + ABSL_CHECK_OK(macro_or_status); // Crash OK + return std::move(*macro_or_status); +} + +} // namespace + +const Macro& CelBlockMacro() { + static const absl::NoDestructor macro(MakeCelBlockMacro()); + return *macro; +} + +const Macro& CelIndexMacro() { + static const absl::NoDestructor macro(MakeCelIndexMacro()); + return *macro; +} + +const Macro& CelIterVarMacro() { + static const absl::NoDestructor macro(MakeCelIterVarMacro()); + return *macro; +} + +const Macro& CelAccuVarMacro() { + static const absl::NoDestructor macro(MakeCelAccuVarMacro()); + return *macro; +} + +absl::Status RegisterTestMacros(MacroRegistry& registry) { + CEL_RETURN_IF_ERROR(registry.RegisterMacro(CelBlockMacro())); + CEL_RETURN_IF_ERROR(registry.RegisterMacro(CelIndexMacro())); + CEL_RETURN_IF_ERROR(registry.RegisterMacro(CelIterVarMacro())); + CEL_RETURN_IF_ERROR(registry.RegisterMacro(CelAccuVarMacro())); + return absl::OkStatus(); +} + +ParserLibrary TestMacrosParserLibrary() { + return ParserLibrary{ + "cel.test.test_macros", + [](ParserBuilder& builder) -> absl::Status { + CEL_RETURN_IF_ERROR(builder.AddMacro(CelBlockMacro())); + CEL_RETURN_IF_ERROR(builder.AddMacro(CelIndexMacro())); + CEL_RETURN_IF_ERROR(builder.AddMacro(CelIterVarMacro())); + CEL_RETURN_IF_ERROR(builder.AddMacro(CelAccuVarMacro())); + return absl::OkStatus(); + }, + }; +} + +} // namespace cel::test diff --git a/testutil/test_macros.h b/testutil/test_macros.h new file mode 100644 index 000000000..cc988ceb1 --- /dev/null +++ b/testutil/test_macros.h @@ -0,0 +1,36 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_TESTUTIL_TEST_MACROS_H_ +#define THIRD_PARTY_CEL_CPP_TESTUTIL_TEST_MACROS_H_ + +#include "absl/status/status.h" +#include "parser/macro.h" +#include "parser/macro_registry.h" +#include "parser/parser_interface.h" + +namespace cel::test { + +const Macro& CelBlockMacro(); +const Macro& CelIndexMacro(); +const Macro& CelIterVarMacro(); +const Macro& CelAccuVarMacro(); + +absl::Status RegisterTestMacros(MacroRegistry& registry); + +ParserLibrary TestMacrosParserLibrary(); + +} // namespace cel::test + +#endif // THIRD_PARTY_CEL_CPP_TESTUTIL_TEST_MACROS_H_ diff --git a/tools/BUILD b/tools/BUILD index ceb2befc5..8b6ba13d9 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -74,41 +74,6 @@ cc_test( ], ) -cc_library( - name = "flatbuffers_backed_impl", - srcs = [ - "flatbuffers_backed_impl.cc", - ], - hdrs = [ - "flatbuffers_backed_impl.h", - ], - deps = [ - "//eval/public:cel_value", - "@com_github_google_flatbuffers//:flatbuffers", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", - ], -) - -cc_test( - name = "flatbuffers_backed_impl_test", - size = "small", - srcs = [ - "flatbuffers_backed_impl_test.cc", - ], - data = [ - "//tools/testdata:flatbuffers_reflection_out", - ], - deps = [ - ":flatbuffers_backed_impl", - "//internal:status_macros", - "//internal:testing", - "@com_github_google_flatbuffers//:flatbuffers", - ], -) - cc_library( name = "navigable_ast", srcs = ["navigable_ast.cc"], @@ -204,6 +169,56 @@ cc_library( ], ) +cc_library( + name = "proto_to_predicate", + srcs = ["proto_to_predicate.cc"], + hdrs = ["proto_to_predicate.h"], + deps = [ + "//common:ast", + "//common:expr", + "//common:expr_factory", + "//common:operators", + "//internal:status_macros", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "proto_to_predicate_test", + srcs = ["proto_to_predicate_test.cc"], + deps = [ + ":cel_unparser", + ":proto_to_predicate", + "//common:ast", + "//common:ast_proto", + "//common:value", + "//env:config", + "//env:env_runtime", + "//env:env_yaml", + "//env:runtime_std_extensions", + "//eval/testutil:test_message_cc_proto", + "//extensions/protobuf:value", + "//internal:status_macros", + "//internal:testing", + "//internal:testing_descriptor_pool", + "//parser", + "//runtime", + "//runtime:activation", + "//tools/testdata:test_policy_cc_proto", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + "@com_google_protobuf//:protobuf", + ], +) + cc_test( name = "descriptor_pool_builder_test", srcs = ["descriptor_pool_builder_test.cc"], diff --git a/tools/branch_coverage.cc b/tools/branch_coverage.cc index 00ab7cb5a..b5bba3ffe 100644 --- a/tools/branch_coverage.cc +++ b/tools/branch_coverage.cc @@ -71,7 +71,7 @@ struct OtherNode { // Representation for coverage of an AST node. struct CoverageNode { int evaluate_count; - absl::variant kind; + std::variant kind; }; const Type* absl_nullable FindCheckerType(const CheckedExpr& expr, diff --git a/tools/cel_unparser.cc b/tools/cel_unparser.cc index 28a1187bb..741d91208 100644 --- a/tools/cel_unparser.cc +++ b/tools/cel_unparser.cc @@ -150,6 +150,8 @@ class Unparser { // - a ternary conditional operator bool IsBinaryOrTernaryOperator(const Expr& expr); + bool IsLogicalOperator(absl::string_view op); + template void Print(Ts&&... args) { absl::StrAppend(&output_, std::forward(args)...); @@ -436,6 +438,24 @@ absl::Status Unparser::VisitUnary(const Expr::Call& expr, absl::Status Unparser::VisitBinary(const Expr::Call& expr, const std::string& op) { + if (expr.args_size() < 2) { + return absl::InvalidArgumentError( + absl::StrCat("Unexpected binary: ", expr.ShortDebugString())); + } + + const auto& fun = expr.function(); + if (IsLogicalOperator(fun)) { + for (int i = 0; i < expr.args_size(); ++i) { + if (i > 0) { + Print(kSpace, op, kSpace); + } + const auto& arg = expr.args(i); + bool arg_paren = IsComplexOperatorWithRespectTo(arg, fun); + CEL_RETURN_IF_ERROR(VisitMaybeNested(arg, arg_paren)); + } + return absl::OkStatus(); + } + if (expr.args_size() != 2) { return absl::InvalidArgumentError( absl::StrCat("Unexpected binary: ", expr.ShortDebugString())); @@ -443,7 +463,6 @@ absl::Status Unparser::VisitBinary(const Expr::Call& expr, const auto& lhs = expr.args(0); const auto& rhs = expr.args(1); - const auto& fun = expr.function(); // add parens if the current operator is lower precedence than the lhs expr // operator. @@ -549,6 +568,10 @@ bool Unparser::IsBinaryOrTernaryOperator(const Expr& expr) { IsOperatorSamePrecedence(CelOperator::CONDITIONAL, expr); } +bool Unparser::IsLogicalOperator(absl::string_view op) { + return op == CelOperator::LOGICAL_AND || op == CelOperator::LOGICAL_OR; +} + } // namespace absl::StatusOr Unparse(const Expr& expr, diff --git a/tools/cel_unparser_test.cc b/tools/cel_unparser_test.cc index 4cba4ce4d..aca6e91fd 100644 --- a/tools/cel_unparser_test.cc +++ b/tools/cel_unparser_test.cc @@ -67,6 +67,22 @@ INSTANTIATE_TEST_SUITE_P( {// Empty Expr error {"", absl::InvalidArgumentError("Unsupported Expr")}, + // Logical operators with too few arguments (single argument) + { + R"pb( + call_expr { + function: "_&&_" + args { const_expr { bool_value: true } } + })pb", + absl::InvalidArgumentError("Unexpected binary")}, + { + R"pb( + call_expr { + function: "_||_" + args { const_expr { bool_value: true } } + })pb", + absl::InvalidArgumentError("Unexpected binary")}, + // Constants {"const_expr{}", absl::InvalidArgumentError("Unsupported Constant")}, {"const_expr{bool_value: true}", "true"}, @@ -619,6 +635,7 @@ TEST_P(UnparserTestTextExpr, Test) { options.add_macro_calls = true; options.enable_optional_syntax = true; options.enable_quoted_identifiers = true; + options.enable_variadic_logical_operators = true; ASSERT_OK_AND_ASSIGN(ParsedExpr result, Parse(GetParam().expr, "unparser", options)); @@ -779,6 +796,8 @@ INSTANTIATE_TEST_SUITE_P( {"has(a.`b.c`)", ""}, {"a.`b/c`", ""}, {"a.?`b/c`", ""}, + {"a && b && c && d", ""}, + {"a || b || c || d", ""}, })); } // namespace diff --git a/tools/flatbuffers_backed_impl.cc b/tools/flatbuffers_backed_impl.cc index 10c0b1cb8..2ee226859 100644 --- a/tools/flatbuffers_backed_impl.cc +++ b/tools/flatbuffers_backed_impl.cc @@ -127,7 +127,7 @@ class ObjectStringIndexedMapImpl : public CelMap { arena_, **it, schema_, object_, arena_)); } } - return absl::nullopt; + return std::nullopt; } absl::StatusOr ListKeys() const override { return &keys_; } @@ -188,7 +188,7 @@ absl::optional FlatBuffersMapImpl::operator[]( } auto field = keys_.fields->LookupByKey(cel_key.StringOrDie().value().data()); if (field == nullptr) { - return absl::nullopt; + return std::nullopt; } switch (field->type()->base_type()) { case reflection::Byte: @@ -323,15 +323,15 @@ absl::optional FlatBuffersMapImpl::operator[]( } default: // Unsupported vector base types - return absl::nullopt; + return std::nullopt; } break; } default: // Unsupported types: enums, unions, arrays - return absl::nullopt; + return std::nullopt; } - return absl::nullopt; + return std::nullopt; } const CelMap* CreateFlatBuffersBackedObject(const uint8_t* flatbuf, diff --git a/tools/flatbuffers_backed_impl.h b/tools/flatbuffers_backed_impl.h index 7051ef5d5..bf02436ad 100644 --- a/tools/flatbuffers_backed_impl.h +++ b/tools/flatbuffers_backed_impl.h @@ -9,6 +9,10 @@ namespace api { namespace expr { namespace runtime { +// Reference implementation of CelMap backed by a flatbuffer table. +// +// To avoid taking a dependency on flatbuffers, this is not included in the +// bazel module or built by public CI. class FlatBuffersMapImpl : public CelMap { public: FlatBuffersMapImpl(const flatbuffers::Table& table, diff --git a/tools/proto_to_predicate.cc b/tools/proto_to_predicate.cc new file mode 100644 index 000000000..01f35fd05 --- /dev/null +++ b/tools/proto_to_predicate.cc @@ -0,0 +1,470 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tools/proto_to_predicate.h" + +#include +#include +#include +#include +#include +#include + +#include "google/protobuf/descriptor.pb.h" +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "common/expr.h" +#include "common/expr_factory.h" +#include "common/operators.h" +#include "internal/status_macros.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" +#include "google/protobuf/reflection.h" + +namespace cel::tools { + +using ::google::api::expr::common::CelOperator; +using ::google::protobuf::FieldDescriptor; +using ::google::protobuf::Message; +using ::google::protobuf::Reflection; + +class ProtoToPredicateBuilder final : private ExprFactory { + public: + ProtoToPredicateBuilder() : id_(1) {} + + absl::StatusOr Build(absl::string_view input_name, + const Message& message) { + std::vector predicates; + Expr base_expr = NewIdent(NextId(), input_name); + + CEL_RETURN_IF_ERROR(Walk(message, base_expr, predicates)); + Expr root = LogicalAnd(predicates); + return Ast(std::move(root), std::move(source_info_)); + } + + absl::StatusOr Build(absl::string_view input_name, + absl::Span messages) { + if (messages.empty()) { + return Ast(NewBoolConst(NextId(), true), std::move(source_info_)); + } + + std::vector message_asts; + message_asts.reserve(messages.size()); + for (const auto* message : messages) { + std::vector predicates; + Expr base_expr = NewIdent(NextId(), input_name); + + CEL_RETURN_IF_ERROR(Walk(*message, base_expr, predicates)); + message_asts.push_back(LogicalAnd(predicates)); + } + + return Ast(LogicalOr(message_asts), std::move(source_info_)); + } + + private: + // Retrieves the "match_path" string option from the field options if + // defined, returning an empty string otherwise. + std::string GetMatchPath(const ::google::protobuf::FieldDescriptor* field) { + const ::google::protobuf::Message& options = field->options(); + const ::google::protobuf::Reflection* refl = options.GetReflection(); + std::vector fields; + refl->ListFields(options, &fields); + for (const auto* f : fields) { + if (f->name() == "match_path") { + return refl->GetString(options, f); + } + } + return ""; + } + + // Parses a dot-separated string representation of a path (e.g. "dest.region") + // and builds a corresponding select chain AST. + Expr ParseAndBuildPath(absl::string_view path_str) { + std::vector parts = absl::StrSplit(path_str, '.'); + Expr e = NewIdent(NextId(), parts[0]); + for (size_t i = 1; i < parts.size(); ++i) { + e = NewSelect(NextId(), std::move(e), parts[i]); + } + return e; + } + + // Returns either the path specified by the "match_path" annotation, + // or the default path derived from the field name. + Expr GetFieldPath(const Expr& base_expr, + const ::google::protobuf::FieldDescriptor* field) { + std::string match_path_val = GetMatchPath(field); + if (!match_path_val.empty()) { + return ParseAndBuildPath(match_path_val); + } + return NewSelect(NextId(), base_expr, field->name()); + } + ExprId NextId() { return id_++; } + + // --------------------------------------------------------------------------- + // Field value extraction + // --------------------------------------------------------------------------- + + // Converts a singular field value to a CEL constant expression. + Expr PrimitiveToExpr(ExprId expr_id, const Message& message, + const Reflection* reflection, + const FieldDescriptor* field) { + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + return NewIntConst(expr_id, reflection->GetInt32(message, field)); + case FieldDescriptor::CPPTYPE_INT64: + return NewIntConst(expr_id, reflection->GetInt64(message, field)); + case FieldDescriptor::CPPTYPE_UINT32: + return NewUintConst(expr_id, reflection->GetUInt32(message, field)); + case FieldDescriptor::CPPTYPE_UINT64: + return NewUintConst(expr_id, reflection->GetUInt64(message, field)); + case FieldDescriptor::CPPTYPE_DOUBLE: + return NewDoubleConst(expr_id, reflection->GetDouble(message, field)); + case FieldDescriptor::CPPTYPE_FLOAT: + return NewDoubleConst(expr_id, reflection->GetFloat(message, field)); + case FieldDescriptor::CPPTYPE_BOOL: + return NewBoolConst(expr_id, reflection->GetBool(message, field)); + case FieldDescriptor::CPPTYPE_ENUM: + return NewIntConst(expr_id, reflection->GetEnumValue(message, field)); + case FieldDescriptor::CPPTYPE_STRING: { + std::string str_val = reflection->GetString(message, field); + if (field->type() == FieldDescriptor::TYPE_BYTES) { + return NewBytesConst(expr_id, std::move(str_val)); + } + return NewStringConst(expr_id, std::move(str_val)); + } + default: + // Log a warning as message should be handled by Walk. + ABSL_LOG(WARNING) << "PrimitiveToExpr: Unhandled field type: " + << FieldDescriptor::TypeName(field->type()); + break; + } + return NewNullConst(expr_id); + } + + Expr PrimitiveToExpr(const Message& message, const Reflection* reflection, + const FieldDescriptor* field) { + return PrimitiveToExpr(NextId(), message, reflection, field); + } + + // Converts a repeated field element to a CEL constant expression. + Expr RepeatedPrimitiveToExpr(const Message& message, + const Reflection* reflection, + const FieldDescriptor* field, int index) { + const ExprId id = NextId(); + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + return NewIntConst(id, + reflection->GetRepeatedInt32(message, field, index)); + case FieldDescriptor::CPPTYPE_INT64: + return NewIntConst(id, + reflection->GetRepeatedInt64(message, field, index)); + case FieldDescriptor::CPPTYPE_UINT32: + return NewUintConst( + id, reflection->GetRepeatedUInt32(message, field, index)); + case FieldDescriptor::CPPTYPE_UINT64: + return NewUintConst( + id, reflection->GetRepeatedUInt64(message, field, index)); + case FieldDescriptor::CPPTYPE_DOUBLE: + return NewDoubleConst( + id, reflection->GetRepeatedDouble(message, field, index)); + case FieldDescriptor::CPPTYPE_FLOAT: + return NewDoubleConst( + id, reflection->GetRepeatedFloat(message, field, index)); + case FieldDescriptor::CPPTYPE_BOOL: + return NewBoolConst(id, + reflection->GetRepeatedBool(message, field, index)); + case FieldDescriptor::CPPTYPE_ENUM: + return NewIntConst( + id, reflection->GetRepeatedEnumValue(message, field, index)); + case FieldDescriptor::CPPTYPE_STRING: { + std::string str_val = + reflection->GetRepeatedString(message, field, index); + if (field->type() == FieldDescriptor::TYPE_BYTES) { + return NewBytesConst(id, std::move(str_val)); + } + return NewStringConst(id, std::move(str_val)); + } + default: + break; + } + return NewNullConst(id); + } + + // --------------------------------------------------------------------------- + // Expression construction helpers + // --------------------------------------------------------------------------- + + // Creates a binary operator call: `lhs rhs`. + Expr ConstructBinaryOp(absl::string_view op, Expr lhs, Expr rhs) { + std::vector args = {std::move(lhs), std::move(rhs)}; + return NewCall(NextId(), op, std::move(args)); + } + + Expr ConstructEquality(Expr lhs, Expr rhs) { + return ConstructBinaryOp(CelOperator::EQUALS, std::move(lhs), + std::move(rhs)); + } + + Expr LogicalOr(std::vector& exprs) { + return LogicalOp(CelOperator::LOGICAL_OR, exprs); + } + + Expr LogicalAnd(std::vector& exprs) { + return LogicalOp(CelOperator::LOGICAL_AND, exprs); + } + + // Left-folds a vector of expressions with a binary operator. + // Requires: `exprs` is non-empty. + Expr LogicalOp(absl::string_view op, std::vector& exprs) { + if (exprs.empty()) { + return NewBoolConst(NextId(), true); + } + if (exprs.size() == 1) { + return std::move(exprs[0]); + } + return NewCall(NextId(), op, std::move(exprs)); + } + + // --------------------------------------------------------------------------- + // Map field predicate (extracted from Walk for readability) + // --------------------------------------------------------------------------- + + // Builds the predicate for a map field to assert that all key-value pairs + // specified in the policy are present in the input map field: + // "key" in input.map && input.map["key"] == value + absl::Status WalkMapField(const Reflection* reflection, + const Message& message, + const FieldDescriptor* field, const Expr& base_expr, + int size, std::vector& predicates) { + const FieldDescriptor* const key_field = + field->message_type()->FindFieldByName("key"); + const FieldDescriptor* const value_field = + field->message_type()->FindFieldByName("value"); + + Expr map_path = GetFieldPath(base_expr, field); + + struct MapEntry { + const Message* message; + }; + std::vector entries; + entries.reserve(size); + for (int i = 0; i < size; ++i) { + entries.push_back({&reflection->GetRepeatedMessage(message, field, i)}); + } + + if (!entries.empty()) { + const Reflection* const entry_ref = entries[0].message->GetReflection(); + std::sort(entries.begin(), entries.end(), + [entry_ref, key_field](const MapEntry& a, const MapEntry& b) { + switch (key_field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + return entry_ref->GetInt32(*a.message, key_field) < + entry_ref->GetInt32(*b.message, key_field); + case FieldDescriptor::CPPTYPE_INT64: + return entry_ref->GetInt64(*a.message, key_field) < + entry_ref->GetInt64(*b.message, key_field); + case FieldDescriptor::CPPTYPE_UINT32: + return entry_ref->GetUInt32(*a.message, key_field) < + entry_ref->GetUInt32(*b.message, key_field); + case FieldDescriptor::CPPTYPE_UINT64: + return entry_ref->GetUInt64(*a.message, key_field) < + entry_ref->GetUInt64(*b.message, key_field); + case FieldDescriptor::CPPTYPE_BOOL: + return !entry_ref->GetBool(*a.message, key_field) && + entry_ref->GetBool(*b.message, key_field); + case FieldDescriptor::CPPTYPE_STRING: + return entry_ref->GetString(*a.message, key_field) < + entry_ref->GetString(*b.message, key_field); + default: + return false; + } + }); + } + + std::vector map_checks; + map_checks.reserve(size); + for (const auto& entry : entries) { + const Message& entry_msg = *entry.message; + const Reflection* const entry_ref = entry_msg.GetReflection(); + + Expr key_expr = PrimitiveToExpr(entry_msg, entry_ref, key_field); + + // Represents `"key" in input.map` to assert the key exists. + Expr in_check = NewCall(NextId(), CelOperator::IN, + std::vector{key_expr, map_path}); + // Represents `input.map["key"]` to lookup the value. + Expr lookup_path = NewCall(NextId(), CelOperator::INDEX, + std::vector{map_path, key_expr}); + + if (value_field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + const Message& value_msg = + entry_ref->GetMessage(entry_msg, value_field); + std::vector val_predicates; + CEL_RETURN_IF_ERROR(Walk(value_msg, lookup_path, val_predicates)); + + if (!val_predicates.empty()) { + // Represents `"key" in input.map && (nested message fields check...)` + map_checks.push_back(std::move(in_check)); + map_checks.insert(map_checks.end(), + std::make_move_iterator(val_predicates.begin()), + std::make_move_iterator(val_predicates.end())); + } else { + // Represents `"key" in input.map` if nested message is empty. + map_checks.push_back(std::move(in_check)); + } + } else { + Expr value_expr = PrimitiveToExpr(entry_msg, entry_ref, value_field); + // Represents `input.map["key"] == value` + Expr eq_check = + ConstructEquality(std::move(lookup_path), std::move(value_expr)); + + // Represents `"key" in input.map && input.map["key"] == value` + map_checks.push_back(std::move(in_check)); + map_checks.push_back(std::move(eq_check)); + } + } + + predicates.push_back(LogicalAnd(map_checks)); + return absl::OkStatus(); + } + + // --------------------------------------------------------------------------- + // Repeated field predicate (extracted from Walk for readability) + // --------------------------------------------------------------------------- + + // Builds predicates for a repeated field: + // - Repeated Messages are mapped to a logical OR (||) of the generated + // predicates for each message. + // - Repeated Primitives are mapped either to: + // - `lhs in [values]` if a "match_path" option is specified. + // - `value in input.field` conjoined with && for each value otherwise. + absl::Status WalkRepeatedField(const Reflection* reflection, + const Message& message, + const FieldDescriptor* field, + const Expr& base_expr, int size, + std::vector& predicates) { + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + std::vector message_asts; + message_asts.reserve(size); + for (int i = 0; i < size; ++i) { + const Message& sub_message = + reflection->GetRepeatedMessage(message, field, i); + std::vector sub_predicates; + Expr sub_base = GetFieldPath(base_expr, field); + CEL_RETURN_IF_ERROR(Walk(sub_message, sub_base, sub_predicates)); + message_asts.push_back(LogicalAnd(sub_predicates)); + } + // Represents alternate message predicates conjoined with OR: `msg_1 || + // msg_2 || ...` + predicates.push_back(LogicalOr(message_asts)); + return absl::OkStatus(); + } + + std::vector elements; + elements.reserve(size); + for (int i = 0; i < size; ++i) { + elements.push_back(NewListElement( + RepeatedPrimitiveToExpr(message, reflection, field, i))); + } + Expr literal_list = NewList(NextId(), std::move(elements)); + + std::string match_path_val = GetMatchPath(field); + if (!match_path_val.empty()) { + Expr lhs = ParseAndBuildPath(match_path_val); + // Represents `lhs in [values]` check (e.g. `dest.region in ["us-east", + // "us-west"]`). + predicates.push_back( + NewCall(NextId(), CelOperator::IN, + std::vector{std::move(lhs), std::move(literal_list)})); + return absl::OkStatus(); + } + + Expr map_path = NewSelect(NextId(), base_expr, field->name()); + std::vector element_checks; + element_checks.reserve(size); + for (int i = 0; i < size; ++i) { + Expr elem_expr = RepeatedPrimitiveToExpr(message, reflection, field, i); + // Represents `value in input.field` check. + Expr in_check = + NewCall(NextId(), CelOperator::IN, + std::vector{std::move(elem_expr), map_path}); + element_checks.push_back(std::move(in_check)); + } + // Represents `"val1" in input.list && "val2" in input.list && ...` + predicates.push_back(LogicalAnd(element_checks)); + + return absl::OkStatus(); + } + + // --------------------------------------------------------------------------- + // Recursive message walk + // --------------------------------------------------------------------------- + + absl::Status Walk(const Message& message, const Expr& base_expr, + std::vector& predicates) { + const Reflection* const reflection = message.GetReflection(); + std::vector fields; + reflection->ListFields(message, &fields); + + for (const auto* field : fields) { + if (field->is_map()) { + const int size = reflection->FieldSize(message, field); + if (size > 0) { + CEL_RETURN_IF_ERROR(WalkMapField(reflection, message, field, + base_expr, size, predicates)); + } + } else if (field->is_repeated()) { + const int size = reflection->FieldSize(message, field); + if (size > 0) { + CEL_RETURN_IF_ERROR(WalkRepeatedField(reflection, message, field, + base_expr, size, predicates)); + } + } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + const Message& sub_message = reflection->GetMessage(message, field); + Expr field_path = GetFieldPath(base_expr, field); + CEL_RETURN_IF_ERROR(Walk(sub_message, field_path, predicates)); + } else { + // Primitive field: base_expr.field == + Expr field_path = GetFieldPath(base_expr, field); + predicates.push_back( + ConstructEquality(std::move(field_path), + PrimitiveToExpr(message, reflection, field))); + } + } + return absl::OkStatus(); + } + + ExprId id_; + SourceInfo source_info_; +}; + +absl::StatusOr ProtoToPredicateAst(absl::string_view input_name, + const ::google::protobuf::Message& message) { + ProtoToPredicateBuilder builder; + return builder.Build(input_name, message); +} + +absl::StatusOr ProtoToPredicateAst( + absl::string_view input_name, + absl::Span messages) { + ProtoToPredicateBuilder builder; + return builder.Build(input_name, messages); +} + +} // namespace cel::tools diff --git a/tools/proto_to_predicate.h b/tools/proto_to_predicate.h new file mode 100644 index 000000000..ed01cb1e8 --- /dev/null +++ b/tools/proto_to_predicate.h @@ -0,0 +1,48 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_TOOLS_PROTO_TO_PREDICATE_H_ +#define THIRD_PARTY_CEL_CPP_TOOLS_PROTO_TO_PREDICATE_H_ + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "google/protobuf/message.h" + +namespace cel::tools { + +// Translates a Protocol Buffer message into a CEL AST representing a predicate. +// +// NOTE: The protocol message schemas used for policy definition should use +// `proto2` or `editions` (and not `proto3` implicit presence) to ensure correct +// behavior, as this library relies on field presence (via reflection) to +// identify which fields are explicitly set by the policy. +absl::StatusOr ProtoToPredicateAst(absl::string_view input_name, + const ::google::protobuf::Message& message); + +// Translates a list of Protocol Buffer messages into a CEL AST representing a +// conjoined or alternate predicate. +// +// NOTE: The protocol message schemas used for policy definition should use +// `proto2` or `editions` (and not `proto3` implicit presence) to ensure correct +// behavior, as this library relies on field presence (via reflection) to +// identify which fields are explicitly set by the policy. +absl::StatusOr ProtoToPredicateAst( + absl::string_view input_name, + absl::Span messages); + +} // namespace cel::tools + +#endif // THIRD_PARTY_CEL_CPP_TOOLS_PROTO_TO_PREDICATE_H_ diff --git a/tools/proto_to_predicate_test.cc b/tools/proto_to_predicate_test.cc new file mode 100644 index 000000000..2e7b94f6f --- /dev/null +++ b/tools/proto_to_predicate_test.cc @@ -0,0 +1,690 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tools/proto_to_predicate.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "common/ast_proto.h" +#include "common/value.h" +#include "env/config.h" +#include "env/env_runtime.h" +#include "env/env_yaml.h" +#include "env/runtime_std_extensions.h" +#include "eval/testutil/test_message.pb.h" +#include "extensions/protobuf/value.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "parser/parser.h" +#include "runtime/activation.h" +#include "runtime/runtime.h" +#include "tools/cel_unparser.h" +#include "tools/testdata/test_policy.pb.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/json/json.h" +#include "google/protobuf/message.h" +#include "google/protobuf/text_format.h" + +namespace cel::tools { +namespace { + +using ::absl_testing::IsOk; +using ::google::api::expr::runtime::TestMessage; + +constexpr absl::string_view kEnvYaml = R"( +name: "test" +extensions: + - name: "bindings" + - name: "optional" +variables: + - name: "input" + type: "google.api.expr.runtime.TestMessage" +)"; + +TestMessage ParseTestMessage(absl::string_view textproto) { + TestMessage msg; + google::protobuf::TextFormat::ParseFromString(textproto, &msg); + return msg; +} + +absl::StatusOr EvaluatePredicate(const cel::Ast& ast, + const TestMessage& input) { + auto descriptor_pool = cel::internal::GetSharedTestingDescriptorPool(); + + CEL_ASSIGN_OR_RETURN(cel::Config config, + cel::EnvConfigFromYaml(std::string(kEnvYaml))); + + cel::EnvRuntime env_runtime; + env_runtime.SetDescriptorPool(descriptor_pool); + cel::RegisterStandardExtensions(env_runtime); + env_runtime.SetConfig(config); + + CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + env_runtime.NewRuntime()); + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime->CreateProgram(std::make_unique(ast))); + + google::protobuf::Arena arena; + cel::Activation activation; + CEL_ASSIGN_OR_RETURN( + cel::Value val, cel::extensions::ProtoMessageToValue( + input, descriptor_pool.get(), + google::protobuf::MessageFactory::generated_factory(), &arena)); + activation.InsertOrAssignValue("input", val); + + CEL_ASSIGN_OR_RETURN(cel::Value result, + program->Evaluate(&arena, activation)); + if (!result.IsBool()) { + return absl::InvalidArgumentError( + "Predicate evaluate result must be a boolean value."); + } + return result.GetBool(); +} + +struct TestCase { + std::string name; + std::vector input_textprotos; + std::string expected_unparsed; + std::string eval_textproto; + bool expected_eval_result = true; + // If true, skip the eval step of the test. This is useful for tests where + // the expected expression does not share the same type structure as the + // input proto, such as empty messages. + bool skip_eval = false; +}; + +class ProtoToPredicateTest : public ::testing::TestWithParam {}; + +TEST_P(ProtoToPredicateTest, ConformanceTests) { + const TestCase& param = GetParam(); + + std::vector input_messages; + input_messages.reserve(param.input_textprotos.size()); + for (const auto& proto_str : param.input_textprotos) { + input_messages.push_back(ParseTestMessage(proto_str)); + } + + std::vector ptr_messages; + ptr_messages.reserve(input_messages.size()); + for (const auto& msg : input_messages) { + ptr_messages.push_back(&msg); + } + + absl::StatusOr ast_or; + if (input_messages.size() == 1) { + ast_or = ProtoToPredicateAst("input", input_messages[0]); + } else { + ast_or = ProtoToPredicateAst("input", absl::MakeSpan(ptr_messages)); + } + + ASSERT_THAT(ast_or, IsOk()); + cel::Ast ast = std::move(*ast_or); + + cel::expr::ParsedExpr parsed_expr; + ASSERT_THAT(cel::AstToParsedExpr(ast, &parsed_expr), IsOk()); + ASSERT_OK_AND_ASSIGN(auto unparsed, google::api::expr::Unparse(parsed_expr)); + + EXPECT_EQ(unparsed, param.expected_unparsed); + + if (!param.skip_eval) { + TestMessage eval_msg = ParseTestMessage(param.eval_textproto); + ASSERT_OK_AND_ASSIGN(bool eval_result, EvaluatePredicate(ast, eval_msg)); + EXPECT_EQ(eval_result, param.expected_eval_result); + } +} + +INSTANTIATE_TEST_SUITE_P( + ProtoToPredicateSubCases, ProtoToPredicateTest, + testing::Values( + TestCase{ + .name = "EmptyMessageTest", + .input_textprotos = {""}, + .expected_unparsed = "true", + .eval_textproto = "", + }, + TestCase{ + .name = "EmptyMessagesListTest", + .input_textprotos = {}, + .expected_unparsed = "true", + .eval_textproto = "", + }, + TestCase{ + .name = "PrimitivesTest", + .input_textprotos = {R"pb( + int32_value: 42 string_value: "hello" + )pb"}, + .expected_unparsed = + "input.int32_value == 42 && input.string_value == \"hello\"", + .eval_textproto = R"pb( + int32_value: 42 string_value: "hello" + )pb", + }, + TestCase{ + .name = "AllPrimitivesTest", + .input_textprotos = {R"pb( + int32_value: 42 + int64_value: 43 + uint32_value: 44 + uint64_value: 45 + float_value: 46.5 + double_value: 47.5 + bool_value: true + enum_value: TEST_ENUM_1 + string_value: "hello" + bytes_value: "world" + )pb"}, + .expected_unparsed = + "input.int32_value == 42 && input.int64_value == 43 && " + "input.uint32_value == 44u && input.uint64_value == 45u && " + "input.float_value == 46.5 && input.double_value == 47.5 && " + "input.string_value == \"hello\" && " + "input.bytes_value == b\"world\" && " + "input.bool_value == true && " + "input.enum_value == 1", + .eval_textproto = R"pb( + int32_value: 42 + int64_value: 43 + uint32_value: 44 + uint64_value: 45 + float_value: 46.5 + double_value: 47.5 + bool_value: true + enum_value: TEST_ENUM_1 + string_value: "hello" + bytes_value: "world" + )pb", + }, + TestCase{ + .name = "NestedMessageTest", + .input_textprotos = {R"pb( + message_value: { int32_value: 42 } + )pb"}, + .expected_unparsed = "input.message_value.int32_value == 42", + .eval_textproto = R"pb( + message_value: { int32_value: 42 } + )pb", + }, + TestCase{ + .name = "RepeatedFieldTest", + .input_textprotos = {R"pb( + int32_list: [ 1, 2 ] + )pb"}, + .expected_unparsed = + "1 in input.int32_list && 2 in input.int32_list", + .eval_textproto = R"pb( + int32_list: [ 1, 2 ] + )pb", + }, + TestCase{ + .name = "RepeatedFieldSingleElementTest", + .input_textprotos = {R"pb( + int32_list: [ 42 ] + )pb"}, + .expected_unparsed = "42 in input.int32_list", + .eval_textproto = R"pb( + int32_list: [ 42 ] + )pb", + }, + TestCase{ + .name = "RepeatedFieldEmptyTest", + .input_textprotos = {R"pb( + int32_list: [] + )pb"}, + .expected_unparsed = "true", + .eval_textproto = R"pb( + int32_list: [] + )pb", + }, + TestCase{ + .name = "ListFieldEvalNegative", + .input_textprotos = {R"pb( + int32_list: [ 1, 2 ] + )pb"}, + .expected_unparsed = + "1 in input.int32_list && 2 in input.int32_list", + .eval_textproto = R"pb( + int32_list: [ 1, 3 ] + )pb", + .expected_eval_result = false, + }, + TestCase{ + .name = "SingleRepeatedFieldAllPrimitivesTest", + .input_textprotos = {R"pb( + int32_list: [ 42 ] + int64_list: [ 43 ] + uint32_list: [ 44 ] + uint64_list: [ 45 ] + float_list: [ 46.5 ] + double_list: [ 47.5 ] + bool_list: [ true ] + enum_list: [ TEST_ENUM_1 ] + string_list: [ "hello" ] + bytes_list: [ "world" ] + )pb"}, + .expected_unparsed = "42 in input.int32_list && " + "43 in input.int64_list && " + "44u in input.uint32_list && " + "45u in input.uint64_list && " + "46.5 in input.float_list && " + "47.5 in input.double_list && " + "\"hello\" in input.string_list && " + "b\"world\" in input.bytes_list && " + "true in input.bool_list && " + "1 in input.enum_list", + .eval_textproto = R"pb( + int32_list: [ 42 ] + int64_list: [ 43 ] + uint32_list: [ 44 ] + uint64_list: [ 45 ] + float_list: [ 46.5 ] + double_list: [ 47.5 ] + bool_list: [ true ] + enum_list: [ TEST_ENUM_1 ] + string_list: [ "hello" ] + bytes_list: [ "world" ] + )pb", + }, + TestCase{ + .name = "MultipleRepeatedFieldAllPrimitivesTest", + .input_textprotos = {R"pb( + int32_list: [ 42, 142 ] + int64_list: [ 43, 143 ] + uint32_list: [ 44, 144 ] + uint64_list: [ 45, 145 ] + float_list: [ 46.5, 146.5 ] + double_list: [ 47.5, 147.5 ] + bool_list: [ true, false ] + enum_list: [ TEST_ENUM_1, TEST_ENUM_2 ] + string_list: [ "hello", "universe" ] + bytes_list: [ "world", "space" ] + )pb"}, + .expected_unparsed = + "42 in input.int32_list && 142 in input.int32_list && " + "43 in input.int64_list && 143 in input.int64_list && " + "44u in input.uint32_list && 144u in input.uint32_list && " + "45u in input.uint64_list && 145u in input.uint64_list && " + "46.5 in input.float_list && 146.5 in input.float_list && " + "47.5 in input.double_list && 147.5 in input.double_list && " + "\"hello\" in input.string_list && \"universe\" in " + "input.string_list && " + "b\"world\" in input.bytes_list && b\"space\" in " + "input.bytes_list && " + "true in input.bool_list && false in input.bool_list && " + "1 in input.enum_list && 2 in input.enum_list", + .eval_textproto = R"pb( + int32_list: [ 42, 142 ] + int64_list: [ 43, 143 ] + uint32_list: [ 44, 144 ] + uint64_list: [ 45, 145 ] + float_list: [ 46.5, 146.5 ] + double_list: [ 47.5, 147.5 ] + bool_list: [ true, false ] + enum_list: [ TEST_ENUM_1, TEST_ENUM_2 ] + string_list: [ "hello", "universe" ] + bytes_list: [ "world", "space" ] + )pb", + }, + TestCase{ + .name = "MapFieldTest", + .input_textprotos = {R"pb( + string_int32_map: { key: "foo" value: 1 } + string_int32_map: { key: "bar" value: 2 } + )pb"}, + .expected_unparsed = "\"bar\" in input.string_int32_map && " + "input.string_int32_map[\"bar\"] == 2 && " + "\"foo\" in input.string_int32_map && " + "input.string_int32_map[\"foo\"] == 1", + .eval_textproto = R"pb( + string_int32_map: { key: "foo" value: 1 } + string_int32_map: { key: "bar" value: 2 } + )pb", + }, + TestCase{ + .name = "MapFieldEvalNegativeVal", + .input_textprotos = {R"pb( + string_int32_map: { key: "foo" value: 1 } + string_int32_map: { key: "bar" value: 2 } + )pb"}, + .expected_unparsed = "\"bar\" in input.string_int32_map && " + "input.string_int32_map[\"bar\"] == 2 && " + "\"foo\" in input.string_int32_map && " + "input.string_int32_map[\"foo\"] == 1", + .eval_textproto = R"pb( + string_int32_map: { key: "foo" value: 1 } + string_int32_map: { key: "bar" value: 3 } + )pb", + .expected_eval_result = false, + }, + TestCase{ + .name = "MapFieldEvalNegativeNoKey", + .input_textprotos = {R"pb( + string_int32_map: { key: "foo" value: 1 } + string_int32_map: { key: "bar" value: 2 } + )pb"}, + .expected_unparsed = "\"bar\" in input.string_int32_map && " + "input.string_int32_map[\"bar\"] == 2 && " + "\"foo\" in input.string_int32_map && " + "input.string_int32_map[\"foo\"] == 1", + .eval_textproto = R"pb( + string_int32_map: { key: "foo" value: 1 } + )pb", + .expected_eval_result = false, + }, + TestCase{ + .name = "MapFieldIntKeySortingTest", + .input_textprotos = {R"pb( + int32_int32_map: { key: 10 value: 100 } + int32_int32_map: { key: 5 value: 50 } + int32_int32_map: { key: 8 value: 80 } + )pb"}, + .expected_unparsed = "5 in input.int32_int32_map && " + "input.int32_int32_map[5] == 50 && " + "8 in input.int32_int32_map && " + "input.int32_int32_map[8] == 80 && " + "10 in input.int32_int32_map && " + "input.int32_int32_map[10] == 100", + .eval_textproto = R"pb( + int32_int32_map: { key: 5 value: 50 } + int32_int32_map: { key: 8 value: 80 } + int32_int32_map: { key: 10 value: 100 } + )pb", + }, + TestCase{ + .name = "MultipleMessagesTest", + .input_textprotos = {R"pb( + int32_value: 42 + )pb", + R"pb( + int32_value: 41 string_value: "hello" + )pb"}, + .expected_unparsed = + "input.int32_value == 42 || input.int32_value == 41 && " + "input.string_value == \"hello\"", + .eval_textproto = R"pb( + int32_value: 41 string_value: "hello" + )pb", + }, + TestCase{ + .name = "RepeatedMessageFieldTest", + .input_textprotos = {R"pb( + message_list: + [ { int32_value: 42 } + , { int32_value: 43 }] + )pb"}, + .expected_unparsed = "input.message_list.int32_value == 42 || " + "input.message_list.int32_value == 43", + .skip_eval = true, + }, + TestCase{ + .name = "RepeatedMessageSingleElementTest", + .input_textprotos = {R"pb( + message_list: + [ { int32_value: 42 }] + )pb"}, + .expected_unparsed = "input.message_list.int32_value == 42", + .skip_eval = true, + })); + +struct PolicyTestCase { + std::string name; + std::string json_input; + std::string expected_unparsed; +}; + +class PolicyJsonTest : public ::testing::TestWithParam {}; + +TEST_P(PolicyJsonTest, Conformance) { + const PolicyTestCase& param = GetParam(); + + cel::cpp::tools::Policy policy; + google::protobuf::json::ParseOptions options; + options.ignore_unknown_fields = true; + auto status = + google::protobuf::json::JsonStringToMessage(param.json_input, &policy, options); + ASSERT_THAT(status, IsOk()) << "Failed to parse JSON: " << param.json_input; + + absl::StatusOr ast_or; + std::vector ptr_messages; + ptr_messages.reserve(policy.destinations_size()); + for (const auto& dest : policy.destinations()) { + ptr_messages.push_back(&dest); + } + + if (ptr_messages.empty()) { + auto parsed_expr_or = google::api::expr::parser::Parse("false"); + ASSERT_THAT(parsed_expr_or, IsOk()); + auto ast_ptr_or = cel::CreateAstFromParsedExpr(*parsed_expr_or); + ASSERT_THAT(ast_ptr_or, IsOk()); + ast_or = std::move(**ast_ptr_or); + } else if (ptr_messages.size() == 1) { + ast_or = ProtoToPredicateAst("dest", *ptr_messages[0]); + } else { + ast_or = ProtoToPredicateAst("dest", absl::MakeSpan(ptr_messages)); + } + + ASSERT_THAT(ast_or, IsOk()); + cel::Ast ast = std::move(*ast_or); + + cel::expr::ParsedExpr parsed_expr; + ASSERT_THAT(cel::AstToParsedExpr(ast, &parsed_expr), IsOk()); + ASSERT_OK_AND_ASSIGN(auto unparsed, google::api::expr::Unparse(parsed_expr)); + + EXPECT_EQ(unparsed, param.expected_unparsed); +} + +INSTANTIATE_TEST_SUITE_P( + PolicyJsonSubCases, PolicyJsonTest, + testing::Values( + PolicyTestCase{ + .name = "SimpleMatch", + .json_input = + R"({ "destinations": [ { "agent": { "id": "agent-007" } } ] })", + .expected_unparsed = "dest.agent.name == \"agent-007\"", + }, + PolicyTestCase{ + .name = "MultipleFields", + .json_input = + R"({ "destinations": [ { + "tool": { + "name": "admin_tool", + "annotations": { + "read_only_hint": false + } + } + } + ] })", + .expected_unparsed = + "dest.tool.name == \"admin_tool\" && " + "dest.tool.annotations.read_only_hint == false", + }, + PolicyTestCase{ + .name = "RepeatedMessages", + .json_input = + R"({ "destinations": [ + { "agent": { "id": "worker-1" } }, + { "agent": { "id": "worker-2" } }, + ] })", + .expected_unparsed = "dest.agent.name == \"worker-1\" || " + "dest.agent.name == \"worker-2\"", + }, + PolicyTestCase{ + .name = "RepeatedPrimitiveArraySingleElement", + .json_input = + R"({ "destinations": [ { + "tool": { + "role_members": { + "admin": { + "principals": ["alice"] + } + } + } + } ] })", + .expected_unparsed = + "\"admin\" in dest.tool.role_members && " + "\"alice\" in dest.tool.role_members[\"admin\"].principals", + }, + PolicyTestCase{ + .name = "RepeatedArrayEmpty", + .json_input = R"({ "destinations": [ { "tool": { } } ] })", + .expected_unparsed = "true", + }, + PolicyTestCase{ + .name = "AnnotatedSingularFieldInMessage", + .json_input = + R"({ "destinations": [ { + "agent": { + "id": "agent-007", + "location": "us-central1" + } + } ] })", + .expected_unparsed = "dest.agent.name == \"agent-007\" && " + "custom.agent.location == \"us-central1\"", + }, + PolicyTestCase{ + .name = "AnnotatedOneofPrimitiveField", + .json_input = + R"({ "destinations": [ { + "ip": "192.168.1.1" + } ] })", + .expected_unparsed = "custom.ip == \"192.168.1.1\"", + }, + PolicyTestCase{ + .name = "AnnotatedMapFieldPrimitive", + .json_input = + R"({ "destinations": [ { + "tool": { + "annotated_labels": { + "cluster": "us-central1" + } + } + } ] })", + .expected_unparsed = + "\"cluster\" in custom.labels && " + "custom.labels[\"cluster\"] == \"us-central1\"", + }, + PolicyTestCase{ + .name = "AnnotatedMapFieldMessage", + .json_input = + R"({ "destinations": [ { + "tool": { + "annotated_role_members": { + "admin": { + "all_users": true, + "principals": ["alice_user"], + "leader": "alice", + "leaders": ["bob"] + } + } + } + } ] })", + .expected_unparsed = + "\"admin\" in custom.role_members && " + "\"alice_user\" in custom.role_members[\"admin\"].principals " + "&& " + "custom.role_members[\"admin\"].all_users == true && " + "custom.member.leader == \"alice\" && " + "custom.member.leaders in [\"bob\"]", + }, + PolicyTestCase{ + .name = "AnnotatedMessageField", + .json_input = + R"({ "destinations": [ { + "tool": { + "annotated_annotations": { + "read_only_hint": true + } + } + } ] })", + .expected_unparsed = + "custom.tool_annotations.read_only_hint == true", + }, + PolicyTestCase{ + .name = "AnnotatedRepeatedMessageFieldSingle", + .json_input = + R"({ "destinations": [ { + "tool": { + "backup_agents": [ + { "id": "agent-007" } + ] + } + } ] })", + .expected_unparsed = "custom.backup_agents.name == \"agent-007\"", + }, + PolicyTestCase{ + .name = "AnnotatedRepeatedMessageFieldMultiple", + .json_input = + R"({ "destinations": [ { + "tool": { + "backup_agents": [ + { "id": "agent-007" }, + { "id": "agent-008" } + ] + } + } ] })", + .expected_unparsed = + "custom.backup_agents.name == \"agent-007\" || " + "custom.backup_agents.name == \"agent-008\"", + }, + PolicyTestCase{ + .name = "MapEquality", + .json_input = + R"({ "destinations": [ + { "tool": { + "name": "shell", + "labels": { + "cluster": "us-central1", + "project": "dev" + } + } + } ] })", + .expected_unparsed = + "dest.tool.name == \"shell\" && \"cluster\" in " + "dest.tool.labels && dest.tool.labels[\"cluster\"] == " + "\"us-central1\" && \"project\" in dest.tool.labels && " + "dest.tool.labels[\"project\"] == \"dev\"", + }, + PolicyTestCase{ + .name = "NestedMapEquality", + .json_input = + R"({ "destinations": [ + { "tool": { + "role_members": { + "admin": { + "all_users": true + } + } + } } + ] })", + .expected_unparsed = + "\"admin\" in dest.tool.role_members && " + "dest.tool.role_members[\"admin\"].all_users == true", + }, + PolicyTestCase{ + .name = "EmptyPolicy", + .json_input = "{}", + .expected_unparsed = "false", + })); + +} // namespace +} // namespace cel::tools diff --git a/tools/testdata/BUILD b/tools/testdata/BUILD index 493f0ff2f..956a13eaf 100644 --- a/tools/testdata/BUILD +++ b/tools/testdata/BUILD @@ -12,24 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -load( - "@com_github_google_flatbuffers//:build_defs.bzl", - "flatbuffer_library_public", -) +load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") load("@rules_cc//cc:cc_library.bzl", "cc_library") licenses(["notice"]) package(default_visibility = ["//visibility:public"]) -flatbuffer_library_public( - name = "flatbuffers_test", - srcs = ["flatbuffers.fbs"], - outs = ["flatbuffers_generated.h"], - language_flag = "-c", - reflection_name = "flatbuffers_reflection", -) - filegroup( name = "coverage_testdata", srcs = [ @@ -38,11 +28,15 @@ filegroup( ], ) -cc_library( - name = "flatbuffers_test_cc", - srcs = [":flatbuffers_test"], - hdrs = [":flatbuffers_test"], - features = ["-parse_headers"], - linkstatic = True, - deps = ["@com_github_google_flatbuffers//:runtime_cc"], +proto_library( + name = "test_policy_proto", + srcs = ["test_policy.proto"], + visibility = ["//tools:__subpackages__"], + deps = ["@com_google_protobuf//:descriptor_proto"], +) + +cc_proto_library( + name = "test_policy_cc_proto", + visibility = ["//tools:__subpackages__"], + deps = [":test_policy_proto"], ) diff --git a/tools/testdata/map.textproto b/tools/testdata/map.textproto new file mode 100644 index 000000000..f20523d4f --- /dev/null +++ b/tools/testdata/map.textproto @@ -0,0 +1,167 @@ +# proto-file: google/api/expr/checked.proto +# proto-message: CheckedExpr +# has({"key 1": "value 1", "key 2": "value 2", "example_string": "value 3"}.example_string) +type_map { + key: 2 + value { + map_type { + key_type { + primitive: STRING + } + value_type { + primitive: STRING + } + } + } +} +type_map { + key: 4 + value { + primitive: STRING + } +} +type_map { + key: 5 + value { + primitive: STRING + } +} +type_map { + key: 7 + value { + primitive: STRING + } +} +type_map { + key: 8 + value { + primitive: STRING + } +} +type_map { + key: 10 + value { + primitive: STRING + } +} +type_map { + key: 11 + value { + primitive: STRING + } +} +type_map { + key: 13 + value { + primitive: BOOL + } +} +expr { + id: 13 + select_expr { + operand { + id: 2 + struct_expr { + entries { + id: 3 + map_key { + id: 4 + const_expr { + string_value: "key 1" + } + } + value { + id: 5 + const_expr { + string_value: "value 1" + } + } + } + entries { + id: 6 + map_key { + id: 7 + const_expr { + string_value: "key 2" + } + } + value { + id: 8 + const_expr { + string_value: "value 2" + } + } + } + entries { + id: 9 + map_key { + id: 10 + const_expr { + string_value: "example_string" + } + } + value { + id: 11 + const_expr { + string_value: "value 3" + } + } + } + } + } + field: "example_string" + test_only: true + } +} +source_info { + location: "" + line_offsets: 90 + positions { + key: 2 + value: 4 + } + positions { + key: 3 + value: 12 + } + positions { + key: 4 + value: 5 + } + positions { + key: 5 + value: 14 + } + positions { + key: 6 + value: 32 + } + positions { + key: 7 + value: 25 + } + positions { + key: 8 + value: 34 + } + positions { + key: 9 + value: 61 + } + positions { + key: 10 + value: 45 + } + positions { + key: 11 + value: 63 + } + positions { + key: 12 + value: 73 + } + positions { + key: 13 + value: 3 + } +} diff --git a/tools/testdata/map_renumbered.textproto b/tools/testdata/map_renumbered.textproto new file mode 100644 index 000000000..905fe130a --- /dev/null +++ b/tools/testdata/map_renumbered.textproto @@ -0,0 +1,168 @@ +# proto-file: google/api/expr/checked.proto +# proto-message: CheckedExpr +# Renumbered version of map starting from 52. +# has({"key 1": "value 1", "key 2": "value 2", "example_string": "value 3"}.example_string) +type_map { + key: 52 + value { + primitive: STRING + } +} +type_map { + key: 53 + value { + primitive: STRING + } +} +type_map { + key: 54 + value { + primitive: STRING + } +} +type_map { + key: 55 + value { + primitive: STRING + } +} +type_map { + key: 56 + value { + primitive: STRING + } +} +type_map { + key: 57 + value { + primitive: STRING + } +} +type_map { + key: 58 + value { + map_type { + key_type { + primitive: STRING + } + value_type { + primitive: STRING + } + } + } +} +type_map { + key: 62 + value { + primitive: BOOL + } +} +expr { + id: 62 + select_expr { + operand { + id: 58 + struct_expr { + entries { + id: 59 + map_key { + id: 52 + const_expr { + string_value: "key 1" + } + } + value { + id: 53 + const_expr { + string_value: "value 1" + } + } + } + entries { + id: 60 + map_key { + id: 54 + const_expr { + string_value: "key 2" + } + } + value { + id: 55 + const_expr { + string_value: "value 2" + } + } + } + entries { + id: 61 + map_key { + id: 56 + const_expr { + string_value: "example_string" + } + } + value { + id: 57 + const_expr { + string_value: "value 3" + } + } + } + } + } + field: "example_string" + test_only: true + } +} +source_info { + location: "" + line_offsets: 90 + positions { + key: 52 + value: 5 + } + positions { + key: 53 + value: 14 + } + positions { + key: 54 + value: 25 + } + positions { + key: 55 + value: 34 + } + positions { + key: 56 + value: 45 + } + positions { + key: 57 + value: 63 + } + positions { + key: 58 + value: 4 + } + positions { + key: 59 + value: 12 + } + positions { + key: 60 + value: 32 + } + positions { + key: 61 + value: 61 + } + positions { + key: 62 + value: 3 + } + positions { + key: 63 + value: 73 + } +} diff --git a/tools/testdata/test_policy.proto b/tools/testdata/test_policy.proto new file mode 100644 index 000000000..f0ce9fe76 --- /dev/null +++ b/tools/testdata/test_policy.proto @@ -0,0 +1,100 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Test schema representing client-configured policies. +// It is used by the `proto_to_predicate` tool to translate Protobuf policies +// into CEL predicates. +edition = "2023"; + +package cel.cpp.tools; + +import "google/protobuf/descriptor.proto"; + +option cc_enable_arenas = true; + +extend google.protobuf.FieldOptions { + string match_path = 51074; +} + +// Represents the targeted client agent. +message Agent { + string name = 1 [json_name = "id"]; + string location = 2 [(match_path) = "custom.agent.location"]; +} + +// Specifies additional metadata tool annotations. +message ToolAnnotations { + bool read_only_hint = 1; +} + +// Represents a mapped nested message entry value inside map fields. +message Members { + repeated string principals = 1; + + repeated string regions = 2; + + bool all_users = 3; + + bool all_authenticated_users = 4; + + string leader = 5 [(match_path) = "custom.member.leader"]; + + repeated string leaders = 6 [(match_path) = "custom.member.leaders"]; +} + +// Represents a metadata tool block. +message Tool { + // The name of the tool. + string name = 1; + + // Additional metadata annotations for the tool. + ToolAnnotations annotations = 2; + + // A string-to-string map, transpiled as conjoined existence and equality + // checks. + map labels = 3; + + // A map with string keys representing roles and Member instances as values. + map role_members = 4; + + // A map with string keys representing roles and Member instances as values, + // annotated with a match_path. + map annotated_role_members = 5 + [(match_path) = "custom.role_members"]; + + // A string-to-string map, annotated with a match_path. + map annotated_labels = 6 [(match_path) = "custom.labels"]; + + // A non-repeated message field, annotated with a match_path. + ToolAnnotations annotated_annotations = 7 + [(match_path) = "custom.tool_annotations"]; + + // A repeated message field, annotated with a match_path. + repeated Agent backup_agents = 8 [(match_path) = "custom.backup_agents"]; +} + +// Represents a policy mapping destination block. +message Target { + oneof kind { + Agent agent = 1; + Tool tool = 2; + string ip = 3 [(match_path) = "custom.ip"]; + } +} + +// Represents the top-level policy containing multiple alternate destination +// rules. +message Policy { + repeated Target destinations = 1; +}