diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index d8030e644d00a..2ee3fb9811370 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -466,17 +466,6 @@ jobs: -Dprotobuf_BUILD_EXAMPLES=ON vsversion: '2022' cache-prefix: windows-2022-cmake - - name: Windows CMake 2019 - os: windows-2019 - flags: >- - -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF - -Dprotobuf_BUILD_SHARED_LIBS=OFF - -Dprotobuf_BUILD_EXAMPLES=ON - vsversion: '2019' - cache-prefix: windows-2019-cmake - # windows-2019 has python3.7 installed, which is incompatible with the latest gcloud - python-version: '3.9' - continuous-only: true - name: Windows CMake 32-bit os: windows-2022 flags: >- @@ -519,17 +508,6 @@ jobs: arch: ${{ matrix.windows-arch || 'x64' }} vsversion: ${{ matrix.vsversion }} - # Workaround for incompatibility between gcloud and windows-2019 runners. - - name: Install Python - if: ${{ matrix.python-version && (!matrix.continuous-only || inputs.continuous-run) }} - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ matrix.python-version }} - - name: Use custom python for gcloud - if: ${{ matrix.python-version && (!matrix.continuous-only || inputs.continuous-run) }} - run: echo "CLOUDSDK_PYTHON=${Python3_ROOT_DIR}\\python3" >> $GITHUB_ENV - shell: bash - - name: Setup sccache if: ${{ !matrix.continuous-only || inputs.continuous-run }} uses: protocolbuffers/protobuf-ci/sccache@v4 diff --git a/.github/workflows/test_csharp.yml b/.github/workflows/test_csharp.yml index ad55b6f840544..f3eb076b8007e 100644 --- a/.github/workflows/test_csharp.yml +++ b/.github/workflows/test_csharp.yml @@ -50,7 +50,7 @@ jobs: windows: name: Windows - runs-on: windows-2019 + runs-on: windows-2022 steps: - name: Checkout pending changes uses: protocolbuffers/protobuf-ci/checkout@v4 @@ -62,15 +62,6 @@ jobs: with: dotnet-version: '6.0.x' - # Workaround for incompatibility between gcloud and windows-2019 runners. - - name: Install Python - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: '3.9' - - name: Use custom python for gcloud - run: echo "CLOUDSDK_PYTHON=${Python3_ROOT_DIR}\\python3" >> $GITHUB_ENV - shell: bash - - name: Run tests uses: protocolbuffers/protobuf-ci/bash@v4 with: diff --git a/.github/workflows/test_upb.yml b/.github/workflows/test_upb.yml index 174c1b42f6025..6d5ba52e4e26c 100644 --- a/.github/workflows/test_upb.yml +++ b/.github/workflows/test_upb.yml @@ -193,16 +193,16 @@ jobs: - { os: macos-13, python-version: "3.13", architecture: x64, type: 'source', continuous-only: true } # Windows uses the full API up until Python 3.10. - - { os: windows-2019, python-version: "3.9", architecture: x86, type: 'binary', continuous-only: true } - - { os: windows-2019, python-version: "3.10", architecture: x86, type: 'binary', continuous-only: true } - - { os: windows-2019, python-version: "3.11", architecture: x86, type: 'binary', continuous-only: true } - - { os: windows-2019, python-version: "3.12", architecture: x86, type: 'binary', continuous-only: true } - - { os: windows-2019, python-version: "3.13", architecture: x86, type: 'binary', continuous-only: true } - - { os: windows-2019, python-version: "3.9", architecture: x64, type: 'binary' } - - { os: windows-2019, python-version: "3.10", architecture: x64, type: 'binary', continuous-only: true } - - { os: windows-2019, python-version: "3.11", architecture: x64, type: 'binary', continuous-only: true } - - { os: windows-2019, python-version: "3.12", architecture: x64, type: 'binary', continuous-only: true } - - { os: windows-2019, python-version: "3.13", architecture: x64, type: 'binary' } + - { os: windows-2022, python-version: "3.9", architecture: x86, type: 'binary', continuous-only: true } + - { os: windows-2022, python-version: "3.10", architecture: x86, type: 'binary', continuous-only: true } + - { os: windows-2022, python-version: "3.11", architecture: x86, type: 'binary', continuous-only: true } + - { os: windows-2022, python-version: "3.12", architecture: x86, type: 'binary', continuous-only: true } + - { os: windows-2022, python-version: "3.13", architecture: x86, type: 'binary', continuous-only: true } + - { os: windows-2022, python-version: "3.9", architecture: x64, type: 'binary' } + - { os: windows-2022, python-version: "3.10", architecture: x64, type: 'binary', continuous-only: true } + - { os: windows-2022, python-version: "3.11", architecture: x64, type: 'binary', continuous-only: true } + - { os: windows-2022, python-version: "3.12", architecture: x64, type: 'binary', continuous-only: true } + - { os: windows-2022, python-version: "3.13", architecture: x64, type: 'binary' } name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Test Wheels Python ${{ matrix.python-version }} ${{ matrix.os }} ${{ matrix.architecture }} ${{ matrix.type }} needs: build_wheels runs-on: ${{ matrix.os }} diff --git a/MODULE.bazel b/MODULE.bazel index 78b2d2040a692..e8a3ecbc979d8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -44,7 +44,7 @@ bazel_dep(name = "rules_jvm_external", version = "6.7") bazel_dep(name = "rules_kotlin", version = "1.9.6") bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_pkg", version = "1.0.1") -bazel_dep(name = "rules_python", version = "1.0.0") +bazel_dep(name = "rules_python", version = "1.1.0") # Pin to rules_proto to 7.1.0 to avoid toolchain incompatibilities when # --incompatible_enable_proto_toolchain_resolution=true in Bazel 7. diff --git a/bazel/private/proto_info.bzl b/bazel/private/proto_info.bzl index 798af53b22133..ea8b98a3703a5 100644 --- a/bazel/private/proto_info.bzl +++ b/bazel/private/proto_info.bzl @@ -34,12 +34,13 @@ def _from_root(root, repo, relpath): # - with sibling layout: `{root}/package/path` return _join(root, "" if repo.startswith("../") else repo, relpath) -def _create_proto_info(*, srcs, deps, descriptor_set, proto_path = "", workspace_root = "", bin_dir = None, allow_exports = None): +def _create_proto_info(*, srcs, deps, descriptor_set, option_deps = [], proto_path = "", workspace_root = "", bin_dir = None, allow_exports = None): """Constructs ProtoInfo. Args: srcs: ([File]) List of .proto files (possibly under _virtual path) deps: ([ProtoInfo]) List of dependencies + option_deps: ([ProtoInfo]) List of option dependencies descriptor_set: (File) Descriptor set for this Proto proto_path: (str) Path that should be stripped from files in srcs. When stripping is needed, the files should be symlinked into `_virtual_imports/target_name` @@ -61,6 +62,8 @@ def _create_proto_info(*, srcs, deps, descriptor_set, proto_path = "", workspace fail("srcs parameter expects all files to have the same workspace_root: ", workspace_root) if not src.short_path.startswith(src_prefix): fail("srcs parameter expects all files start with %s" % src_prefix) + if not srcs and option_deps: + fail("option_deps parameter should not be set when srcs is empty") if type(descriptor_set) != "File": fail("descriptor_set parameter expected to be a File") if proto_path: @@ -71,9 +74,10 @@ def _create_proto_info(*, srcs, deps, descriptor_set, proto_path = "", workspace if not bin_dir: fail("bin_dir parameter should be set when _virtual_imports are used") + protoc_deps = deps + option_deps transitive_sources = depset( direct = srcs, - transitive = [dep.transitive_sources for dep in deps], + transitive = [dep.transitive_sources for dep in protoc_deps], order = "preorder", ) @@ -82,7 +86,7 @@ def _create_proto_info(*, srcs, deps, descriptor_set, proto_path = "", workspace root_paths = _uniq([src.root.path for src in srcs]) transitive_proto_path = depset( direct = [_empty_to_dot(_from_root(root, workspace_root, proto_path)) for root in root_paths], - transitive = [dep.transitive_proto_path for dep in deps], + transitive = [dep.transitive_proto_path for dep in protoc_deps], ) if srcs: @@ -90,6 +94,9 @@ def _create_proto_info(*, srcs, deps, descriptor_set, proto_path = "", workspace else: check_deps_sources = depset(transitive = [dep.check_deps_sources for dep in deps]) + # Exclude option_deps from transitive descriptor sets. option_dep are needed for options + # interpretation when compiling .proto sources into descriptors, but should no longer be needed + # once descriptor sets are already compiled. transitive_descriptor_sets = depset( direct = [descriptor_set], transitive = [dep.transitive_descriptor_sets for dep in deps], @@ -122,12 +129,13 @@ ProtoInfo, _ = provider( fields = { "direct_sources": "(list[File]) The `.proto` source files from the `srcs` attribute.", "transitive_sources": """(depset[File]) The `.proto` source files from this rule and all - its dependent protocol buffer rules.""", + its `deps` and `option_deps` protocol buffer rules.""", "direct_descriptor_set": """(File) The descriptor set of the direct sources. If no srcs, contains an empty file.""", "transitive_descriptor_sets": """(depset[File]) A set of descriptor set files of all - dependent `proto_library` rules, and this one's. This is not the same as passing - --include_imports to proto-compiler. Will be empty if no dependencies.""", + its `deps` `proto_library` rules, and this one's. Excludes `option_deps`. This is not + the same as passing --include_imports to proto-compiler. Will be empty if no + dependencies.""", "proto_source_root": """(str) The directory relative to which the `.proto` files defined in the `proto_library` are defined. For example, if this is `a/b` and the rule has the file `a/b/c/d.proto` as a source, that source file would be imported as @@ -147,10 +155,10 @@ ProtoInfo, _ = provider( This will make it possible to fix `proto_library` in the future. """, "transitive_proto_path": """(depset(str) A set of `proto_source_root`s collected from the - transitive closure of this rule.""", + transitive closure of this rule's `deps` and `option_deps`.""", "check_deps_sources": """(depset[File]) The `.proto` sources from the 'srcs' attribute. If the library is a proxy library that has no sources, it contains the - `check_deps_sources` from this library's direct deps.""", + `check_deps_sources` from this library's direct `deps`.""", "allow_exports": """(Target) The packages where this proto_library can be exported.""", # Deprecated fields: diff --git a/bazel/private/proto_library_rule.bzl b/bazel/private/proto_library_rule.bzl index aaaf90a7acf3d..642dd3eaecf56 100644 --- a/bazel/private/proto_library_rule.bzl +++ b/bazel/private/proto_library_rule.bzl @@ -15,10 +15,14 @@ load("//bazel/common:proto_common.bzl", "proto_common") load("//bazel/common:proto_info.bzl", "ProtoInfo") load("//bazel/private:toolchain_helpers.bzl", "toolchains") -STRICT_DEPS_FLAG_TEMPLATE = ( - # +DIRECT_DEPS_FLAG_TEMPLATE = ( "--direct_dependencies_violation_msg=" + - "%%s is imported, but %s doesn't directly depend on a proto_library that 'srcs' it." + "%%s is imported, but %s doesn't have direct `deps` on a proto_library that 'srcs' it." +) + +OPTION_DEPS_FLAG_TEMPLATE = ( + "--option_dependencies_violation_msg=" + + "%%s is option imported, but %s doesn't have direct `option_deps` on a proto_library that 'srcs' it." ) def _check_srcs_package(target_package, srcs): @@ -63,6 +67,7 @@ def _proto_library_impl(ctx): _check_srcs_package(ctx.label.package, ctx.attr.srcs) srcs = ctx.files.srcs deps = [dep[ProtoInfo] for dep in ctx.attr.deps] + option_deps = [dep[ProtoInfo] for dep in ctx.attr.option_deps] exports = [dep[ProtoInfo] for dep in ctx.attr.exports] import_prefix = _get_import_prefix(ctx) strip_import_prefix = _get_strip_import_prefix(ctx) @@ -77,17 +82,19 @@ def _proto_library_impl(ctx): proto_path, virtual_srcs = _process_srcs(ctx, srcs, import_prefix, strip_import_prefix) descriptor_set = ctx.actions.declare_file(ctx.label.name + "-descriptor-set.proto.bin") + proto_info = ProtoInfo( srcs = virtual_srcs, deps = deps, descriptor_set = descriptor_set, + option_deps = option_deps, proto_path = proto_path, workspace_root = ctx.label.workspace_root, bin_dir = ctx.bin_dir.path, allow_exports = ctx.attr.allow_exports, ) - _write_descriptor_set(ctx, proto_info, deps, exports, descriptor_set) + _write_descriptor_set(ctx, proto_info, deps, option_deps, exports, descriptor_set) # We assume that the proto sources will not have conflicting artifacts # with the same root relative path @@ -154,13 +161,14 @@ def _symlink_to_virtual_imports(ctx, srcs, import_prefix, strip_import_prefix): virtual_srcs.append(virtual_src) return proto_path, virtual_srcs -def _write_descriptor_set(ctx, proto_info, deps, exports, descriptor_set): +def _write_descriptor_set(ctx, proto_info, deps, option_deps, exports, descriptor_set): """Writes descriptor set.""" if proto_info.direct_sources == []: ctx.actions.write(descriptor_set, "") return - dependencies_descriptor_sets = depset(transitive = [dep.transitive_descriptor_sets for dep in deps]) + # Descriptor sets for transitive `deps` of `deps` and `option_deps`. + dependencies_descriptor_sets = depset(transitive = [dep.transitive_descriptor_sets for dep in deps + option_deps]) args = ctx.actions.args() @@ -171,6 +179,7 @@ def _write_descriptor_set(ctx, proto_info, deps, exports, descriptor_set): strict_deps = ctx.attr._strict_proto_deps[BuildSettingInfo].value if strict_deps: if proto_info.direct_sources: + # Direct sources can be option imported in addition to `deps`. strict_importable_sources = depset( direct = proto_info.direct_sources, transitive = [dep.check_deps_sources for dep in deps], @@ -191,7 +200,23 @@ def _write_descriptor_set(ctx, proto_info, deps, exports, descriptor_set): args.add("--direct_dependencies=") # Set `-direct_dependencies_violation_msg=` - args.add(ctx.label, format = STRICT_DEPS_FLAG_TEMPLATE) + args.add(ctx.label, format = DIRECT_DEPS_FLAG_TEMPLATE) + + # Direct sources can be option imported in addition to `option_deps`. + # `option_deps` can't be set anyways unless `direct_sources` is non-empty. + option_importable_sources = depset( + direct = proto_info.direct_sources, + transitive = [dep.check_deps_sources for dep in option_deps], + ) + args.add_joined( + "--option_dependencies", + option_importable_sources, + map_each = proto_common.get_import_path, + join_with = ":", + ) + + # Set `-option_dependencies_violation_msg=` + args.add(ctx.label, format = OPTION_DEPS_FLAG_TEMPLATE) strict_imports = ctx.attr._strict_public_imports[BuildSettingInfo].value if strict_imports: @@ -206,6 +231,7 @@ def _write_descriptor_set(ctx, proto_info, deps, exports, descriptor_set): map_each = proto_common.get_import_path, join_with = ":", ) + if proto_common.INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION: toolchain = ctx.toolchains[toolchains.PROTO_TOOLCHAIN] if not toolchain: @@ -227,6 +253,7 @@ def _write_descriptor_set(ctx, proto_info, deps, exports, descriptor_set): proto_info, proto_lang_toolchain_info, generated_files = [descriptor_set], + # TODO: Fix protoc to actually use the transitive descriptor sets. additional_inputs = dependencies_descriptor_sets, additional_args = args, ) @@ -283,6 +310,13 @@ This pattern can be used to e.g. export a public api under a persistent name.""" doc = """ The list of other proto_library rules that the target depends upon. A proto_library may only depend on other proto_library +targets. It may not depend on language-specific libraries.""", + ), + "option_deps": attr.label_list( + providers = [ProtoInfo], + doc = """ +The list of other proto_library rules that the target depends upon for options only. +A proto_library may only depend on other proto_library targets. It may not depend on language-specific libraries.""", ), "exports": attr.label_list( diff --git a/benchmarks/BUILD b/benchmarks/BUILD index 9b512933d9bdc..93fd47214f0a6 100644 --- a/benchmarks/BUILD +++ b/benchmarks/BUILD @@ -71,11 +71,11 @@ cc_test( ":benchmark_descriptor_upb_proto_reflection", "//src/google/protobuf", "//src/google/protobuf/json", - "//upb:base", - "//upb:json", - "//upb:mem", - "//upb:reflection", - "//upb:wire", + "//upb/base", + "//upb/json", + "//upb/mem", + "//upb/reflection", + "//upb/wire", "@abseil-cpp//absl/container:flat_hash_set", "@abseil-cpp//absl/log:absl_check", "@google_benchmark//:benchmark_main", diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 4aa04a3d96769..00c317676ff46 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -23,7 +23,7 @@ set(rules_jvm_external-version "6.7") set(rules_kotlin-version "1.9.6") set(rules_license-version "1.0.0") set(rules_pkg-version "1.0.1") -set(rules_python-version "1.0.0") +set(rules_python-version "1.1.0") set(rules_proto-version "7.1.0") set(rules_rust-version "0.56.0") set(rules_ruby-version "0.17.3") diff --git a/editions/BUILD b/editions/BUILD index d24e9bb515abd..d420a9f7af07d 100644 --- a/editions/BUILD +++ b/editions/BUILD @@ -130,6 +130,7 @@ proto_library( name = "test_messages_proto2_editions_proto", testonly = True, srcs = ["golden/test_messages_proto2_editions.proto"], + visibility = ["//net/proto2/util/converter/internal/conformance:__pkg__"], ) cc_proto_library( @@ -187,6 +188,7 @@ proto_library( name = "test_messages_proto3_editions_proto", testonly = True, srcs = ["golden/test_messages_proto3_editions.proto"], + visibility = ["//net/proto2/util/converter/internal/conformance:__pkg__"], deps = [ "//:any_proto", "//:duration_proto", diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index 137987c7942a4..a05e5b6bd7e4e 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -23,7 +23,7 @@ bazel_dep(name = "platforms", version = "0.0.10") bazel_dep(name = "rules_cc", version = "0.0.17") bazel_dep(name = "rules_java", version = "8.6.1") bazel_dep(name = "rules_pkg", version = "1.0.1") -bazel_dep(name = "rules_python", version = "1.0.0") +bazel_dep(name = "rules_python", version = "1.1.0") # For clang-cl configuration cc_configure = use_extension("@rules_cc//cc:extensions.bzl", "cc_configure_extension") diff --git a/hpb/BUILD b/hpb/BUILD new file mode 100644 index 0000000000000..fdde049ec1762 --- /dev/null +++ b/hpb/BUILD @@ -0,0 +1,205 @@ +# Copyright (c) 2009-2021, Google LLC +# All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") +load( + "//hpb/bazel:hpb_proto_library.bzl", + "hpb_proto_library_copts", +) +load( + "//upb/bazel:build_defs.bzl", + "UPB_DEFAULT_CPPOPTS", +) + +package(default_applicable_licenses = ["//:license"]) + +licenses(["notice"]) + +string_flag( + name = "hpb_backend", + build_setting_default = "upb", + values = [ + "upb", + "cpp", + ], + visibility = ["//visibility:public"], +) + +config_setting( + name = "hpb_backend_upb", + flag_values = { + ":hpb_backend": "upb", + }, + visibility = ["//visibility:public"], +) + +config_setting( + name = "hpb_backend_cpp", + flag_values = { + ":hpb_backend": "cpp", + }, + visibility = ["//visibility:public"], +) + +cc_library( + name = "hpb", + hdrs = [ + "hpb.h", + ], + copts = UPB_DEFAULT_CPPOPTS, + defines = select({ + ":hpb_backend_upb": ["HPB_INTERNAL_BACKEND=HPB_INTERNAL_BACKEND_UPB"], + ":hpb_backend_cpp": ["HPB_INTERNAL_BACKEND=HPB_INTERNAL_BACKEND_CPP"], + }), + visibility = ["//visibility:public"], + deps = [ + ":arena", + ":extension", + ":multibackend", + ":ptr", + ":status", + "//hpb/internal", + "//hpb/internal:message_lock", + "//hpb/internal:template_help", + "@abseil-cpp//absl/base:core_headers", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings", + ] + select({ + ":hpb_backend_cpp": [ + "//hpb/backend/cpp", + ], + ":hpb_backend_upb": [ + "//hpb/backend/upb", + "//hpb/backend/upb:interop", + "//upb/mini_table", + "//upb/wire", + ], + }), +) + +cc_library( + name = "multibackend", + hdrs = [ + "multibackend.h", + ], + visibility = ["//visibility:private"], +) + +cc_library( + name = "repeated_field", + hdrs = [ + "repeated_field.h", + "repeated_field_iterator.h", + ], + copts = UPB_DEFAULT_CPPOPTS, + visibility = ["//visibility:public"], + deps = [ + ":hpb", + ":ptr", + "//hpb/backend/upb:interop", + "//hpb/internal:template_help", + "//upb/base", + "//upb/mem", + "//upb/message", + "//upb/message:copy", + "@abseil-cpp//absl/strings", + ], +) + +# Common support code for C++ generated code. +cc_library( + name = "generated_hpb_support", + copts = UPB_DEFAULT_CPPOPTS, + visibility = [ + "//hpb/bazel:__pkg__", + "//src/google/protobuf/compiler/hpb:__pkg__", + ], + deps = [ + ":hpb", + ":repeated_field", + "//upb/mem", + "//upb/message", + ], +) + +hpb_proto_library_copts( + name = "hpb_proto_library_copts", + copts = UPB_DEFAULT_CPPOPTS, + visibility = ["//hpb/bazel:__pkg__"], +) + +cc_test( + name = "repeated_field_iterator_test", + srcs = ["repeated_field_iterator_test.cc"], + deps = [ + ":arena", + ":repeated_field", + "//hpb/backend/upb:interop", + "//upb/base", + "//upb/mem", + "//upb/message", + "@abseil-cpp//absl/strings:string_view", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +cc_library( + name = "requires", + hdrs = ["requires.h"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "ptr", + hdrs = ["ptr.h"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "arena", + hdrs = ["arena.h"], + visibility = ["//visibility:public"], + deps = [ + "//upb/mem", + ], +) + +cc_library( + name = "status", + srcs = ["status.cc"], + hdrs = ["status.h"], + visibility = ["//visibility:public"], + deps = [ + "//upb/wire", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/strings:str_format", + ], +) + +cc_library( + name = "extension", + srcs = ["extension.cc"], + hdrs = ["extension.h"], + visibility = ["//visibility:public"], + deps = [ + ":ptr", + ":status", + "//hpb/backend/upb:interop", + "//hpb/internal:message_lock", + "//hpb/internal:template_help", + "//upb/base", + "//upb/mem", + "//upb/message", + "//upb/mini_table", + "@abseil-cpp//absl/base:core_headers", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings:string_view", + ], +) diff --git a/hpb/backend/cpp/BUILD b/hpb/backend/cpp/BUILD new file mode 100644 index 0000000000000..f90dd5dd95a03 --- /dev/null +++ b/hpb/backend/cpp/BUILD @@ -0,0 +1,21 @@ +# Copyright (c) 2025, Google LLC +# All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +package(default_applicable_licenses = ["//:license"]) + +cc_library( + name = "cpp", + hdrs = ["cpp.h"], + visibility = ["//hpb:__subpackages__"], + deps = [ + "//hpb:arena", + "//hpb:ptr", + "//hpb/internal:template_help", + "//src/google/protobuf:protobuf_lite", + "@abseil-cpp//absl/strings:string_view", + ], +) diff --git a/hpb/backend/cpp/cpp.h b/hpb/backend/cpp/cpp.h index f4afcee10263f..fadc7ad313087 100644 --- a/hpb/backend/cpp/cpp.h +++ b/hpb/backend/cpp/cpp.h @@ -9,9 +9,9 @@ #define GOOGLE_PROTOBUF_HPB_BACKEND_CPP_CPP_H__ #include "absl/strings/string_view.h" -#include "google/protobuf/hpb/arena.h" -#include "google/protobuf/hpb/internal/template_help.h" -#include "google/protobuf/hpb/ptr.h" +#include "hpb/arena.h" +#include "hpb/internal/template_help.h" +#include "hpb/ptr.h" namespace hpb::internal::backend::cpp { diff --git a/hpb/backend/upb/BUILD b/hpb/backend/upb/BUILD new file mode 100644 index 0000000000000..b2240f9e1c198 --- /dev/null +++ b/hpb/backend/upb/BUILD @@ -0,0 +1,62 @@ +# Copyright (c) 2024, Google LLC +# All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +load( + "//upb/bazel:build_defs.bzl", + "UPB_DEFAULT_CPPOPTS", +) + +package(default_applicable_licenses = ["//:license"]) + +cc_library( + name = "upb", + hdrs = ["upb.h"], + visibility = ["//upb:friends"], + deps = [ + ":interop", + "//hpb:arena", + "//hpb:ptr", + "//hpb/internal", + "//hpb/internal:message_lock", + "//hpb/internal:template_help", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings:string_view", + ], +) + +cc_library( + name = "interop", + hdrs = ["interop.h"], + visibility = [ + "//hpb:__subpackages__", + "//src/google/protobuf/compiler:__subpackages__", + ], + deps = [ + "//hpb:ptr", + "//hpb/internal", + "//upb/base", + "//upb/mem", + "//upb/message", + "//upb/mini_table", + "@abseil-cpp//absl/strings:string_view", + ], +) + +cc_test( + name = "interop_test", + srcs = ["interop_test.cc"], + copts = UPB_DEFAULT_CPPOPTS, + deps = [ + ":interop", + "//src/google/protobuf/compiler/hpb/tests:test_model_hpb_proto", + "//src/google/protobuf/compiler/hpb/tests:test_model_upb_proto", + "//upb/mem", + "//upb/message", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) diff --git a/hpb/backend/upb/interop.h b/hpb/backend/upb/interop.h index 7d96b392078e5..e4fb84d16f70d 100644 --- a/hpb/backend/upb/interop.h +++ b/hpb/backend/upb/interop.h @@ -13,8 +13,8 @@ #include #include "absl/strings/string_view.h" -#include "google/protobuf/hpb/internal/internal.h" -#include "google/protobuf/hpb/ptr.h" +#include "hpb/internal/internal.h" +#include "hpb/ptr.h" #include "upb/base/string_view.h" #include "upb/mem/arena.h" #include "upb/message/message.h" diff --git a/hpb/backend/upb/interop_test.cc b/hpb/backend/upb/interop_test.cc index 369e8e8608559..d20c0a26f68a6 100644 --- a/hpb/backend/upb/interop_test.cc +++ b/hpb/backend/upb/interop_test.cc @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#include "google/protobuf/hpb/backend/upb/interop.h" +#include "hpb/backend/upb/interop.h" #include #include "google/protobuf/compiler/hpb/tests/test_model.hpb.h" diff --git a/hpb/backend/upb/upb.h b/hpb/backend/upb/upb.h index 015950c73ef4a..750350a93685e 100644 --- a/hpb/backend/upb/upb.h +++ b/hpb/backend/upb/upb.h @@ -10,26 +10,26 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "google/protobuf/hpb/arena.h" -#include "google/protobuf/hpb/backend/upb/interop.h" -#include "google/protobuf/hpb/internal/internal.h" -#include "google/protobuf/hpb/internal/message_lock.h" -#include "google/protobuf/hpb/internal/template_help.h" -#include "google/protobuf/hpb/ptr.h" +#include "hpb/arena.h" +#include "hpb/backend/upb/interop.h" +#include "hpb/internal/internal.h" +#include "hpb/internal/message_lock.h" +#include "hpb/internal/template_help.h" +#include "hpb/ptr.h" namespace hpb::internal::backend::upb { template typename T::Proxy CreateMessage(Arena& arena) { - return PrivateAccess::CreateMessage(arena.ptr()); + return PrivateAccess::CreateMessage(hpb::interop::upb::UnwrapArena(arena)); } template typename T::Proxy CloneMessage(Ptr message, Arena& arena) { return internal::PrivateAccess::Proxy( internal::DeepClone(interop::upb::GetMessage(message), T::minitable(), - arena.ptr()), - arena.ptr()); + hpb::interop::upb::UnwrapArena(arena)), + hpb::interop::upb::UnwrapArena(arena)); } template @@ -51,7 +51,7 @@ template absl::StatusOr Serialize(PtrOrRaw message, Arena& arena) { return hpb::internal::Serialize(interop::upb::GetMessage(message), interop::upb::GetMiniTable(message), - arena.ptr(), 0); + hpb::interop::upb::UnwrapArena(arena), 0); } } // namespace hpb::internal::backend::upb diff --git a/hpb/bazel/BUILD b/hpb/bazel/BUILD new file mode 100644 index 0000000000000..0954ba1c68e06 --- /dev/null +++ b/hpb/bazel/BUILD @@ -0,0 +1,24 @@ +# Copyright (c) 2009-2021, Google LLC +# All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +package(default_applicable_licenses = ["//:license"]) + +licenses(["notice"]) + +bzl_library( + name = "hpb_proto_library_bzl", + srcs = ["hpb_proto_library.bzl"], + visibility = ["//visibility:public"], + deps = [ + "//bazel:upb_proto_library_bzl", + "//tools/build_defs/proto:proto_common.bzl", + "@bazel_skylib//lib:paths", + "@bazel_tools//tools/cpp:toolchain_utils.bzl", + ], +) diff --git a/hpb/extension.cc b/hpb/extension.cc index 681a72e107ca5..de348588e5bf7 100644 --- a/hpb/extension.cc +++ b/hpb/extension.cc @@ -5,12 +5,12 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#include "google/protobuf/hpb/extension.h" +#include "hpb/extension.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" -#include "google/protobuf/hpb/internal/message_lock.h" -#include "google/protobuf/hpb/status.h" +#include "hpb/internal/message_lock.h" +#include "hpb/status.h" #include "upb/mem/arena.h" #include "upb/message/accessors.h" #include "upb/message/message.h" diff --git a/hpb/extension.h b/hpb/extension.h index 1769cef3bcccd..155a902682f5d 100644 --- a/hpb/extension.h +++ b/hpb/extension.h @@ -15,11 +15,11 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "google/protobuf/hpb/backend/upb/interop.h" -#include "google/protobuf/hpb/internal/message_lock.h" -#include "google/protobuf/hpb/internal/template_help.h" -#include "google/protobuf/hpb/ptr.h" -#include "google/protobuf/hpb/status.h" +#include "hpb/backend/upb/interop.h" +#include "hpb/internal/message_lock.h" +#include "hpb/internal/template_help.h" +#include "hpb/ptr.h" +#include "hpb/status.h" #include "upb/base/string_view.h" #include "upb/mem/arena.h" #include "upb/mem/arena.hpp" diff --git a/hpb/hpb.h b/hpb/hpb.h index 09ee5c6465cfe..2b77290e2cdbc 100644 --- a/hpb/hpb.h +++ b/hpb/hpb.h @@ -13,19 +13,19 @@ #include "absl/base/attributes.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "google/protobuf/hpb/arena.h" -#include "google/protobuf/hpb/extension.h" -#include "google/protobuf/hpb/internal/template_help.h" -#include "google/protobuf/hpb/multibackend.h" -#include "google/protobuf/hpb/ptr.h" -#include "google/protobuf/hpb/status.h" +#include "hpb/arena.h" +#include "hpb/extension.h" +#include "hpb/internal/template_help.h" +#include "hpb/multibackend.h" +#include "hpb/ptr.h" +#include "hpb/status.h" #if HPB_INTERNAL_BACKEND == HPB_INTERNAL_BACKEND_UPB -#include "google/protobuf/hpb/backend/upb/interop.h" -#include "google/protobuf/hpb/backend/upb/upb.h" +#include "hpb/backend/upb/interop.h" +#include "hpb/backend/upb/upb.h" #include "upb/wire/decode.h" #elif HPB_INTERNAL_BACKEND == HPB_INTERNAL_BACKEND_CPP -#include "google/protobuf/hpb/backend/cpp/cpp.h" +#include "hpb/backend/cpp/cpp.h" #else #error hpb backend unknown #endif diff --git a/hpb/internal/BUILD b/hpb/internal/BUILD new file mode 100644 index 0000000000000..618e5ed9d8995 --- /dev/null +++ b/hpb/internal/BUILD @@ -0,0 +1,87 @@ +# Copyright (c) 2024, Google LLC +# All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +load( + "//upb/bazel:build_defs.bzl", + "UPB_DEFAULT_CPPOPTS", +) + +package(default_applicable_licenses = ["//:license"]) + +cc_library( + name = "template_help", + hdrs = ["template_help.h"], + copts = UPB_DEFAULT_CPPOPTS, + visibility = ["//hpb:__subpackages__"], + deps = [ + "//hpb:ptr", + ], +) + +cc_test( + name = "template_help_test", + srcs = ["template_help_test.cc"], + deps = [ + ":template_help", + "//src/google/protobuf/compiler/hpb/tests:test_model_hpb_proto", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +cc_library( + name = "internal", + hdrs = ["internal.h"], + visibility = [ + "//hpb:__subpackages__", + ], + deps = [ + "//upb/mem", + "//upb/message", + ], +) + +cc_library( + name = "message_lock", + srcs = ["message_lock.cc"], + hdrs = [ + "message_lock.h", + ], + copts = UPB_DEFAULT_CPPOPTS, + visibility = [ + "//hpb:__subpackages__", + ], + deps = [ + "//hpb:status", + "//upb/mem", + "//upb/message", + "//upb/message:copy", + "//upb/message:promote", + "//upb/mini_table", + "//upb/wire", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings:string_view", + ], +) + +cc_test( + name = "message_lock_test", + srcs = ["message_lock_test.cc"], + deps = [ + ":message_lock", + "//hpb", + "//hpb:arena", + "//hpb:extension", + "//src/google/protobuf/compiler/hpb/tests:test_model_hpb_proto", + "//upb/mem", + "//upb/mini_table", + "@abseil-cpp//absl/hash", + "@abseil-cpp//absl/log:absl_check", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) diff --git a/hpb/internal/message_lock.cc b/hpb/internal/message_lock.cc index bb5a10d451eb7..819294f3b7ceb 100644 --- a/hpb/internal/message_lock.cc +++ b/hpb/internal/message_lock.cc @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#include "google/protobuf/hpb/internal/message_lock.h" +#include "hpb/internal/message_lock.h" #include #include @@ -13,7 +13,7 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "google/protobuf/hpb/status.h" +#include "hpb/status.h" #include "upb/mem/arena.h" #include "upb/message/accessors.h" #include "upb/message/array.h" diff --git a/hpb/internal/message_lock_test.cc b/hpb/internal/message_lock_test.cc index 4ee396fe09ca4..318761eff72b2 100644 --- a/hpb/internal/message_lock_test.cc +++ b/hpb/internal/message_lock_test.cc @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#include "google/protobuf/hpb/internal/message_lock.h" +#include "hpb/internal/message_lock.h" #include #include @@ -16,9 +16,9 @@ #include "absl/hash/hash.h" #include "absl/log/absl_check.h" #include "google/protobuf/compiler/hpb/tests/test_model.hpb.h" -#include "google/protobuf/hpb/arena.h" -#include "google/protobuf/hpb/extension.h" -#include "google/protobuf/hpb/hpb.h" +#include "hpb/arena.h" +#include "hpb/extension.h" +#include "hpb/hpb.h" #include "upb/mem/arena.hpp" #ifndef ASSERT_OK diff --git a/hpb/internal/template_help.h b/hpb/internal/template_help.h index e4339586692d6..9f61fc6e886c9 100644 --- a/hpb/internal/template_help.h +++ b/hpb/internal/template_help.h @@ -10,7 +10,7 @@ #include -#include "google/protobuf/hpb/ptr.h" +#include "hpb/ptr.h" namespace hpb { namespace internal { diff --git a/hpb/internal/template_help_test.cc b/hpb/internal/template_help_test.cc index 66388557b9642..390e666ec46ab 100644 --- a/hpb/internal/template_help_test.cc +++ b/hpb/internal/template_help_test.cc @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#include "google/protobuf/hpb/internal/template_help.h" +#include "hpb/internal/template_help.h" #include #include "google/protobuf/compiler/hpb/tests/test_model.hpb.h" diff --git a/hpb/repeated_field.h b/hpb/repeated_field.h index f6b840ac9fd20..f18166a12f35f 100644 --- a/hpb/repeated_field.h +++ b/hpb/repeated_field.h @@ -15,10 +15,10 @@ #include #include "absl/strings/string_view.h" -#include "google/protobuf/hpb/backend/upb/interop.h" -#include "google/protobuf/hpb/internal/template_help.h" -#include "google/protobuf/hpb/ptr.h" -#include "google/protobuf/hpb/repeated_field_iterator.h" +#include "hpb/backend/upb/interop.h" +#include "hpb/internal/template_help.h" +#include "hpb/ptr.h" +#include "hpb/repeated_field_iterator.h" #include "upb/base/string_view.h" #include "upb/mem/arena.h" #include "upb/message/array.h" diff --git a/hpb/repeated_field_iterator.h b/hpb/repeated_field_iterator.h index 6bee59317a0d3..4b2c071b95509 100644 --- a/hpb/repeated_field_iterator.h +++ b/hpb/repeated_field_iterator.h @@ -14,8 +14,8 @@ #include #include "absl/strings/string_view.h" -#include "google/protobuf/hpb/backend/upb/interop.h" -#include "google/protobuf/hpb/hpb.h" +#include "hpb/backend/upb/interop.h" +#include "hpb/hpb.h" #include "upb/base/string_view.h" #include "upb/mem/arena.h" #include "upb/message/array.h" diff --git a/hpb/repeated_field_iterator_test.cc b/hpb/repeated_field_iterator_test.cc index f3ba6efc81ef5..0c4d6ce2eca39 100644 --- a/hpb/repeated_field_iterator_test.cc +++ b/hpb/repeated_field_iterator_test.cc @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#include "google/protobuf/hpb/repeated_field_iterator.h" +#include "hpb/repeated_field_iterator.h" #include #include @@ -18,7 +18,10 @@ #include #include #include "absl/strings/string_view.h" -#include "google/protobuf/hpb/hpb.h" +#include "hpb/arena.h" +#include "hpb/backend/upb/interop.h" +#include "upb/base/descriptor_constants.h" +#include "upb/mem/arena.h" #include "upb/message/array.h" using ::testing::ElementsAre; @@ -49,13 +52,13 @@ struct IteratorTestPeer { template static StringRef MakeStringRefProxy(upb_Array* arr, hpb::Arena& arena) { - return StringRef({arr, arena.ptr(), 0}); + return StringRef({arr, hpb::interop::upb::UnwrapArena(arena), 0}); } template static StringIterator MakeStringIterator(upb_Array* arr, hpb::Arena& arena) { - return StringIterator({arr, arena.ptr()}); + return StringIterator({arr, hpb::interop::upb::UnwrapArena(arena), 0}); } }; @@ -229,18 +232,20 @@ TEST(ScalarIteratorTest, IteratorBasedAlgorithmsWork) { } const char* CloneString(hpb::Arena& arena, absl::string_view str) { - char* data = (char*)upb_Arena_Malloc(arena.ptr(), str.size()); + char* data = (char*)upb_Arena_Malloc(hpb::interop::upb::UnwrapArena(arena), + str.size()); memcpy(data, str.data(), str.size()); return data; } upb_Array* MakeStringArray(hpb::Arena& arena, const std::vector& input) { - upb_Array* arr = upb_Array_New(arena.ptr(), kUpb_CType_String); + upb_Array* arr = + upb_Array_New(hpb::interop::upb::UnwrapArena(arena), kUpb_CType_String); for (absl::string_view str : input) { upb_MessageValue message_value; message_value.str_val = upb_StringView_FromDataAndSize(CloneString(arena, str), str.size()); - upb_Array_Append(arr, message_value, arena.ptr()); + upb_Array_Append(arr, message_value, hpb::interop::upb::UnwrapArena(arena)); } return arr; } diff --git a/hpb/status.cc b/hpb/status.cc index 4d5c3946dd8da..559e86f73ad16 100644 --- a/hpb/status.cc +++ b/hpb/status.cc @@ -5,13 +5,12 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#include "google/protobuf/hpb/status.h" +#include "hpb/status.h" #include #include "absl/status/status.h" #include "absl/strings/str_format.h" -#include "absl/types/source_location.h" #include "upb/wire/decode.h" #include "upb/wire/encode.h" diff --git a/hpb/status.h b/hpb/status.h index 8abe29e97d3c0..202bdbbf0d9da 100644 --- a/hpb/status.h +++ b/hpb/status.h @@ -11,7 +11,6 @@ #include #include "absl/status/status.h" -#include "absl/types/source_location.h" #include "upb/wire/decode.h" #include "upb/wire/encode.h" diff --git a/hpb_generator/BUILD b/hpb_generator/BUILD new file mode 100644 index 0000000000000..95cdaa9daa8a0 --- /dev/null +++ b/hpb_generator/BUILD @@ -0,0 +1,139 @@ +# Copyright (c) 2009-2021, Google LLC +# All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +load("//bazel/toolchains:proto_lang_toolchain.bzl", "proto_lang_toolchain") +load( + "//upb/bazel:build_defs.bzl", + "UPB_DEFAULT_CPPOPTS", +) + +package(default_applicable_licenses = ["//:license"]) + +licenses(["notice"]) + +cc_binary( + name = "protoc-gen-hpb", + srcs = [ + "protoc-gen-hpb.cc", + ], + copts = UPB_DEFAULT_CPPOPTS, + malloc = "@bazel_tools//tools/cpp:malloc", + visibility = ["//visibility:public"], + deps = [ + ":generator", + "//base", + "//src/google/protobuf/compiler:plugin", + "@abseil-cpp//absl/flags:flag", + ], +) + +proto_lang_toolchain( + name = "toolchain", + command_line = select({ + "//hpb:hpb_backend_upb": "--hpb_out=$(OUT)", + "//hpb:hpb_backend_cpp": "--hpb_out=backend=cpp:$(OUT)", + }), + output_files = "multiple", + plugin = ":protoc-gen-hpb", + plugin_format_flag = "--plugin=protoc-gen-hpb=%s", + progress_message = "Generating hpb protocol buffers", + runtime = "//hpb:generated_hpb_support", + visibility = ["//hpb/bazel:__pkg__"], +) + +cc_library( + name = "generator", + srcs = [ + "gen_accessors.cc", + "gen_accessors.h", + "gen_enums.cc", + "gen_enums.h", + "gen_extensions.cc", + "gen_extensions.h", + "gen_messages.cc", + "gen_messages.h", + "gen_repeated_fields.cc", + "gen_repeated_fields.h", + "generator.cc", + ], + hdrs = [ + "generator.h", + ], + copts = UPB_DEFAULT_CPPOPTS, + visibility = ["//visibility:private"], + deps = [ + ":context", + ":gen_utils", + ":keywords", + ":names", + "//src/google/protobuf", + "//src/google/protobuf/compiler:code_generator", + "//src/google/protobuf/compiler:code_generator_lite", + "//src/google/protobuf/compiler:plugin", + "//upb_generator:common", + "//upb_generator:file_layout", + "//upb_generator/c:names", + "//upb_generator/minitable:names", + "@abseil-cpp//absl/container:flat_hash_set", + "@abseil-cpp//absl/strings", + ], +) + +cc_library( + name = "keywords", + srcs = ["keywords.cc"], + hdrs = ["keywords.h"], + visibility = ["//visibility:private"], + deps = [ + "@abseil-cpp//absl/container:flat_hash_set", + "@abseil-cpp//absl/strings", + ], +) + +cc_library( + name = "gen_utils", + srcs = ["gen_utils.cc"], + hdrs = ["gen_utils.h"], + visibility = ["//visibility:public"], + deps = [ + "//src/google/protobuf", + "//src/google/protobuf/compiler:code_generator", + "@abseil-cpp//absl/log:absl_log", + "@abseil-cpp//absl/strings", + ], +) + +cc_library( + name = "names", + srcs = ["names.cc"], + hdrs = ["names.h"], + visibility = ["//visibility:private"], + deps = [ + ":context", + ":keywords", + "//src/google/protobuf", + "//src/google/protobuf/compiler:code_generator", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + ], +) + +cc_library( + name = "context", + hdrs = ["context.h"], + deps = [ + "//src/google/protobuf", + "//src/google/protobuf/io", + "//src/google/protobuf/io:printer", + "//upb/reflection", + "//upb_generator/common:cpp_to_upb_def", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:source_location", + "@abseil-cpp//absl/types:span", + ], +) diff --git a/hpb_generator/gen_messages.cc b/hpb_generator/gen_messages.cc index fc8e251130798..aec5ab44c2104 100644 --- a/hpb_generator/gen_messages.cc +++ b/hpb_generator/gen_messages.cc @@ -260,7 +260,7 @@ void WriteModelPublicDeclaration( $0(upb_Message* msg, upb_Arena* arena) : $0Access() { msg_ = ($1*)msg; - arena_ = owned_arena_.ptr(); + arena_ = ::hpb::interop::upb::UnwrapArena(owned_arena_); upb_Arena_Fuse(arena_, arena); } ::hpb::Arena owned_arena_; @@ -398,15 +398,15 @@ void WriteMessageImplementation( ctx.EmitLegacy( R"cc( $0::$0() : $0Access() { - arena_ = owned_arena_.ptr(); + arena_ = ::hpb::interop::upb::UnwrapArena(owned_arena_); msg_ = $1_new(arena_); } $0::$0(const $0& from) : $0Access() { - arena_ = owned_arena_.ptr(); + arena_ = ::hpb::interop::upb::UnwrapArena(owned_arena_); msg_ = ($1*)::hpb::internal::DeepClone(UPB_UPCAST(from.msg_), &$2, arena_); } $0::$0(const CProxy& from) : $0Access() { - arena_ = owned_arena_.ptr(); + arena_ = ::hpb::interop::upb::UnwrapArena(owned_arena_); msg_ = ($1*)::hpb::internal::DeepClone( ::hpb::interop::upb::GetMessage(&from), &$2, arena_); } @@ -416,12 +416,12 @@ void WriteMessageImplementation( msg_ = ($1*)::hpb::interop::upb::GetMessage(&m); } $0& $0::operator=(const $3& from) { - arena_ = owned_arena_.ptr(); + arena_ = ::hpb::interop::upb::UnwrapArena(owned_arena_); msg_ = ($1*)::hpb::internal::DeepClone(UPB_UPCAST(from.msg_), &$2, arena_); return *this; } $0& $0::operator=(const CProxy& from) { - arena_ = owned_arena_.ptr(); + arena_ = ::hpb::interop::upb::UnwrapArena(owned_arena_); msg_ = ($1*)::hpb::internal::DeepClone( ::hpb::interop::upb::GetMessage(&from), &$2, arena_); return *this; diff --git a/hpb_generator/generator.cc b/hpb_generator/generator.cc index b205ecb0a97e9..53190edaae570 100644 --- a/hpb_generator/generator.cc +++ b/hpb_generator/generator.cc @@ -92,7 +92,7 @@ void WriteHeader(const protobuf::FileDescriptor* file, Context& ctx) { #ifndef $filename$_HPB_PROTO_H_ #define $filename$_HPB_PROTO_H_ -#include "google/protobuf/hpb/repeated_field.h" +#include "hpb/repeated_field.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" diff --git a/hpb_generator/tests/BUILD b/hpb_generator/tests/BUILD new file mode 100644 index 0000000000000..bab52a6e778f5 --- /dev/null +++ b/hpb_generator/tests/BUILD @@ -0,0 +1,12 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +# todo: re-add post migration + +package(default_applicable_licenses = ["//:license"]) + +# todo: re-add post migration diff --git a/hpb_generator/tests/extension_test.cc b/hpb_generator/tests/extension_test.cc index e0bcd96d11ebe..ca777e36a7b4c 100644 --- a/hpb_generator/tests/extension_test.cc +++ b/hpb_generator/tests/extension_test.cc @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#include "google/protobuf/hpb/extension.h" +#include "hpb/extension.h" #include #include @@ -16,9 +16,9 @@ #include "google/protobuf/compiler/hpb/tests/child_model.hpb.h" #include "google/protobuf/compiler/hpb/tests/test_extension.hpb.h" #include "google/protobuf/compiler/hpb/tests/test_model.hpb.h" -#include "google/protobuf/hpb/arena.h" -#include "google/protobuf/hpb/hpb.h" -#include "google/protobuf/hpb/requires.h" +#include "hpb/arena.h" +#include "hpb/hpb.h" +#include "hpb/requires.h" namespace { using ::hpb::internal::Requires; diff --git a/hpb_generator/tests/multibackend_test.cc b/hpb_generator/tests/multibackend_test.cc index c69f338195d0c..ee1586f119105 100644 --- a/hpb_generator/tests/multibackend_test.cc +++ b/hpb_generator/tests/multibackend_test.cc @@ -7,8 +7,8 @@ #include #include "google/protobuf/compiler/hpb/tests/test_model.hpb.h" -#include "google/protobuf/hpb/arena.h" -#include "google/protobuf/hpb/hpb.h" +#include "hpb/arena.h" +#include "hpb/hpb.h" namespace { using ::hpb_unittest::protos::TestModel; diff --git a/hpb_generator/tests/repeated_test.cc b/hpb_generator/tests/repeated_test.cc index 362d9d8af9e3f..347ea6f5f6765 100644 --- a/hpb_generator/tests/repeated_test.cc +++ b/hpb_generator/tests/repeated_test.cc @@ -10,7 +10,7 @@ #include "google/protobuf/compiler/hpb/tests/child_model.hpb.h" #include "google/protobuf/compiler/hpb/tests/test_extension.hpb.h" #include "google/protobuf/compiler/hpb/tests/test_model.hpb.h" -#include "google/protobuf/hpb/requires.h" +#include "hpb/requires.h" namespace { diff --git a/hpb_generator/tests/test_generated.cc b/hpb_generator/tests/test_generated.cc index a5bacfc34388a..366b784619e7d 100644 --- a/hpb_generator/tests/test_generated.cc +++ b/hpb_generator/tests/test_generated.cc @@ -18,11 +18,11 @@ #include "google/protobuf/compiler/hpb/tests/test_enum.hpb.h" #include "google/protobuf/compiler/hpb/tests/test_extension.hpb.h" #include "google/protobuf/compiler/hpb/tests/test_model.hpb.h" -#include "google/protobuf/hpb/arena.h" -#include "google/protobuf/hpb/backend/upb/interop.h" -#include "google/protobuf/hpb/hpb.h" -#include "google/protobuf/hpb/ptr.h" -#include "google/protobuf/hpb/requires.h" +#include "hpb/arena.h" +#include "hpb/backend/upb/interop.h" +#include "hpb/hpb.h" +#include "hpb/ptr.h" +#include "hpb/requires.h" namespace { diff --git a/hpb_generator/tests/test_hpb_bzl_alias.cc b/hpb_generator/tests/test_hpb_bzl_alias.cc index 64a7a8e8bdc12..0d3c4ae790b8e 100644 --- a/hpb_generator/tests/test_hpb_bzl_alias.cc +++ b/hpb_generator/tests/test_hpb_bzl_alias.cc @@ -7,8 +7,8 @@ #include #include "google/protobuf/compiler/hpb/tests/set_alias.hpb.h" -#include "google/protobuf/hpb/arena.h" -#include "google/protobuf/hpb/hpb.h" +#include "hpb/arena.h" +#include "hpb/hpb.h" namespace { using hpb_unittest::protos::Child; diff --git a/java/core/src/main/java/com/google/protobuf/CodedOutputStreamWriter.java b/java/core/src/main/java/com/google/protobuf/CodedOutputStreamWriter.java index a20a579e3e2a3..59aa41ffab3bd 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedOutputStreamWriter.java +++ b/java/core/src/main/java/com/google/protobuf/CodedOutputStreamWriter.java @@ -1025,11 +1025,11 @@ private void writeDeterministicMap( switch (metadata.keyType) { case BOOL: V value; - if ((value = map.get(Boolean.FALSE)) != null) { + if ((value = map.get(false)) != null) { writeDeterministicBooleanMapEntry( fieldNumber, /* key= */ false, value, (MapEntryLite.Metadata) metadata); } - if ((value = map.get(Boolean.TRUE)) != null) { + if ((value = map.get(true)) != null) { writeDeterministicBooleanMapEntry( fieldNumber, /* key= */ true, value, (MapEntryLite.Metadata) metadata); } diff --git a/java/core/src/main/java/com/google/protobuf/ExtensionRegistryLite.java b/java/core/src/main/java/com/google/protobuf/ExtensionRegistryLite.java index 46a13b4b9dac3..3d20793a8e9fb 100644 --- a/java/core/src/main/java/com/google/protobuf/ExtensionRegistryLite.java +++ b/java/core/src/main/java/com/google/protobuf/ExtensionRegistryLite.java @@ -141,7 +141,7 @@ public final void add(final GeneratedMessageLite.GeneratedExtension extens * i.e. {@link GeneratedMessageLite.GeneratedExtension}. */ public final void add(ExtensionLite extension) { - if (GeneratedMessageLite.GeneratedExtension.class.isAssignableFrom(extension.getClass())) { + if (extension instanceof GeneratedMessageLite.GeneratedExtension) { add((GeneratedMessageLite.GeneratedExtension) extension); } if (!Protobuf.assumeLiteRuntime && ExtensionRegistryFactory.isFullRegistry(this)) { diff --git a/java/core/src/main/java/com/google/protobuf/IterableByteBufferInputStream.java b/java/core/src/main/java/com/google/protobuf/IterableByteBufferInputStream.java index c5739c56dddeb..943157dd11324 100644 --- a/java/core/src/main/java/com/google/protobuf/IterableByteBufferInputStream.java +++ b/java/core/src/main/java/com/google/protobuf/IterableByteBufferInputStream.java @@ -14,30 +14,38 @@ import java.nio.ByteBuffer; import java.util.Iterator; -class IterableByteBufferInputStream extends InputStream { +final class IterableByteBufferInputStream extends InputStream { /** The {@link Iterator} with type {@link ByteBuffer} of {@code input} */ private Iterator iterator; + /** The current ByteBuffer; */ private ByteBuffer currentByteBuffer; + /** The number of ByteBuffers in the input data. */ private int dataSize; + /** * Current {@code ByteBuffer}'s index * *

If index equals dataSize, then all the data in the InputStream has been consumed */ private int currentIndex; + /** The current position for current ByteBuffer */ private int currentByteBufferPos; + /** Whether current ByteBuffer has an array */ private boolean hasArray; + /** * If the current ByteBuffer is unsafe-direct based, currentArray is null; otherwise should be the * array inside ByteBuffer. */ private byte[] currentArray; + /** Current ByteBuffer's array offset */ private int currentArrayOffset; + /** * If the current ByteBuffer is unsafe-direct based, currentAddress is the start address of this * ByteBuffer; otherwise should be zero. diff --git a/objectivec/DevTools/check_version_stamps.sh b/objectivec/DevTools/check_version_stamps.sh index 8b6d496a06744..184215d4e848f 100755 --- a/objectivec/DevTools/check_version_stamps.sh +++ b/objectivec/DevTools/check_version_stamps.sh @@ -28,7 +28,7 @@ check_constant() { # Collect version from generator sources. local GeneratorVersion=$( \ cat "${GeneratorSrc}" \ - | sed -n -e "s:const int32_t ${ConstantName} = \([0-9]*\);:\1:p" + | sed -n -e "s:^const int32_t ${ConstantName} = \([0-9]*\);:\1:p" ) if [[ -z "${GeneratorVersion}" ]] ; then die "Failed to find ${ConstantName} in the generator source (${GeneratorSrc})." @@ -37,7 +37,7 @@ check_constant() { # Collect version from runtime sources. local RuntimeVersion=$( \ cat "${RuntimeSrc}" \ - | sed -n -e "s:#define ${ConstantName} \([0-9]*\):\1:p" + | sed -n -e "s:^#define ${ConstantName} \([0-9]*\):\1:p" ) if [[ -z "${RuntimeVersion}" ]] ; then die "Failed to find ${ConstantName} in the runtime source (${RuntimeSrc})." diff --git a/objectivec/Tests/GPBObjectiveCPlusPlusTest.mm b/objectivec/Tests/GPBObjectiveCPlusPlusTest.mm index 96351e0dcad65..80ec66afe9ba3 100644 --- a/objectivec/Tests/GPBObjectiveCPlusPlusTest.mm +++ b/objectivec/Tests/GPBObjectiveCPlusPlusTest.mm @@ -24,11 +24,6 @@ // Sanity check the conditions of the test within the Xcode project. #if !__cplusplus #error This isn't compiled as Objective C++? -#elif __cplusplus >= 201103L -// If this trips, it means the Xcode default might have change (or someone -// edited the testing project) and it might be time to revisit the GPB_ENUM -// define in GPBBootstrap.h. -#warning Did the Xcode default for C++ spec change? #endif // Dummy XCTest. diff --git a/objectivec/Tests/GPBTestUtilities.m b/objectivec/Tests/GPBTestUtilities.m index 99a270c2c8d99..3f4f39f86314e 100644 --- a/objectivec/Tests/GPBTestUtilities.m +++ b/objectivec/Tests/GPBTestUtilities.m @@ -33,7 +33,7 @@ - (instancetype)initWithUint32_gpbtu:(uint32_t)value { } + (instancetype)gpbtu_dataWithEmbeddedNulls { - char bytes[6] = "\1\0\2\3\0\5"; + char bytes[6] = {'\1', '\0', '\2', '\3', '\0', '\5'}; return [self dataWithBytes:bytes length:sizeof(bytes)]; } @end diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index f30f492a1deaf..888af41ee9add 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -6955,18 +6955,6 @@ static const upb_MiniTableEnum* _upb_MiniTableSubs_EnumByField( return subs[field->UPB_PRIVATE(submsg_index)].UPB_PRIVATE(subenum); } -UPB_NORETURN static void* _upb_Decoder_ErrorJmp(upb_Decoder* d, - upb_DecodeStatus status) { - UPB_ASSERT(status != kUpb_DecodeStatus_Ok); - d->status = status; - UPB_LONGJMP(d->err, 1); -} - -const char* _upb_FastDecoder_ErrorJmp2(upb_Decoder* d) { - UPB_LONGJMP(d->err, 1); - return NULL; -} - static void _upb_Decoder_VerifyUtf8(upb_Decoder* d, const char* buf, int len) { if (!_upb_Decoder_VerifyUtf8Inline(buf, len)) { _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); @@ -7618,18 +7606,6 @@ static const char* _upb_Decoder_DecodeToSubMessage( return ptr; } -UPB_NOINLINE -const char* _upb_Decoder_CheckRequired(upb_Decoder* d, const char* ptr, - const upb_Message* msg, - const upb_MiniTable* m) { - UPB_ASSERT(m->UPB_PRIVATE(required_count)); - if (UPB_UNLIKELY(d->options & kUpb_DecodeOption_CheckRequired)) { - d->missing_required = - !UPB_PRIVATE(_upb_Message_IsInitializedShallow)(msg, m); - } - return ptr; -} - static const char* upb_Decoder_SkipField(upb_Decoder* d, const char* ptr, uint32_t tag) { int field_number = tag >> 3; @@ -8232,13 +8208,6 @@ static upb_DecodeStatus _upb_Decoder_DecodeTop(struct upb_Decoder* d, return kUpb_DecodeStatus_Ok; } -UPB_NOINLINE -const char* _upb_Decoder_IsDoneFallback(upb_EpsCopyInputStream* e, - const char* ptr, int overrun) { - return _upb_EpsCopyInputStream_IsDoneFallbackInline( - e, ptr, overrun, _upb_Decoder_BufferFlipCallback); -} - static upb_DecodeStatus upb_Decoder_Decode(upb_Decoder* const decoder, const char* const buf, upb_Message* const msg, @@ -8451,8 +8420,9 @@ static char* encode_fixed32(char* ptr, upb_encstate* e, uint32_t val) { #define UPB_PB_VARINT_MAX_LEN 10 -// Need gnu extended inline asm -#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +// Need gnu extended inline asm; msan can't instrument stores in inline assembly +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) && \ + !UPB_HAS_FEATURE(memory_sanitizer) #define UPB_ARM64_ASM #endif @@ -17443,6 +17413,36 @@ upb_ServiceDef* _upb_ServiceDefs_New(upb_DefBuilder* ctx, int n, return s; } + + +// Must be last. + +UPB_NOINLINE +const char* _upb_Decoder_CheckRequired(upb_Decoder* d, const char* ptr, + const upb_Message* msg, + const upb_MiniTable* m) { + UPB_ASSERT(m->UPB_PRIVATE(required_count)); + if (UPB_UNLIKELY(d->options & kUpb_DecodeOption_CheckRequired)) { + d->missing_required = + !UPB_PRIVATE(_upb_Message_IsInitializedShallow)(msg, m); + } + return ptr; +} + +UPB_NORETURN void* _upb_Decoder_ErrorJmp(upb_Decoder* d, + upb_DecodeStatus status) { + UPB_ASSERT(status != kUpb_DecodeStatus_Ok); + d->status = status; + UPB_LONGJMP(d->err, 1); +} + +UPB_NOINLINE +const char* _upb_Decoder_IsDoneFallback(upb_EpsCopyInputStream* e, + const char* ptr, int overrun) { + return _upb_EpsCopyInputStream_IsDoneFallbackInline( + e, ptr, overrun, _upb_Decoder_BufferFlipCallback); +} + // This should #undef all macros #defined in def.inc #undef UPB_SIZE diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index 92fc353b6ead2..627cf284cd3a9 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -5683,6 +5683,8 @@ bool upb_Message_SetMapEntry(upb_Map* map, const upb_MiniTable* mini_table, #ifndef UPB_MINI_TABLE_DECODE_H_ #define UPB_MINI_TABLE_DECODE_H_ +#include + #ifndef UPB_MINI_TABLE_SUB_H_ #define UPB_MINI_TABLE_SUB_H_ @@ -5877,7 +5879,7 @@ UPB_API_INLINE upb_MiniTableExtension* upb_MiniTableExtension_Build( UPB_API_INLINE upb_MiniTableExtension* upb_MiniTableExtension_BuildMessage( const char* data, size_t len, const upb_MiniTable* extendee, - upb_MiniTable* submsg, upb_Arena* arena, upb_Status* status) { + const upb_MiniTable* submsg, upb_Arena* arena, upb_Status* status) { upb_MiniTableSub sub = upb_MiniTableSub_FromMessage(submsg); return _upb_MiniTableExtension_Build( data, len, extendee, sub, kUpb_MiniTablePlatform_Native, arena, status); @@ -5885,7 +5887,7 @@ UPB_API_INLINE upb_MiniTableExtension* upb_MiniTableExtension_BuildMessage( UPB_API_INLINE upb_MiniTableExtension* upb_MiniTableExtension_BuildEnum( const char* data, size_t len, const upb_MiniTable* extendee, - upb_MiniTableEnum* subenum, upb_Arena* arena, upb_Status* status) { + const upb_MiniTableEnum* subenum, upb_Arena* arena, upb_Status* status) { upb_MiniTableSub sub = upb_MiniTableSub_FromEnum(subenum); return _upb_MiniTableExtension_Build( data, len, extendee, sub, kUpb_MiniTablePlatform_Native, arena, status); @@ -15617,21 +15619,6 @@ UPB_INLINE void _upb_Decoder_Trace(upb_Decoder* d, char event) { #endif }; -/* Error function that will abort decoding with longjmp(). We can't declare this - * UPB_NORETURN, even though it is appropriate, because if we do then compilers - * will "helpfully" refuse to tailcall to it - * (see: https://stackoverflow.com/a/55657013), which will defeat a major goal - * of our optimizations. That is also why we must declare it in a separate file, - * otherwise the compiler will see that it calls longjmp() and deduce that it is - * noreturn. */ -const char* _upb_FastDecoder_ErrorJmp2(upb_Decoder* d); - -UPB_INLINE -const char* _upb_FastDecoder_ErrorJmp(upb_Decoder* d, upb_DecodeStatus status) { - d->status = status; - return _upb_FastDecoder_ErrorJmp2(d); -} - UPB_INLINE bool _upb_Decoder_VerifyUtf8Inline(const char* ptr, int len) { return utf8_range_IsValid(ptr, len); @@ -15663,10 +15650,13 @@ UPB_INLINE bool _upb_Decoder_IsDone(upb_Decoder* d, const char** ptr) { &d->input, ptr, &_upb_Decoder_IsDoneFallback); } +UPB_NORETURN void* _upb_Decoder_ErrorJmp(upb_Decoder* d, + upb_DecodeStatus status); + UPB_INLINE const char* _upb_Decoder_BufferFlipCallback( upb_EpsCopyInputStream* e, const char* old_end, const char* new_start) { upb_Decoder* d = (upb_Decoder*)e; - if (!old_end) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); + if (!old_end) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); return new_start; } diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 31d5506d33d8a..fb607859bd553 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -141,9 +141,9 @@ def protobuf_deps(): if not native.existing_rule("rules_python"): http_archive( name = "rules_python", - sha256 = "4f7e2aa1eb9aa722d96498f5ef514f426c1f55161c3c9ae628c857a7128ceb07", - strip_prefix = "rules_python-1.0.0", - url = "https://github.com/bazelbuild/rules_python/releases/download/1.0.0/rules_python-1.0.0.tar.gz", + sha256 = "9c6e26911a79fbf510a8f06d8eedb40f412023cf7fa6d1461def27116bff022c", + strip_prefix = "rules_python-1.1.0", + url = "https://github.com/bazelbuild/rules_python/releases/download/1.1.0/rules_python-1.1.0.tar.gz", ) if not native.existing_rule("system_python"): @@ -155,9 +155,9 @@ def protobuf_deps(): if not native.existing_rule("rules_jvm_external"): http_archive( name = "rules_jvm_external", - strip_prefix = "rules_jvm_external-6.3", - sha256 = "c18a69d784bcd851be95897ca0eca0b57dc86bb02e62402f15736df44160eb02", - url = "https://github.com/bazelbuild/rules_jvm_external/releases/download/6.3/rules_jvm_external-6.3.tar.gz", + strip_prefix = "rules_jvm_external-6.7", + sha256 = "a1e351607f04fed296ba33c4977d3fe2a615ed50df7896676b67aac993c53c18", + url = "https://github.com/bazelbuild/rules_jvm_external/releases/download/6.7/rules_jvm_external-6.7.tar.gz", ) if not native.existing_rule("rules_pkg"): diff --git a/python/BUILD.bazel b/python/BUILD.bazel index f1c384a8f2191..e8c94aecff63e 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -148,18 +148,18 @@ py_extension( deps = [ "//src/google/protobuf:descriptor_upb_reflection_proto", "//third_party/utf8_range", - "//upb:base", - "//upb:eps_copy_input_stream", - "//upb:message", - "//upb:message_compare", - "//upb:message_copy", - "//upb:port", - "//upb:reflection", - "//upb:text", - "//upb:wire_reader", + "//upb/base", "//upb/hash", + "//upb/message", + "//upb/message:compare", + "//upb/message:copy", + "//upb/port", + "//upb/reflection", + "//upb/text", "//upb/util:def_to_proto", "//upb/util:required_fields", + "//upb/wire:eps_copy_input_stream", + "//upb/wire:reader", "@abseil-cpp//absl/base:core_headers", ], ) diff --git a/python/descriptor_containers.c b/python/descriptor_containers.c index 47a00e4d9b2af..00aee2dc6bd46 100644 --- a/python/descriptor_containers.c +++ b/python/descriptor_containers.c @@ -24,6 +24,10 @@ static PyObject* PyUpb_DescriptorMap_Repr(PyObject* _self) { return ret; } +#define CHECK_TYPE(obj, state_member) \ + assert(PyUpb_ModuleState_MaybeGet() == NULL || /* During shutdown. */ \ + Py_TYPE(obj) == PyUpb_ModuleState_Get()->state_member) + // ----------------------------------------------------------------------------- // ByNameIterator // ----------------------------------------------------------------------------- @@ -39,7 +43,7 @@ typedef struct { } PyUpb_ByNameIterator; static PyUpb_ByNameIterator* PyUpb_ByNameIterator_Self(PyObject* obj) { - assert(Py_TYPE(obj) == PyUpb_ModuleState_Get()->by_name_iterator_type); + CHECK_TYPE(obj, by_name_iterator_type); return (PyUpb_ByNameIterator*)obj; } @@ -101,7 +105,7 @@ typedef struct { } PyUpb_ByNumberIterator; static PyUpb_ByNumberIterator* PyUpb_ByNumberIterator_Self(PyObject* obj) { - assert(Py_TYPE(obj) == PyUpb_ModuleState_Get()->by_number_iterator_type); + CHECK_TYPE(obj, by_number_iterator_type); return (PyUpb_ByNumberIterator*)obj; } @@ -162,7 +166,7 @@ typedef struct { } PyUpb_GenericSequence; PyUpb_GenericSequence* PyUpb_GenericSequence_Self(PyObject* obj) { - assert(Py_TYPE(obj) == PyUpb_ModuleState_Get()->generic_sequence_type); + CHECK_TYPE(obj, generic_sequence_type); return (PyUpb_GenericSequence*)obj; } @@ -357,7 +361,7 @@ typedef struct { } PyUpb_ByNameMap; PyUpb_ByNameMap* PyUpb_ByNameMap_Self(PyObject* obj) { - assert(Py_TYPE(obj) == PyUpb_ModuleState_Get()->by_name_map_type); + CHECK_TYPE(obj, by_name_map_type); return (PyUpb_ByNameMap*)obj; } @@ -573,7 +577,7 @@ typedef struct { } PyUpb_ByNumberMap; PyUpb_ByNumberMap* PyUpb_ByNumberMap_Self(PyObject* obj) { - assert(Py_TYPE(obj) == PyUpb_ModuleState_Get()->by_number_map_type); + CHECK_TYPE(obj, by_number_map_type); return (PyUpb_ByNumberMap*)obj; } diff --git a/python/google/protobuf/internal/reflection_test.py b/python/google/protobuf/internal/reflection_test.py index 150e48abb8ed8..4a9dba797b8aa 100755 --- a/python/google/protobuf/internal/reflection_test.py +++ b/python/google/protobuf/internal/reflection_test.py @@ -3144,7 +3144,7 @@ def testFieldProperties(self): with self.assertRaises(AttributeError) as e: # Try to access the descriptor of the field 'optional_int32' cls.optional_int32.DESCRIPTOR - self.assertEquals('optional_int32', str(e.exception)) + self.assertEqual('optional_int32', str(e.exception)) else: self.assertIs( cls.optional_int32.DESCRIPTOR, diff --git a/ruby/.yardopts b/ruby/.yardopts new file mode 100644 index 0000000000000..ed0dbff549e4d --- /dev/null +++ b/ruby/.yardopts @@ -0,0 +1,4 @@ +--no-private +lib/stubs/*.rb +lib/google/protobuf/well_known_types.rb +lib/google/protobuf.rb diff --git a/ruby/README.md b/ruby/README.md index e5b5bb3cf59d4..75ee34679e194 100644 --- a/ruby/README.md +++ b/ruby/README.md @@ -26,9 +26,8 @@ main `README` file. The version of `protoc` included in the latest release supports the `--ruby_out` option to generate Ruby code. A simple example of using the Ruby extension follows. More extensive -documentation may be found in the RubyDoc comments (`call-seq` tags) in the -source, and we plan to release separate, more detailed, documentation at a -later date. +documentation may be found on +[rubydoc.info](https://www.rubydoc.info/gems/google-protobuf). ```ruby require 'google/protobuf' diff --git a/ruby/Rakefile b/ruby/Rakefile index 36254664fa263..862aaf862c615 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -206,7 +206,11 @@ Rake::TestTask.new(:gc_test => ENV['BAZEL'] == 'true' ? [] : :build) do |t| t.test_files = FileList["tests/gc_test.rb"] end -task :build => [:clean, :genproto, :copy_third_party, :compile, :"ffi-protobuf:default"] +task :generate_stubs do + load './generate_stubs.rb' +end + +task :build => [:clean, :genproto, :copy_third_party, :compile, :generate_stubs, :"ffi-protobuf:default"] Rake::Task[:gem].enhance [:copy_third_party, :genproto] task :default => [:build] diff --git a/ruby/ext/google/protobuf_c/defs.c b/ruby/ext/google/protobuf_c/defs.c index fd17093891fe7..e771e44722638 100644 --- a/ruby/ext/google/protobuf_c/defs.c +++ b/ruby/ext/google/protobuf_c/defs.c @@ -93,9 +93,15 @@ const upb_DefPool* DescriptorPool_GetSymtab(VALUE desc_pool_rb) { return pool->symtab; } +/** + * ruby-doc: DescriptorPool + * + * A DescriptorPool is the registry of all known Protobuf descriptor objects. + * + */ + /* - * call-seq: - * DescriptorPool.new => pool + * ruby-doc: DescriptorPool.new * * Creates a new, empty, descriptor pool. */ @@ -112,10 +118,14 @@ static VALUE DescriptorPool_alloc(VALUE klass) { } /* - * call-seq: - * DescriptorPool.add_serialized_file(serialized_file_proto) + * ruby-doc: DescriptorPool#add_serialized_file + * + * Adds the given serialized + * {https://protobuf.com/docs/descriptors#file-descriptors FileDescriptorProto} + * to the pool. * - * Adds the given serialized FileDescriptorProto to the pool. + * @param serialized_file_proto [String] + * @return [FileDescriptor] */ VALUE DescriptorPool_add_serialized_file(VALUE _self, VALUE serialized_file_proto) { @@ -143,11 +153,14 @@ VALUE DescriptorPool_add_serialized_file(VALUE _self, } /* - * call-seq: - * DescriptorPool.lookup(name) => descriptor + * ruby-doc: DescriptorPool#lookup * - * Finds a Descriptor, EnumDescriptor, FieldDescriptor or ServiceDescriptor by + * Finds a {Descriptor}, {EnumDescriptor}, + * {FieldDescriptor} or {ServiceDescriptor} by * name and returns it, or nil if none exists with the given name. + * + * @param name [String] + * @return [Descriptor,EnumDescriptor,FieldDescriptor,ServiceDescriptor] */ static VALUE DescriptorPool_lookup(VALUE _self, VALUE name) { DescriptorPool* self = ruby_to_DescriptorPool(_self); @@ -187,13 +200,14 @@ static VALUE DescriptorPool_lookup(VALUE _self, VALUE name) { } /* - * call-seq: - * DescriptorPool.generated_pool => descriptor_pool + * ruby-doc: DescriptorPool.generated_pool * - * Class method that returns the global DescriptorPool. This is a singleton into - * which generated-code message and enum types are registered. The user may also - * register types in this pool for convenience so that they do not have to hold - * a reference to a private pool instance. + * Class method that returns the global {DescriptorPool}. This is a singleton + * into which generated-code message and enum types are registered. The user may + * also register types in this pool for convenience so that they do not have to + * hold a reference to a private pool instance. + * + * @return [DescriptorPool] */ static VALUE DescriptorPool_generated_pool(VALUE _self) { return generated_pool; @@ -290,8 +304,13 @@ static VALUE decode_options(VALUE self, const char* option_type, int size, } /* - * call-seq: - * Descriptor.new => descriptor + * ruby-doc: Descriptor + * + * A Descriptor provides information about a given Protobuf definition. + */ + +/* + * ruby-doc: Descriptor.initialize * * Creates a new, empty, message type descriptor. At a minimum, its name must be * set before it is added to a pool. It cannot be used to create messages until @@ -329,10 +348,11 @@ static VALUE Descriptor_initialize(VALUE _self, VALUE cookie, } /* - * call-seq: - * Descriptor.file_descriptor + * ruby-doc: Descriptor#file_descriptor * - * Returns the FileDescriptor object this message belongs to. + * Returns the {FileDescriptor} object this message belongs to. + * + * @return [FileDescriptor] */ static VALUE Descriptor_file_descriptor(VALUE _self) { Descriptor* self = ruby_to_Descriptor(_self); @@ -341,11 +361,12 @@ static VALUE Descriptor_file_descriptor(VALUE _self) { } /* - * call-seq: - * Descriptor.name => name + * ruby-doc: Descriptor#name * * Returns the name of this message type as a fully-qualified string (e.g., * My.Package.MessageType). + * + * @return [String] */ static VALUE Descriptor_name(VALUE _self) { Descriptor* self = ruby_to_Descriptor(_self); @@ -353,10 +374,12 @@ static VALUE Descriptor_name(VALUE _self) { } /* - * call-seq: - * Descriptor.each(&block) + * ruby-doc: Descriptor#each * * Iterates over fields in this message type, yielding to the block on each one. + * + * @yield [FieldDescriptor] + * @return [nil] */ static VALUE Descriptor_each(VALUE _self) { Descriptor* self = ruby_to_Descriptor(_self); @@ -371,11 +394,13 @@ static VALUE Descriptor_each(VALUE _self) { } /* - * call-seq: - * Descriptor.lookup(name) => FieldDescriptor + * ruby-doc: Descriptor#lookup * * Returns the field descriptor for the field with the given name, if present, * or nil if none. + * + * @param name [String] + * @return [FieldDescriptor] */ static VALUE Descriptor_lookup(VALUE _self, VALUE name) { Descriptor* self = ruby_to_Descriptor(_self); @@ -388,11 +413,13 @@ static VALUE Descriptor_lookup(VALUE _self, VALUE name) { } /* - * call-seq: - * Descriptor.each_oneof(&block) => nil + * ruby-doc: Descriptor#each_oneof * * Invokes the given block for each oneof in this message type, passing the - * corresponding OneofDescriptor. + * corresponding {OneofDescriptor}. + * + * @yield [OneofDescriptor] + * @return [nil] */ static VALUE Descriptor_each_oneof(VALUE _self) { Descriptor* self = ruby_to_Descriptor(_self); @@ -407,11 +434,13 @@ static VALUE Descriptor_each_oneof(VALUE _self) { } /* - * call-seq: - * Descriptor.lookup_oneof(name) => OneofDescriptor + * ruby-doc: Descriptor#lookup_oneof * * Returns the oneof descriptor for the oneof with the given name, if present, * or nil if none. + * + * @param name [String] + * @return [OneofDescriptor] */ static VALUE Descriptor_lookup_oneof(VALUE _self, VALUE name) { Descriptor* self = ruby_to_Descriptor(_self); @@ -424,10 +453,11 @@ static VALUE Descriptor_lookup_oneof(VALUE _self, VALUE name) { } /* - * call-seq: - * Descriptor.msgclass => message_klass + * ruby-doc: Descriptor#msgclass * * Returns the Ruby class created for this message type. + * + * @return [Class] */ static VALUE Descriptor_msgclass(VALUE _self) { Descriptor* self = ruby_to_Descriptor(_self); @@ -438,10 +468,13 @@ static VALUE Descriptor_msgclass(VALUE _self) { } /* - * call-seq: - * Descriptor.options => options + * ruby-doc: Descriptor#options * - * Returns the `MessageOptions` for this `Descriptor`. + * Returns the + * {https://github.com/protocolbuffers/protobuf/blob/v30.2/src/google/protobuf/descriptor.proto#L571 + * MessageOptions} for this {Descriptor}. + * + * @return [MessageOptions] */ static VALUE Descriptor_options(VALUE _self) { Descriptor* self = ruby_to_Descriptor(_self); @@ -458,10 +491,13 @@ static VALUE Descriptor_options(VALUE _self) { } /* - * call-seq: - * Descriptor.to_proto => DescriptorProto + * ruby-doc: Descriptor#to_proto * - * Returns the `DescriptorProto` of this `Descriptor`. + * Returns the + * {https://github.com/protocolbuffers/protobuf/blob/v30.2/src/google/protobuf/descriptor.proto#L147 + * DescriptorProto} of this {Descriptor}. + * + * @return [DescriptorProto] */ static VALUE Descriptor_to_proto(VALUE _self) { Descriptor* self = ruby_to_Descriptor(_self); @@ -534,9 +570,15 @@ static VALUE FileDescriptor_alloc(VALUE klass) { return ret; } +/** + * ruby-doc: FileDescriptor + * + * A FileDescriptor provides information about all Protobuf definitions in a + * particular file. + */ + /* - * call-seq: - * FileDescriptor.new => file + * ruby-doc: FileDescriptor#initialize * * Returns a new file descriptor. May * to a builder. @@ -557,10 +599,11 @@ static VALUE FileDescriptor_initialize(VALUE _self, VALUE cookie, } /* - * call-seq: - * FileDescriptor.name => name + * ruby-doc: FileDescriptor#name * * Returns the name of the file. + * + * @return [String] */ static VALUE FileDescriptor_name(VALUE _self) { FileDescriptor* self = ruby_to_FileDescriptor(_self); @@ -569,10 +612,13 @@ static VALUE FileDescriptor_name(VALUE _self) { } /* - * call-seq: - * FileDescriptor.options => options + * ruby-doc: FileDescriptor#options + * + * Returns the + * {https://github.com/protocolbuffers/protobuf/blob/v30.2/src/google/protobuf/descriptor.proto#L442 + * FileOptions} for this {FileDescriptor}. * - * Returns the `FileOptions` for this `FileDescriptor`. + * @return [FileOptions] */ static VALUE FileDescriptor_options(VALUE _self) { FileDescriptor* self = ruby_to_FileDescriptor(_self); @@ -587,10 +633,13 @@ static VALUE FileDescriptor_options(VALUE _self) { } /* - * call-seq: - * FileDescriptor.to_proto => FileDescriptorProto + * ruby-doc: FileDescriptor#to_proto + * + * Returns the + * {https://github.com/protocolbuffers/protobuf/blob/v30.2/src/google/protobuf/descriptor.proto#L104 + * FileDescriptorProto} of this {FileDescriptor}. * - * Returns the `FileDescriptorProto` of this `FileDescriptor`. + * @return [FileDescriptorProto] */ static VALUE FileDescriptor_to_proto(VALUE _self) { FileDescriptor* self = ruby_to_FileDescriptor(_self); @@ -651,9 +700,15 @@ static FieldDescriptor* ruby_to_FieldDescriptor(VALUE val) { return ret; } +/** + * ruby-doc: FieldDescriptor + * + * A FieldDescriptor provides information about the Protobuf definition of a + * field inside a {Descriptor}. + */ + /* - * call-seq: - * FieldDescriptor.new => field + * ruby-doc: FieldDescriptor#initialize * * Returns a new field descriptor. Its name, type, etc. must be set before it is * added to a message type. @@ -687,10 +742,11 @@ static VALUE FieldDescriptor_initialize(VALUE _self, VALUE cookie, } /* - * call-seq: - * FieldDescriptor.name => name + * ruby-doc: FieldDescriptor#name * * Returns the name of this field. + * + * @return [String] */ static VALUE FieldDescriptor_name(VALUE _self) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -755,14 +811,15 @@ static VALUE descriptortype_to_ruby(upb_FieldType type) { } /* - * call-seq: - * FieldDescriptor.type => type + * ruby-doc: FieldDescriptor#type * * Returns this field's type, as a Ruby symbol, or nil if not yet set. * * Valid field types are: * :int32, :int64, :uint32, :uint64, :float, :double, :bool, :string, * :bytes, :message. + * + * @return [Symbol] */ static VALUE FieldDescriptor__type(VALUE _self) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -770,10 +827,11 @@ static VALUE FieldDescriptor__type(VALUE _self) { } /* - * call-seq: - * FieldDescriptor.default => default + * ruby-doc: FieldDescriptor#default * * Returns this field's default, as a Ruby object, or nil if not yet set. + * + * @return [Object,nil] */ static VALUE FieldDescriptor_default(VALUE _self) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -788,10 +846,11 @@ static VALUE FieldDescriptor_default(VALUE _self) { } /* - * call-seq: - * FieldDescriptor.has_presence? => bool + * ruby-doc: FieldDescriptor.has_presence? * * Returns whether this field tracks presence. + * + * @return [Boolean] */ static VALUE FieldDescriptor_has_presence(VALUE _self) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -799,10 +858,11 @@ static VALUE FieldDescriptor_has_presence(VALUE _self) { } /* - * call-seq: - * FieldDescriptor.required? => bool + * ruby-doc: FieldDescriptor#required? * * Returns whether this is a required field. + * + * @return [Boolean] */ static VALUE FieldDescriptor_is_required(VALUE _self) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -810,10 +870,10 @@ static VALUE FieldDescriptor_is_required(VALUE _self) { } /* - * call-seq: - * FieldDescriptor.repeated? => bool + * ruby-doc: FieldDescriptor#repeated? * * Returns whether this is a repeated field. + * @return [Boolean] */ static VALUE FieldDescriptor_is_repeated(VALUE _self) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -821,10 +881,10 @@ static VALUE FieldDescriptor_is_repeated(VALUE _self) { } /* - * call-seq: - * FieldDescriptor.is_packed? => bool + * ruby-doc: FieldDescriptor#is_packed? * * Returns whether this is a repeated field that uses packed encoding. + * @return [Boolean] */ static VALUE FieldDescriptor_is_packed(VALUE _self) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -832,10 +892,11 @@ static VALUE FieldDescriptor_is_packed(VALUE _self) { } /* - * call-seq: - * FieldDescriptor.json_name => json_name + * ruby-doc: FieldDescriptor#json_name * * Returns this field's json_name, as a Ruby string, or nil if not yet set. + * + * @return [String,nil] */ static VALUE FieldDescriptor_json_name(VALUE _self) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -845,15 +906,14 @@ static VALUE FieldDescriptor_json_name(VALUE _self) { } /* - * DEPRECATED: Use repeated? or required? instead. - * - * call-seq: - * FieldDescriptor.label => label + * ruby-doc: FieldDescriptor#label * * Returns this field's label (i.e., plurality), as a Ruby symbol. - * * Valid field labels are: - * :optional, :repeated + * :optional, :repeated + * + * @return [Symbol] + * @deprecated Use {#repeated?} or {#required?} instead. */ static VALUE FieldDescriptor_label(VALUE _self) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -873,10 +933,11 @@ static VALUE FieldDescriptor_label(VALUE _self) { } /* - * call-seq: - * FieldDescriptor.number => number + * ruby-doc: FieldDescriptor#number * * Returns the tag number for this field. + * + * @return [Integer] */ static VALUE FieldDescriptor_number(VALUE _self) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -884,13 +945,14 @@ static VALUE FieldDescriptor_number(VALUE _self) { } /* - * call-seq: - * FieldDescriptor.submsg_name => submsg_name + * ruby-doc: FieldDescriptor#submsg_name * * Returns the name of the message or enum type corresponding to this field, if * it is a message or enum field (respectively), or nil otherwise. This type * name will be resolved within the context of the pool to which the containing * message type is added. + * + * @return [String,nil] */ static VALUE FieldDescriptor_submsg_name(VALUE _self) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -907,13 +969,14 @@ static VALUE FieldDescriptor_submsg_name(VALUE _self) { } /* - * call-seq: - * FieldDescriptor.subtype => message_or_enum_descriptor + * ruby-doc: FieldDescriptor#subtype * * Returns the message or enum descriptor corresponding to this field's type if * it is a message or enum field, respectively, or nil otherwise. Cannot be * called *until* the containing message type is added to a pool (and thus * resolved). + * + * @return [Descriptor,EnumDescriptor,nil] */ static VALUE FieldDescriptor_subtype(VALUE _self) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -930,11 +993,13 @@ static VALUE FieldDescriptor_subtype(VALUE _self) { } /* - * call-seq: - * FieldDescriptor.get(message) => value + * ruby-doc: FieldDescriptor#get * * Returns the value set for this field on the given message. Raises an * exception if message is of the wrong type. + * + * @param message [AbstractMessage] + * @return [Object] */ static VALUE FieldDescriptor_get(VALUE _self, VALUE msg_rb) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -950,11 +1015,13 @@ static VALUE FieldDescriptor_get(VALUE _self, VALUE msg_rb) { } /* - * call-seq: - * FieldDescriptor.has?(message) => boolean + * ruby-doc: FieldDescriptor.has? * * Returns whether the value is set on the given message. Raises an * exception when calling for fields that do not have presence. + * + * @param message [AbstractMessage] + * @return [Boolean] */ static VALUE FieldDescriptor_has(VALUE _self, VALUE msg_rb) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -971,10 +1038,12 @@ static VALUE FieldDescriptor_has(VALUE _self, VALUE msg_rb) { } /* - * call-seq: - * FieldDescriptor.clear(message) + * ruby-doc: FieldDescriptor#clear * * Clears the field from the message if it's set. + * + * @param message [AbstractMessage] + * @return [nil] */ static VALUE FieldDescriptor_clear(VALUE _self, VALUE msg_rb) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -990,12 +1059,14 @@ static VALUE FieldDescriptor_clear(VALUE _self, VALUE msg_rb) { } /* - * call-seq: - * FieldDescriptor.set(message, value) + * ruby-doc: FieldDescriptor#set * * Sets the value corresponding to this field to the given value on the given * message. Raises an exception if message is of the wrong type. Performs the * ordinary type-checks for field setting. + * + * @param message [AbstractMessage] + * @param value [Object] */ static VALUE FieldDescriptor_set(VALUE _self, VALUE msg_rb, VALUE value) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -1015,10 +1086,13 @@ static VALUE FieldDescriptor_set(VALUE _self, VALUE msg_rb, VALUE value) { } /* - * call-seq: - * FieldDescriptor.options => options + * ruby-doc: FieldDescriptor#options + * + * Returns the + * {https://github.com/protocolbuffers/protobuf/blob/v30.2/src/google/protobuf/descriptor.proto#L656 + * FieldOptions} for this {FieldDescriptor}. * - * Returns the `FieldOptions` for this `FieldDescriptor`. + * @return [FieldOptions] */ static VALUE FieldDescriptor_options(VALUE _self) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -1034,10 +1108,13 @@ static VALUE FieldDescriptor_options(VALUE _self) { } /* - * call-seq: - * FieldDescriptor.to_proto => FieldDescriptorProto + * ruby-doc: FieldDescriptor#to_proto + * + * Returns the + * {https://github.com/protocolbuffers/protobuf/blob/v30.2/src/google/protobuf/descriptor.proto#L236 + * FieldDescriptorProto} of this {FieldDescriptor}. * - * Returns the `FieldDescriptorProto` of this `FieldDescriptor`. + * @return [FieldDescriptorProto] */ static VALUE FieldDescriptor_to_proto(VALUE _self) { FieldDescriptor* self = ruby_to_FieldDescriptor(_self); @@ -1110,9 +1187,15 @@ static OneofDescriptor* ruby_to_OneofDescriptor(VALUE val) { return ret; } +/** + * ruby-doc: OneofDescriptor + * + * A OneofDescriptor provides information about the Protobuf definition of a + * oneof inside a {Descriptor}. + */ + /* - * call-seq: - * OneofDescriptor.new => oneof_descriptor + * ruby-doc: OneofDescriptor#initialize * * Creates a new, empty, oneof descriptor. The oneof may only be modified prior * to being added to a message descriptor which is subsequently added to a pool. @@ -1147,10 +1230,11 @@ static VALUE OneofDescriptor_initialize(VALUE _self, VALUE cookie, } /* - * call-seq: - * OneofDescriptor.name => name + * ruby-doc: OneofDescriptor#name * * Returns the name of this oneof. + * + * @return [String] */ static VALUE OneofDescriptor_name(VALUE _self) { OneofDescriptor* self = ruby_to_OneofDescriptor(_self); @@ -1158,10 +1242,12 @@ static VALUE OneofDescriptor_name(VALUE _self) { } /* - * call-seq: - * OneofDescriptor.each(&block) => nil + * ruby-doc: OneofDescriptor#each * * Iterates through fields in this oneof, yielding to the block on each one. + * + * @yield [FieldDescriptor] + * @return [nil] */ static VALUE OneofDescriptor_each(VALUE _self) { OneofDescriptor* self = ruby_to_OneofDescriptor(_self); @@ -1176,10 +1262,13 @@ static VALUE OneofDescriptor_each(VALUE _self) { } /* - * call-seq: - * OneofDescriptor.options => options + * ruby-doc: OneofDescriptor#options + * + * Returns the + * {https://github.com/protocolbuffers/protobuf/blob/v30.2/src/google/protobuf/descriptor.proto#L824 + * OneofOptions} for this {OneofDescriptor}. * - * Returns the `OneofOptions` for this `OneofDescriptor`. + * @return [OneofOptions] */ static VALUE OneOfDescriptor_options(VALUE _self) { OneofDescriptor* self = ruby_to_OneofDescriptor(_self); @@ -1195,10 +1284,13 @@ static VALUE OneOfDescriptor_options(VALUE _self) { } /* - * call-seq: - * OneofDescriptor.to_proto => OneofDescriptorProto + * ruby-doc: OneofDescriptor#to_proto + * + * Returns the + * {https://github.com/protocolbuffers/protobuf/blob/v30.2/src/google/protobuf/descriptor.proto#L343 + * OneofDescriptorProto} of this {OneofDescriptor}. * - * Returns the `OneofDescriptorProto` of this `OneofDescriptor`. + * @return [OneofDescriptorProto] */ static VALUE OneOfDescriptor_to_proto(VALUE _self) { OneofDescriptor* self = ruby_to_OneofDescriptor(_self); @@ -1275,6 +1367,13 @@ const upb_EnumDef* EnumDescriptor_GetEnumDef(VALUE enum_desc_rb) { return desc->enumdef; } +/** + * ruby-doc: EnumDescriptor + * + * An EnumDescriptor provides information about the Protobuf definition of an + * enum inside a {Descriptor}. + */ + /* * call-seq: * EnumDescriptor.new(c_only_cookie, ptr) => EnumDescriptor @@ -1297,10 +1396,11 @@ static VALUE EnumDescriptor_initialize(VALUE _self, VALUE cookie, } /* - * call-seq: - * EnumDescriptor.file_descriptor + * ruby-doc: EnumDescriptor#file_descriptor * - * Returns the FileDescriptor object this enum belongs to. + * Returns the {FileDescriptor} object this enum belongs to. + * + * @return [FileDescriptor] */ static VALUE EnumDescriptor_file_descriptor(VALUE _self) { EnumDescriptor* self = ruby_to_EnumDescriptor(_self); @@ -1309,10 +1409,11 @@ static VALUE EnumDescriptor_file_descriptor(VALUE _self) { } /* - * call-seq: - * EnumDescriptor.is_closed? => bool + * ruby-doc: EnumDescriptor#is_closed? * * Returns whether this enum is open or closed. + * + * @return [Boolean] */ static VALUE EnumDescriptor_is_closed(VALUE _self) { EnumDescriptor* self = ruby_to_EnumDescriptor(_self); @@ -1320,10 +1421,11 @@ static VALUE EnumDescriptor_is_closed(VALUE _self) { } /* - * call-seq: - * EnumDescriptor.name => name + * ruby-doc: EnumDescriptor#name * * Returns the name of this enum type. + * + * @return [String] */ static VALUE EnumDescriptor_name(VALUE _self) { EnumDescriptor* self = ruby_to_EnumDescriptor(_self); @@ -1331,11 +1433,13 @@ static VALUE EnumDescriptor_name(VALUE _self) { } /* - * call-seq: - * EnumDescriptor.lookup_name(name) => value + * ruby-doc: EnumDescriptor#lookup_name * * Returns the numeric value corresponding to the given key name (as a Ruby * symbol), or nil if none. + * + * @param name [Symbol] + * @return [Integer,nil] */ static VALUE EnumDescriptor_lookup_name(VALUE _self, VALUE name) { EnumDescriptor* self = ruby_to_EnumDescriptor(_self); @@ -1350,11 +1454,13 @@ static VALUE EnumDescriptor_lookup_name(VALUE _self, VALUE name) { } /* - * call-seq: - * EnumDescriptor.lookup_value(name) => value + * ruby-doc: EnumDescriptor#lookup_value * * Returns the key name (as a Ruby symbol) corresponding to the integer value, * or nil if none. + * + * @param name [Integer] + * @return [Symbol,nil] */ static VALUE EnumDescriptor_lookup_value(VALUE _self, VALUE number) { EnumDescriptor* self = ruby_to_EnumDescriptor(_self); @@ -1369,11 +1475,13 @@ static VALUE EnumDescriptor_lookup_value(VALUE _self, VALUE number) { } /* - * call-seq: - * EnumDescriptor.each(&block) + * ruby-doc: EnumDescriptor#each * * Iterates over key => value mappings in this enum's definition, yielding to * the block with (key, value) arguments for each one. + * + * @yield [Symbol, Integer] + * @return [nil] */ static VALUE EnumDescriptor_each(VALUE _self) { EnumDescriptor* self = ruby_to_EnumDescriptor(_self); @@ -1390,10 +1498,11 @@ static VALUE EnumDescriptor_each(VALUE _self) { } /* - * call-seq: - * EnumDescriptor.enummodule => module + * ruby-doc: EnumDescriptor#enummodule * * Returns the Ruby module corresponding to this enum type. + * + * @return [Module] */ static VALUE EnumDescriptor_enummodule(VALUE _self) { EnumDescriptor* self = ruby_to_EnumDescriptor(_self); @@ -1404,10 +1513,13 @@ static VALUE EnumDescriptor_enummodule(VALUE _self) { } /* - * call-seq: - * EnumDescriptor.options => options + * ruby-doc: EnumDescriptor#options * - * Returns the `EnumOptions` for this `EnumDescriptor`. + * Returns the + * {https://github.com/protocolbuffers/protobuf/blob/v30.2/src/google/protobuf/descriptor.proto#L838 + * EnumOptions} for this {EnumDescriptor}. + * + * @return [EnumOptions] */ static VALUE EnumDescriptor_options(VALUE _self) { EnumDescriptor* self = ruby_to_EnumDescriptor(_self); @@ -1422,10 +1534,12 @@ static VALUE EnumDescriptor_options(VALUE _self) { } /* - * call-seq: - * EnumDescriptor.to_proto => EnumDescriptorProto + * ruby-doc: EnumDescriptor#to_proto * - * Returns the `EnumDescriptorProto` of this `EnumDescriptor`. + * Returns the + * {https://github.com/protocolbuffers/protobuf/blob/v30.2/src/google/protobuf/descriptor.proto#L349 + * EnumDescriptorProto} of this {EnumDescriptor}. + * @return [EnumDescriptorProto] */ static VALUE EnumDescriptor_to_proto(VALUE _self) { EnumDescriptor* self = ruby_to_EnumDescriptor(_self); @@ -1503,6 +1617,13 @@ static VALUE ServiceDescriptor_alloc(VALUE klass) { return ret; } +/** + * ruby-doc: ServiceDescriptor + * + * A ServiceDescriptor provides information about the Protobuf definition of an + * RPC service. + */ + /* * call-seq: * ServiceDescriptor.new(c_only_cookie, ptr) => ServiceDescriptor @@ -1525,10 +1646,11 @@ static VALUE ServiceDescriptor_initialize(VALUE _self, VALUE cookie, } /* - * call-seq: - * ServiceDescriptor.name => name + * ruby-doc: ServiceDescriptor#name * * Returns the name of this service. + * + * @return [String] */ static VALUE ServiceDescriptor_name(VALUE _self) { ServiceDescriptor* self = ruby_to_ServiceDescriptor(_self); @@ -1536,10 +1658,10 @@ static VALUE ServiceDescriptor_name(VALUE _self) { } /* - * call-seq: - * ServiceDescriptor.file_descriptor + * ruby-doc: ServiceDescriptor#file_descriptor * - * Returns the FileDescriptor object this service belongs to. + * Returns the {FileDescriptor} object this service belongs to. + * @return [FileDescriptor] */ static VALUE ServiceDescriptor_file_descriptor(VALUE _self) { ServiceDescriptor* self = ruby_to_ServiceDescriptor(_self); @@ -1548,10 +1670,12 @@ static VALUE ServiceDescriptor_file_descriptor(VALUE _self) { } /* - * call-seq: - * ServiceDescriptor.each(&block) + * ruby-doc: ServiceDescriptor#each * * Iterates over methods in this service, yielding to the block on each one. + * + * @yield [MethodDescriptor] + * @return [nil] */ static VALUE ServiceDescriptor_each(VALUE _self) { ServiceDescriptor* self = ruby_to_ServiceDescriptor(_self); @@ -1566,10 +1690,13 @@ static VALUE ServiceDescriptor_each(VALUE _self) { } /* - * call-seq: - * ServiceDescriptor.options => options + * ruby-doc: ServiceDescriptor#options * - * Returns the `ServiceOptions` for this `ServiceDescriptor`. + * Returns the + * {https://github.com/protocolbuffers/protobuf/blob/v30.2/src/google/protobuf/descriptor.proto#L901 + * ServiceOptions} for this {ServiceDescriptor}. + * + * @return [ServiceOptions] */ static VALUE ServiceDescriptor_options(VALUE _self) { ServiceDescriptor* self = ruby_to_ServiceDescriptor(_self); @@ -1586,10 +1713,13 @@ static VALUE ServiceDescriptor_options(VALUE _self) { } /* - * call-seq: - * ServiceDescriptor.to_proto => ServiceDescriptorProto + * ruby-doc: ServiceDescriptor#to_proto * - * Returns the `ServiceDescriptorProto` of this `ServiceDescriptor`. + * Returns the + * {https://github.com/protocolbuffers/protobuf/blob/v30.2/src/google/protobuf/descriptor.proto#L386 + * ServiceDescriptorProto} of this {ServiceDescriptor}. + * + * @return [ServiceDescriptorProto] */ static VALUE ServiceDescriptor_to_proto(VALUE _self) { ServiceDescriptor* self = ruby_to_ServiceDescriptor(_self); @@ -1662,6 +1792,13 @@ static VALUE MethodDescriptor_alloc(VALUE klass) { return ret; } +/** + * ruby-doc: MethodDescriptor + * + * A MethodDescriptor provides information about the Protobuf definition of a + * method inside an RPC service. + */ + /* * call-seq: * MethodDescriptor.new(c_only_cookie, ptr) => MethodDescriptor @@ -1684,10 +1821,11 @@ static VALUE MethodDescriptor_initialize(VALUE _self, VALUE cookie, } /* - * call-seq: - * MethodDescriptor.name => name + * ruby-doc: MethodDescriptor#name * * Returns the name of this method + * + * @return [String] */ static VALUE MethodDescriptor_name(VALUE _self) { MethodDescriptor* self = ruby_to_MethodDescriptor(_self); @@ -1695,10 +1833,13 @@ static VALUE MethodDescriptor_name(VALUE _self) { } /* - * call-seq: - * MethodDescriptor.options => options + * ruby-doc: MethodDescriptor#options * - * Returns the `MethodOptions` for this `MethodDescriptor`. + * Returns the + * {https://github.com/protocolbuffers/protobuf/blob/v30.2/src/google/protobuf/descriptor.proto#L927 + * MethodOptions} for this {MethodDescriptor}. + * + * @return [MethodOptions] */ static VALUE MethodDescriptor_options(VALUE _self) { MethodDescriptor* self = ruby_to_MethodDescriptor(_self); @@ -1715,10 +1856,11 @@ static VALUE MethodDescriptor_options(VALUE _self) { } /* - * call-seq: - * MethodDescriptor.input_type => Descriptor + * ruby-doc: MethodDescriptor#input_type * - * Returns the `Descriptor` for the request message type of this method + * Returns the {Descriptor} for the request message type of this method + * + * @return [Descriptor] */ static VALUE MethodDescriptor_input_type(VALUE _self) { MethodDescriptor* self = ruby_to_MethodDescriptor(_self); @@ -1727,10 +1869,11 @@ static VALUE MethodDescriptor_input_type(VALUE _self) { } /* - * call-seq: - * MethodDescriptor.output_type => Descriptor + * ruby-doc: MethodDescriptor#output_type + * + * Returns the {Descriptor} for the response message type of this method * - * Returns the `Descriptor` for the response message type of this method + * @return [Descriptor] */ static VALUE MethodDescriptor_output_type(VALUE _self) { MethodDescriptor* self = ruby_to_MethodDescriptor(_self); @@ -1739,10 +1882,11 @@ static VALUE MethodDescriptor_output_type(VALUE _self) { } /* - * call-seq: - * MethodDescriptor.client_streaming => bool + * ruby-doc: MethodDescriptor#client_streaming * * Returns whether or not this is a streaming request method + * + * @return [Boolean] */ static VALUE MethodDescriptor_client_streaming(VALUE _self) { MethodDescriptor* self = ruby_to_MethodDescriptor(_self); @@ -1750,10 +1894,13 @@ static VALUE MethodDescriptor_client_streaming(VALUE _self) { } /* - * call-seq: - * MethodDescriptor.to_proto => MethodDescriptorProto + * ruby-doc: MethodDescriptor#to_proto + * + * Returns the + * {https://github.com/protocolbuffers/protobuf/blob/v30.2/src/google/protobuf/descriptor.proto#L394 + * MethodDescriptorProto} of this {MethodDescriptor}. * - * Returns the `MethodDescriptorProto` of this `MethodDescriptor`. + * @return [MethodDescriptorProto] */ static VALUE MethodDescriptor_to_proto(VALUE _self) { MethodDescriptor* self = ruby_to_MethodDescriptor(_self); @@ -1771,10 +1918,11 @@ static VALUE MethodDescriptor_to_proto(VALUE _self) { } /* - * call-seq: - * MethodDescriptor.server_streaming => bool + * ruby-doc: MethodDescriptor#server_streaming * * Returns whether or not this is a streaming response method + * + * @return [Boolean] */ static VALUE MethodDescriptor_server_streaming(VALUE _self) { MethodDescriptor* self = ruby_to_MethodDescriptor(_self); diff --git a/ruby/ext/google/protobuf_c/map.c b/ruby/ext/google/protobuf_c/map.c index af4a742622e18..4ad399569a713 100644 --- a/ruby/ext/google/protobuf_c/map.c +++ b/ruby/ext/google/protobuf_c/map.c @@ -236,10 +236,15 @@ static VALUE Map_merge_into_self(VALUE _self, VALUE hashmap) { return _self; } +/** + * ruby-doc: Map + * + * This class represents a Protobuf Map. It is largely automatically transformed + * to and from a Ruby hash. + */ + /* - * call-seq: - * Map.new(key_type, value_type, value_typeclass = nil, init_hashmap = {}) - * => new map + * ruby-doc: Map#initialize * * Allocates a new Map container. This constructor may be called with 2, 3, or 4 * arguments. The first two arguments are always present and are symbols (taking @@ -265,6 +270,13 @@ static VALUE Map_merge_into_self(VALUE _self, VALUE hashmap) { * shallow-copied into the new Map: the original map is unmodified, but * references to underlying objects will be shared if the value type is a * message type. + * + * @param key_type [Symbol] + * @param value_type [Symbol] + * @param value_typeclass [Class,Module] + * @paramdefault value_typeclass nil + * @param init_hashmap [Hash,Map] + * @paramdefault init_hashmap {} */ static VALUE Map_init(int argc, VALUE* argv, VALUE _self) { Map* self = ruby_to_Map(_self); @@ -311,12 +323,14 @@ static VALUE Map_init(int argc, VALUE* argv, VALUE _self) { } /* - * call-seq: - * Map.each(&block) + * ruby-doc: Map#each * * Invokes &block on each |key, value| pair in the map, in unspecified order. * Note that Map also includes Enumerable; map thus acts like a normal Ruby * sequence. + * + * @yield [Object, Object] + * @return [nil] */ static VALUE Map_each(VALUE _self) { Map* self = ruby_to_Map(_self); @@ -333,10 +347,11 @@ static VALUE Map_each(VALUE _self) { } /* - * call-seq: - * Map.keys => [list_of_keys] + * ruby-doc: Map#keys * * Returns the list of keys contained in the map, in unspecified order. + * + * @return [Array] */ static VALUE Map_keys(VALUE _self) { Map* self = ruby_to_Map(_self); @@ -353,10 +368,11 @@ static VALUE Map_keys(VALUE _self) { } /* - * call-seq: - * Map.values => [list_of_values] + * ruby-doc: Map#values * * Returns the list of values contained in the map, in unspecified order. + * + * @return [Array] */ static VALUE Map_values(VALUE _self) { Map* self = ruby_to_Map(_self); @@ -373,11 +389,13 @@ static VALUE Map_values(VALUE _self) { } /* - * call-seq: - * Map.[](key) => value + * ruby-doc: Map#[] * * Accesses the element at the given key. Throws an exception if the key type is * incorrect. Returns nil when the key is not present in the map. + * + * @param key [Object] + * @return [Object] */ static VALUE Map_index(VALUE _self, VALUE key) { Map* self = ruby_to_Map(_self); @@ -393,12 +411,15 @@ static VALUE Map_index(VALUE _self, VALUE key) { } /* - * call-seq: - * Map.[]=(key, value) => value + * ruby-doc: Map#[]= * * Inserts or overwrites the value at the given key with the given new value. * Throws an exception if the key type is incorrect. Returns the new value that * was just inserted. + * + * @param key [Object] + * @param value [Object] + * @return [Object] */ static VALUE Map_index_set(VALUE _self, VALUE key, VALUE val) { Map* self = ruby_to_Map(_self); @@ -414,11 +435,13 @@ static VALUE Map_index_set(VALUE _self, VALUE key, VALUE val) { } /* - * call-seq: - * Map.has_key?(key) => bool + * ruby-doc: Map#has_key? * * Returns true if the given key is present in the map. Throws an exception if * the key has the wrong type. + * + * @param key [Object] + * @return [Boolean] */ static VALUE Map_has_key(VALUE _self, VALUE key) { Map* self = ruby_to_Map(_self); @@ -433,11 +456,13 @@ static VALUE Map_has_key(VALUE _self, VALUE key) { } /* - * call-seq: - * Map.delete(key) => old_value + * ruby-doc: Map#delete * * Deletes the value at the given key, if any, returning either the old value or * nil if none was present. Throws an exception if the key is of the wrong type. + * + * @param key [Object] + * @return [Object] */ static VALUE Map_delete(VALUE _self, VALUE key) { upb_Map* map = Map_GetMutable(_self); @@ -455,10 +480,11 @@ static VALUE Map_delete(VALUE _self, VALUE key) { } /* - * call-seq: - * Map.clear + * ruby-doc: Map#clear * * Removes all entries from the map. + * + * @return [nil] */ static VALUE Map_clear(VALUE _self) { upb_Map_Clear(Map_GetMutable(_self)); @@ -466,10 +492,11 @@ static VALUE Map_clear(VALUE _self) { } /* - * call-seq: - * Map.length + * ruby-doc: Map#length * * Returns the number of entries (key-value pairs) in the map. + * + * @return [Integer] */ static VALUE Map_length(VALUE _self) { Map* self = ruby_to_Map(_self); @@ -477,11 +504,12 @@ static VALUE Map_length(VALUE _self) { } /* - * call-seq: - * Map.dup => new_map + * ruby-doc: Map#dup * * Duplicates this map with a shallow copy. References to all non-primitive * element objects (e.g., submessages) are shared. + * + * @return [Map] */ static VALUE Map_dup(VALUE _self) { Map* self = ruby_to_Map(_self); @@ -502,8 +530,7 @@ static VALUE Map_dup(VALUE _self) { } /* - * call-seq: - * Map.==(other) => boolean + * ruby-doc: Map#== * * Compares this map to another. Maps are equal if they have identical key sets, * and for each key, the values in both maps compare equal. Elements are @@ -513,6 +540,9 @@ static VALUE Map_dup(VALUE _self) { * Maps with dissimilar key types or value types/typeclasses are never equal, * even if value comparison (for example, between integers and floats) would * have otherwise indicated that every element has equal value. + * + * @param other [Map] + * @return [Boolean] */ VALUE Map_eq(VALUE _self, VALUE _other) { Map* self = ruby_to_Map(_self); @@ -560,12 +590,13 @@ VALUE Map_eq(VALUE _self, VALUE _other) { } /* - * call-seq: - * Map.frozen? => bool + * ruby-doc: Map#frozen? * * Returns true if the map is frozen in either Ruby or the underlying * representation. Freezes the Ruby map object if it is not already frozen in * Ruby but it is frozen in the underlying representation. + * + * @return [Boolean] */ VALUE Map_frozen(VALUE _self) { Map* self = ruby_to_Map(_self); @@ -580,11 +611,12 @@ VALUE Map_frozen(VALUE _self) { } /* - * call-seq: - * Map.freeze => self + * ruby-doc: Map#freeze * * Freezes the map object. We have to intercept this so we can freeze the * underlying representation, not just the Ruby wrapper. + * + * @return [self] */ VALUE Map_freeze(VALUE _self) { Map* self = ruby_to_Map(_self); @@ -637,10 +669,11 @@ VALUE Map_EmptyFrozen(const upb_FieldDef* f) { } /* - * call-seq: - * Map.hash => hash_value + * ruby-doc: Map#hash * * Returns a hash value based on this map's contents. + * + * @return [Integer] */ VALUE Map_hash(VALUE _self) { Map* self = ruby_to_Map(_self); @@ -658,10 +691,11 @@ VALUE Map_hash(VALUE _self) { } /* - * call-seq: - * Map.to_h => {} + * ruby-doc: Map#to_h * * Returns a Ruby Hash object containing all the values within the map + * + * @return [Hash] */ VALUE Map_to_h(VALUE _self) { Map* self = ruby_to_Map(_self); @@ -669,12 +703,13 @@ VALUE Map_to_h(VALUE _self) { } /* - * call-seq: - * Map.inspect => string + * ruby-doc: Map#inspect * * Returns a string representing this map's elements. It will be formatted as * "{key => value, key => value, ...}", with each key and value string * representation computed by its own #inspect method. + * + * @return [String] */ VALUE Map_inspect(VALUE _self) { Map* self = ruby_to_Map(_self); @@ -687,13 +722,15 @@ VALUE Map_inspect(VALUE _self) { } /* - * call-seq: - * Map.merge(other_map) => map + * ruby-doc: Map#merge * * Copies key/value pairs from other_map into a copy of this map. If a key is * set in other_map and this map, the value from other_map overwrites the value * in the new copy of this map. Returns the new copy of this map with merged * contents. + * + * @param other_map [Map] + * @return [Map] */ static VALUE Map_merge(VALUE _self, VALUE hashmap) { VALUE dupped = Map_dup(_self); diff --git a/ruby/ext/google/protobuf_c/message.c b/ruby/ext/google/protobuf_c/message.c index 4146640f27d47..62ee3122ed6ab 100644 --- a/ruby/ext/google/protobuf_c/message.c +++ b/ruby/ext/google/protobuf_c/message.c @@ -420,11 +420,10 @@ static VALUE Message_field_accessor(VALUE _self, const upb_FieldDef* f, } /* - * call-seq: - * Message.method_missing(*args) + * ruby-doc: AbstractMessage * - * Provides accessors and setters and methods to clear and check for presence of - * message fields according to their field names. + * The {AbstractMessage} class is the parent class for all Protobuf messages, + * and is generated from C code. * * For any field whose name does not conflict with a built-in method, an * accessor is provided with the same name as the field, and a setter is @@ -653,16 +652,12 @@ void Message_InitFromValue(upb_Message* msg, const upb_MessageDef* m, VALUE val, } /* - * call-seq: - * Message.new(kwargs) => new_message + * ruby-doc: AbstractMessage#initialize * * Creates a new instance of the given message class. Keyword arguments may be * provided with keywords corresponding to field names. * - * Note that no literal Message class exists. Only concrete classes per message - * type exist, as provided by the #msgclass method on Descriptors after they - * have been added to a pool. The method definitions described here on the - * Message class are provided on each concrete message class. + * @param kwargs the list of field keys and values. */ static VALUE Message_initialize(int argc, VALUE* argv, VALUE _self) { Message* self = ruby_to_Message(_self); @@ -684,10 +679,11 @@ static VALUE Message_initialize(int argc, VALUE* argv, VALUE _self) { } /* - * call-seq: - * Message.dup => new_message + * ruby-doc: AbstractMessage#dup * * Performs a shallow copy of this message and returns the new copy. + * + * @return [AbstractMessage] */ static VALUE Message_dup(VALUE _self) { Message* self = ruby_to_Message(_self); @@ -700,13 +696,15 @@ static VALUE Message_dup(VALUE _self) { } /* - * call-seq: - * Message.==(other) => boolean + * ruby-doc: AbstractMessage#== * * Performs a deep comparison of this message with another. Messages are equal * if they have the same type and if each field is equal according to the :== * method's semantics (a more efficient comparison may actually be done if the * field is of a primitive type). + * + * @param other [AbstractMessage] + * @return [Boolean] */ static VALUE Message_eq(VALUE _self, VALUE _other) { if (CLASS_OF(_self) != CLASS_OF(_other)) return Qfalse; @@ -735,10 +733,11 @@ uint64_t Message_Hash(const upb_Message* msg, const upb_MessageDef* m, } /* - * call-seq: - * Message.hash => hash_value + * ruby-doc: AbstractMessage#hash * * Returns a hash value that represents this message's field values. + * + * @return [Integer] */ static VALUE Message_hash(VALUE _self) { Message* self = ruby_to_Message(_self); @@ -749,12 +748,13 @@ static VALUE Message_hash(VALUE _self) { } /* - * call-seq: - * Message.inspect => string + * ruby-doc: AbstractMessage#inspect * * Returns a human-readable string representing this message. It will be * formatted as "". Each * field's value is represented according to its own #inspect method. + * + * @return [String] */ static VALUE Message_inspect(VALUE _self) { Message* self = ruby_to_Message(_self); @@ -830,10 +830,11 @@ VALUE Scalar_CreateHash(upb_MessageValue msgval, TypeInfo type_info) { } /* - * call-seq: - * Message.to_h => {} + * ruby-doc: AbstractMessage#to_h * * Returns the message as a Ruby Hash object, with keys as symbols. + * + * @return [Hash] */ static VALUE Message_to_h(VALUE _self) { Message* self = ruby_to_Message(_self); @@ -841,12 +842,13 @@ static VALUE Message_to_h(VALUE _self) { } /* - * call-seq: - * Message.frozen? => bool + * ruby-doc: AbstractMessage#frozen? * * Returns true if the message is frozen in either Ruby or the underlying * representation. Freezes the Ruby message object if it is not already frozen * in Ruby but it is frozen in the underlying representation. + * + * @return [Boolean] */ VALUE Message_frozen(VALUE _self) { Message* self = ruby_to_Message(_self); @@ -861,11 +863,12 @@ VALUE Message_frozen(VALUE _self) { } /* - * call-seq: - * Message.freeze => self + * ruby-doc: AbstractMessage#freeze * * Freezes the message object. We have to intercept this so we can freeze the * underlying representation, not just the Ruby wrapper. + * + * @return [self] */ VALUE Message_freeze(VALUE _self) { Message* self = ruby_to_Message(_self); @@ -882,11 +885,13 @@ VALUE Message_freeze(VALUE _self) { } /* - * call-seq: - * Message.[](index) => value + * ruby-doc: AbstractMessage#[] * * Accesses a field's value by field name. The provided field name should be a * string. + * + * @param index [Integer] + * @return [Object] */ static VALUE Message_index(VALUE _self, VALUE field_name) { Message* self = ruby_to_Message(_self); @@ -903,11 +908,14 @@ static VALUE Message_index(VALUE _self, VALUE field_name) { } /* - * call-seq: - * Message.[]=(index, value) + * ruby-doc: AbstractMessage#[]= * * Sets a field's value by field name. The provided field name should be a * string. + * + * @param index [Integer] + * @param value [Object] + * @return [nil] */ static VALUE Message_index_set(VALUE _self, VALUE field_name, VALUE value) { Message* self = ruby_to_Message(_self); @@ -929,14 +937,16 @@ static VALUE Message_index_set(VALUE _self, VALUE field_name, VALUE value) { } /* - * call-seq: - * MessageClass.decode(data, options) => message + * ruby-doc: AbstractMessage.decode * * Decodes the given data (as a string containing bytes in protocol buffers wire * format) under the interpretation given by this message class's definition * and returns a message object with the corresponding field values. - * @param options [Hash] options for the decoder - * recursion_limit: set to maximum decoding depth for message (default is 64) + * @param data [String] + * @param options [Hash] + * @option recursion_limit [Integer] set to maximum decoding depth for message + * (default is 64) + * @return [AbstractMessage] */ static VALUE Message_decode(int argc, VALUE* argv, VALUE klass) { VALUE data = argv[0]; @@ -989,16 +999,17 @@ VALUE Message_decode_bytes(int size, const char* bytes, int options, } /* - * call-seq: - * MessageClass.decode_json(data, options = {}) => message + * ruby-doc: AbstractMessage.decode_json * * Decodes the given data (as a string containing bytes in protocol buffers wire * format) under the interpretration given by this message class's definition * and returns a message object with the corresponding field values. * - * @param options [Hash] options for the decoder - * ignore_unknown_fields: set true to ignore unknown fields (default is to - * raise an error) + * @param data [String] + * @param options [Hash] + * @option ignore_unknown_fields [Boolean] set true to ignore unknown fields + * (default is to raise an error) + * @return [AbstractMessage] */ static VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) { VALUE data = argv[0]; @@ -1057,13 +1068,16 @@ static VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) { } /* - * call-seq: - * MessageClass.encode(msg, options) => bytes + * ruby-doc: AbstractMessage.encode * * Encodes the given message object to its serialized form in protocol buffers * wire format. - * @param options [Hash] options for the encoder - * recursion_limit: set to maximum encoding depth for message (default is 64) + * + * @param msg [AbstractMessage] + * @param options [Hash] + * @option recursion_limit [Integer] set to maximum encoding depth for message + * (default is 64) + * @return [String] */ static VALUE Message_encode(int argc, VALUE* argv, VALUE klass) { Message* msg = ruby_to_Message(argv[0]); @@ -1110,14 +1124,17 @@ static VALUE Message_encode(int argc, VALUE* argv, VALUE klass) { } /* - * call-seq: - * MessageClass.encode_json(msg, options = {}) => json_string + * ruby-doc: AbstractMessage.encode_json * * Encodes the given message object into its serialized JSON representation. - * @param options [Hash] options for the decoder - * preserve_proto_fieldnames: set true to use original fieldnames (default is - * to camelCase) emit_defaults: set true to emit 0/false values (default is to - * omit them) + * + * @param msg [AbstractMessage] + * @param options [Hash] + * @option preserve_proto_fieldnames [Boolean] set true to use original + * fieldnames (default is to camelCase) + * @option emit_defaults [Boolean] set true to emit 0/false values (default is + * to omit them) + * @return [String] */ static VALUE Message_encode_json(int argc, VALUE* argv, VALUE klass) { Message* msg = ruby_to_Message(argv[0]); @@ -1185,11 +1202,12 @@ static VALUE Message_encode_json(int argc, VALUE* argv, VALUE klass) { } /* - * call-seq: - * Message.descriptor => descriptor + * ruby-doc: AbstractMessage.descriptor * * Class method that returns the Descriptor instance corresponding to this * message class's type. + * + * @return [Descriptor] */ static VALUE Message_descriptor(VALUE klass) { return rb_ivar_get(klass, descriptor_instancevar_interned); @@ -1212,12 +1230,26 @@ VALUE build_class_from_descriptor(VALUE descriptor) { return klass; } +/* ruby-doc: Enum + * + * There isn't really a concrete `Enum` module generated by Protobuf. Instead, + * you can use this documentation as an indicator of methods that are defined on + * each `Enum` module that is generated. E.g. if you have: + * + * enum my_enum_type + * + * in your Proto file and generate Ruby code, a module + * called `MyEnumType` will be generated with the following methods available. + */ + /* - * call-seq: - * Enum.lookup(number) => name + * ruby-doc: Enum.lookup * * This module method, provided on each generated enum module, looks up an enum * value by number and returns its name as a Ruby symbol, or nil if not found. + * + * @param number [Integer] + * @return [String] */ static VALUE enum_lookup(VALUE self, VALUE number) { int32_t num = NUM2INT(number); @@ -1232,11 +1264,13 @@ static VALUE enum_lookup(VALUE self, VALUE number) { } /* - * call-seq: - * Enum.resolve(name) => number + * ruby-doc: Enum.resolve * * This module method, provided on each generated enum module, looks up an enum * value by name (as a Ruby symbol) and returns its name, or nil if not found. + * + * @param name [String] + * @return [Integer] */ static VALUE enum_resolve(VALUE self, VALUE sym) { const char* name = rb_id2name(SYM2ID(sym)); @@ -1251,11 +1285,13 @@ static VALUE enum_resolve(VALUE self, VALUE sym) { } /* - * call-seq: - * Enum.descriptor + * ruby-doc: Enum.descriptor * * This module method, provided on each generated enum module, returns the - * EnumDescriptor corresponding to this enum type. + * {EnumDescriptor} corresponding to this enum type. + * + * @return [EnumDescriptor] + * */ static VALUE enum_descriptor(VALUE self) { return rb_ivar_get(self, descriptor_instancevar_interned); diff --git a/ruby/ext/google/protobuf_c/repeated_field.c b/ruby/ext/google/protobuf_c/repeated_field.c index 2d19c1645fc5a..ed193466c85b5 100644 --- a/ruby/ext/google/protobuf_c/repeated_field.c +++ b/ruby/ext/google/protobuf_c/repeated_field.c @@ -192,13 +192,20 @@ static VALUE RepeatedField_subarray(RepeatedField* self, long beg, long len) { return ary; } +/** + * ruby-doc: RepeatedField + * + */ + /* - * call-seq: - * RepeatedField.each(&block) + * ruby-doc: RepeatedField#each * * Invokes the block once for each element of the repeated field. RepeatedField * also includes Enumerable; combined with this method, the repeated field thus * acts like an ordinary Ruby sequence. + * + * @yield [Object] + * @return [self] */ static VALUE RepeatedField_each(VALUE _self) { RepeatedField* self = ruby_to_RepeatedField(_self); @@ -214,10 +221,12 @@ static VALUE RepeatedField_each(VALUE _self) { } /* - * call-seq: - * RepeatedField.[](index) => value + * ruby-doc: RepeatedField#[] * * Accesses the element at the given index. Returns nil on out-of-bounds + * + * @param index [Integer] + * @return [Object,nil] */ static VALUE RepeatedField_index(int argc, VALUE* argv, VALUE _self) { RepeatedField* self = ruby_to_RepeatedField(_self); @@ -262,11 +271,14 @@ static VALUE RepeatedField_index(int argc, VALUE* argv, VALUE _self) { } /* - * call-seq: - * RepeatedField.[]=(index, value) + * ruby-doc: RepeatedField#[]= * * Sets the element at the given index. On out-of-bounds assignments, extends * the array and fills the hole (if any) with default values. + * + * @param index [Integer] + * @param value [Object + * @return [nil] */ static VALUE RepeatedField_index_set(VALUE _self, VALUE _index, VALUE val) { RepeatedField* self = ruby_to_RepeatedField(_self); @@ -296,10 +308,12 @@ static VALUE RepeatedField_index_set(VALUE _self, VALUE _index, VALUE val) { } /* - * call-seq: - * RepeatedField.push(value, ...) + * ruby-doc: RepeatedField#push * * Adds a new element to the repeated field. + * + * @param value [Object] + * @return [self] */ static VALUE RepeatedField_push_vararg(int argc, VALUE* argv, VALUE _self) { RepeatedField* self = ruby_to_RepeatedField(_self); @@ -317,10 +331,12 @@ static VALUE RepeatedField_push_vararg(int argc, VALUE* argv, VALUE _self) { } /* - * call-seq: - * RepeatedField.<<(value) + * ruby-doc: RepeatedField#<< * * Adds a new element to the repeated field. + * + * @param value [Object] + * @return [self] */ static VALUE RepeatedField_push(VALUE _self, VALUE val) { RepeatedField* self = ruby_to_RepeatedField(_self); @@ -355,10 +371,12 @@ static VALUE RepeatedField_pop_one(VALUE _self) { } /* - * call-seq: - * RepeatedField.replace(list) + * ruby-doc: RepeatedField#replace * * Replaces the contents of the repeated field with the given list of elements. + * + * @param list [Array] + * @return [Array] */ static VALUE RepeatedField_replace(VALUE _self, VALUE list) { RepeatedField* self = ruby_to_RepeatedField(_self); @@ -376,10 +394,11 @@ static VALUE RepeatedField_replace(VALUE _self, VALUE list) { } /* - * call-seq: - * RepeatedField.clear + * ruby-doc: RepeatedField#clear * * Clears (removes all elements from) this repeated field. + * + * @return [self] */ static VALUE RepeatedField_clear(VALUE _self) { RepeatedField* self = ruby_to_RepeatedField(_self); @@ -389,10 +408,11 @@ static VALUE RepeatedField_clear(VALUE _self) { } /* - * call-seq: - * RepeatedField.length + * ruby-doc: RepeatedField#length * * Returns the length of this repeated field. + * + * @return [Integer] */ static VALUE RepeatedField_length(VALUE _self) { RepeatedField* self = ruby_to_RepeatedField(_self); @@ -400,11 +420,12 @@ static VALUE RepeatedField_length(VALUE _self) { } /* - * call-seq: - * RepeatedField.dup => repeated_field + * ruby-doc: RepeatedField#dup * * Duplicates this repeated field with a shallow copy. References to all * non-primitive element objects (e.g., submessages) are shared. + * + * @return [RepeatedField] */ static VALUE RepeatedField_dup(VALUE _self) { RepeatedField* self = ruby_to_RepeatedField(_self); @@ -426,11 +447,12 @@ static VALUE RepeatedField_dup(VALUE _self) { } /* - * call-seq: - * RepeatedField.to_ary => array + * ruby-doc: RepeatedField#to_ary * * Used when converted implicitly into array, e.g. compared to an Array. * Also called as a fallback of Object#to_a + * + * @return [Array] */ VALUE RepeatedField_to_ary(VALUE _self) { RepeatedField* self = ruby_to_RepeatedField(_self); @@ -448,8 +470,7 @@ VALUE RepeatedField_to_ary(VALUE _self) { } /* - * call-seq: - * RepeatedField.==(other) => boolean + * ruby-doc: RepeatedField#== * * Compares this repeated field to another. Repeated fields are equal if their * element types are equal, their lengths are equal, and each element is equal. @@ -459,6 +480,9 @@ VALUE RepeatedField_to_ary(VALUE _self) { * Repeated fields with dissimilar element types are never equal, even if value * comparison (for example, between integers and floats) would have otherwise * indicated that every element has equal value. + * + * @param other [RepeatedField] + * @return [Boolean] */ VALUE RepeatedField_eq(VALUE _self, VALUE _other) { RepeatedField* self; @@ -495,12 +519,13 @@ VALUE RepeatedField_eq(VALUE _self, VALUE _other) { } /* - * call-seq: - * RepeatedField.frozen? => bool + * ruby-doc: RepeatedField#frozen? * * Returns true if the repeated field is frozen in either Ruby or the underlying * representation. Freezes the Ruby repeated field object if it is not already * frozen in Ruby but it is frozen in the underlying representation. + * + * @return [Boolean] */ VALUE RepeatedField_frozen(VALUE _self) { RepeatedField* self = ruby_to_RepeatedField(_self); @@ -515,11 +540,12 @@ VALUE RepeatedField_frozen(VALUE _self) { } /* - * call-seq: - * RepeatedField.freeze => self + * ruby-doc: RepeatedField#freeze * * Freezes the repeated field object. We have to intercept this so we can freeze * the underlying representation, not just the Ruby wrapper. + * + * @return [self] */ VALUE RepeatedField_freeze(VALUE _self) { RepeatedField* self = ruby_to_RepeatedField(_self); @@ -541,10 +567,11 @@ VALUE RepeatedField_freeze(VALUE _self) { } /* - * call-seq: - * RepeatedField.hash => hash_value + * ruby-doc: RepeatedField#hash * * Returns a hash value computed from this repeated field's elements. + * + * @return [Integer] */ VALUE RepeatedField_hash(VALUE _self) { RepeatedField* self = ruby_to_RepeatedField(_self); @@ -560,12 +587,14 @@ VALUE RepeatedField_hash(VALUE _self) { } /* - * call-seq: - * RepeatedField.+(other) => repeated field + * ruby-doc: RepeatedField#+ * * Returns a new repeated field that contains the concatenated list of this * repeated field's elements and other's elements. The other (second) list may * be either another repeated field or a Ruby array. + * + * @param other [Array,RepeatedField] + * @return [RepeatedField] */ VALUE RepeatedField_plus(VALUE _self, VALUE list) { VALUE dupped_ = RepeatedField_dup(_self); @@ -605,10 +634,12 @@ VALUE RepeatedField_plus(VALUE _self, VALUE list) { } /* - * call-seq: - * RepeatedField.concat(other) => self + * ruby-doc: RepeatedField#concat * * concats the passed in array to self. Returns a Ruby array. + * + * @param other [RepeatedField] + * @return [Array] */ VALUE RepeatedField_concat(VALUE _self, VALUE list) { int i; @@ -621,15 +652,21 @@ VALUE RepeatedField_concat(VALUE _self, VALUE list) { } /* - * call-seq: - * RepeatedField.new(type, type_class = nil, initial_elems = []) + * ruby-doc: RepeatedField#initialize * * Creates a new repeated field. The provided type must be a Ruby symbol, and - * can take on the same values as those accepted by FieldDescriptor#type=. If + * can take on the same values as those accepted by {FieldDescriptor#type=}. If * the type is :message or :enum, type_class must be non-nil, and must be the - * Ruby class or module returned by Descriptor#msgclass or - * EnumDescriptor#enummodule, respectively. An initial list of elements may also - * be provided. + * Ruby class or module returned by {Descriptor#msgclass} or + * {EnumDescriptor#enummodule}, respectively. An initial list of elements may + * also be provided. + * + * @param type [Symbol] + * @param type_class [Class, Module] + * @paramdefault type_class nil + * @param initial_elems [Array] + * @paramdefault initial_elems [] + * @return [RepeatedField] */ VALUE RepeatedField_init(int argc, VALUE* argv, VALUE _self) { RepeatedField* self = ruby_to_RepeatedField(_self); diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index bd18f59723a3a..8a1adf70b93ab 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -6955,18 +6955,6 @@ static const upb_MiniTableEnum* _upb_MiniTableSubs_EnumByField( return subs[field->UPB_PRIVATE(submsg_index)].UPB_PRIVATE(subenum); } -UPB_NORETURN static void* _upb_Decoder_ErrorJmp(upb_Decoder* d, - upb_DecodeStatus status) { - UPB_ASSERT(status != kUpb_DecodeStatus_Ok); - d->status = status; - UPB_LONGJMP(d->err, 1); -} - -const char* _upb_FastDecoder_ErrorJmp2(upb_Decoder* d) { - UPB_LONGJMP(d->err, 1); - return NULL; -} - static void _upb_Decoder_VerifyUtf8(upb_Decoder* d, const char* buf, int len) { if (!_upb_Decoder_VerifyUtf8Inline(buf, len)) { _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); @@ -7618,18 +7606,6 @@ static const char* _upb_Decoder_DecodeToSubMessage( return ptr; } -UPB_NOINLINE -const char* _upb_Decoder_CheckRequired(upb_Decoder* d, const char* ptr, - const upb_Message* msg, - const upb_MiniTable* m) { - UPB_ASSERT(m->UPB_PRIVATE(required_count)); - if (UPB_UNLIKELY(d->options & kUpb_DecodeOption_CheckRequired)) { - d->missing_required = - !UPB_PRIVATE(_upb_Message_IsInitializedShallow)(msg, m); - } - return ptr; -} - static const char* upb_Decoder_SkipField(upb_Decoder* d, const char* ptr, uint32_t tag) { int field_number = tag >> 3; @@ -8232,13 +8208,6 @@ static upb_DecodeStatus _upb_Decoder_DecodeTop(struct upb_Decoder* d, return kUpb_DecodeStatus_Ok; } -UPB_NOINLINE -const char* _upb_Decoder_IsDoneFallback(upb_EpsCopyInputStream* e, - const char* ptr, int overrun) { - return _upb_EpsCopyInputStream_IsDoneFallbackInline( - e, ptr, overrun, _upb_Decoder_BufferFlipCallback); -} - static upb_DecodeStatus upb_Decoder_Decode(upb_Decoder* const decoder, const char* const buf, upb_Message* const msg, @@ -8451,8 +8420,9 @@ static char* encode_fixed32(char* ptr, upb_encstate* e, uint32_t val) { #define UPB_PB_VARINT_MAX_LEN 10 -// Need gnu extended inline asm -#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +// Need gnu extended inline asm; msan can't instrument stores in inline assembly +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) && \ + !UPB_HAS_FEATURE(memory_sanitizer) #define UPB_ARM64_ASM #endif @@ -16986,6 +16956,36 @@ google_protobuf_ServiceDescriptorProto* upb_ServiceDef_ToProto(const upb_Service return upb_ToProto_ConvertServiceDef(&ctx, s); } + + +// Must be last. + +UPB_NOINLINE +const char* _upb_Decoder_CheckRequired(upb_Decoder* d, const char* ptr, + const upb_Message* msg, + const upb_MiniTable* m) { + UPB_ASSERT(m->UPB_PRIVATE(required_count)); + if (UPB_UNLIKELY(d->options & kUpb_DecodeOption_CheckRequired)) { + d->missing_required = + !UPB_PRIVATE(_upb_Message_IsInitializedShallow)(msg, m); + } + return ptr; +} + +UPB_NORETURN void* _upb_Decoder_ErrorJmp(upb_Decoder* d, + upb_DecodeStatus status) { + UPB_ASSERT(status != kUpb_DecodeStatus_Ok); + d->status = status; + UPB_LONGJMP(d->err, 1); +} + +UPB_NOINLINE +const char* _upb_Decoder_IsDoneFallback(upb_EpsCopyInputStream* e, + const char* ptr, int overrun) { + return _upb_EpsCopyInputStream_IsDoneFallbackInline( + e, ptr, overrun, _upb_Decoder_BufferFlipCallback); +} + // This should #undef all macros #defined in def.inc #undef UPB_SIZE diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index 0754b52f88c37..847ecadc04405 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -5685,6 +5685,8 @@ bool upb_Message_SetMapEntry(upb_Map* map, const upb_MiniTable* mini_table, #ifndef UPB_MINI_TABLE_DECODE_H_ #define UPB_MINI_TABLE_DECODE_H_ +#include + #ifndef UPB_MINI_TABLE_SUB_H_ #define UPB_MINI_TABLE_SUB_H_ @@ -5879,7 +5881,7 @@ UPB_API_INLINE upb_MiniTableExtension* upb_MiniTableExtension_Build( UPB_API_INLINE upb_MiniTableExtension* upb_MiniTableExtension_BuildMessage( const char* data, size_t len, const upb_MiniTable* extendee, - upb_MiniTable* submsg, upb_Arena* arena, upb_Status* status) { + const upb_MiniTable* submsg, upb_Arena* arena, upb_Status* status) { upb_MiniTableSub sub = upb_MiniTableSub_FromMessage(submsg); return _upb_MiniTableExtension_Build( data, len, extendee, sub, kUpb_MiniTablePlatform_Native, arena, status); @@ -5887,7 +5889,7 @@ UPB_API_INLINE upb_MiniTableExtension* upb_MiniTableExtension_BuildMessage( UPB_API_INLINE upb_MiniTableExtension* upb_MiniTableExtension_BuildEnum( const char* data, size_t len, const upb_MiniTable* extendee, - upb_MiniTableEnum* subenum, upb_Arena* arena, upb_Status* status) { + const upb_MiniTableEnum* subenum, upb_Arena* arena, upb_Status* status) { upb_MiniTableSub sub = upb_MiniTableSub_FromEnum(subenum); return _upb_MiniTableExtension_Build( data, len, extendee, sub, kUpb_MiniTablePlatform_Native, arena, status); @@ -15619,21 +15621,6 @@ UPB_INLINE void _upb_Decoder_Trace(upb_Decoder* d, char event) { #endif }; -/* Error function that will abort decoding with longjmp(). We can't declare this - * UPB_NORETURN, even though it is appropriate, because if we do then compilers - * will "helpfully" refuse to tailcall to it - * (see: https://stackoverflow.com/a/55657013), which will defeat a major goal - * of our optimizations. That is also why we must declare it in a separate file, - * otherwise the compiler will see that it calls longjmp() and deduce that it is - * noreturn. */ -const char* _upb_FastDecoder_ErrorJmp2(upb_Decoder* d); - -UPB_INLINE -const char* _upb_FastDecoder_ErrorJmp(upb_Decoder* d, upb_DecodeStatus status) { - d->status = status; - return _upb_FastDecoder_ErrorJmp2(d); -} - UPB_INLINE bool _upb_Decoder_VerifyUtf8Inline(const char* ptr, int len) { return utf8_range_IsValid(ptr, len); @@ -15665,10 +15652,13 @@ UPB_INLINE bool _upb_Decoder_IsDone(upb_Decoder* d, const char** ptr) { &d->input, ptr, &_upb_Decoder_IsDoneFallback); } +UPB_NORETURN void* _upb_Decoder_ErrorJmp(upb_Decoder* d, + upb_DecodeStatus status); + UPB_INLINE const char* _upb_Decoder_BufferFlipCallback( upb_EpsCopyInputStream* e, const char* old_end, const char* new_start) { upb_Decoder* d = (upb_Decoder*)e; - if (!old_end) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); + if (!old_end) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); return new_start; } diff --git a/ruby/generate_stubs.rb b/ruby/generate_stubs.rb new file mode 100644 index 0000000000000..b7d074c71f141 --- /dev/null +++ b/ruby/generate_stubs.rb @@ -0,0 +1,95 @@ +# This script parses the comments in the Ruby C extension files and translates them into Ruby +# stubs. It looks for an instance of `ruby-doc:` and takes the following string as the object +# under documentation (a class, instance method, or class method). +# The intention of generating these files is for use by YARD to auto-generate documentation +# here: https://www.rubydoc.info/gems/google-protobuf/ +# +# Because these comments are essentially "pretending" to be Ruby, we had to add one additional +# YARD tag, `@paramdefault`, which is not a standard YARD tag. This tag is used to specify default +# values for parameters. + +require 'erb' + +files = Dir.glob('ext/**/*.c') +classes = {} +MethodDefn = Struct.new(:name, :body, keyword_init: true) do + def params + params = body.scan(/@param (\S+) /).map { |p| p[0]} + defaults = body.scan(/@paramdefault (\S+) (\S+)/).map { |p| [p[0], p[1]]}.to_h + params = params.map { |p| p == 'kwargs' ? '**kwargs' : p } + params = params.map { |p| defaults[p] ? "#{p}=#{defaults[p]}" : p } + if body.include?('@yield') + params.push('&block') + end + params + end +end + +Defn = Struct.new(:name, :instance_meths, :class_meths, :body) do + def initialize(*args, **kwargs) + super + self.instance_meths ||= [] + self.class_meths ||= [] + end +end + +files.each do |file| + defs = File.read(file).scan(/ruby-doc:(.*?)\n(.*?)\*\//m) + defs.each do |definition| + name = definition[0].strip + body = definition[1].strip. + gsub(/^\s*\*/, ''). + gsub(/\n/m, "\n # ").strip + if name.include?('.') + klass, method_name = name.split('.') + method = MethodDefn.new(name: method_name, body: body) + classes[klass] ||= Defn.new(name: klass) + classes[klass].class_meths.push(method) + elsif name.include?('#') + klass, method_name = name.split('#') + method = MethodDefn.new(name: method_name, body: body) + classes[klass] ||= Defn.new(name: klass) + classes[klass].instance_meths.push(method) + else + classes[name] = Defn.new(name: name, body: body) + end + end +end + +# copied from ActiveSupport::Inflector +def underscore(str) + regex = /(?:(?<=([A-Za-z\d]))|\b)((?=a)b)(?=\b|[^a-z])/ + str.gsub(regex) { "#{$1 && '_' }#{$2.downcase}" }. + gsub(/([A-Z])(?=[A-Z][a-z])|([a-z\d])(?=[A-Z])/) { ($1 || $2) << "_" }. + downcase +end + +class_erb = <<-ERB +# This file was generated by generate_stubs.rb +# Do not edit this file directly. + +<%- if defn.body -%> +<%= defn.body.gsub(' #', '#').gsub('# ', '# ') %> +<%- end -%> +class Google::Protobuf::<%= name %> +<%- defn.instance_meths.each do |meth| -%> + + <%= meth.body.gsub('# ', '# ') %> + def <%= meth.name %><% if meth.params.any? %>(<%= meth.params.join(', ') %>)<% end %>; end +<%- end -%> +<%- defn.class_meths.each do |meth| -%> + + <%= meth.body.gsub('# ', '# ') %> + def self.<%= meth.name %><% if meth.params.any? %>(<%= meth.params.join(', ') %>)<% end %>; end +<%- end -%> + +end # class Google::Protobuf::<%= name %> +ERB + +classes.each do |name, defn| + file = File.join('lib/stubs', "#{underscore(name)}.rb") + File.open(file, 'w') do |f| + result = ERB.new(class_erb, trim_mode: '-').result(binding) + f.puts result + end +end diff --git a/ruby/lib/stubs/.keep b/ruby/lib/stubs/.keep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/rust/bazel/aspects.bzl b/rust/bazel/aspects.bzl index 9294a87dbdf74..ff23da4e0fb33 100644 --- a/rust/bazel/aspects.bzl +++ b/rust/bazel/aspects.bzl @@ -371,12 +371,22 @@ def _rust_proto_aspect_common(target, ctx, is_upb): build_info = None, ) + extra_dep_variant_infos = [ + DepVariantInfo( + crate_info = dep[CrateInfo] if CrateInfo in dep else None, + dep_info = dep[DepInfo] if DepInfo in dep else None, + cc_info = dep[CcInfo] if CcInfo in dep else None, + build_info = None, + ) + for dep in ctx.attr._extra_deps + ] + dep_variant_info = _compile_rust( ctx = ctx, attr = ctx.rule.attr, src = entry_point_rs_output, extra_srcs = rs_gencode, - deps = [dep_variant_info_for_runtime] + dep_variant_info_for_native_gencode + dep_variant_infos, + deps = [dep_variant_info_for_runtime] + dep_variant_info_for_native_gencode + dep_variant_infos + extra_dep_variant_infos, runtime = runtime, ) return [RustProtoInfo( @@ -423,6 +433,10 @@ def _make_proto_library_aspect(is_upb): "_extra_rustc_flags": attr.label( default = Label("@rules_rust//:extra_rustc_flags"), ), + "_extra_deps": attr.label_list( + default = [ + ], + ), "_process_wrapper": attr.label( doc = "A process wrapper for running rustc on all platforms.", default = Label("@rules_rust//util/process_wrapper"), diff --git a/rust/cpp.rs b/rust/cpp.rs index bd8710a080830..910d3c6e71375 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -9,9 +9,10 @@ use crate::__internal::{Enum, MatcherEq, Private, SealedInternal}; use crate::{ - AsMut, AsView, Clear, ClearAndParse, IntoProxied, Map, MapIter, MapMut, MapView, Message, Mut, - MutProxied, ParseError, ProtoBytes, ProtoStr, ProtoString, Proxied, ProxiedInMapValue, - ProxiedInRepeated, Repeated, RepeatedMut, RepeatedView, View, + AsMut, AsView, Clear, ClearAndParse, CopyFrom, IntoProxied, Map, MapIter, MapMut, MapView, + MergeFrom, Message, Mut, MutProxied, OwnedMessageInterop, ParseError, ProtoBytes, ProtoStr, + ProtoString, Proxied, ProxiedInMapValue, ProxiedInRepeated, Repeated, RepeatedMut, + RepeatedView, TakeFrom, View, }; use core::fmt::Debug; use paste::paste; @@ -1407,6 +1408,52 @@ impl ClearAndParse for T { } } +impl TakeFrom for T +where + Self: CopyFrom + AsMut, + for<'a> Mut<'a, ::MutProxied>: Clear, +{ + fn take_from(&mut self, mut src: impl AsMut) { + let mut src = src.as_mut(); + // TODO: b/393559271 - Optimize this copy out. + CopyFrom::copy_from(self, AsView::as_view(&src)); + Clear::clear(&mut src); + } +} + +impl CopyFrom for T +where + Self: AsMut, + for<'a> View<'a, Self::Proxied>: CppGetRawMessage, + for<'a> Mut<'a, Self::Proxied>: CppGetRawMessageMut, +{ + fn copy_from(&mut self, src: impl AsView) { + unsafe { + proto2_rust_Message_copy_from( + self.as_mut().get_raw_message_mut(Private), + src.as_view().get_raw_message(Private), + ); + } + } +} + +impl MergeFrom for T +where + Self: AsMut, + for<'a> View<'a, Self::Proxied>: CppGetRawMessage, + for<'a> Mut<'a, Self::Proxied>: CppGetRawMessageMut, +{ + fn merge_from(&mut self, src: impl AsView) { + // SAFETY: self and src are both valid `T`s. + unsafe { + proto2_rust_Message_merge_from( + self.as_mut().get_raw_message_mut(Private), + src.as_view().get_raw_message(Private), + ); + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/rust/upb.rs b/rust/upb.rs index 406bceb4eb0d8..78e859cae6736 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -9,9 +9,9 @@ use crate::__internal::{Enum, MatcherEq, Private, SealedInternal}; use crate::{ - AsView, Clear, ClearAndParse, IntoProxied, Map, MapIter, MapMut, MapView, Message, Mut, - ParseError, ProtoBytes, ProtoStr, ProtoString, Proxied, ProxiedInMapValue, ProxiedInRepeated, - Repeated, RepeatedMut, RepeatedView, View, + AsMut, AsView, Clear, ClearAndParse, CopyFrom, IntoProxied, Map, MapIter, MapMut, MapView, + MergeFrom, Message, Mut, ParseError, ProtoBytes, ProtoStr, ProtoString, Proxied, + ProxiedInMapValue, ProxiedInRepeated, Repeated, RepeatedMut, RepeatedView, TakeFrom, View, }; use std::fmt::Debug; use std::marker::PhantomData; @@ -99,7 +99,19 @@ pub struct OwnedMessageInner { arena: Arena, } +impl Default for OwnedMessageInner { + fn default() -> Self { + Self::new() + } +} + impl OwnedMessageInner { + pub fn new() -> Self { + let arena = Arena::new(); + let ptr = MessagePtr::new(&arena).expect("alloc should never fail"); + Self { ptr, arena } + } + /// # Safety /// - The underlying pointer must of type `T` and be allocated on `arena`. pub unsafe fn wrap_raw(raw: RawMessage, arena: Arena) -> Self { @@ -222,6 +234,14 @@ impl<'msg, T: Message + AssociatedMiniTable> Clone for MessageViewInner<'msg, T> impl<'msg, T: Message + AssociatedMiniTable> Copy for MessageViewInner<'msg, T> {} impl<'msg, T: Message + AssociatedMiniTable> MessageViewInner<'msg, T> { + /// # Safety + /// - The underlying pointer must live as long as `'msg`. + pub unsafe fn wrap(ptr: MessagePtr) -> Self { + // SAFETY: + // - Caller guaranteed `raw` is valid + MessageViewInner { ptr, _phantom: PhantomData } + } + /// # Safety /// - The underlying pointer must of type `T` and live as long as `'msg`. pub unsafe fn wrap_raw(raw: RawMessage) -> Self { @@ -1172,3 +1192,55 @@ where clear_and_parse_helper(self, data, 0) } } + +impl TakeFrom for T +where + Self: CopyFrom + AsMut, + for<'a> Mut<'a, ::MutProxied>: Clear, +{ + fn take_from(&mut self, mut src: impl AsMut) { + let mut src = src.as_mut(); + // TODO: b/393559271 - Optimize this copy out. + CopyFrom::copy_from(self, AsView::as_view(&src)); + Clear::clear(&mut src); + } +} + +impl CopyFrom for T +where + Self: AsView + AssociatedMiniTable + UpbGetArena + UpbGetMessagePtr, + for<'a> View<'a, Self::Proxied>: UpbGetMessagePtr, +{ + fn copy_from(&mut self, src: impl AsView) { + // SAFETY: self and src are both valid `T`s associated with + // `Self::mini_table()`. + unsafe { + assert!(upb_Message_DeepCopy( + self.get_raw_message(Private), + src.as_view().get_raw_message(Private), + ::mini_table(), + self.get_arena(Private).raw() + )); + } + } +} + +impl MergeFrom for T +where + Self: AsView + AssociatedMiniTable + UpbGetArena + UpbGetMessagePtr, + for<'a> View<'a, Self::Proxied>: UpbGetMessagePtr, +{ + fn merge_from(&mut self, src: impl AsView) { + // SAFETY: self and src are both valid `T`s. + unsafe { + assert!(upb_Message_MergeFrom( + self.get_raw_message(Private), + src.as_view().get_raw_message(Private), + ::mini_table(), + // Use a nullptr for the ExtensionRegistry. + std::ptr::null(), + self.get_arena(Private).raw() + )); + } + } +} diff --git a/rust/upb/BUILD b/rust/upb/BUILD index f4dd676139284..9c289692cc113 100644 --- a/rust/upb/BUILD +++ b/rust/upb/BUILD @@ -21,12 +21,17 @@ rust_library( "text.rs", "wire.rs", ], + proc_macro_deps = [ + "@crate_index//:paste", + ], rustc_flags = ["--cfg=bzl"], visibility = [ "//rust:__subpackages__", "//src/google/protobuf:__subpackages__", ], - deps = ["//rust/upb/sys"], + deps = [ + "//rust/upb/sys", + ], ) rust_test( diff --git a/rust/upb/message.rs b/rust/upb/message.rs index 439ddfb240b4f..c42778ba6ed68 100644 --- a/rust/upb/message.rs +++ b/rust/upb/message.rs @@ -5,11 +5,13 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use super::mini_table::MiniTableFieldPtr; -use super::sys::{message::array as sys_array, message::message as sys_msg}; +use super::sys::{ + message::array as sys_array, message::message as sys_msg, mini_table::mini_table as sys_mt, +}; use super::StringView; use super::{Arena, AssociatedMiniTable}; use core::marker::PhantomData; +use paste::paste; pub type RawMessage = sys_msg::RawMessage; @@ -33,6 +35,39 @@ impl MessagePtr { } } +macro_rules! scalar_accessors { + ( $ty:ty, $rust_ty_name:ident, $upb_ty_name:ident) => { + paste! { + impl MessagePtr { + /// # Safety + /// - `self` must be legally dereferenceable. + /// - The field at `index` must be a $ty field. + pub unsafe fn [< get_ $rust_ty_name _at_index >] (self, index: u32, default_value: $ty) -> $ty { + let f = unsafe { sys_mt::upb_MiniTable_GetFieldByIndex(T::mini_table(), index) }; + unsafe { sys_msg::[< upb_Message_Get $upb_ty_name >](self.raw, f, default_value) } + } + + /// # Safety + /// - `self` must be legally dereferenceable to a mutable message. + /// - The field at `index` must be a $ty field. + pub unsafe fn [< set_base_field_ $rust_ty_name _at_index >] (self, index: u32, value: $ty) { + let f = unsafe { sys_mt::upb_MiniTable_GetFieldByIndex(T::mini_table(), index) }; + unsafe { sys_msg::[< upb_Message_SetBaseField $upb_ty_name >](self.raw, f, value) } + } + } + } + }; +} + +scalar_accessors!(bool, bool, Bool); +scalar_accessors!(i32, i32, Int32); +scalar_accessors!(i64, i64, Int64); +scalar_accessors!(u32, u32, UInt32); +scalar_accessors!(u64, u64, UInt64); +scalar_accessors!(f32, f32, Float); +scalar_accessors!(f64, f64, Double); +scalar_accessors!(StringView, string, String); + impl MessagePtr { /// Constructs a new mutable message pointer. /// Will only return None if arena allocation fails. @@ -71,89 +106,64 @@ impl MessagePtr { /// # Safety /// - `self` must be a legally dereferenceable. - /// - `f` must be a bool field. - pub unsafe fn get_bool(self, f: MiniTableFieldPtr, default_value: bool) -> bool { - unsafe { sys_msg::upb_Message_GetBool(self.raw, f.raw, default_value) } - } - - /// # Safety - /// - `self` must be a legally dereferenceable. - /// - `f` must be a bool field. - pub unsafe fn get_i32(self, f: MiniTableFieldPtr, default_value: i32) -> i32 { - unsafe { sys_msg::upb_Message_GetInt32(self.raw, f.raw, default_value) } - } - - /// # Safety - /// - `self` must be a legally dereferenceable. - /// - `f` must be a int64 field. - pub unsafe fn get_i64(self, f: MiniTableFieldPtr, default_value: i64) -> i64 { - unsafe { sys_msg::upb_Message_GetInt64(self.raw, f.raw, default_value) } - } - - /// # Safety - /// - `self` must be a legally dereferenceable. - /// - `f` must be a uint32 field. - pub unsafe fn get_u32(self, f: MiniTableFieldPtr, default_value: u32) -> u32 { - unsafe { sys_msg::upb_Message_GetUInt32(self.raw, f.raw, default_value) } + /// - `index` must be within bounds of `T::mini_table()`. + pub unsafe fn clear_field_at_index(self, index: u32) { + let f = unsafe { sys_mt::upb_MiniTable_GetFieldByIndex(T::mini_table(), index) }; + unsafe { sys_msg::upb_Message_ClearBaseField(self.raw, f) } } /// # Safety /// - `self` must be a legally dereferenceable. - /// - `f` must be a uint64 field. - pub unsafe fn get_u64(self, f: MiniTableFieldPtr, default_value: u64) -> u64 { - unsafe { sys_msg::upb_Message_GetUInt64(self.raw, f.raw, default_value) } - } - - /// # Safety - /// - `self` must be a legally dereferenceable. - /// - `f` must be a float field. - pub unsafe fn get_f32(self, f: MiniTableFieldPtr, default_value: f32) -> f32 { - unsafe { sys_msg::upb_Message_GetFloat(self.raw, f.raw, default_value) } + /// - `index` must be within bounds of `T::mini_table()`. + pub unsafe fn has_field_at_index(self, index: u32) -> bool { + let f = unsafe { sys_mt::upb_MiniTable_GetFieldByIndex(T::mini_table(), index) }; + unsafe { sys_msg::upb_Message_HasBaseField(self.raw, f) } } + /// Returns a constant message pointer, or None if the field is not present. + /// /// # Safety /// - `self` must be a legally dereferenceable. - /// - `f` must be a double field. - pub unsafe fn get_f64(self, f: MiniTableFieldPtr, default_value: f64) -> f64 { - unsafe { sys_msg::upb_Message_GetDouble(self.raw, f.raw, default_value) } - } + /// - The field at `index` must be a message field. + pub unsafe fn get_message_at_index(self, index: u32) -> Option> { + let f = unsafe { sys_mt::upb_MiniTable_GetFieldByIndex(T::mini_table(), index) }; - /// # Safety - /// - `self` must be a legally dereferenceable. - /// - `f` must be a string field. - pub unsafe fn get_string( - self, - f: MiniTableFieldPtr, - default_value: StringView, - ) -> StringView { - unsafe { sys_msg::upb_Message_GetString(self.raw, f.raw, default_value) } + let raw = unsafe { sys_msg::upb_Message_GetMessage(self.raw, f) }; + raw.map(|raw| MessagePtr { raw, _phantom: PhantomData }) } /// Returns an constant message pointer, or None if the field is not present. /// /// # Safety /// - `self` must be a legally dereferenceable. - /// - `f` must be a message field. - pub unsafe fn get_message(self, f: MiniTableFieldPtr) -> Option> { - let raw = unsafe { sys_msg::upb_Message_GetMessage(self.raw, f.raw) }; - raw.map(|raw| MessagePtr { raw, _phantom: PhantomData }) + /// - The field at `index` must be a message field of type `ChildT`. + /// - Caller must ensure that `value` outlives `self` (typically by being on the same arena). + pub unsafe fn set_base_field_message_at_index( + self, + index: u32, + value: MessagePtr, + ) { + let f = unsafe { sys_mt::upb_MiniTable_GetFieldByIndex(T::mini_table(), index) }; + unsafe { sys_msg::upb_Message_SetBaseFieldMessage(self.raw, f, value.raw) } } /// Returns a mutable message pointer, creating the field if it is not present. /// /// # Safety - /// - `self` must be a legally deferencable. - /// - `f` must be a message field. - pub unsafe fn get_or_create_mutable_message( + /// - `self` must be a legally deferencable to a mutable message. + /// - The field at `index` must be a message field. + pub unsafe fn get_or_create_mutable_message_at_index( self, - f: MiniTableFieldPtr, + index: u32, arena: &Arena, ) -> Option> { + let f = unsafe { sys_mt::upb_MiniTable_GetFieldByIndex(T::mini_table(), index) }; + let raw = unsafe { sys_msg::upb_Message_GetOrCreateMutableMessage( self.raw, T::mini_table(), - f.raw, + f, arena.raw(), ) }; @@ -164,24 +174,25 @@ impl MessagePtr { /// /// # Safety /// - `self` must be a legally dereferenceable. - /// - `f` must be a repeated field. - pub unsafe fn get_array(self, f: MiniTableFieldPtr) -> Option { - let raw = unsafe { sys_msg::upb_Message_GetArray(self.raw, f.raw) }; + /// - The field at `index` must be a repeated field. + pub unsafe fn get_array_at_index(self, index: u32) -> Option { + let f = unsafe { sys_mt::upb_MiniTable_GetFieldByIndex(T::mini_table(), index) }; + let raw = unsafe { sys_msg::upb_Message_GetArray(self.raw, f) }; raw } /// Returns a mutable pointer to an array. Will only return None if arena allocation fails. /// /// # Safety - /// - `self` must be a legally dereferenceable. - /// - `f` must be a repeated field. - pub unsafe fn get_or_create_mutable_array( + /// - `self` must be a legally dereferenceable to a mutable message. + /// - The field at `index` must be a repeated field. + pub unsafe fn get_or_create_mutable_array_at_index( self, - f: MiniTableFieldPtr, + index: u32, arena: &Arena, ) -> Option { - let raw = - unsafe { sys_msg::upb_Message_GetOrCreateMutableArray(self.raw, f.raw, arena.raw()) }; + let f = unsafe { sys_mt::upb_MiniTable_GetFieldByIndex(T::mini_table(), index) }; + let raw = unsafe { sys_msg::upb_Message_GetOrCreateMutableArray(self.raw, f, arena.raw()) }; raw } } diff --git a/rust/upb/sys/BUILD b/rust/upb/sys/BUILD index 9fb3ec1caff73..858fd7dea8559 100644 --- a/rust/upb/sys/BUILD +++ b/rust/upb/sys/BUILD @@ -50,12 +50,12 @@ cc_library( name = "upb_c_api", srcs = ["upb_api.c"], deps = [ - "//upb:mem", - "//upb:message", - "//upb:message_compare", - "//upb:message_copy", - "//upb:mini_table", + "//upb/mem", + "//upb/message", + "//upb/message:compare", + "//upb/message:copy", "//upb/mini_descriptor", + "//upb/mini_table", "//upb/text:debug", ], ) diff --git a/src/file_lists.cmake b/src/file_lists.cmake index f89c679bb2ed2..894e23d6a1ada 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -698,6 +698,7 @@ set(libupb_srcs ${protobuf_SOURCE_DIR}/upb/wire/decode_fast/select.c ${protobuf_SOURCE_DIR}/upb/wire/encode.c ${protobuf_SOURCE_DIR}/upb/wire/eps_copy_input_stream.c + ${protobuf_SOURCE_DIR}/upb/wire/internal/decoder.c ${protobuf_SOURCE_DIR}/upb/wire/reader.c ) diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index b83199ffa3c30..12fce836e8e2d 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -2096,6 +2096,7 @@ cc_test( deps = [ ":cc_test_protos", ":protobuf", + ":protobuf_lite", ":test_util", "//src/google/protobuf/stubs", "//src/google/protobuf/testing", diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index 6228e84ee733e..a6f2e4a07abe2 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -146,10 +146,10 @@ class PROTOBUF_EXPORT Any final : public ::google::protobuf::Message } static bool GetAnyFieldDescriptors( const ::google::protobuf::Message& message, - const ::google::protobuf::FieldDescriptor* PROTOBUF_NULLABLE* PROTOBUF_NONNULL - type_url_field, - const ::google::protobuf::FieldDescriptor* PROTOBUF_NULLABLE* PROTOBUF_NONNULL - value_field); + const ::google::protobuf::FieldDescriptor * PROTOBUF_NULLABLE * + PROTOBUF_NONNULL type_url_field, + const ::google::protobuf::FieldDescriptor * PROTOBUF_NULLABLE * + PROTOBUF_NONNULL value_field); template < typename T, class = typename std::enable_iffile(); + ASSERT_FALSE(generator.Generate(file, parameter, generator_context, &error)); EXPECT_EQ(error, absl::StrCat("Unknown generator option: ", parameter)); } diff --git a/src/google/protobuf/compiler/cpp/field_generators/map_field.cc b/src/google/protobuf/compiler/cpp/field_generators/map_field.cc index 8652ddb07ced7..d3f4123b53421 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/map_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/map_field.cc @@ -5,6 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd +#include #include #include #include @@ -19,6 +20,7 @@ #include "google/protobuf/compiler/cpp/options.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/io/printer.h" +#include "google/protobuf/port.h" // Must be included last. #include "google/protobuf/port_def.inc" @@ -174,6 +176,7 @@ class Map : public FieldGeneratorBase { void GenerateByteSize(io::Printer* p) const override; private: + const FieldDescriptor* key_; const FieldDescriptor* val_; const Options* opts_; diff --git a/src/google/protobuf/compiler/cpp/field_generators/string_view_field.cc b/src/google/protobuf/compiler/cpp/field_generators/string_view_field.cc index c97be2abedc47..5acb84e57b87d 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/string_view_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/string_view_field.cc @@ -565,9 +565,15 @@ void SingularStringView::GenerateConstexprAggregateInitializer( /*decltype($field_$)*/ {nullptr, false}, )cc"); } else if (use_micro_string()) { - p->Emit(R"cc( - /*decltype($field_$)*/ {}, - )cc"); + if (EmptyDefault()) { + p->Emit(R"cc( + /*decltype($field_$)*/ {}, + )cc"); + } else { + p->Emit(R"cc( + /*decltype($field_$)*/ {$classname$::$default_variable_field$}, + )cc"); + } } else { p->Emit(R"cc( /*decltype($field_$)*/ { diff --git a/src/google/protobuf/compiler/cpp/file_unittest.cc b/src/google/protobuf/compiler/cpp/file_unittest.cc index c46cdd4e4c3ab..58c07e717fd55 100644 --- a/src/google/protobuf/compiler/cpp/file_unittest.cc +++ b/src/google/protobuf/compiler/cpp/file_unittest.cc @@ -146,6 +146,7 @@ TEST(FileTest, TopologicallyOrderedDescriptors) { "TestVerifyBigFieldNumberUint32", "TestRequiredOneof", "TestRequired", + "TestRequired.MapFieldEntry", "TestOneof2", "TestNestedMessageHasBits.NestedMessage", "TestNestedGroupExtensionOuter.Layer1OptionalGroup", diff --git a/src/google/protobuf/compiler/cpp/generator.cc b/src/google/protobuf/compiler/cpp/generator.cc index 5607ed92b4f4d..9dced4f9a4819 100644 --- a/src/google/protobuf/compiler/cpp/generator.cc +++ b/src/google/protobuf/compiler/cpp/generator.cc @@ -97,6 +97,8 @@ bool CppGenerator::Generate(const FileDescriptor* file, const std::string& parameter, GeneratorContext* generator_context, std::string* error) const { + ABSL_DCHECK_NE(file, nullptr); + std::vector> options; ParseGeneratorParameter(parameter, &options); diff --git a/src/google/protobuf/compiler/cpp/helpers.cc b/src/google/protobuf/compiler/cpp/helpers.cc index e800b9c051fff..52eb10d433eab 100644 --- a/src/google/protobuf/compiler/cpp/helpers.cc +++ b/src/google/protobuf/compiler/cpp/helpers.cc @@ -1339,18 +1339,23 @@ bool IsV2EnabledForMessage(const Descriptor* descriptor, } #ifdef PROTOBUF_INTERNAL_V2_EXPERIMENT +bool IsV2CodegenEnabled(const Options& options) { + return !options.opensource_runtime && !options.bootstrap; +} + bool ShouldGenerateV2Code(const Descriptor* descriptor, const Options& options) { - return !options.opensource_runtime && !options.bootstrap && + return IsV2CodegenEnabled(options) && !HasSimpleBaseClass(descriptor, options); } bool IsEligibleForV2Batching(const FieldDescriptor* field) { // Non-message fields whose numbers fit into 2B should be considered for // batching although the actual batching depends on the current batching, the - // payload size, etc. + // payload size, etc. Oneof fields are not eligible for batching because they + // are handled separately. return field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE && - !field->is_map() && + field->real_containing_oneof() == nullptr && !field->is_map() && field->number() < std::numeric_limits::max(); } @@ -1909,15 +1914,15 @@ bool GetBootstrapBasename(const Options& options, absl::string_view basename, } static const auto* bootstrap_mapping = - // TODO Replace these with string_view once we remove - // StringPiece. - new absl::flat_hash_map{ + new absl::flat_hash_map{ {"net/proto2/proto/descriptor", "third_party/protobuf/descriptor"}, {"third_party/protobuf/cpp_features", "third_party/protobuf/cpp_features"}, {"third_party/protobuf/compiler/plugin", "third_party/protobuf/compiler/plugin"}, + {"third_party/protobuf/internal_options", + "third_party/protobuf/internal_options_bootstrap"}, {"net/proto2/compiler/proto/profile", "net/proto2/compiler/proto/profile_bootstrap"}, }; @@ -1926,7 +1931,7 @@ bool GetBootstrapBasename(const Options& options, absl::string_view basename, *bootstrap_basename = std::string(basename); return false; } else { - *bootstrap_basename = iter->second; + *bootstrap_basename = std::string(iter->second); return true; } } diff --git a/src/google/protobuf/compiler/cpp/helpers.h b/src/google/protobuf/compiler/cpp/helpers.h index ea91e1100aa27..857bfe29e6a26 100644 --- a/src/google/protobuf/compiler/cpp/helpers.h +++ b/src/google/protobuf/compiler/cpp/helpers.h @@ -515,6 +515,7 @@ bool IsV2EnabledForMessage(const Descriptor* descriptor, const Options& options); #ifdef PROTOBUF_INTERNAL_V2_EXPERIMENT +bool IsV2CodegenEnabled(const Options& options); bool ShouldGenerateV2Code(const Descriptor* descriptor, const Options& options); // Returns true if a field can be batched. diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 3553a4ff445f4..d8f9f56b657be 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -37,6 +37,7 @@ #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" #include "absl/types/span.h" #include "google/protobuf/compiler/cpp/enum.h" #include "google/protobuf/compiler/cpp/extension.h" @@ -611,8 +612,7 @@ std::vector ClassVars(const Descriptor* desc, Options opts) { return vars; } - -} // anonymous namespace +} // namespace // =================================================================== @@ -1743,10 +1743,10 @@ void MessageGenerator::GenerateAnyMethodDefinition(io::Printer* p) { } static bool GetAnyFieldDescriptors( const $pb$::Message& message, - const $pb$::FieldDescriptor* $nullable$* $nonnull$ - type_url_field, - const $pb$::FieldDescriptor* $nullable$* $nonnull$ - value_field); + const $pb$::FieldDescriptor * $nullable$ * + $nonnull$ type_url_field, + const $pb$::FieldDescriptor * $nullable$ * + $nonnull$ value_field); template < typename T, class = typename std::enable_if +void MessageGenerator::EmitOneofFields(io::Printer* p, const T& emitter) const { + // Fields inside a oneof don't use _has_bits_ so we count them in a + // separate pass. + for (auto oneof : OneOfRange(descriptor_)) { + p->Emit({{"oneof_name", oneof->name()}, + {"oneof_case_name", absl::AsciiStrToUpper(oneof->name())}, + {"case_per_field", + [&] { + for (auto field : FieldRange(oneof)) { + PrintFieldComment(Formatter{p}, field, options_); + p->Emit({{"field_name", + UnderscoresToCamelCase(field->name(), true)}, + {"field_byte_size", [&] { emitter(p, field); }}}, + R"cc( + case k$field_name$: { + $field_byte_size$; + break; + } + )cc"); + } + }}}, + R"cc( + switch (this_.$oneof_name$_case()) { + $case_per_field$; + case $oneof_case_name$_NOT_SET: { + break; + } + } + )cc"); + } +} + void MessageGenerator::GenerateByteSizeV2(io::Printer* p) { } diff --git a/src/google/protobuf/compiler/cpp/message.h b/src/google/protobuf/compiler/cpp/message.h index 01c8388bedb77..73e936bb8a47e 100644 --- a/src/google/protobuf/compiler/cpp/message.h +++ b/src/google/protobuf/compiler/cpp/message.h @@ -221,6 +221,8 @@ class MessageGenerator { std::vector&& fields) const; void EmitCheckAndSerializeField(const FieldDescriptor* field, io::Printer* p) const; + template + void EmitOneofFields(io::Printer* p, const T& emitter) const; const Descriptor* descriptor_; int index_in_file_messages_; diff --git a/src/google/protobuf/compiler/cpp/unittest.inc b/src/google/protobuf/compiler/cpp/unittest.inc index 9df60aed52616..ac7c7863de2e4 100644 --- a/src/google/protobuf/compiler/cpp/unittest.inc +++ b/src/google/protobuf/compiler/cpp/unittest.inc @@ -2120,7 +2120,7 @@ TEST(HELPERS_TEST_NAME, TestSCCAnalysis) { MessageSCCAnalyzer scc_analyzer((Options())); const SCC* scc = scc_analyzer.GetSCC(msg.GetDescriptor()); MessageAnalysis result = scc_analyzer.GetSCCAnalysis(scc); - EXPECT_EQ(result.is_recursive, false); + EXPECT_EQ(result.is_recursive, true); EXPECT_EQ(result.contains_required, true); EXPECT_EQ(result.contains_cord, false); EXPECT_EQ(result.contains_extension, false); diff --git a/src/google/protobuf/compiler/java/options.h b/src/google/protobuf/compiler/java/options.h index 0328af4c9c4ec..ab409679cef0b 100644 --- a/src/google/protobuf/compiler/java/options.h +++ b/src/google/protobuf/compiler/java/options.h @@ -26,7 +26,8 @@ struct Options { enforce_lite(false), annotate_code(false), strip_nonfunctional_codegen(false), - jvm_dsl(true) {} + jvm_dsl(true), + dsl_use_concrete_types(false) {} bool generate_immutable_code; bool generate_mutable_code; @@ -50,6 +51,10 @@ struct Options { // If true, generate JVM-specific DSL code. This defaults to true for // compatibility with the old behavior. bool jvm_dsl; + + // If true, the generated DSL code will only utilize concrete types, never + // referring to the OrBuilder interfaces. + bool dsl_use_concrete_types; }; } // namespace java diff --git a/src/google/protobuf/compiler/kotlin/generator.cc b/src/google/protobuf/compiler/kotlin/generator.cc index da2ee921f6816..d6760272765f3 100644 --- a/src/google/protobuf/compiler/kotlin/generator.cc +++ b/src/google/protobuf/compiler/kotlin/generator.cc @@ -68,6 +68,7 @@ bool KotlinGenerator::Generate(const FileDescriptor* file, file_options.strip_nonfunctional_codegen = true; } else if (option.first == "no_jvm_dsl") { file_options.jvm_dsl = false; + file_options.dsl_use_concrete_types = true; } else { *error = absl::StrCat("Unknown generator option: ", option.first); return false; diff --git a/src/google/protobuf/compiler/kotlin/message.cc b/src/google/protobuf/compiler/kotlin/message.cc index 9026dfaee17db..c5c840e1c9e27 100644 --- a/src/google/protobuf/compiler/kotlin/message.cc +++ b/src/google/protobuf/compiler/kotlin/message.cc @@ -38,6 +38,8 @@ MessageGenerator::MessageGenerator(const Descriptor* descriptor, lite_(!java::HasDescriptorMethods(descriptor_->file(), context->EnforceLite())), jvm_dsl_(!lite_ || context->options().jvm_dsl), + dsl_use_concrete_types_(lite_ && + context->options().dsl_use_concrete_types), field_generators_(java::FieldGeneratorMap(descriptor_)) { for (int i = 0; i < descriptor_->field_count(); i++) { if (java::IsRealOneof(descriptor_->field(i))) { @@ -196,9 +198,9 @@ void MessageGenerator::GenerateOrNull(io::Printer* printer) const { "@kotlin.Deprecated(message = \"Field $name$ is deprecated\")\n", "name", context_->GetFieldGeneratorInfo(field)->name); } - if (jvm_dsl_) { - // On the JVM, we can use `FooOrBuilder`, and it saves code size to - // generate only one method instead of two. + if (!dsl_use_concrete_types_) { + // We can use `FooOrBuilder`, and it saves code size to generate only one + // method instead of two. printer->Print( "public val $full_classname$OrBuilder.$camelcase_name$OrNull: " "$full_name$?\n" @@ -212,8 +214,8 @@ void MessageGenerator::GenerateOrNull(io::Printer* printer) const { name_resolver_->GetImmutableClassName(field->message_type())), "name", context_->GetFieldGeneratorInfo(field)->capitalized_name); } else { - // Non-JVM platforms don't have `FooOrBuilder`, so we generate `Foo` - // and `Foo.Builder` methods. + // We don't have `FooOrBuilder`, so we generate `Foo` and `Foo.Builder` + // methods. printer->Print( "public val $full_classname$.$camelcase_name$OrNull: " "$full_name$?\n" diff --git a/src/google/protobuf/compiler/kotlin/message.h b/src/google/protobuf/compiler/kotlin/message.h index 9c10476618f52..c6c72c01d5f08 100644 --- a/src/google/protobuf/compiler/kotlin/message.h +++ b/src/google/protobuf/compiler/kotlin/message.h @@ -38,6 +38,7 @@ class MessageGenerator { absl::btree_map oneofs_; bool lite_; bool jvm_dsl_; + bool dsl_use_concrete_types_; java::FieldGeneratorMap field_generators_; diff --git a/src/google/protobuf/compiler/rust/accessors/repeated_field.cc b/src/google/protobuf/compiler/rust/accessors/repeated_field.cc index e47678ad3ac70..ffb71ec01b695 100644 --- a/src/google/protobuf/compiler/rust/accessors/repeated_field.cc +++ b/src/google/protobuf/compiler/rust/accessors/repeated_field.cc @@ -39,11 +39,9 @@ void RepeatedField::InMsgImpl(Context& ctx, const FieldDescriptor& field, ctx.Emit(R"rs( pub fn $field$($view_self$) -> $pb$::RepeatedView<$view_lifetime$, $RsType$> { unsafe { - let f = $pbr$::upb_MiniTable_GetFieldByIndex( - ::mini_table(), - $upb_mt_field_index$); - $pbr$::upb_Message_GetArray( - self.raw_msg(), f) + self.inner.ptr().get_array_at_index( + $upb_mt_field_index$ + ) }.map_or_else( $pbr$::empty_array::<$RsType$>, |raw| unsafe { @@ -74,14 +72,10 @@ void RepeatedField::InMsgImpl(Context& ctx, const FieldDescriptor& field, ctx.Emit({}, R"rs( pub fn $field$_mut(&mut self) -> $pb$::RepeatedMut<'_, $RsType$> { unsafe { - let f = $pbr$::upb_MiniTable_GetFieldByIndex( - ::mini_table(), - $upb_mt_field_index$); - let raw_array = $pbr$::upb_Message_GetOrCreateMutableArray( - self.raw_msg(), - f, - self.arena().raw(), - ).unwrap(); + let raw_array = self.inner.ptr_mut().get_or_create_mutable_array_at_index( + $upb_mt_field_index$, + self.arena() + ).expect("alloc should not fail"); $pb$::RepeatedMut::from_inner( $pbi$::Private, $pbr$::InnerRepeatedMut::new( diff --git a/src/google/protobuf/compiler/rust/accessors/singular_cord.cc b/src/google/protobuf/compiler/rust/accessors/singular_cord.cc index 6c25e9030914a..dea7b05a1f39d 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_cord.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_cord.cc @@ -112,11 +112,9 @@ void SingularCord::InMsgImpl(Context& ctx, const FieldDescriptor& field, } else { ctx.Emit(R"rs( let view = unsafe { - let f = $pbr$::upb_MiniTable_GetFieldByIndex( - ::mini_table(), - $upb_mt_field_index$); - $pbr$::upb_Message_GetString( - self.raw_msg(), f, ($default_value$).into()) + self.inner.ptr().get_string_at_index( + $upb_mt_field_index$, ($default_value$).into() + ) }; $transform_borrowed$ )rs"); @@ -153,13 +151,10 @@ void SingularCord::InMsgImpl(Context& ctx, const FieldDescriptor& field, parent_arena.fuse(&arena); unsafe { - let f = $pbr$::upb_MiniTable_GetFieldByIndex( - ::mini_table(), - $upb_mt_field_index$); - $pbr$::upb_Message_SetBaseFieldString( - self.inner.raw(), - f, - view); + self.inner.ptr_mut().set_base_field_string_at_index( + $upb_mt_field_index$, + view, + ); } )rs"); } diff --git a/src/google/protobuf/compiler/rust/accessors/singular_message.cc b/src/google/protobuf/compiler/rust/accessors/singular_message.cc index 015e63d69f7dd..83b4ca2b891a4 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_message.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_message.cc @@ -46,15 +46,12 @@ void SingularMessage::InMsgImpl(Context& ctx, const FieldDescriptor& field, if (ctx.is_upb()) { ctx.Emit(R"rs( let submsg = unsafe { - let f = $pbr$::upb_MiniTable_GetFieldByIndex( - ::mini_table(), - $upb_mt_field_index$); - $pbr$::upb_Message_GetMessage(self.raw_msg(), f) + self.inner.ptr().get_message_at_index($upb_mt_field_index$) }; //~ For upb, getters return null if the field is unset, so we need //~ to check for null and return the default instance manually. //~ Note that a nullptr received from upb manifests as Option::None - let raw = submsg.unwrap_or($pbr$::ScratchSpace::zeroed_block()); + let raw = submsg.map(|ptr| ptr.raw()).unwrap_or($pbr$::ScratchSpace::zeroed_block()); let inner = unsafe { $pbr$::MessageViewInner::wrap_raw(raw) }; $msg_type$View::new($pbi$::Private, inner) )rs"); @@ -92,16 +89,15 @@ void SingularMessage::InMsgImpl(Context& ctx, const FieldDescriptor& field, )rs"); } else { ctx.Emit({}, R"rs( - let raw_msg = unsafe { - let mt = ::mini_table(); - let f = $pbr$::upb_MiniTable_GetFieldByIndex(mt, $upb_mt_field_index$); - $pbr$::upb_Message_GetOrCreateMutableMessage( - self.raw_msg(), mt, f, self.arena().raw()).unwrap() + let ptr = unsafe { + self.inner.ptr_mut().get_or_create_mutable_message_at_index( + $upb_mt_field_index$, self.arena() + ).unwrap() }; $msg_type$Mut::from_parent( $pbi$::Private, self.as_message_mut_inner($pbi$::Private), - raw_msg) + ptr.raw()) )rs"); } }}, @@ -124,19 +120,16 @@ void SingularMessage::InMsgImpl(Context& ctx, const FieldDescriptor& field, // The message and arena are dropped after the setter. The // memory remains allocated as we fuse the arena with the // parent message's arena. - let mut msg = val.into_proxied($pbi$::Private); + let mut child = val.into_proxied($pbi$::Private); self.inner .arena() - .fuse($pbr$::UpbGetArena::get_arena(&mut msg, $pbi$::Private)); + .fuse($pbr$::UpbGetArena::get_arena(&mut child, $pbi$::Private)); + let child_ptr = $pbr$::UpbGetMessagePtrMut::get_ptr_mut(&mut child, $pbi$::Private); unsafe { - let f = $pbr$::upb_MiniTable_GetFieldByIndex( - ::mini_table(), - $upb_mt_field_index$); - $pbr$::upb_Message_SetBaseFieldMessage( - self.inner.raw(), - f, - $pbr$::UpbGetMessagePtrMut::get_raw_message_mut(&mut msg, $pbi$::Private)); + self.inner.ptr_mut().set_base_field_message_at_index( + $upb_mt_field_index$, child_ptr + ); } )rs"); } else { diff --git a/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc b/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc index 91b500e89729d..cd8e0ea9c3567 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc @@ -26,26 +26,27 @@ namespace rust { namespace { -// The upb function to use for the get/set functions, eg `Int32` for the -// functions `upb_Message_GetInt32` and upb_Message_SetInt32`. +// The type name to use for the get/set methods on `MessagePtr`, e.g. `i32` +// for `MessagePtr::get_i32_at_index()` and +// `MessagePtr::set_base_field_i32_at_index()`. std::string UpbCTypeNameForFunctions(const FieldDescriptor& field) { switch (field.cpp_type()) { case FieldDescriptor::CPPTYPE_INT32: - return "Int32"; + return "i32"; case FieldDescriptor::CPPTYPE_INT64: - return "Int64"; + return "i64"; case FieldDescriptor::CPPTYPE_UINT32: - return "UInt32"; + return "u32"; case FieldDescriptor::CPPTYPE_UINT64: - return "UInt64"; + return "u64"; case FieldDescriptor::CPPTYPE_DOUBLE: - return "Double"; + return "f64"; case FieldDescriptor::CPPTYPE_FLOAT: - return "Float"; + return "f32"; case FieldDescriptor::CPPTYPE_BOOL: - return "Bool"; + return "bool"; case FieldDescriptor::CPPTYPE_ENUM: - return "Int32"; + return "i32"; case FieldDescriptor::CPPTYPE_STRING: case FieldDescriptor::CPPTYPE_MESSAGE: // Handled by a different file. @@ -73,7 +74,7 @@ void SingularScalar::InMsgImpl(Context& ctx, const FieldDescriptor& field, {"Scalar", RsTypePath(ctx, field)}, {"default_value", DefaultValue(ctx, field)}, {"upb_mt_field_index", UpbMiniTableFieldIndex(field)}, - {"upb_fn_type_name", UpbCTypeNameForFunctions(field)}, + {"type_name", UpbCTypeNameForFunctions(field)}, {"getter", [&] { if (ctx.is_cpp()) { @@ -87,18 +88,15 @@ void SingularScalar::InMsgImpl(Context& ctx, const FieldDescriptor& field, R"rs( pub fn $field$($view_self$) -> $Scalar$ { unsafe { - let mt = ::mini_table(); - let f = $pbr$::upb_MiniTable_GetFieldByIndex( - mt, $upb_mt_field_index$); - // TODO: b/361751487: This .into() and .try_into() is only // here for the enum<->i32 case, we should avoid it for // other primitives where the types naturally match // perfectly (and do an unchecked conversion for // i32->enum types, since even for closed enums we trust // upb to only return one of the named values). - $pbr$::upb_Message_Get$upb_fn_type_name$( - self.raw_msg(), f, ($default_value$).into()).try_into().unwrap() + self.inner.ptr().get_$type_name$_at_index( + $upb_mt_field_index$, ($default_value$).into() + ).try_into().unwrap() } } )rs"); @@ -117,15 +115,13 @@ void SingularScalar::InMsgImpl(Context& ctx, const FieldDescriptor& field, ctx.Emit(R"rs( pub fn set_$raw_field_name$(&mut self, val: $Scalar$) { unsafe { - let mt = ::mini_table(); - let f = $pbr$::upb_MiniTable_GetFieldByIndex( - mt, $upb_mt_field_index$); // TODO: b/361751487: This .into() is only here // here for the enum<->i32 case, we should avoid it for // other primitives where the types naturally match - // perfectly. - $pbr$::upb_Message_SetBaseField$upb_fn_type_name$( - self.raw_msg(), f, val.into()); + //perfectly. + self.inner.ptr_mut().set_base_field_$type_name$_at_index( + $upb_mt_field_index$, val.into() + ) } } )rs"); diff --git a/src/google/protobuf/compiler/rust/accessors/singular_string.cc b/src/google/protobuf/compiler/rust/accessors/singular_string.cc index 9d80efbd1a4c0..344dbb7ae0e99 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_string.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_string.cc @@ -65,11 +65,9 @@ void SingularString::InMsgImpl(Context& ctx, const FieldDescriptor& field, ctx.Emit(R"rs( pub fn $field$($view_self$) -> $pb$::View<$view_lifetime$, $proxied_type$> { let str_view = unsafe { - let f = $pbr$::upb_MiniTable_GetFieldByIndex( - ::mini_table(), - $upb_mt_field_index$); - $pbr$::upb_Message_GetString( - self.raw_msg(), f, ($default_value$).into()) + self.inner.ptr().get_string_at_index( + $upb_mt_field_index$, ($default_value$).into() + ) }; $transform_view$ })rs"); @@ -98,13 +96,10 @@ void SingularString::InMsgImpl(Context& ctx, const FieldDescriptor& field, parent_arena.fuse(&arena); unsafe { - let f = $pbr$::upb_MiniTable_GetFieldByIndex( - ::mini_table(), - $upb_mt_field_index$); - $pbr$::upb_Message_SetBaseFieldString( - self.inner.raw(), - f, - view); + self.inner.ptr_mut().set_base_field_string_at_index( + $upb_mt_field_index$, + view, + ); } )rs"); } diff --git a/src/google/protobuf/compiler/rust/accessors/with_presence.cc b/src/google/protobuf/compiler/rust/accessors/with_presence.cc index d5e9b18c9f5fd..8974e2626cf3b 100644 --- a/src/google/protobuf/compiler/rust/accessors/with_presence.cc +++ b/src/google/protobuf/compiler/rust/accessors/with_presence.cc @@ -49,10 +49,7 @@ void WithPresenceAccessorsInMsgImpl(Context& ctx, const FieldDescriptor& field, R"rs( pub fn has_$raw_field_name$($view_self$) -> bool { unsafe { - let f = $pbr$::upb_MiniTable_GetFieldByIndex( - ::mini_table(), - $upb_mt_field_index$); - $pbr$::upb_Message_HasBaseField(self.raw_msg(), f) + self.inner.ptr().has_field_at_index($upb_mt_field_index$) } } )rs"); @@ -72,10 +69,9 @@ void WithPresenceAccessorsInMsgImpl(Context& ctx, const FieldDescriptor& field, R"rs( pub fn clear_$raw_field_name$(&mut self) { unsafe { - let mt = ::mini_table(); - let f = $pbr$::upb_MiniTable_GetFieldByIndex( - mt, $upb_mt_field_index$); - $pbr$::upb_Message_ClearBaseField(self.raw_msg(), f); + self.inner.ptr().clear_field_at_index( + $upb_mt_field_index$ + ); } })rs"); } diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index a4c3e37ee118d..e14de096a88d5 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -14,6 +14,8 @@ #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/compiler/cpp/names.h" @@ -48,14 +50,7 @@ void MessageNew(Context& ctx, const Descriptor& msg) { case Kernel::kUpb: ctx.Emit(R"rs( - let arena = $pbr$::Arena::new(); - let raw = unsafe { - $pbr$::upb_Message_New( - ::mini_table(), - arena.raw()).unwrap() - }; - let inner = unsafe { $pbr$::OwnedMessageInner::::wrap_raw(raw, arena) }; - Self { inner } + Self { inner: $pbr$::OwnedMessageInner::::new() } )rs"); return; } @@ -386,83 +381,6 @@ void UpbGeneratedMessageTraitImpls(Context& ctx, const Descriptor& msg, )rs"); } -void MessageMutTakeCopyMergeFrom(Context& ctx, const Descriptor& msg) { - switch (ctx.opts().kernel) { - case Kernel::kCpp: - ctx.Emit(R"rs( - impl $pb$::TakeFrom for $Msg$Mut<'_> { - fn take_from(&mut self, mut src: impl $pb$::AsMut) { - //~ TODO: b/393559271 - Optimize this copy out. - let mut src = src.as_mut(); - $pb$::CopyFrom::copy_from(self, $pb$::AsView::as_view(&src)); - $pb$::Clear::clear(&mut src); - } - } - - impl $pb$::CopyFrom for $Msg$Mut<'_> { - fn copy_from(&mut self, src: impl $pb$::AsView) { - unsafe { $pbr$::proto2_rust_Message_copy_from(self.raw_msg(), src.as_view().raw_msg()) }; - } - } - - impl $pb$::MergeFrom for $Msg$Mut<'_> { - fn merge_from(&mut self, src: impl $pb$::AsView) { - // SAFETY: self and src are both valid `$Msg$`s. - unsafe { - $pbr$::proto2_rust_Message_merge_from(self.raw_msg(), src.as_view().raw_msg()); - } - } - } - )rs"); - return; - case Kernel::kUpb: - ctx.Emit( - R"rs( - impl $pb$::TakeFrom for $Msg$Mut<'_> { - fn take_from(&mut self, mut src: impl $pb$::AsMut) { - let mut src = src.as_mut(); - //~ TODO: b/393559271 - Optimize this copy out. - $pb$::CopyFrom::copy_from(self, $pb$::AsView::as_view(&src)); - $pb$::Clear::clear(&mut src); - } - } - - impl $pb$::CopyFrom for $Msg$Mut<'_> { - fn copy_from(&mut self, src: impl $pb$::AsView) { - // SAFETY: self and src are both valid `$Msg$`s associated with - // `Self::mini_table()`. - unsafe { - assert!( - $pbr$::upb_Message_DeepCopy( - self.raw_msg(), - src.as_view().raw_msg(), - ::mini_table(), - self.arena().raw()) - ); - } - } - } - - impl $pb$::MergeFrom for $Msg$Mut<'_> { - fn merge_from(&mut self, src: impl $pb$::AsView) { - // SAFETY: self and src are both valid `$Msg$`s. - unsafe { - assert!( - $pbr$::upb_Message_MergeFrom(self.raw_msg(), - src.as_view().raw_msg(), - ::mini_table(), - // Use a nullptr for the ExtensionRegistry. - $std$::ptr::null(), - self.arena().raw()) - ); - } - } - } - )rs"); - return; - } -} - void MessageProxiedInRepeated(Context& ctx, const Descriptor& msg) { switch (ctx.opts().kernel) { case Kernel::kCpp: @@ -703,8 +621,6 @@ void GenerateRs(Context& ctx, const Descriptor& msg, const upb::DefPool& pool) { {"Msg::serialize", [&] { MessageSerialize(ctx, msg); }}, {"Msg::drop", [&] { MessageDrop(ctx, msg); }}, {"Msg::debug", [&] { MessageDebug(ctx, msg); }}, - {"MsgMut::take_copy_merge_from", - [&] { MessageMutTakeCopyMergeFrom(ctx, msg); }}, {"default_instance_impl", [&] { GenerateDefaultInstanceImpl(ctx, msg); }}, {"accessor_fns", @@ -849,27 +765,6 @@ void GenerateRs(Context& ctx, const Descriptor& msg, const upb::DefPool& pool) { } } - impl $pb$::TakeFrom for $Msg$ { - fn take_from(&mut self, src: impl $pb$::AsMut) { - let mut m = self.as_mut(); - $pb$::TakeFrom::take_from(&mut m, src) - } - } - - impl $pb$::CopyFrom for $Msg$ { - fn copy_from(&mut self, src: impl $pb$::AsView) { - let mut m = self.as_mut(); - $pb$::CopyFrom::copy_from(&mut m, src) - } - } - - impl $pb$::MergeFrom for $Msg$ { - fn merge_from<'src>(&mut self, src: impl $pb$::AsView) { - let mut m = self.as_mut(); - $pb$::MergeFrom::merge_from(&mut m, src) - } - } - impl $pb$::Serialize for $Msg$ { fn serialize(&self) -> $Result$, $pb$::SerializeError> { $pb$::AsView::as_view(self).serialize() @@ -1002,8 +897,6 @@ void GenerateRs(Context& ctx, const Descriptor& msg, const upb::DefPool& pool) { } } - $MsgMut::take_copy_merge_from$ - #[allow(dead_code)] impl<'msg> $Msg$Mut<'msg> { #[doc(hidden)] diff --git a/src/google/protobuf/compiler/rust/naming.cc b/src/google/protobuf/compiler/rust/naming.cc index 647eadf4c2a50..185aa45a0700e 100644 --- a/src/google/protobuf/compiler/rust/naming.cc +++ b/src/google/protobuf/compiler/rust/naming.cc @@ -392,6 +392,21 @@ std::string ScreamingSnakeToUpperCamelCase(absl::string_view input) { return result; } +std::string CrubitCcSymbolName(const Descriptor& msg) { + // To support forward declares of C++ types, Crubit requires that the symbol + // literal is spelled identical to the one used in the generated bindings. + // This requires some string mangling here to make them match. + std::string cpp_name = cpp::QualifiedClassName(&msg); + if (absl::StartsWith(cpp_name, "::")) { + cpp_name = cpp_name.substr(2); + } + cpp_name = absl::StrReplaceAll(cpp_name, + {{"::", " :: "}, {"<", " < "}, {">", " > "}}); + absl::StripTrailingAsciiWhitespace(&cpp_name); + + return cpp_name; +} + MultiCasePrefixStripper::MultiCasePrefixStripper(absl::string_view prefix) : prefixes_{ std::string(prefix), diff --git a/src/google/protobuf/compiler/rust/naming.h b/src/google/protobuf/compiler/rust/naming.h index 3ddcbe1f47462..eaae5fa00e011 100644 --- a/src/google/protobuf/compiler/rust/naming.h +++ b/src/google/protobuf/compiler/rust/naming.h @@ -124,6 +124,10 @@ std::string SnakeToUpperCamelCase(absl::string_view input); // Converts a SCREAMING_SNAKE_CASE string to an UpperCamelCase string. std::string ScreamingSnakeToUpperCamelCase(absl::string_view input); +// Converts a Descriptor's expected C++ fully qualified name to the symbol +// literal that Crubit expects to be used in the generated bindings. +std::string CrubitCcSymbolName(const Descriptor& msg); + // Given a fixed prefix, this will repeatedly strip provided // string_views if they start with the prefix, the prefix in UpperCamel, or // the prefix in snake_case. diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index eb3b114f698d0..f967c459d587e 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -8595,6 +8595,7 @@ void DescriptorBuilder::ValidateOptions(const FieldDescriptor* field, ValidateFieldFeatures(field, proto); + if (field->file()->edition() >= Edition::EDITION_2024 && field->has_legacy_proto_ctype()) { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, @@ -10710,6 +10711,10 @@ bool IsLazilyInitializedFile(absl::string_view filename) { filename == "google/protobuf/cpp_features.proto") { return true; } + if (filename == "third_party/protobuf/internal_options.proto" || + filename == "google/protobuf/internal_options.proto") { + return true; + } return filename == "net/proto2/proto/descriptor.proto" || filename == "google/protobuf/descriptor.proto"; } diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 826f1278acb10..b0719a8aa88f5 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -2864,6 +2864,23 @@ MapIterator Reflection::MapEnd(Message* message, return iter; } +ConstMapIterator Reflection::ConstMapBegin(const Message* message, + const FieldDescriptor* field) const { + USAGE_CHECK(IsMapFieldInApi(field), ConstMapBegin, + "Field is not a map field."); + ConstMapIterator iter(message, field); + GetRaw(*message, field).ConstMapBegin(&iter); + return iter; +} + +ConstMapIterator Reflection::ConstMapEnd(const Message* message, + const FieldDescriptor* field) const { + USAGE_CHECK(IsMapFieldInApi(field), ConstMapEnd, "Field is not a map field."); + ConstMapIterator iter(message, field); + GetRaw(*message, field).ConstMapEnd(&iter); + return iter; +} + int Reflection::MapSize(const Message& message, const FieldDescriptor* field) const { USAGE_CHECK(IsMapFieldInApi(field), MapSize, "Field is not a map field."); @@ -3455,13 +3472,17 @@ void* Reflection::RepeatedFieldData(Message* message, MapFieldBase* Reflection::MutableMapData(Message* message, const FieldDescriptor* field) const { USAGE_CHECK(IsMapFieldInApi(field), GetMapData, "Field is not a map field."); - return MutableRaw(message, field); + auto* map = MutableRaw(message, field); + map->MutableAccess(); + return map; } const MapFieldBase* Reflection::GetMapData(const Message& message, const FieldDescriptor* field) const { USAGE_CHECK(IsMapFieldInApi(field), GetMapData, "Field is not a map field."); - return &(GetRaw(message, field)); + const auto* map = &(GetRaw(message, field)); + map->ConstAccess(); + return map; } template diff --git a/src/google/protobuf/internal_options.proto b/src/google/protobuf/internal_options.proto new file mode 100644 index 0000000000000..90984d796917b --- /dev/null +++ b/src/google/protobuf/internal_options.proto @@ -0,0 +1,36 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +edition = "2023"; + +package pb; + +import "google/protobuf/descriptor.proto"; + +// Internal options for testing only. +// These are used to turn on/off certain features that are not normally +// controlled from the .proto file, but we want to be able to control them for +// unit tests of said features. + +message InternalOptionsForce { + enum Enum { + UNSPECIFIED = 0; + FORCE_ON = 1; + FORCE_OFF = 2; + } +} + +message InternalFieldOptionsCpp { +} + +message InternalFieldOptions { + InternalFieldOptionsCpp cpp = 1; +} + +extend google.protobuf.FieldOptions { + InternalFieldOptions internal_field_options = 535801550; +} diff --git a/src/google/protobuf/map.h b/src/google/protobuf/map.h index de938c202e457..d755c7364c08d 100644 --- a/src/google/protobuf/map.h +++ b/src/google/protobuf/map.h @@ -641,7 +641,6 @@ class MapFieldBaseForParse { mutable std::atomic payload_{}; const void* prototype_as_void_; - PROTOBUF_TSAN_DECLARE_MEMBER; }; // The value might be of different signedness, so use memcpy to extract it. @@ -663,6 +662,18 @@ struct KeyNode : NodeBase { decltype(auto) key() const { return ReadKey(GetVoidKey()); } }; +inline map_index_t Hash(absl::string_view k, void* salt) { + // Note: we could potentially also use CRC32-based hashing here. + return absl::HashOf(k, salt); +} +inline map_index_t Hash(uint64_t k, void* salt) { + if constexpr (!HasCrc32()) return absl::HashOf(k, salt); + uintptr_t salt_int = reinterpret_cast(salt); + // Note: Crc32(salt_int, k) causes the random iteration order test to fail so + // we also rotate. + return Crc32(salt_int, absl::rotr(k, salt_int & 0x3f)); +} + // KeyMapBase is a chaining hash map. // The implementation doesn't need the full generality of unordered_map, // and it doesn't have it. More bells and whistles can be added as needed. @@ -929,8 +940,7 @@ class KeyMapBase : public UntypedMapBase { } map_index_t BucketNumber(typename TS::ViewType k) const { - return static_cast(absl::HashOf(k, table_) & - (num_buckets_ - 1)); + return Hash(k, table_) & (num_buckets_ - 1); } }; diff --git a/src/google/protobuf/map_field.cc b/src/google/protobuf/map_field.cc index da55d74bcbfc7..179a6b221fa4a 100644 --- a/src/google/protobuf/map_field.cc +++ b/src/google/protobuf/map_field.cc @@ -99,7 +99,9 @@ bool MapFieldBase::DeleteMapValue(const MapKey& map_key) { void MapFieldBase::ClearMapNoSync() { GetMapRaw().ClearTable(true); } -void MapFieldBase::SetMapIteratorValue(MapIterator* map_iter) const { +template +void MapFieldBase::SetMapIteratorValue( + MapIteratorBase* map_iter) const { if (map_iter->iter_.Equals(UntypedMapBase::EndIterator())) return; const UntypedMapBase& map = *map_iter->iter_.m_; @@ -142,18 +144,32 @@ void MapFieldBase::MapEnd(MapIterator* map_iter) const { map_iter->iter_ = UntypedMapBase::EndIterator(); } -bool MapFieldBase::EqualIterator(const MapIterator& a, - const MapIterator& b) const { +void MapFieldBase::ConstMapBegin(ConstMapIterator* map_iter) const { + map_iter->iter_ = GetMap().begin(); + SetMapIteratorValue(map_iter); +} + +void MapFieldBase::ConstMapEnd(ConstMapIterator* map_iter) const { + map_iter->iter_ = UntypedMapBase::EndIterator(); +} + +template +bool MapFieldBase::EqualIterator(const MapIteratorBase& a, + const MapIteratorBase& b) const { return a.iter_.Equals(b.iter_); } -void MapFieldBase::IncreaseIterator(MapIterator* map_iter) const { +template +void MapFieldBase::IncreaseIterator( + MapIteratorBase* map_iter) const { map_iter->iter_.PlusPlus(); SetMapIteratorValue(map_iter); } -void MapFieldBase::CopyIterator(MapIterator* this_iter, - const MapIterator& that_iter) const { +template +void MapFieldBase::CopyIterator( + MapIteratorBase* this_iter, + const MapIteratorBase& that_iter) const { this_iter->iter_ = that_iter.iter_; this_iter->key_.SetType(that_iter.key_.type()); // MapValueRef::type() fails when containing data is null. However, if @@ -310,8 +326,8 @@ void MapFieldBase::SyncRepeatedFieldWithMapNoLock() { RepeatedPtrField& rep = payload().repeated_field; rep.Clear(); - MapIterator it(this, descriptor); - MapIterator end(this, descriptor); + ConstMapIterator it(this, descriptor); + ConstMapIterator end(this, descriptor); it.iter_ = GetMapRaw().begin(); SetMapIteratorValue(&it); @@ -345,7 +361,7 @@ void MapFieldBase::SyncRepeatedFieldWithMapNoLock() { Unreachable(); } - const MapValueRef& map_val = it.GetValueRef(); + const MapValueConstRef& map_val = it.GetValueRef(); switch (val_des->cpp_type()) { case FieldDescriptor::CPPTYPE_STRING: reflection->SetString(new_entry, val_des, @@ -496,6 +512,62 @@ bool MapFieldBase::InsertOrLookupMapValue(const MapKey& map_key, } } // namespace internal + +template +MapIteratorBase::MapIteratorBase(MessageT* message, + const FieldDescriptor* field) { + const Reflection* reflection = message->GetReflection(); + if constexpr (kIsMutable) { + map_ = reflection->MutableMapData(message, field); + } else { + map_ = reflection->GetMapData(*message, field); + } + key_.SetType(field->message_type()->map_key()->cpp_type()); + value_.SetType(field->message_type()->map_value()->cpp_type()); +} + +template +MapIteratorBase& MapIteratorBase::operator=( + const MapIteratorBase& other) { + map_ = other.map_; + map_->CopyIterator(this, other); + return *this; +} + +template +bool MapIteratorBase::operator==( + const MapIteratorBase& other) const { + return map_->EqualIterator(*this, other); +} + +template +typename MapIteratorBase::DerivedIterator& +MapIteratorBase::operator++() { + map_->IncreaseIterator(this); + return static_cast(*this); +} + +template +typename MapIteratorBase::DerivedIterator +MapIteratorBase::operator++(int) { + // iter_ is copied from Map<...>::iterator, no need to + // copy from its self again. Use the same implementation + // with operator++() + map_->IncreaseIterator(this); + return *static_cast(this); +} + +template +MapIteratorBase::MapIteratorBase(MapFieldBase* map, + const Descriptor* descriptor) { + map_ = map; + key_.SetType(descriptor->map_key()->cpp_type()); + value_.SetType(descriptor->map_value()->cpp_type()); +} + +template class MapIteratorBase; +template class MapIteratorBase; + } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/map_field.h b/src/google/protobuf/map_field.h index f493383033a58..c976cfbcb6ff9 100644 --- a/src/google/protobuf/map_field.h +++ b/src/google/protobuf/map_field.h @@ -11,15 +11,10 @@ #include #include #include -#include #include #include #include -#include -#include "absl/base/attributes.h" -#include "absl/base/config.h" -#include "absl/hash/hash.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/strings/string_view.h" @@ -30,7 +25,6 @@ #include "google/protobuf/generated_message_util.h" #include "google/protobuf/internal_visibility.h" #include "google/protobuf/map.h" -#include "google/protobuf/map_field_lite.h" #include "google/protobuf/map_type_handler.h" #include "google/protobuf/message.h" #include "google/protobuf/message_lite.h" @@ -49,8 +43,15 @@ namespace google { namespace protobuf { class DynamicMessage; +template +class MapIteratorBase; +class ConstMapIterator; class MapIterator; +namespace internal { +class MapFieldBase; +} + // Microsoft compiler complains about non-virtual destructor, // even when the destructor is private. #ifdef _MSC_VER @@ -228,7 +229,8 @@ class PROTOBUF_EXPORT MapKey { template friend class internal::TypeDefinedMapFieldBase; friend class internal::MapFieldBase; - friend class MapIterator; + template + friend class MapIteratorBase; template friend auto AbslHashValue(H state, const MapKey& key) { @@ -339,11 +341,16 @@ class PROTOBUF_EXPORT MapFieldBase : public MapFieldBaseForParse { // Sync Map with repeated field and returns the size of map. int size() const; void Clear(); - void SetMapIteratorValue(MapIterator* map_iter) const; + template + void SetMapIteratorValue(MapIteratorBase* map_iter) const; void MapBegin(MapIterator* map_iter) const; void MapEnd(MapIterator* map_iter) const; - bool EqualIterator(const MapIterator& a, const MapIterator& b) const; + void ConstMapBegin(ConstMapIterator* map_iter) const; + void ConstMapEnd(ConstMapIterator* map_iter) const; + template + bool EqualIterator(const MapIteratorBase& a, + const MapIteratorBase& b) const; // Returns the number of bytes used by the repeated field, excluding // sizeof(*this) @@ -482,15 +489,20 @@ class PROTOBUF_EXPORT MapFieldBase : public MapFieldBaseForParse { // type helper for key and value. Call these help methods to deal with // different types. Real helper methods are implemented in // TypeDefinedMapFieldBase. + template + friend class google::protobuf::MapIteratorBase; friend class google::protobuf::MapIterator; // Copy the map<...>::iterator from other_iterator to // this_iterator. - void CopyIterator(MapIterator* this_iter, const MapIterator& that_iter) const; + template + void CopyIterator(MapIteratorBase* this_iter, + const MapIteratorBase& that_iter) const; // IncreaseIterator() is called by operator++() of MapIterator only. // It implements the ++ operator of MapIterator. - void IncreaseIterator(MapIterator* map_iter) const; + template + void IncreaseIterator(MapIteratorBase* map_iter) const; bool LookupMapValueNoSync(const MapKey& map_key, MapValueConstRef* val) const; static ReflectionPayload* ToPayload(TaggedPtr p) { @@ -709,7 +721,8 @@ class PROTOBUF_EXPORT MapValueConstRef { friend class internal::MapField; template friend class internal::TypeDefinedMapFieldBase; - friend class google::protobuf::MapIterator; + template + friend class google::protobuf::MapIteratorBase; friend class Reflection; friend class internal::MapFieldBase; @@ -777,45 +790,35 @@ class PROTOBUF_EXPORT MapValueRef final : public MapValueConstRef { #undef TYPE_CHECK -class PROTOBUF_EXPORT MapIterator { +template +class PROTOBUF_EXPORT MapIteratorBase { + using MessageT = + std::conditional_t; + using MapFieldBase = std::conditional_t; + using ValueRef = + std::conditional_t; + using DerivedIterator = + std::conditional_t; + public: - MapIterator(Message* message, const FieldDescriptor* field) { - const Reflection* reflection = message->GetReflection(); - map_ = reflection->MutableMapData(message, field); - key_.SetType(field->message_type()->map_key()->cpp_type()); - value_.SetType(field->message_type()->map_value()->cpp_type()); - } - MapIterator(const MapIterator& other) { *this = other; } - MapIterator& operator=(const MapIterator& other) { - map_ = other.map_; - map_->CopyIterator(this, other); - return *this; - } - friend bool operator==(const MapIterator& a, const MapIterator& b) { - return a.map_->EqualIterator(a, b); - } - friend bool operator!=(const MapIterator& a, const MapIterator& b) { - return !a.map_->EqualIterator(a, b); - } - MapIterator& operator++() { - map_->IncreaseIterator(this); - return *this; - } - MapIterator operator++(int) { - // iter_ is copied from Map<...>::iterator, no need to - // copy from its self again. Use the same implementation - // with operator++() - map_->IncreaseIterator(this); - return *this; + MapIteratorBase(MessageT* message, const FieldDescriptor* field); + MapIteratorBase(const MapIteratorBase& other) { *this = other; } + + MapIteratorBase& operator=(const MapIteratorBase& other); + + bool operator==(const MapIteratorBase& other) const; + friend bool operator!=(const MapIteratorBase& a, const MapIteratorBase& b) { + return !(a == b); } + + DerivedIterator& operator++(); + DerivedIterator operator++(int); + const MapKey& GetKey() { return key_; } - const MapValueRef& GetValueRef() { return value_; } - MapValueRef* MutableValueRef() { - map_->SetMapDirty(); - return &value_; - } + const ValueRef& GetValueRef() { return value_; } - private: + protected: template friend class internal::TypeDefinedMapFieldBase; template friend class internal::MapField; friend class internal::MapFieldBase; - template - friend struct internal::MapDynamicFieldInfo; - MapIterator(internal::MapFieldBase* map, const Descriptor* descriptor) { - map_ = map; - key_.SetType(descriptor->map_key()->cpp_type()); - value_.SetType(descriptor->map_value()->cpp_type()); - } + MapIteratorBase(MapFieldBase* map, const Descriptor* descriptor); internal::UntypedMapIterator iter_; // Point to a MapField to call helper methods implemented in MapField. // MapIterator does not own this object. - internal::MapFieldBase* map_; + MapFieldBase* map_; MapKey key_; - MapValueRef value_; + ValueRef value_; +}; + +extern template class MapIteratorBase; +extern template class MapIteratorBase; + +class PROTOBUF_EXPORT ConstMapIterator final + : public MapIteratorBase { + friend class internal::MapFieldBase; + template + friend struct internal::MapDynamicFieldInfo; + + public: + ConstMapIterator(const google::protobuf::Message* message, const FieldDescriptor* field) + : MapIteratorBase(message, field) {} + + private: + ConstMapIterator(const internal::MapFieldBase* map, + const Descriptor* descriptor) + : MapIteratorBase(map, descriptor) {} +}; + +class PROTOBUF_EXPORT MapIterator final + : public MapIteratorBase { + friend class internal::MapFieldBase; + template + friend struct internal::MapDynamicFieldInfo; + + public: + MapIterator(google::protobuf::Message* message, const FieldDescriptor* field) + : MapIteratorBase(message, field) {} + + MapValueRef* MutableValueRef() { + map_->SetMapDirty(); + return &value_; + } + + private: + MapIterator(internal::MapFieldBase* map, const Descriptor* descriptor) + : MapIteratorBase(map, descriptor) {} }; namespace internal { @@ -845,6 +881,7 @@ template <> struct is_internal_map_value_type : std::true_type {}; template <> struct is_internal_map_value_type : std::true_type {}; + } // namespace internal } // namespace protobuf diff --git a/src/google/protobuf/map_test.cc b/src/google/protobuf/map_test.cc index 0e14bfc5db8fb..02e7a25673c54 100644 --- a/src/google/protobuf/map_test.cc +++ b/src/google/protobuf/map_test.cc @@ -11,7 +11,9 @@ #include #include #include +#include #include +#include #include #include #include diff --git a/src/google/protobuf/map_test.inc b/src/google/protobuf/map_test.inc index 78a56e7b757b9..27624503e44e1 100644 --- a/src/google/protobuf/map_test.inc +++ b/src/google/protobuf/map_test.inc @@ -1110,8 +1110,10 @@ TEST_F(MapImplTest, DeleteMapValues) { auto* field = descriptor->field(i); if (!field->is_map()) continue; - MapIterator it = reflection_tester.MapBegin(message, field->name()); - MapIterator end = reflection_tester.MapEnd(message, field->name()); + ConstMapIterator it = + reflection_tester.ConstMapBegin(message, field->name()); + ConstMapIterator end = + reflection_tester.ConstMapEnd(message, field->name()); // It's unordered, so search for it. if (it == end) continue; auto next = it; @@ -1160,8 +1162,10 @@ TEST_F(MapImplTest, DeleteMapValues) { auto* field = descriptor->field(i); if (!field->is_map()) continue; - MapIterator it = reflection_tester.MapBegin(message, field->name()); - MapIterator end = reflection_tester.MapEnd(message, field->name()); + ConstMapIterator it = + reflection_tester.ConstMapBegin(message, field->name()); + ConstMapIterator end = + reflection_tester.ConstMapEnd(message, field->name()); while (it != end) { auto next = it; @@ -1179,8 +1183,10 @@ TEST_F(MapImplTest, CopyAssignMapIterator) { TestMap message; MapReflectionTester reflection_tester(UNITTEST::TestMap::descriptor()); reflection_tester.SetMapFieldsViaMapReflection(&message); - MapIterator it1 = reflection_tester.MapBegin(&message, "map_int32_int32"); - MapIterator it2 = reflection_tester.MapEnd(&message, "map_int32_int32"); + ConstMapIterator it1 = + reflection_tester.ConstMapBegin(&message, "map_int32_int32"); + ConstMapIterator it2 = + reflection_tester.ConstMapEnd(&message, "map_int32_int32"); it2 = it1; EXPECT_EQ(it1.GetKey().GetInt32Value(), it2.GetKey().GetInt32Value()); } diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index ee2a5acf20f4d..27569978f53fe 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -131,6 +131,7 @@ class MessageFactory; // Defined in other files. class AssignDescriptorsHelper; +class ConstMapIterator; class DynamicMessageFactory; class GeneratedMessageReflectionTestHelper; class MapKey; @@ -1189,12 +1190,25 @@ class PROTOBUF_EXPORT Reflection final { // real element. Mutation to the field may invalidate the iterator. MapIterator MapEnd(Message* message, const FieldDescriptor* field) const; + // Returns a ConstMapIterator referring to the first element in the map field. + // If the map field is empty, this function returns the same as + // reflection::ConstMapEnd. Mutation to the field may invalidate the iterator. + ConstMapIterator ConstMapBegin(const Message* message, + const FieldDescriptor* field) const; + + // Returns a ConstMapIterator referring to the theoretical element that would + // follow the last element in the map field. It does not point to any + // real element. Mutation to the field may invalidate the iterator. + ConstMapIterator ConstMapEnd(const Message* message, + const FieldDescriptor* field) const; + // Get the number of pair of a map field. The result may be // different from FieldSize which can have duplicate keys. int MapSize(const Message& message, const FieldDescriptor* field) const; // Help method for MapIterator. - friend class MapIterator; + template + friend class MapIteratorBase; friend class WireFormatForMapFieldTest; internal::MapFieldBase* MutableMapData(Message* message, const FieldDescriptor* field) const; diff --git a/src/google/protobuf/micro_string.h b/src/google/protobuf/micro_string.h index 50177d2520f66..651494fc3ab6d 100644 --- a/src/google/protobuf/micro_string.h +++ b/src/google/protobuf/micro_string.h @@ -152,7 +152,7 @@ class PROTOBUF_EXPORT MicroString { // To match the LazyString API. auto get() const { return payload.view(); } }; - explicit constexpr MicroString(const UnownedPayload& unowned_input) + constexpr MicroString(const UnownedPayload& unowned_input) // NOLINT : rep_(const_cast(unowned_input.for_tag + kIsLargeRepTag)) {} // Like the constructor above, but for DynamicMessage where we don't have a diff --git a/src/google/protobuf/port.h b/src/google/protobuf/port.h index 77066bcfcd7b4..7b3422cbf2310 100644 --- a/src/google/protobuf/port.h +++ b/src/google/protobuf/port.h @@ -762,6 +762,27 @@ PROTOBUF_EXPORT constexpr BoundsCheckMode GetBoundsCheckMode() { } +#if defined(__x86_64__) && defined(__SSE4_2__) + +constexpr bool HasCrc32() { return true; } +inline uint32_t Crc32(uint32_t crc, uint64_t v) { + return __builtin_ia32_crc32di(crc, v); +} + +#elif defined(__ARM_FEATURE_CRC32) + +constexpr bool HasCrc32() { return true; } +inline uint32_t Crc32(uint32_t crc, uint64_t v) { + return __builtin_arm_crc32cd(crc, v); +} + +#else + +constexpr bool HasCrc32() { return false; } +inline uint32_t Crc32(uint32_t, uint64_t) { return 0; } + +#endif + } // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/reflection_ops.cc b/src/google/protobuf/reflection_ops.cc index dbf5ff0b7a6b4..7fae4bdc74ca0 100644 --- a/src/google/protobuf/reflection_ops.cc +++ b/src/google/protobuf/reflection_ops.cc @@ -197,9 +197,10 @@ bool ReflectionOps::IsInitialized(const Message& message, bool check_fields, const MapFieldBase* map_field = reflection->GetMapData(message, field); if (map_field->IsMapValid()) { - MapIterator it(const_cast(&message), field); - MapIterator end_map(const_cast(&message), field); - for (map_field->MapBegin(&it), map_field->MapEnd(&end_map); + ConstMapIterator it(&message, field); + ConstMapIterator end_map(&message, field); + for (map_field->ConstMapBegin(&it), + map_field->ConstMapEnd(&end_map); it != end_map; ++it) { if (!it.GetValueRef().GetMessageValue().IsInitialized()) { return false; @@ -278,9 +279,9 @@ bool ReflectionOps::IsInitialized(const Message& message) { const MapFieldBase* map_field = reflection->GetMapData(message, field); if (map_field->IsMapValid()) { - MapIterator iter(const_cast(&message), field); - MapIterator end(const_cast(&message), field); - for (map_field->MapBegin(&iter), map_field->MapEnd(&end); + ConstMapIterator iter(&message, field); + ConstMapIterator end(&message, field); + for (map_field->ConstMapBegin(&iter), map_field->ConstMapEnd(&end); iter != end; ++iter) { if (!iter.GetValueRef().GetMessageValue().IsInitialized()) { return false; diff --git a/src/google/protobuf/reflection_ops_unittest.cc b/src/google/protobuf/reflection_ops_unittest.cc index 5efe8dbf05b60..fee690d74e829 100644 --- a/src/google/protobuf/reflection_ops_unittest.cc +++ b/src/google/protobuf/reflection_ops_unittest.cc @@ -11,9 +11,15 @@ #include "google/protobuf/reflection_ops.h" +#include +#include +#include +#include + #include #include "absl/strings/str_join.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/generated_message_util.h" #include "google/protobuf/test_util.h" #include "google/protobuf/unittest.pb.h" #include "google/protobuf/unittest_import.pb.h" @@ -324,20 +330,28 @@ TEST(ReflectionOpsTest, IsInitialized) { unittest::TestRequired message; EXPECT_FALSE(ReflectionOps::IsInitialized(message)); - EXPECT_FALSE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_FALSE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); message.set_a(1); EXPECT_FALSE(ReflectionOps::IsInitialized(message)); - EXPECT_FALSE(ReflectionOps::IsInitialized(message, true, true)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_FALSE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); message.set_b(2); EXPECT_FALSE(ReflectionOps::IsInitialized(message)); - EXPECT_FALSE(ReflectionOps::IsInitialized(message, true, true)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_FALSE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); message.set_c(3); EXPECT_TRUE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); } TEST(ReflectionOpsTest, ForeignIsInitialized) { @@ -346,35 +360,44 @@ TEST(ReflectionOpsTest, ForeignIsInitialized) { // Starts out initialized because the foreign message is itself an optional // field. EXPECT_TRUE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); // Once we create that field, the message is no longer initialized. message.mutable_optional_message(); EXPECT_FALSE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_FALSE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_FALSE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); // Initialize it. Now we're initialized. message.mutable_optional_message()->set_a(1); message.mutable_optional_message()->set_b(2); message.mutable_optional_message()->set_c(3); EXPECT_TRUE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); // Add a repeated version of the message. No longer initialized. unittest::TestRequired* sub_message = message.add_repeated_message(); EXPECT_FALSE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_FALSE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_FALSE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); // Initialize that repeated version. sub_message->set_a(1); sub_message->set_b(2); sub_message->set_c(3); EXPECT_TRUE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); } TEST(ReflectionOpsTest, ExtensionIsInitialized) { @@ -383,62 +406,128 @@ TEST(ReflectionOpsTest, ExtensionIsInitialized) { // Starts out initialized because the foreign message is itself an optional // field. EXPECT_TRUE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); // Once we create that field, the message is no longer initialized. message.MutableExtension(unittest::TestRequired::single); EXPECT_FALSE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_FALSE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_FALSE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); // Initialize it. Now we're initialized. message.MutableExtension(unittest::TestRequired::single)->set_a(1); message.MutableExtension(unittest::TestRequired::single)->set_b(2); message.MutableExtension(unittest::TestRequired::single)->set_c(3); EXPECT_TRUE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); // Add a repeated version of the message. No longer initialized. message.AddExtension(unittest::TestRequired::multi); EXPECT_FALSE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_FALSE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_FALSE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); // Initialize that repeated version. message.MutableExtension(unittest::TestRequired::multi, 0)->set_a(1); message.MutableExtension(unittest::TestRequired::multi, 0)->set_b(2); message.MutableExtension(unittest::TestRequired::multi, 0)->set_c(3); EXPECT_TRUE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); } TEST(ReflectionOpsTest, OneofIsInitialized) { unittest::TestRequiredOneof message; EXPECT_TRUE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); message.mutable_foo_message(); EXPECT_FALSE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_FALSE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_FALSE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); message.set_foo_int(1); EXPECT_TRUE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); message.mutable_foo_message(); EXPECT_FALSE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_FALSE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_FALSE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); message.mutable_foo_message()->set_required_double(0.1); EXPECT_TRUE(ReflectionOps::IsInitialized(message)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, true, false)); - EXPECT_TRUE(ReflectionOps::IsInitialized(message, false, true)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); +} + +TEST(ReflectionOpsTest, EmptyMapIsInitialized) { + unittest::TestRequired message; + message.set_a(1); + message.set_b(2); + message.set_c(3); + message.mutable_map_field(); + EXPECT_TRUE(ReflectionOps::IsInitialized(message)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); +} + +TEST(ReflectionOpsTest, MapIsNotInitializedIfValuesAreNotInitialized) { + unittest::TestRequired message; + message.set_a(1); + message.set_b(2); + message.set_c(3); + + unittest::TestRequired sub_message; + message.mutable_map_field()->insert({"key", std::move(sub_message)}); + EXPECT_FALSE(ReflectionOps::IsInitialized(message)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_FALSE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); +} + +TEST(ReflectionOpsTest, MapIsInitializedIfValuesAreInitialized) { + unittest::TestRequired message; + message.set_a(1); + message.set_b(2); + message.set_c(3); + + unittest::TestRequired sub_message; + sub_message.set_a(1); + sub_message.set_b(2); + sub_message.set_c(3); + message.mutable_map_field()->insert({"key", std::move(sub_message)}); + EXPECT_TRUE(ReflectionOps::IsInitialized(message)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/true, + /*check_descendants=*/false)); + EXPECT_TRUE(ReflectionOps::IsInitialized(message, /*check_fields=*/false, + /*check_descendants=*/true)); } static std::string FindInitializationErrors(const Message& message) { diff --git a/src/google/protobuf/reflection_tester.cc b/src/google/protobuf/reflection_tester.cc index d48adaaf422a8..ff57334582df1 100644 --- a/src/google/protobuf/reflection_tester.cc +++ b/src/google/protobuf/reflection_tester.cc @@ -7,10 +7,15 @@ #include "google/protobuf/reflection_tester.h" #include +#include +#include #include +#include #include #include "absl/container/flat_hash_map.h" +#include "absl/strings/string_view.h" +#include "google/protobuf/descriptor.h" #include "google/protobuf/map_field.h" #include "google/protobuf/message.h" @@ -635,6 +640,18 @@ MapIterator MapReflectionTester::MapEnd(Message* message, return reflection->MapEnd(message, F(field_name)); } +ConstMapIterator MapReflectionTester::ConstMapBegin( + const Message* message, absl::string_view field_name) { + const Reflection* reflection = message->GetReflection(); + return reflection->ConstMapBegin(message, F(field_name)); +} + +ConstMapIterator MapReflectionTester::ConstMapEnd( + const Message* message, absl::string_view field_name) { + const Reflection* reflection = message->GetReflection(); + return reflection->ConstMapEnd(message, F(field_name)); +} + int MapReflectionTester::MapSize(const Message& message, absl::string_view field_name) { const Reflection* reflection = message.GetReflection(); @@ -765,7 +782,7 @@ void MapReflectionTester::RemoveLastMapsViaReflection(Message* message) { std::vector output; reflection->ListFields(*message, &output); - for (int i = 0; i < output.size(); ++i) { + for (size_t i = 0; i < output.size(); ++i) { const FieldDescriptor* field = output[i]; if (!field->is_repeated()) continue; reflection->RemoveLast(message, field); @@ -777,7 +794,7 @@ void MapReflectionTester::ReleaseLastMapsViaReflection(Message* message) { std::vector output; reflection->ListFields(*message, &output); - for (int i = 0; i < output.size(); ++i) { + for (size_t i = 0; i < output.size(); ++i) { const FieldDescriptor* field = output[i]; if (!field->is_repeated()) continue; if (field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) continue; @@ -793,7 +810,7 @@ void MapReflectionTester::SwapMapsViaReflection(Message* message) { const Reflection* reflection = message->GetReflection(); std::vector output; reflection->ListFields(*message, &output); - for (int i = 0; i < output.size(); ++i) { + for (size_t i = 0; i < output.size(); ++i) { const FieldDescriptor* field = output[i]; if (!field->is_repeated()) continue; reflection->SwapElements(message, field, 0, 1); @@ -1406,8 +1423,9 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( map[0] = 0; map[1] = 1; int size = 0; - for (MapIterator iter = reflection->MapBegin(message, F("map_int32_int32")); - iter != reflection->MapEnd(message, F("map_int32_int32")); + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_int32_int32")); + iter != reflection->ConstMapEnd(message, F("map_int32_int32")); ++iter, ++size) { // Check const methods do not invalidate map. message->DebugString(); @@ -1424,8 +1442,10 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( absl::flat_hash_map map; map[0] = 0; map[1] = 1; - for (MapIterator iter = reflection->MapBegin(message, F("map_int64_int64")); - iter != reflection->MapEnd(message, F("map_int64_int64")); ++iter) { + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_int64_int64")); + iter != reflection->ConstMapEnd(message, F("map_int64_int64")); + ++iter) { EXPECT_EQ(map[iter.GetKey().GetInt64Value()], iter.GetValueRef().GetInt64Value()); } @@ -1434,9 +1454,10 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( absl::flat_hash_map map; map[0] = 0; map[1] = 1; - for (MapIterator iter = - reflection->MapBegin(message, F("map_uint32_uint32")); - iter != reflection->MapEnd(message, F("map_uint32_uint32")); ++iter) { + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_uint32_uint32")); + iter != reflection->ConstMapEnd(message, F("map_uint32_uint32")); + ++iter) { EXPECT_EQ(map[iter.GetKey().GetUInt32Value()], iter.GetValueRef().GetUInt32Value()); } @@ -1445,9 +1466,10 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( absl::flat_hash_map map; map[0] = 0; map[1] = 1; - for (MapIterator iter = - reflection->MapBegin(message, F("map_uint64_uint64")); - iter != reflection->MapEnd(message, F("map_uint64_uint64")); ++iter) { + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_uint64_uint64")); + iter != reflection->ConstMapEnd(message, F("map_uint64_uint64")); + ++iter) { EXPECT_EQ(map[iter.GetKey().GetUInt64Value()], iter.GetValueRef().GetUInt64Value()); } @@ -1456,9 +1478,10 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( absl::flat_hash_map map; map[0] = 0; map[1] = 1; - for (MapIterator iter = - reflection->MapBegin(message, F("map_sint32_sint32")); - iter != reflection->MapEnd(message, F("map_sint32_sint32")); ++iter) { + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_sint32_sint32")); + iter != reflection->ConstMapEnd(message, F("map_sint32_sint32")); + ++iter) { EXPECT_EQ(map[iter.GetKey().GetInt32Value()], iter.GetValueRef().GetInt32Value()); } @@ -1467,9 +1490,10 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( absl::flat_hash_map map; map[0] = 0; map[1] = 1; - for (MapIterator iter = - reflection->MapBegin(message, F("map_sint64_sint64")); - iter != reflection->MapEnd(message, F("map_sint64_sint64")); ++iter) { + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_sint64_sint64")); + iter != reflection->ConstMapEnd(message, F("map_sint64_sint64")); + ++iter) { EXPECT_EQ(map[iter.GetKey().GetInt64Value()], iter.GetValueRef().GetInt64Value()); } @@ -1478,9 +1502,9 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( absl::flat_hash_map map; map[0] = 0; map[1] = 1; - for (MapIterator iter = - reflection->MapBegin(message, F("map_fixed32_fixed32")); - iter != reflection->MapEnd(message, F("map_fixed32_fixed32")); + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_fixed32_fixed32")); + iter != reflection->ConstMapEnd(message, F("map_fixed32_fixed32")); ++iter) { EXPECT_EQ(map[iter.GetKey().GetUInt32Value()], iter.GetValueRef().GetUInt32Value()); @@ -1490,9 +1514,9 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( absl::flat_hash_map map; map[0] = 0; map[1] = 1; - for (MapIterator iter = - reflection->MapBegin(message, F("map_fixed64_fixed64")); - iter != reflection->MapEnd(message, F("map_fixed64_fixed64")); + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_fixed64_fixed64")); + iter != reflection->ConstMapEnd(message, F("map_fixed64_fixed64")); ++iter) { EXPECT_EQ(map[iter.GetKey().GetUInt64Value()], iter.GetValueRef().GetUInt64Value()); @@ -1502,9 +1526,9 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( absl::flat_hash_map map; map[0] = 0; map[1] = 1; - for (MapIterator iter = - reflection->MapBegin(message, F("map_sfixed32_sfixed32")); - iter != reflection->MapEnd(message, F("map_sfixed32_sfixed32")); + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_sfixed32_sfixed32")); + iter != reflection->ConstMapEnd(message, F("map_sfixed32_sfixed32")); ++iter) { EXPECT_EQ(map[iter.GetKey().GetInt32Value()], iter.GetValueRef().GetInt32Value()); @@ -1514,8 +1538,10 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( absl::flat_hash_map map; map[0] = 0.0; map[1] = 1.0; - for (MapIterator iter = reflection->MapBegin(message, F("map_int32_float")); - iter != reflection->MapEnd(message, F("map_int32_float")); ++iter) { + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_int32_float")); + iter != reflection->ConstMapEnd(message, F("map_int32_float")); + ++iter) { EXPECT_EQ(map[iter.GetKey().GetInt32Value()], iter.GetValueRef().GetFloatValue()); } @@ -1524,9 +1550,10 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( absl::flat_hash_map map; map[0] = 0.0; map[1] = 1.0; - for (MapIterator iter = - reflection->MapBegin(message, F("map_int32_double")); - iter != reflection->MapEnd(message, F("map_int32_double")); ++iter) { + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_int32_double")); + iter != reflection->ConstMapEnd(message, F("map_int32_double")); + ++iter) { EXPECT_EQ(map[iter.GetKey().GetInt32Value()], iter.GetValueRef().GetDoubleValue()); } @@ -1535,8 +1562,9 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( std::array map; map[0] = false; map[1] = true; - for (MapIterator iter = reflection->MapBegin(message, F("map_bool_bool")); - iter != reflection->MapEnd(message, F("map_bool_bool")); ++iter) { + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_bool_bool")); + iter != reflection->ConstMapEnd(message, F("map_bool_bool")); ++iter) { EXPECT_EQ(map[iter.GetKey().GetBoolValue() ? 1 : 0], iter.GetValueRef().GetBoolValue()); } @@ -1546,9 +1574,9 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( map[long_string()] = long_string(); map[long_string_2()] = long_string_2(); int size = 0; - for (MapIterator iter = - reflection->MapBegin(message, F("map_string_string")); - iter != reflection->MapEnd(message, F("map_string_string")); + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_string_string")); + iter != reflection->ConstMapEnd(message, F("map_string_string")); ++iter, ++size) { // Check const methods do not invalidate map. message->DebugString(); @@ -1565,8 +1593,10 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( absl::flat_hash_map map; map[0] = long_string(); map[1] = long_string_2(); - for (MapIterator iter = reflection->MapBegin(message, F("map_int32_bytes")); - iter != reflection->MapEnd(message, F("map_int32_bytes")); ++iter) { + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_int32_bytes")); + iter != reflection->ConstMapEnd(message, F("map_int32_bytes")); + ++iter) { EXPECT_EQ(map[iter.GetKey().GetInt32Value()], iter.GetValueRef().GetStringValue()); } @@ -1575,8 +1605,10 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( absl::flat_hash_map map; map[0] = map_enum_bar_; map[1] = map_enum_baz_; - for (MapIterator iter = reflection->MapBegin(message, F("map_int32_enum")); - iter != reflection->MapEnd(message, F("map_int32_enum")); ++iter) { + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_int32_enum")); + iter != reflection->ConstMapEnd(message, F("map_int32_enum")); + ++iter) { EXPECT_EQ(map[iter.GetKey().GetInt32Value()]->number(), iter.GetValueRef().GetEnumValue()); } @@ -1586,9 +1618,10 @@ void MapReflectionTester::ExpectMapFieldsSetViaReflectionIterator( map[0] = 0; map[1] = 1; int size = 0; - for (MapIterator iter = - reflection->MapBegin(message, F("map_int32_foreign_message")); - iter != reflection->MapEnd(message, F("map_int32_foreign_message")); + for (ConstMapIterator iter = + reflection->ConstMapBegin(message, F("map_int32_foreign_message")); + iter != + reflection->ConstMapEnd(message, F("map_int32_foreign_message")); ++iter, ++size) { // Check const methods do not invalidate map. message->DebugString(); @@ -1630,40 +1663,41 @@ void MapReflectionTester::ExpectClearViaReflection(const Message& message) { void MapReflectionTester::ExpectClearViaReflectionIterator(Message* message) { const Reflection* reflection = message->GetReflection(); - EXPECT_TRUE(reflection->MapBegin(message, F("map_int32_int32")) == - reflection->MapEnd(message, F("map_int32_int32"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_int64_int64")) == - reflection->MapEnd(message, F("map_int64_int64"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_uint32_uint32")) == - reflection->MapEnd(message, F("map_uint32_uint32"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_uint64_uint64")) == - reflection->MapEnd(message, F("map_uint64_uint64"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_sint32_sint32")) == - reflection->MapEnd(message, F("map_sint32_sint32"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_sint64_sint64")) == - reflection->MapEnd(message, F("map_sint64_sint64"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_fixed32_fixed32")) == - reflection->MapEnd(message, F("map_fixed32_fixed32"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_fixed64_fixed64")) == - reflection->MapEnd(message, F("map_fixed64_fixed64"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_sfixed32_sfixed32")) == - reflection->MapEnd(message, F("map_sfixed32_sfixed32"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_sfixed64_sfixed64")) == - reflection->MapEnd(message, F("map_sfixed64_sfixed64"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_int32_float")) == - reflection->MapEnd(message, F("map_int32_float"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_int32_double")) == - reflection->MapEnd(message, F("map_int32_double"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_bool_bool")) == - reflection->MapEnd(message, F("map_bool_bool"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_string_string")) == - reflection->MapEnd(message, F("map_string_string"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_int32_bytes")) == - reflection->MapEnd(message, F("map_int32_bytes"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_int32_enum")) == - reflection->MapEnd(message, F("map_int32_enum"))); - EXPECT_TRUE(reflection->MapBegin(message, F("map_int32_foreign_message")) == - reflection->MapEnd(message, F("map_int32_foreign_message"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_int32_int32")) == + reflection->ConstMapEnd(message, F("map_int32_int32"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_int64_int64")) == + reflection->ConstMapEnd(message, F("map_int64_int64"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_uint32_uint32")) == + reflection->ConstMapEnd(message, F("map_uint32_uint32"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_uint64_uint64")) == + reflection->ConstMapEnd(message, F("map_uint64_uint64"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_sint32_sint32")) == + reflection->ConstMapEnd(message, F("map_sint32_sint32"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_sint64_sint64")) == + reflection->ConstMapEnd(message, F("map_sint64_sint64"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_fixed32_fixed32")) == + reflection->ConstMapEnd(message, F("map_fixed32_fixed32"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_fixed64_fixed64")) == + reflection->ConstMapEnd(message, F("map_fixed64_fixed64"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_sfixed32_sfixed32")) == + reflection->ConstMapEnd(message, F("map_sfixed32_sfixed32"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_sfixed64_sfixed64")) == + reflection->ConstMapEnd(message, F("map_sfixed64_sfixed64"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_int32_float")) == + reflection->ConstMapEnd(message, F("map_int32_float"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_int32_double")) == + reflection->ConstMapEnd(message, F("map_int32_double"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_bool_bool")) == + reflection->ConstMapEnd(message, F("map_bool_bool"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_string_string")) == + reflection->ConstMapEnd(message, F("map_string_string"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_int32_bytes")) == + reflection->ConstMapEnd(message, F("map_int32_bytes"))); + EXPECT_TRUE(reflection->ConstMapBegin(message, F("map_int32_enum")) == + reflection->ConstMapEnd(message, F("map_int32_enum"))); + EXPECT_TRUE( + reflection->ConstMapBegin(message, F("map_int32_foreign_message")) == + reflection->ConstMapEnd(message, F("map_int32_foreign_message"))); } } // namespace protobuf diff --git a/src/google/protobuf/reflection_tester.h b/src/google/protobuf/reflection_tester.h index 9ac05b9d64449..33056fb37afe3 100644 --- a/src/google/protobuf/reflection_tester.h +++ b/src/google/protobuf/reflection_tester.h @@ -50,6 +50,10 @@ class MapReflectionTester { absl::string_view field_name, int index); MapIterator MapBegin(Message* message, absl::string_view field_name); MapIterator MapEnd(Message* message, absl::string_view field_name); + ConstMapIterator ConstMapBegin(const Message* message, + absl::string_view field_name); + ConstMapIterator ConstMapEnd(const Message* message, + absl::string_view field_name); int MapSize(const Message& message, absl::string_view field_name); static std::optional LookupMapValue( diff --git a/src/google/protobuf/reflection_visit_field_info.h b/src/google/protobuf/reflection_visit_field_info.h index 31e918aa7d7a6..8cffdb98f3709 100644 --- a/src/google/protobuf/reflection_visit_field_info.h +++ b/src/google/protobuf/reflection_visit_field_info.h @@ -1379,16 +1379,12 @@ struct MapDynamicFieldInfo { template static void VisitElementsImpl(T& msg, const Reflection*, const FieldDescriptor* field, - const MapFieldBase& const_map_field, - Callback&& cb, Rank0) { - // Unfortunately, we have to const_cast here because MapIterator only takes - // a mutable MapFieldBase pointer. This is still safe because value iterator - // is not mutable. - MapFieldBase* map_field = const_cast(&const_map_field); + const MapFieldBase& map_field, Callback&& cb, + Rank0) { const Descriptor* descriptor = field->message_type(); - MapIterator begin(map_field, descriptor), end(map_field, descriptor); - const_map_field.MapBegin(&begin); - const_map_field.MapEnd(&end); + ConstMapIterator begin(&map_field, descriptor), end(&map_field, descriptor); + map_field.ConstMapBegin(&begin); + map_field.ConstMapEnd(&end); for (auto it = begin; it != end; ++it) { MapDynamicFieldVisitKey(it.GetKey(), it.GetValueRef(), cb); diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index 2a1dd681fa060..a9e1575c5eb2b 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -2475,7 +2475,7 @@ class MapFieldPrinterHelper { const FieldDescriptor* field); static void CopyKey(const MapKey& key, Message* message, const FieldDescriptor* field_desc); - static void CopyValue(const MapValueRef& value, Message* message, + static void CopyValue(const MapValueConstRef& value, Message* message, const FieldDescriptor* field_desc); }; @@ -2502,10 +2502,8 @@ MapEntries MapFieldPrinterHelper::SortMap(const Message& message, reflection->GetMessageFactory()->GetPrototype(map_entry_desc); all_entries.reserve(reflection->MapSize(message, field)); owned_entries.reserve(reflection->MapSize(message, field)); - for (MapIterator iter = - reflection->MapBegin(const_cast(&message), field); - iter != reflection->MapEnd(const_cast(&message), field); - ++iter) { + for (ConstMapIterator iter = reflection->ConstMapBegin(&message, field); + iter != reflection->ConstMapEnd(&message, field); ++iter) { std::unique_ptr map_entry_message = absl::WrapUnique(prototype->New()); CopyKey(iter.GetKey(), map_entry_message.get(), map_entry_desc->field(0)); @@ -2553,7 +2551,7 @@ void MapFieldPrinterHelper::CopyKey(const MapKey& key, Message* message, } } -void MapFieldPrinterHelper::CopyValue(const MapValueRef& value, +void MapFieldPrinterHelper::CopyValue(const MapValueConstRef& value, Message* message, const FieldDescriptor* field_desc) { const Reflection* reflection = message->GetReflection(); diff --git a/src/google/protobuf/unittest.proto b/src/google/protobuf/unittest.proto index 806b76d0802ee..0b5e5a9ab8483 100644 --- a/src/google/protobuf/unittest.proto +++ b/src/google/protobuf/unittest.proto @@ -779,6 +779,8 @@ message TestRequired { // Add an optional child message to make this non-trivial for go/pdlazy. ForeignMessage optional_foreign = 34; + + map map_field = 35; } message TestRequiredForeign { diff --git a/src/google/protobuf/util/message_differencer.cc b/src/google/protobuf/util/message_differencer.cc index ef4be9f447853..1bb622eb2bdad 100644 --- a/src/google/protobuf/util/message_differencer.cc +++ b/src/google/protobuf/util/message_differencer.cc @@ -1095,10 +1095,8 @@ bool MessageDifferencer::CompareMapFieldByMapReflection( } // First pass: check whether the same keys are present. - for (MapIterator it = reflection1->MapBegin(const_cast(&message1), - map_field), - it_end = reflection1->MapEnd(const_cast(&message1), - map_field); + for (ConstMapIterator it = reflection1->ConstMapBegin(&message1, map_field), + it_end = reflection1->ConstMapEnd(&message1, map_field); it != it_end; ++it) { if (!reflection2->ContainsMapKey(message2, map_field, it.GetKey())) { return false; @@ -1110,10 +1108,9 @@ bool MessageDifferencer::CompareMapFieldByMapReflection( switch (val_des->cpp_type()) { #define HANDLE_TYPE(CPPTYPE, METHOD, COMPAREMETHOD) \ case FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - for (MapIterator it = reflection1->MapBegin( \ - const_cast(&message1), map_field), \ - it_end = reflection1->MapEnd( \ - const_cast(&message1), map_field); \ + for (ConstMapIterator \ + it = reflection1->ConstMapBegin(&message1, map_field), \ + it_end = reflection1->ConstMapEnd(&message1, map_field); \ it != it_end; ++it) { \ MapValueConstRef value2; \ reflection2->LookupMapValue(message2, map_field, it.GetKey(), &value2); \ @@ -1136,11 +1133,9 @@ bool MessageDifferencer::CompareMapFieldByMapReflection( HANDLE_TYPE(ENUM, EnumValue, Int32); #undef HANDLE_TYPE case FieldDescriptor::CPPTYPE_MESSAGE: { - for (MapIterator it = reflection1->MapBegin( - const_cast(&message1), map_field); - it != - reflection1->MapEnd(const_cast(&message1), map_field); - ++it) { + for (ConstMapIterator it = + reflection1->ConstMapBegin(&message1, map_field); + it != reflection1->ConstMapEnd(&message1, map_field); ++it) { if (!reflection2->ContainsMapKey(message2, map_field, it.GetKey())) { return false; } diff --git a/src/google/protobuf/wire_format.cc b/src/google/protobuf/wire_format.cc index e473331d0122c..0eaa637afdc13 100644 --- a/src/google/protobuf/wire_format.cc +++ b/src/google/protobuf/wire_format.cc @@ -1138,10 +1138,8 @@ class MapKeySorter { const Reflection* reflection, const FieldDescriptor* field) { std::vector sorted_key_list; - for (MapIterator it = - reflection->MapBegin(const_cast(&message), field); - it != reflection->MapEnd(const_cast(&message), field); - ++it) { + for (ConstMapIterator it = reflection->ConstMapBegin(&message, field); + it != reflection->ConstMapEnd(&message, field); ++it) { sorted_key_list.push_back(it.GetKey()); } MapKeyComparator comparator; @@ -1239,11 +1237,9 @@ uint8_t* WireFormat::InternalSerializeField(const FieldDescriptor* field, InternalSerializeMapEntry(field, *it, map_value, target, stream); } } else { - for (MapIterator it = message_reflection->MapBegin( - const_cast(&message), field); - it != - message_reflection->MapEnd(const_cast(&message), field); - ++it) { + for (ConstMapIterator it = + message_reflection->ConstMapBegin(&message, field); + it != message_reflection->ConstMapEnd(&message, field); ++it) { target = InternalSerializeMapEntry(field, it.GetKey(), it.GetValueRef(), target, stream); } @@ -1621,12 +1617,12 @@ size_t WireFormat::FieldDataOnlyByteSize(const FieldDescriptor* field, const MapFieldBase* map_field = message_reflection->GetMapData(message, field); if (map_field->IsMapValid()) { - MapIterator iter(const_cast(&message), field); - MapIterator end(const_cast(&message), field); + ConstMapIterator iter(&message, field); + ConstMapIterator end(&message, field); const FieldDescriptor* key_field = field->message_type()->field(0); const FieldDescriptor* value_field = field->message_type()->field(1); - for (map_field->MapBegin(&iter), map_field->MapEnd(&end); iter != end; - ++iter) { + for (map_field->ConstMapBegin(&iter), map_field->ConstMapEnd(&end); + iter != end; ++iter) { size_t size = kMapEntryTagByteSize; size += MapKeyDataOnlyByteSize(key_field, iter.GetKey()); size += MapValueRefDataOnlyByteSize(value_field, iter.GetValueRef()); diff --git a/upb/BUILD b/upb/BUILD index a71055345f03c..eb05144fe1555 100644 --- a/upb/BUILD +++ b/upb/BUILD @@ -77,13 +77,13 @@ cc_library( "//upb_generator:__subpackages__", ], deps = [ - ":base", - ":mem", - ":message", - ":mini_descriptor", - ":mini_table", - ":wire", + "//upb/base", + "//upb/mem", + "//upb/message", "//upb/message:internal", + "//upb/mini_descriptor", + "//upb/mini_table", + "//upb/wire", ] + select({ ":fasttable_enabled_setting": [ "//upb/wire/decode_fast:field_parsers", diff --git a/upb/base/BUILD b/upb/base/BUILD index 198131a6a5fbb..15ee832ec32cb 100644 --- a/upb/base/BUILD +++ b/upb/base/BUILD @@ -24,7 +24,7 @@ cc_library( copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [ - "//upb:port", + "//upb/port", ], ) @@ -36,7 +36,7 @@ cc_library( ], copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], - deps = ["//upb:port"], + deps = ["//upb/port"], ) cc_test( diff --git a/upb/conformance/BUILD b/upb/conformance/BUILD index 0d19e7f9c8e35..203b6fb198f95 100644 --- a/upb/conformance/BUILD +++ b/upb/conformance/BUILD @@ -70,12 +70,12 @@ cc_binary( ":test_messages_proto3_upbdefs", "//editions:test_messages_proto2_editions_upbdefs", "//editions:test_messages_proto3_editions_upbdefs", - "//upb:base", - "//upb:json", - "//upb:port", - "//upb:reflection", - "//upb:text", - "//upb:wire", + "//upb/base", + "//upb/json", + "//upb/port", + "//upb/reflection", + "//upb/text", + "//upb/wire", ], ) @@ -124,12 +124,12 @@ cc_binary( ":test_messages_proto3_upbdefs", "//editions:test_messages_proto2_editions_upbdefs", "//editions:test_messages_proto3_editions_upbdefs", - "//upb:base", - "//upb:json", - "//upb:port", - "//upb:reflection", - "//upb:text", - "//upb:wire", + "//upb/base", + "//upb/json", + "//upb/port", + "//upb/reflection", + "//upb/text", + "//upb/wire", ], ) diff --git a/upb/hash/BUILD b/upb/hash/BUILD index 39ae661516a34..8527061890a3f 100644 --- a/upb/hash/BUILD +++ b/upb/hash/BUILD @@ -22,10 +22,10 @@ cc_library( copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [ - "//upb:base", - "//upb:mem", - "//upb:port", + "//upb/base", "//upb/base:internal", + "//upb/mem", + "//upb/port", ], ) @@ -34,8 +34,8 @@ cc_test( srcs = ["test.cc"], deps = [ ":hash", - "//upb:mem", - "//upb:port", + "//upb/mem", + "//upb/port", "@abseil-cpp//absl/container:flat_hash_map", "@googletest//:gtest", "@googletest//:gtest_main", diff --git a/upb/io/BUILD b/upb/io/BUILD index d0f6c9afcfe24..f390cf6558f04 100644 --- a/upb/io/BUILD +++ b/upb/io/BUILD @@ -4,8 +4,8 @@ cc_library( name = "string", hdrs = ["string.h"], deps = [ - "//upb:mem", - "//upb:port", + "//upb/mem", + "//upb/port", ], ) @@ -16,10 +16,10 @@ cc_library( deps = [ ":string", ":zero_copy_stream", - "//upb:base", - "//upb:mem", - "//upb:port", + "//upb/base", "//upb/lex", + "//upb/mem", + "//upb/port", ], ) @@ -30,9 +30,9 @@ cc_library( "zero_copy_output_stream.h", ], deps = [ - "//upb:base", - "//upb:mem", - "//upb:port", + "//upb/base", + "//upb/mem", + "//upb/port", ], ) @@ -51,8 +51,8 @@ cc_library( ], deps = [ ":zero_copy_stream", - "//upb:mem", - "//upb:port", + "//upb/mem", + "//upb/port", ], ) @@ -62,7 +62,7 @@ cc_test( srcs = ["string_test.cc"], deps = [ ":string", - "//upb:mem", + "//upb/mem", "@googletest//:gtest", "@googletest//:gtest_main", ], @@ -77,9 +77,9 @@ cc_test( ":string", ":tokenizer", ":zero_copy_stream", - "//upb:mem", - "//upb:port", "//upb/lex", + "//upb/mem", + "//upb/port", "@abseil-cpp//absl/strings", "@abseil-cpp//absl/strings:str_format", "@googletest//:gtest", @@ -96,8 +96,8 @@ cc_test( deps = [ ":chunked_stream", ":zero_copy_stream", - "//upb:base", - "//upb:mem", + "//upb/base", + "//upb/mem", "@googletest//:gtest", "@googletest//:gtest_main", ], diff --git a/upb/json/BUILD b/upb/json/BUILD index 04fef51174a6d..37afb0552673e 100644 --- a/upb/json/BUILD +++ b/upb/json/BUILD @@ -23,14 +23,14 @@ cc_library( copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [ - "//upb:base", - "//upb:mem", - "//upb:message", - "//upb:mini_table", - "//upb:port", - "//upb:reflection", - "//upb:wire", + "//upb/base", "//upb/lex", + "//upb/mem", + "//upb/message", + "//upb/mini_table", + "//upb/port", + "//upb/reflection", + "//upb/wire", ], ) @@ -42,9 +42,9 @@ cc_test( ":test_upb_proto", ":test_upb_proto_reflection", "//:struct_upb_proto", - "//upb:base", - "//upb:mem", - "//upb:reflection", + "//upb/base", + "//upb/mem", + "//upb/reflection", "@googletest//:gtest", "@googletest//:gtest_main", ], @@ -58,9 +58,9 @@ cc_test( ":test_upb_proto", ":test_upb_proto_reflection", "//:struct_upb_proto", - "//upb:base", - "//upb:mem", - "//upb:reflection", + "//upb/base", + "//upb/mem", + "//upb/reflection", "@googletest//:gtest", "@googletest//:gtest_main", ], diff --git a/upb/lex/BUILD b/upb/lex/BUILD index 0ae91b27c602e..ff927f3477aee 100644 --- a/upb/lex/BUILD +++ b/upb/lex/BUILD @@ -25,7 +25,7 @@ cc_library( ], copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], - deps = ["//upb:port"], + deps = ["//upb/port"], ) cc_test( diff --git a/upb/mem/BUILD b/upb/mem/BUILD index 30941c0e337d8..d4e6962f58f20 100644 --- a/upb/mem/BUILD +++ b/upb/mem/BUILD @@ -24,7 +24,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":internal", - "//upb:port", + "//upb/port", ], ) @@ -36,7 +36,7 @@ cc_library( copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [ - "//upb:port", + "//upb/port", ], ) @@ -45,7 +45,7 @@ cc_test( srcs = ["arena_test.cc"], deps = [ ":mem", - "//upb:port", + "//upb/port", "@abseil-cpp//absl/base:core_headers", "@abseil-cpp//absl/cleanup", "@abseil-cpp//absl/container:flat_hash_map", diff --git a/upb/message/BUILD b/upb/message/BUILD index d3dcb9c7f893d..a3280c607e3ad 100644 --- a/upb/message/BUILD +++ b/upb/message/BUILD @@ -44,13 +44,13 @@ cc_library( deps = [ ":internal", ":types", - "//upb:base", - "//upb:mem", - "//upb:mini_table", - "//upb:port", + "//upb/base", "//upb/base:internal", "//upb/hash", + "//upb/mem", + "//upb/mini_table", "//upb/mini_table:internal", + "//upb/port", ], ) @@ -63,8 +63,8 @@ cc_library( visibility = ["//upb:__subpackages__"], deps = [ ":message", - "//upb:base", - "//upb:mini_table", + "//upb/base", + "//upb/mini_table", ], ) @@ -88,13 +88,13 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":types", - "//upb:base", - "//upb:mem", - "//upb:mini_table", - "//upb:port", + "//upb/base", "//upb/base:internal", "//upb/hash", + "//upb/mem", + "//upb/mini_table", "//upb/mini_table:internal", + "//upb/port", ], ) @@ -111,8 +111,8 @@ cc_library( deps = [ ":internal", ":message", - "//upb:mini_table", - "//upb:port", + "//upb/mini_table", + "//upb/port", ], ) @@ -132,13 +132,13 @@ cc_library( ":internal", ":iterator", ":message", - "//upb:base", - "//upb:eps_copy_input_stream", - "//upb:mem", - "//upb:mini_table", - "//upb:port", - "//upb:wire_reader", + "//upb/base", + "//upb/mem", + "//upb/mini_table", "//upb/mini_table:internal", + "//upb/port", + "//upb/wire:eps_copy_input_stream", + "//upb/wire:reader", ], ) @@ -157,13 +157,13 @@ cc_library( deps = [ ":internal", ":message", - "//upb:base", - "//upb:mem", - "//upb:mini_table", - "//upb:port", - "//upb:wire", + "//upb/base", "//upb/base:internal", + "//upb/mem", + "//upb/mini_table", "//upb/mini_table:internal", + "//upb/port", + "//upb/wire", ], ) @@ -180,13 +180,13 @@ cc_library( deps = [ ":internal", ":message", - "//upb:base", - "//upb:eps_copy_input_stream", - "//upb:mem", - "//upb:mini_table", - "//upb:port", - "//upb:wire", - "//upb:wire_reader", + "//upb/base", + "//upb/mem", + "//upb/mini_table", + "//upb/port", + "//upb/wire", + "//upb/wire:eps_copy_input_stream", + "//upb/wire:reader", ], ) @@ -196,15 +196,15 @@ cc_test( deps = [ ":copy", ":internal", - "//upb:base", - "//upb:mem", - "//upb:message", - "//upb:mini_table", - "//upb:port", - "//upb:wire", + ":message", + "//upb/base", + "//upb/mem", + "//upb/mini_table", + "//upb/port", "//upb/test:test_messages_proto2_upb_minitable", "//upb/test:test_messages_proto2_upb_proto", "//upb/test:test_upb_proto", + "//upb/wire", "@googletest//:gtest", "@googletest//:gtest_main", ], @@ -219,7 +219,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":message", - "//upb:port", + "//upb/port", ], ) @@ -232,9 +232,9 @@ cc_library( copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [ - "//upb:base", - "//upb:port", + "//upb/base", "//upb/hash", + "//upb/port", ], ) @@ -269,18 +269,18 @@ cc_test( deps = [ ":message", "//src/google/protobuf", - "//upb:base", - "//upb:mem", - "//upb:mini_descriptor", - "//upb:mini_table", - "//upb:port", - "//upb:wire", + "//upb/base", + "//upb/mem", + "//upb/mini_descriptor", "//upb/mini_descriptor:internal", + "//upb/mini_table", + "//upb/port", "//upb/test:test_messages_proto2_upb_minitable", "//upb/test:test_messages_proto2_upb_proto", "//upb/test:test_messages_proto3_upb_minitable", "//upb/test:test_messages_proto3_upb_proto", "//upb/test:test_upb_proto", + "//upb/wire", "@abseil-cpp//absl/container:flat_hash_set", "@googletest//:gtest", "@googletest//:gtest_main", @@ -292,8 +292,8 @@ cc_test( srcs = ["array_test.cc"], deps = [ ":message", - "//upb:base", - "//upb:mem", + "//upb/base", + "//upb/mem", "@googletest//:gtest", "@googletest//:gtest_main", ], @@ -306,13 +306,13 @@ cc_test( ":compare", ":internal", ":message", - "//upb:base", - "//upb:mem", - "//upb:port", - "//upb:wire", - "//upb:wire_reader", + "//upb/base", "//upb/base:internal", + "//upb/mem", + "//upb/port", "//upb/test:test_messages_proto2_upb_proto", + "//upb/wire", + "//upb/wire:reader", "//upb/wire/test_util:wire_message", "@googletest//:gtest", "@googletest//:gtest_main", @@ -327,14 +327,14 @@ cc_test( ":internal", ":message", "//src/google/protobuf", - "//upb:base", - "//upb:mem", - "//upb:mini_table", - "//upb:port", - "//upb:wire", + "//upb/base", + "//upb/mem", + "//upb/mini_table", + "//upb/port", "//upb/test:test_messages_proto2_upb_minitable", "//upb/test:test_messages_proto2_upb_proto", "//upb/test:test_upb_proto", + "//upb/wire", "@abseil-cpp//absl/container:flat_hash_set", "@googletest//:gtest", "@googletest//:gtest_main", @@ -346,8 +346,8 @@ cc_test( srcs = ["map_test.cc"], deps = [ ":message", - "//upb:base", - "//upb:mem", + "//upb/base", + "//upb/mem", "@googletest//:gtest", "@googletest//:gtest_main", ], @@ -362,16 +362,16 @@ cc_test( ":message", ":promote", "//src/google/protobuf", - "//upb:base", - "//upb:mem", - "//upb:mini_descriptor", - "//upb:mini_table", - "//upb:wire", + "//upb/base", + "//upb/mem", + "//upb/mini_descriptor", "//upb/mini_descriptor:internal", + "//upb/mini_table", "//upb/test:test_messages_proto2_upb_proto", "//upb/test:test_messages_proto3_upb_proto", "//upb/test:test_proto_upb_minitable", "//upb/test:test_upb_proto", + "//upb/wire", "@abseil-cpp//absl/container:flat_hash_set", "@abseil-cpp//absl/strings:string_view", "@googletest//:gtest", @@ -387,26 +387,26 @@ cc_test( args = ["--fuzztest_stack_limit_kb=256"], copts = UPB_DEFAULT_CPPOPTS, deps = [ + ":compare", ":internal", ":message", ":message_test_upb_minitable_proto", ":message_test_upb_proto", ":message_test_upb_proto_reflection", - "//upb:base", - "//upb:eps_copy_input_stream", - "//upb:json", - "//upb:mem", - "//upb:message_compare", - "//upb:mini_descriptor", - "//upb:mini_table", - "//upb:port", - "//upb:reflection", - "//upb:wire", - "//upb:wire_reader", + "//upb/base", + "//upb/json", + "//upb/mem", + "//upb/mini_descriptor", + "//upb/mini_table", "//upb/mini_table:internal", + "//upb/port", + "//upb/reflection", "//upb/test:fuzz_util", "//upb/test:test_messages_proto3_upb_proto", "//upb/text:debug", + "//upb/wire", + "//upb/wire:eps_copy_input_stream", + "//upb/wire:reader", "//upb/wire:writer", "@abseil-cpp//absl/cleanup", "@abseil-cpp//absl/numeric:bits", @@ -460,9 +460,9 @@ cc_test( ":utf8_test_proto2_upb_proto", ":utf8_test_upb_minitable_proto", ":utf8_test_upb_proto", - "//upb:base", - "//upb:mem", - "//upb:wire", + "//upb/base", + "//upb/mem", + "//upb/wire", "@googletest//:gtest", "@googletest//:gtest_main", ], diff --git a/upb/mini_descriptor/BUILD b/upb/mini_descriptor/BUILD index 8d6a970020754..27fa4facb38ba 100644 --- a/upb/mini_descriptor/BUILD +++ b/upb/mini_descriptor/BUILD @@ -22,13 +22,13 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":internal", - "//upb:base", - "//upb:mem", - "//upb:mini_table", - "//upb:port", + "//upb/base", "//upb/base:internal", + "//upb/mem", "//upb/message:types", + "//upb/mini_table", "//upb/mini_table:internal", + "//upb/port", ] + select({ "//upb:fasttable_enabled_setting": [ "//upb/wire/decode_fast:function_array", @@ -55,9 +55,9 @@ cc_library( copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [ - "//upb:base", - "//upb:port", + "//upb/base", "//upb/base:internal", + "//upb/port", ], ) @@ -69,13 +69,13 @@ cc_test( ":internal", ":mini_descriptor", "//src/google/protobuf", - "//upb:base", - "//upb:mem", - "//upb:message", - "//upb:mini_table", - "//upb:port", - "//upb:wire", + "//upb/base", + "//upb/mem", + "//upb/message", "//upb/message:internal", + "//upb/mini_table", + "//upb/port", + "//upb/wire", "@abseil-cpp//absl/container:flat_hash_set", "@googletest//:gtest", "@googletest//:gtest_main", diff --git a/upb/mini_descriptor/decode.h b/upb/mini_descriptor/decode.h index b5ea16858ac00..d22fd21a2b0d6 100644 --- a/upb/mini_descriptor/decode.h +++ b/upb/mini_descriptor/decode.h @@ -8,9 +8,13 @@ #ifndef UPB_MINI_TABLE_DECODE_H_ #define UPB_MINI_TABLE_DECODE_H_ +#include + #include "upb/base/status.h" #include "upb/mem/arena.h" +#include "upb/mini_table/enum.h" #include "upb/mini_table/extension.h" +#include "upb/mini_table/message.h" #include "upb/mini_table/sub.h" // Export the newer headers, for legacy users. New users should include the @@ -81,7 +85,7 @@ UPB_API_INLINE upb_MiniTableExtension* upb_MiniTableExtension_Build( UPB_API_INLINE upb_MiniTableExtension* upb_MiniTableExtension_BuildMessage( const char* data, size_t len, const upb_MiniTable* extendee, - upb_MiniTable* submsg, upb_Arena* arena, upb_Status* status) { + const upb_MiniTable* submsg, upb_Arena* arena, upb_Status* status) { upb_MiniTableSub sub = upb_MiniTableSub_FromMessage(submsg); return _upb_MiniTableExtension_Build( data, len, extendee, sub, kUpb_MiniTablePlatform_Native, arena, status); @@ -89,7 +93,7 @@ UPB_API_INLINE upb_MiniTableExtension* upb_MiniTableExtension_BuildMessage( UPB_API_INLINE upb_MiniTableExtension* upb_MiniTableExtension_BuildEnum( const char* data, size_t len, const upb_MiniTable* extendee, - upb_MiniTableEnum* subenum, upb_Arena* arena, upb_Status* status) { + const upb_MiniTableEnum* subenum, upb_Arena* arena, upb_Status* status) { upb_MiniTableSub sub = upb_MiniTableSub_FromEnum(subenum); return _upb_MiniTableExtension_Build( data, len, extendee, sub, kUpb_MiniTablePlatform_Native, arena, status); diff --git a/upb/mini_table/BUILD b/upb/mini_table/BUILD index 8b0f54d7c779e..a609f0f15a6ac 100644 --- a/upb/mini_table/BUILD +++ b/upb/mini_table/BUILD @@ -36,10 +36,10 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":internal", - "//upb:base", - "//upb:mem", - "//upb:port", + "//upb/base", "//upb/hash", + "//upb/mem", + "//upb/port", ], ) @@ -60,11 +60,11 @@ cc_library( copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [ - "//upb:base", - "//upb:mem", - "//upb:port", + "//upb/base", "//upb/hash", + "//upb/mem", "//upb/message:types", + "//upb/port", ], ) @@ -80,10 +80,10 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":mini_table", - "//upb:base", - "//upb:mem", - "//upb:port", + "//upb/base", "//upb/hash", + "//upb/mem", + "//upb/port", ], ) @@ -91,7 +91,7 @@ cc_test( name = "compat_test", srcs = ["compat_test.cc"], deps = [ - "//upb:mini_table_compat", + ":compat", "//upb/test:test_messages_proto2_upb_minitable", "//upb/test:test_messages_proto3_upb_minitable", "//upb/test:test_upb_proto", @@ -117,10 +117,10 @@ cc_binary( testonly = 1, srcs = ["message_benchmark.cc"], deps = [ + ":compat", ":message_benchmark_upb_minitable_proto", ":mini_table", - "//upb:mini_table_compat", - "//upb:port", + "//upb/port", "@abseil-cpp//absl/random", "@google_benchmark//:benchmark_main", "@googletest//:gtest", diff --git a/upb/reflection/BUILD b/upb/reflection/BUILD index d7c2c310151c1..0e0617ca16053 100644 --- a/upb/reflection/BUILD +++ b/upb/reflection/BUILD @@ -44,13 +44,13 @@ bootstrap_cc_library( copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [ - "//upb:base", - "//upb:mem", - "//upb:message", - "//upb:mini_descriptor", - "//upb:mini_table", - "//upb:port", + "//upb/base", + "//upb/mem", + "//upb/message", "//upb/message:internal", + "//upb/mini_descriptor", + "//upb/mini_table", + "//upb/port", ], ) @@ -115,19 +115,19 @@ bootstrap_cc_library( copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [ - "//upb:base", - "//upb:mem", - "//upb:message", - "//upb:message_copy", - "//upb:mini_descriptor", - "//upb:mini_table", - "//upb:port", - "//upb:wire", + "//upb/base", "//upb/base:internal", "//upb/hash", + "//upb/mem", + "//upb/message", + "//upb/message:copy", "//upb/message:internal", + "//upb/mini_descriptor", "//upb/mini_descriptor:internal", + "//upb/mini_table", "//upb/mini_table:internal", + "//upb/port", + "//upb/wire", ], ) @@ -139,13 +139,16 @@ cc_library( ], copts = UPB_DEFAULT_COPTS, textual_hdrs = [ - "//upb:port_inc", + "//upb/port:inc", + ], + visibility = [ + "//upb:__pkg__", + "//upb:friends", ], - visibility = ["//upb:__pkg__"], deps = [ ":internal", - "//upb:mem", - "//upb:mini_descriptor", + "//upb/mem", + "//upb/mini_descriptor", ], ) @@ -160,13 +163,13 @@ cc_test( deps = [ ":descriptor_upb_proto", ":internal", - "//upb:base", - "//upb:mem", - "//upb:mini_descriptor", - "//upb:mini_table", - "//upb:port", - "//upb:reflection", + ":reflection", + "//upb/base", "//upb/hash", + "//upb/mem", + "//upb/mini_descriptor", + "//upb/mini_table", + "//upb/port", "@abseil-cpp//absl/strings", "@googletest//:gtest", "@googletest//:gtest_main", diff --git a/upb/test/BUILD b/upb/test/BUILD index c435a2956e395..87a8e74f20d6f 100644 --- a/upb/test/BUILD +++ b/upb/test/BUILD @@ -163,12 +163,12 @@ cc_library( hdrs = ["fuzz_util.h"], visibility = ["//upb:__subpackages__"], deps = [ - "//upb:base", - "//upb:mem", - "//upb:message", - "//upb:mini_descriptor", - "//upb:mini_table", - "//upb:port", + "//upb/base", + "//upb/mem", + "//upb/message", + "//upb/mini_descriptor", + "//upb/mini_table", + "//upb/port", ], ) @@ -179,7 +179,7 @@ cc_test( deps = [ ":proto3_test_upb_proto", ":proto3_test_upb_proto_reflection", - "//upb:reflection", + "//upb/reflection", "@googletest//:gtest", "@googletest//:gtest_main", ], @@ -192,9 +192,9 @@ cc_test( deps = [ ":editions_test_upb_c_proto", ":editions_test_upb_proto_reflection", - "//upb:base", - "//upb:mem", - "//upb:reflection", + "//upb/base", + "//upb/mem", + "//upb/reflection", "@googletest//:gtest", "@googletest//:gtest_main", ], @@ -207,13 +207,13 @@ cc_test( deps = [ ":test_messages_proto2_upb_minitable", ":test_messages_proto2_upb_proto", - "//upb:base", - "//upb:mem", - "//upb:message", - "//upb:message_compare", - "//upb:mini_table", - "//upb:wire", + "//upb/base", + "//upb/mem", "//upb/mem:internal", + "//upb/message", + "//upb/message:compare", + "//upb/mini_table", + "//upb/wire", "@googletest//:gtest", "@googletest//:gtest_main", ], @@ -228,10 +228,10 @@ cc_test( ":test_cpp_upb_proto_reflection", ":timestamp_upb_proto", ":timestamp_upb_proto_reflection", - "//upb:base", - "//upb:json", - "//upb:port", - "//upb:reflection", + "//upb/base", + "//upb/json", + "//upb/port", + "//upb/reflection", "@googletest//:gtest", "@googletest//:gtest_main", ], @@ -247,10 +247,10 @@ cc_test( ":test_messages_proto2_upb_proto", ":test_messages_proto3_upb_proto", ":test_upb_proto", - "//upb:base", - "//upb:mem", - "//upb:message", - "//upb:port", + "//upb/base", + "//upb/mem", + "//upb/message", + "//upb/port", "@googletest//:gtest", "@googletest//:gtest_main", ], @@ -266,7 +266,7 @@ cc_test( ":proto3_test_upb_proto", ":test_messages_proto2_upb_minitable", ":test_upb_proto", - "//upb:mini_table", + "//upb/mini_table", "@googletest//:gtest", "@googletest//:gtest_main", ], diff --git a/upb/text/BUILD b/upb/text/BUILD index 5a59239dcd48c..bc99dc0c4fa15 100644 --- a/upb/text/BUILD +++ b/upb/text/BUILD @@ -22,15 +22,15 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":internal", - "//upb:base", - "//upb:eps_copy_input_stream", - "//upb:message", - "//upb:port", - "//upb:reflection", + "//upb/base", "//upb/hash", "//upb/lex", + "//upb/message", "//upb/message:internal", "//upb/message:types", + "//upb/port", + "//upb/reflection", + "//upb/wire:eps_copy_input_stream", ], ) @@ -47,15 +47,15 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":internal", - "//upb:base", - "//upb:message", - "//upb:mini_table", - "//upb:port", + "//upb/base", "//upb/lex", + "//upb/message", "//upb/message:internal", "//upb/message:iterator", "//upb/message:types", + "//upb/mini_table", "//upb/mini_table:internal", + "//upb/port", ], ) @@ -72,13 +72,13 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//third_party/utf8_range", - "//upb:base", - "//upb:eps_copy_input_stream", - "//upb:message", - "//upb:port", - "//upb:wire_reader", + "//upb/base", "//upb/lex", + "//upb/message", "//upb/message:internal", + "//upb/port", + "//upb/wire:eps_copy_input_stream", + "//upb/wire:reader", ], ) @@ -89,16 +89,16 @@ cc_test( ], deps = [ ":debug", - "//upb:base", - "//upb:eps_copy_input_stream", - "//upb:mem", - "//upb:message", - "//upb:mini_table", - "//upb:port", - "//upb:wire_reader", + "//upb/base", + "//upb/mem", + "//upb/message", "//upb/message:internal", + "//upb/mini_table", + "//upb/port", "//upb/test:test_proto_upb_minitable", "//upb/test:test_upb_proto", + "//upb/wire:eps_copy_input_stream", + "//upb/wire:reader", "@abseil-cpp//absl/log:absl_log", "@googletest//:gtest", "@googletest//:gtest_main", diff --git a/upb/util/BUILD b/upb/util/BUILD index d24ec8f514ecc..f8b9b7d60ccba 100644 --- a/upb/util/BUILD +++ b/upb/util/BUILD @@ -18,9 +18,9 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//src/google/protobuf:descriptor_upb_c_proto", - "//upb:base", - "//upb:port", - "//upb:reflection", + "//upb/base", + "//upb/port", + "//upb/reflection", "//upb/reflection:internal", ], ) @@ -54,8 +54,8 @@ cc_library( "//src/google/protobuf", "//src/google/protobuf:descriptor_upb_c_proto", "//src/google/protobuf/util:differencer", - "//upb:base", - "//upb:mem", + "//upb/base", + "//upb/mem", "//upb/reflection:internal", "@googletest//:gtest", ], @@ -75,9 +75,9 @@ cc_test( "//src/google/protobuf:descriptor_upb_c_proto", "//src/google/protobuf:descriptor_upb_reflection_proto", "//src/google/protobuf/util:differencer", - "//upb:base", - "//upb:mem", - "//upb:reflection", + "//upb/base", + "//upb/mem", + "//upb/reflection", "//upb/test:parse_text_proto", "@abseil-cpp//absl/strings", "@googletest//:gtest", @@ -93,11 +93,11 @@ cc_library( hdrs = ["required_fields.h"], visibility = ["//visibility:public"], deps = [ - "//upb:base", - "//upb:mem", - "//upb:message", - "//upb:port", - "//upb:reflection", + "//upb/base", + "//upb/mem", + "//upb/message", + "//upb/port", + "//upb/reflection", ], ) @@ -141,10 +141,10 @@ cc_test( ":required_fields", ":required_fields_test_upb_proto", ":required_fields_test_upb_proto_reflection", - "//upb:base", - "//upb:json", - "//upb:mem", - "//upb:reflection", + "//upb/base", + "//upb/json", + "//upb/mem", + "//upb/reflection", "//upb/reflection:internal", "@abseil-cpp//absl/strings", "@googletest//:gtest", diff --git a/upb/wire/BUILD b/upb/wire/BUILD index 6ff947a634547..449ef880c7a25 100644 --- a/upb/wire/BUILD +++ b/upb/wire/BUILD @@ -11,6 +11,9 @@ package(default_applicable_licenses = ["//:license"]) cc_library( name = "decoder", + srcs = [ + "internal/decoder.c", + ], hdrs = [ "decode.h", "internal/decoder.h", @@ -23,12 +26,13 @@ cc_library( deps = [ ":eps_copy_input_stream", "//third_party/utf8_range", - "//upb:mem", - "//upb:message", - "//upb:mini_table", - "//upb:port", + "//upb/mem", "//upb/mem:internal", + "//upb/message", + "//upb/message:internal", + "//upb/mini_table", "//upb/mini_table:internal", + "//upb/port", ], ) @@ -51,18 +55,18 @@ cc_library( ":reader", ":writer", "//third_party/utf8_range", - "//upb:base", - "//upb:mem", - "//upb:message", - "//upb:mini_table", - "//upb:port", + "//upb/base", "//upb/base:internal", "//upb/hash", + "//upb/mem", "//upb/mem:internal", + "//upb/message", "//upb/message:internal", "//upb/message:iterator", "//upb/message:types", + "//upb/mini_table", "//upb/mini_table:internal", + "//upb/port", ] + select({ "//upb:fasttable_enabled_setting": [ "//upb/wire/decode_fast:dispatch", @@ -77,11 +81,11 @@ cc_test( copts = UPB_DEFAULT_CPPOPTS, deps = [ ":wire", - "//upb:mem", - "//upb:message", - "//upb:mini_table", - "//upb:port", + "//upb/mem", + "//upb/message", "//upb/message:message_cc", + "//upb/mini_table", + "//upb/port", "//upb/wire/decode_fast:combinations", "//upb/wire/test_util:field_types", "//upb/wire/test_util:make_mini_table", @@ -99,11 +103,11 @@ cc_binary( srcs = ["decode_benchmark.cc"], copts = UPB_DEFAULT_CPPOPTS, deps = [ - "//upb:mem", - "//upb:message", - "//upb:mini_table", - "//upb:port", - "//upb:wire", + ":wire", + "//upb/mem", + "//upb/message", + "//upb/mini_table", + "//upb/port", "//upb/wire/decode_fast:combinations", "//upb/wire/test_util:field_types", "//upb/wire/test_util:make_mini_table", @@ -120,7 +124,7 @@ cc_library( hdrs = ["internal/reader.h"], copts = UPB_DEFAULT_COPTS, visibility = ["//upb:__pkg__"], - deps = ["//upb:port"], + deps = ["//upb/port"], ) cc_library( @@ -135,8 +139,8 @@ cc_library( deps = [ ":eps_copy_input_stream", ":reader_internal", - "//upb:port", "//upb/base:internal", + "//upb/port", ], ) @@ -151,7 +155,7 @@ cc_library( "//upb/message:__pkg__", ], deps = [ - "//upb:port", + "//upb/port", ], ) @@ -161,11 +165,11 @@ cc_library( hdrs = ["byte_size.h"], visibility = ["//visibility:public"], deps = [ - "//upb:mem", - "//upb:message", - "//upb:mini_table", - "//upb:port", - "//upb:wire", + ":wire", + "//upb/mem", + "//upb/message", + "//upb/mini_table", + "//upb/port", ], ) @@ -174,9 +178,9 @@ cc_test( srcs = ["byte_size_test.cc"], deps = [ ":byte_size", - "//upb:base", - "//upb:mem", - "//upb:mini_table", + "//upb/base", + "//upb/mem", + "//upb/mini_table", "//upb/test:test_messages_proto2_upb_minitable", "//upb/test:test_messages_proto2_upb_proto", "@googletest//:gtest", @@ -190,8 +194,8 @@ cc_library( hdrs = ["eps_copy_input_stream.h"], visibility = ["//visibility:public"], deps = [ - "//upb:mem", - "//upb:port", + "//upb/mem", + "//upb/port", ], ) @@ -200,7 +204,7 @@ cc_test( srcs = ["eps_copy_input_stream_test.cc"], deps = [ ":eps_copy_input_stream", - "//upb:mem", + "//upb/mem", "@googletest//:gtest", "@googletest//:gtest_main", ], diff --git a/upb/wire/decode.c b/upb/wire/decode.c index 6b75cf7dd53f1..481482df797e3 100644 --- a/upb/wire/decode.c +++ b/upb/wire/decode.c @@ -119,18 +119,6 @@ static const upb_MiniTableEnum* _upb_MiniTableSubs_EnumByField( return subs[field->UPB_PRIVATE(submsg_index)].UPB_PRIVATE(subenum); } -UPB_NORETURN static void* _upb_Decoder_ErrorJmp(upb_Decoder* d, - upb_DecodeStatus status) { - UPB_ASSERT(status != kUpb_DecodeStatus_Ok); - d->status = status; - UPB_LONGJMP(d->err, 1); -} - -const char* _upb_FastDecoder_ErrorJmp2(upb_Decoder* d) { - UPB_LONGJMP(d->err, 1); - return NULL; -} - static void _upb_Decoder_VerifyUtf8(upb_Decoder* d, const char* buf, int len) { if (!_upb_Decoder_VerifyUtf8Inline(buf, len)) { _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); @@ -782,18 +770,6 @@ static const char* _upb_Decoder_DecodeToSubMessage( return ptr; } -UPB_NOINLINE -const char* _upb_Decoder_CheckRequired(upb_Decoder* d, const char* ptr, - const upb_Message* msg, - const upb_MiniTable* m) { - UPB_ASSERT(m->UPB_PRIVATE(required_count)); - if (UPB_UNLIKELY(d->options & kUpb_DecodeOption_CheckRequired)) { - d->missing_required = - !UPB_PRIVATE(_upb_Message_IsInitializedShallow)(msg, m); - } - return ptr; -} - static const char* upb_Decoder_SkipField(upb_Decoder* d, const char* ptr, uint32_t tag) { int field_number = tag >> 3; @@ -1396,13 +1372,6 @@ static upb_DecodeStatus _upb_Decoder_DecodeTop(struct upb_Decoder* d, return kUpb_DecodeStatus_Ok; } -UPB_NOINLINE -const char* _upb_Decoder_IsDoneFallback(upb_EpsCopyInputStream* e, - const char* ptr, int overrun) { - return _upb_EpsCopyInputStream_IsDoneFallbackInline( - e, ptr, overrun, _upb_Decoder_BufferFlipCallback); -} - static upb_DecodeStatus upb_Decoder_Decode(upb_Decoder* const decoder, const char* const buf, upb_Message* const msg, diff --git a/upb/wire/decode_fast/BUILD b/upb/wire/decode_fast/BUILD index 53aa99c721671..2a669d9aef82d 100644 --- a/upb/wire/decode_fast/BUILD +++ b/upb/wire/decode_fast/BUILD @@ -33,17 +33,17 @@ cc_library( ":combinations", ":data", ":dispatch", - "//upb:base", - "//upb:mem", - "//upb:message", - "//upb:mini_table", - "//upb:port", - "//upb:wire_reader", + "//upb/base", "//upb/base:internal", + "//upb/mem", + "//upb/message", "//upb/message:internal", "//upb/message:types", + "//upb/mini_table", + "//upb/port", "//upb/wire:decoder", "//upb/wire:eps_copy_input_stream", + "//upb/wire:reader", ], ) @@ -54,10 +54,10 @@ cc_library( copts = FASTTABLE_COPTS, visibility = ["//upb/wire:__pkg__"], deps = [ - "//upb:message", - "//upb:mini_table", - "//upb:port", + "//upb/message", + "//upb/mini_table", "//upb/mini_table:internal", + "//upb/port", "//upb/wire:decoder", "//upb/wire:eps_copy_input_stream", ], @@ -76,12 +76,12 @@ cc_library( deps = [ ":combinations", ":data", - "//upb:base", - "//upb:mini_table", - "//upb:port", - "//upb:wire_reader", + "//upb/base", "//upb/base:internal", + "//upb/mini_table", "//upb/mini_table:internal", + "//upb/port", + "//upb/wire:reader", "@abseil-cpp//absl/strings", ], ) @@ -107,9 +107,9 @@ cc_library( "//upb/wire:__subpackages__", ], deps = [ - "//upb:port", - "//upb:wire_reader", "//upb/base:internal", + "//upb/port", + "//upb/wire:reader", ], ) @@ -122,7 +122,7 @@ cc_library( "//upb/wire:__subpackages__", ], deps = [ - "//upb:port", + "//upb/port", ], ) @@ -138,8 +138,8 @@ cc_library( deps = [ ":combinations", ":field_parsers", - "//upb:port", "//upb/mini_table:internal", + "//upb/port", ], ) diff --git a/upb/wire/decode_fast/dispatch.c b/upb/wire/decode_fast/dispatch.c index ebdb89c6e9fcd..bac147cd674f9 100644 --- a/upb/wire/decode_fast/dispatch.c +++ b/upb/wire/decode_fast/dispatch.c @@ -45,3 +45,8 @@ UPB_NOINLINE UPB_PRESERVE_NONE const char* upb_DecodeFast_MessageIsDoneFallback( UPB_UNREACHABLE(); } } + +const char* _upb_FastDecoder_ErrorJmp2(upb_Decoder* d) { + UPB_LONGJMP(d->err, 1); + return NULL; +} diff --git a/upb/wire/decode_fast/dispatch.h b/upb/wire/decode_fast/dispatch.h index 09c19553d6589..c3b96ec4c3643 100644 --- a/upb/wire/decode_fast/dispatch.h +++ b/upb/wire/decode_fast/dispatch.h @@ -187,6 +187,21 @@ typedef enum { const char* upb_DecodeFast_IsDoneFallback(UPB_PARSE_PARAMS); +/* Error function that will abort decoding with longjmp(). We can't declare this + * UPB_NORETURN, even though it is appropriate, because if we do then compilers + * will "helpfully" refuse to tailcall to it + * (see: https://stackoverflow.com/a/55657013), which will defeat a major goal + * of our optimizations. That is also why we must declare it in a separate file, + * otherwise the compiler will see that it calls longjmp() and deduce that it is + * noreturn. */ +const char* _upb_FastDecoder_ErrorJmp2(upb_Decoder* d); + +UPB_INLINE +const char* _upb_FastDecoder_ErrorJmp(upb_Decoder* d, upb_DecodeStatus status) { + d->status = status; + return _upb_FastDecoder_ErrorJmp2(d); +} + #define UPB_DECODEFAST_NEXTMAYBEPACKED(next, func_unpacked, func_packed) \ if (UPB_UNLIKELY(next != kUpb_DecodeFastNext_TailCallDispatch)) { \ switch (next) { \ diff --git a/upb/wire/encode.c b/upb/wire/encode.c index bb778618c95e8..06e83aab4faa7 100644 --- a/upb/wire/encode.c +++ b/upb/wire/encode.c @@ -141,8 +141,9 @@ static char* encode_fixed32(char* ptr, upb_encstate* e, uint32_t val) { #define UPB_PB_VARINT_MAX_LEN 10 -// Need gnu extended inline asm -#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +// Need gnu extended inline asm; msan can't instrument stores in inline assembly +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) && \ + !UPB_HAS_FEATURE(memory_sanitizer) #define UPB_ARM64_ASM #endif diff --git a/upb/wire/internal/decoder.c b/upb/wire/internal/decoder.c new file mode 100644 index 0000000000000..9f0599a62b13c --- /dev/null +++ b/upb/wire/internal/decoder.c @@ -0,0 +1,43 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "upb/wire/internal/decoder.h" + +#include "upb/message/internal/accessors.h" +#include "upb/message/message.h" +#include "upb/mini_table/message.h" +#include "upb/wire/decode.h" +#include "upb/wire/eps_copy_input_stream.h" + +// Must be last. +#include "upb/port/def.inc" + +UPB_NOINLINE +const char* _upb_Decoder_CheckRequired(upb_Decoder* d, const char* ptr, + const upb_Message* msg, + const upb_MiniTable* m) { + UPB_ASSERT(m->UPB_PRIVATE(required_count)); + if (UPB_UNLIKELY(d->options & kUpb_DecodeOption_CheckRequired)) { + d->missing_required = + !UPB_PRIVATE(_upb_Message_IsInitializedShallow)(msg, m); + } + return ptr; +} + +UPB_NORETURN void* _upb_Decoder_ErrorJmp(upb_Decoder* d, + upb_DecodeStatus status) { + UPB_ASSERT(status != kUpb_DecodeStatus_Ok); + d->status = status; + UPB_LONGJMP(d->err, 1); +} + +UPB_NOINLINE +const char* _upb_Decoder_IsDoneFallback(upb_EpsCopyInputStream* e, + const char* ptr, int overrun) { + return _upb_EpsCopyInputStream_IsDoneFallbackInline( + e, ptr, overrun, _upb_Decoder_BufferFlipCallback); +} diff --git a/upb/wire/internal/decoder.h b/upb/wire/internal/decoder.h index f1bdd13997fd2..8f5299d973237 100644 --- a/upb/wire/internal/decoder.h +++ b/upb/wire/internal/decoder.h @@ -112,21 +112,6 @@ UPB_INLINE void _upb_Decoder_Trace(upb_Decoder* d, char event) { #endif }; -/* Error function that will abort decoding with longjmp(). We can't declare this - * UPB_NORETURN, even though it is appropriate, because if we do then compilers - * will "helpfully" refuse to tailcall to it - * (see: https://stackoverflow.com/a/55657013), which will defeat a major goal - * of our optimizations. That is also why we must declare it in a separate file, - * otherwise the compiler will see that it calls longjmp() and deduce that it is - * noreturn. */ -const char* _upb_FastDecoder_ErrorJmp2(upb_Decoder* d); - -UPB_INLINE -const char* _upb_FastDecoder_ErrorJmp(upb_Decoder* d, upb_DecodeStatus status) { - d->status = status; - return _upb_FastDecoder_ErrorJmp2(d); -} - UPB_INLINE bool _upb_Decoder_VerifyUtf8Inline(const char* ptr, int len) { return utf8_range_IsValid(ptr, len); @@ -158,10 +143,13 @@ UPB_INLINE bool _upb_Decoder_IsDone(upb_Decoder* d, const char** ptr) { &d->input, ptr, &_upb_Decoder_IsDoneFallback); } +UPB_NORETURN void* _upb_Decoder_ErrorJmp(upb_Decoder* d, + upb_DecodeStatus status); + UPB_INLINE const char* _upb_Decoder_BufferFlipCallback( upb_EpsCopyInputStream* e, const char* old_end, const char* new_start) { upb_Decoder* d = (upb_Decoder*)e; - if (!old_end) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); + if (!old_end) _upb_Decoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); return new_start; } diff --git a/upb/wire/test_util/BUILD b/upb/wire/test_util/BUILD index 1110c956e0d2b..c43ce685b102a 100644 --- a/upb/wire/test_util/BUILD +++ b/upb/wire/test_util/BUILD @@ -20,13 +20,13 @@ cc_library( "//upb:__subpackages__", ], deps = [ - "//upb:base", - "//upb:mem", - "//upb:mini_descriptor", - "//upb:mini_table", - "//upb:port", - "//upb:wire_reader", + "//upb/base", + "//upb/mem", + "//upb/mini_descriptor", "//upb/mini_descriptor:internal", + "//upb/mini_table", + "//upb/port", + "//upb/wire:reader", "//upb/wire/decode_fast:combinations", "//upb/wire/decode_fast:data", "@abseil-cpp//absl/log", @@ -44,7 +44,7 @@ cc_library( ], deps = [ ":wire_message", - "//upb:base", + "//upb/base", "//upb/wire/decode_fast:combinations", "@abseil-cpp//absl/base", "@abseil-cpp//absl/strings:string_view", @@ -62,8 +62,8 @@ cc_library( "//upb:__subpackages__", ], deps = [ - "//upb:wire_reader", "//upb/base:internal", + "//upb/wire:reader", "@abseil-cpp//absl/log:absl_check", "@abseil-cpp//absl/strings:string_view", ], diff --git a/upb_generator/BUILD b/upb_generator/BUILD index ba8cde0702cd6..bb885d777717a 100644 --- a/upb_generator/BUILD +++ b/upb_generator/BUILD @@ -44,8 +44,8 @@ bootstrap_cc_library( visibility = ["//upb:friend_generators"], deps = [ "//src/google/protobuf:port", - "//upb:mini_table", - "//upb:port", + "//upb/mini_table", + "//upb/port", "@abseil-cpp//absl/strings", ], ) @@ -66,10 +66,10 @@ bootstrap_cc_library( copts = UPB_DEFAULT_CPPOPTS, visibility = ["//visibility:public"], deps = [ - "//upb:base", - "//upb:mini_descriptor", - "//upb:mini_table", - "//upb:port", + "//upb/base", + "//upb/mini_descriptor", + "//upb/mini_table", + "//upb/port", "@abseil-cpp//absl/container:flat_hash_map", "@abseil-cpp//absl/strings", ], @@ -96,9 +96,9 @@ bootstrap_cc_library( deps = [ "//src/google/protobuf", "//src/google/protobuf/compiler:code_generator_lite", - "//upb:base", - "//upb:mem", - "//upb:port", + "//upb/base", + "//upb/mem", + "//upb/port", "@abseil-cpp//absl/container:flat_hash_map", "@abseil-cpp//absl/container:flat_hash_set", "@abseil-cpp//absl/log:absl_check", diff --git a/upb_generator/bootstrap_compiler.bzl b/upb_generator/bootstrap_compiler.bzl index be686938d775d..32a70ab37d9f1 100644 --- a/upb_generator/bootstrap_compiler.bzl +++ b/upb_generator/bootstrap_compiler.bzl @@ -222,7 +222,7 @@ def bootstrap_upb_proto_library( defines = ["UPB_BOOTSTRAP_STAGE=0"], deps = [ "//upb:generated_code_support", - "//upb:mini_table", + "//upb/mini_table", ] + [dep + "_stage0" for dep in deps], **kwargs ) diff --git a/upb_generator/c/BUILD b/upb_generator/c/BUILD index fa89e26b8c50f..185e133f0b4f4 100644 --- a/upb_generator/c/BUILD +++ b/upb_generator/c/BUILD @@ -57,11 +57,11 @@ bootstrap_cc_binary( ":names_internal", "//src/google/protobuf/compiler:code_generator", "//src/google/protobuf/compiler:plugin", - "//upb:base", - "//upb:mem", - "//upb:mini_table", - "//upb:port", - "//upb:wire_reader", + "//upb/base", + "//upb/mem", + "//upb/mini_table", + "//upb/port", + "//upb/wire:reader", "//upb_generator/common:names", "//upb_generator/minitable:names", "//upb_generator/minitable:names_internal", diff --git a/upb_generator/common/BUILD b/upb_generator/common/BUILD index 78bf1727d5329..9860da9ce827f 100644 --- a/upb_generator/common/BUILD +++ b/upb_generator/common/BUILD @@ -32,10 +32,10 @@ cc_library( deps = [ "//src/google/protobuf", "//src/google/protobuf:descriptor_upb_c_proto", - "//upb:base", - "//upb:mem", - "//upb:mini_table", - "//upb:reflection", + "//upb/base", + "//upb/mem", + "//upb/mini_table", + "//upb/reflection", "@abseil-cpp//absl/container:flat_hash_set", "@abseil-cpp//absl/log:absl_check", ], diff --git a/upb_generator/minitable/BUILD b/upb_generator/minitable/BUILD index 1389f6bbbb9d3..0cdf29bc41d2d 100644 --- a/upb_generator/minitable/BUILD +++ b/upb_generator/minitable/BUILD @@ -33,12 +33,12 @@ bootstrap_cc_library( ":names_internal", "//src/google/protobuf/compiler:code_generator", "//src/google/protobuf/compiler:code_generator_lite", - "//upb:base", - "//upb:mem", - "//upb:mini_table", - "//upb:port", - "//upb:wire_reader", + "//upb/base", + "//upb/mem", + "//upb/mini_table", "//upb/mini_table:internal", + "//upb/port", + "//upb/wire:reader", "//upb/wire/decode_fast:select", "//upb_generator/common:names", "@abseil-cpp//absl/container:flat_hash_map", @@ -63,7 +63,7 @@ cc_library( ], deps = [ ":names_internal", - "//upb:port", + "//upb/port", "@abseil-cpp//absl/strings", ], ) @@ -106,9 +106,9 @@ bootstrap_cc_binary( "//src/google/protobuf/compiler:code_generator", "//src/google/protobuf/compiler:code_generator_lite", "//src/google/protobuf/compiler:plugin", - "//upb:base", - "//upb:mem", - "//upb:port", + "//upb/base", + "//upb/mem", + "//upb/port", "//upb_generator/common:names", "@abseil-cpp//absl/container:flat_hash_set", "@abseil-cpp//absl/log:absl_check", diff --git a/upb_generator/reflection/BUILD b/upb_generator/reflection/BUILD index 2c113f272893e..63e82116dbb15 100644 --- a/upb_generator/reflection/BUILD +++ b/upb_generator/reflection/BUILD @@ -17,7 +17,7 @@ cc_library( srcs = ["names.cc"], hdrs = ["names.h"], deps = [ - "//upb:port", + "//upb/port", "//upb_generator/common:names", "@abseil-cpp//absl/strings", ], @@ -45,9 +45,9 @@ cc_binary( "//src/google/protobuf/compiler:plugin", "//src/google/protobuf/io", "//src/google/protobuf/io:printer", - "//upb:mem", - "//upb:port", - "//upb:reflection", + "//upb/mem", + "//upb/port", + "//upb/reflection", "//upb/util:def_to_proto", "//upb_generator:common", "//upb_generator:file_layout", @@ -72,6 +72,6 @@ proto_lang_toolchain( plugin = ":protoc-gen-upbdefs", plugin_format_flag = "--plugin=protoc-gen-upbdefs=%s", progress_message = "Generating upb protos", - runtime = "//upb:generated_reflection_support", + runtime = "//upb/reflection:generated_reflection_support", visibility = ["//bazel:__pkg__"], )