Add optional FlatBuffers messages and code generation by AnDiXL · Pull Request #3558 · apache/brpc · GitHub
Skip to content

Add optional FlatBuffers messages and code generation #3558

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AnDiXL wants to merge 5 commits into
base: master
Choose a base branch
from
+4,572 −16
Open

Add optional FlatBuffers messages and code generation #3558

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
36f2fea
Add FlatBuffers message and service support
Oct 14, 2026
3f5b5b0
Add FlatBuffers service generator and acceptance tests
Oct 14, 2026
da15f28
Verify FlatBuffers support in Linux and macOS CI
Oct 14, 2026
e60ca66
Document FlatBuffers support
Oct 14, 2026
1afbbff
Document using other FlatBuffers versions
Oct 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions .github/workflows/ci-linux.yml
View file
Open in desktop
28 changes: 28 additions & 0 deletions .github/workflows/ci-macos.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,34 @@ concurrency:
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
flatbuffers-codegen:
runs-on: macos-latest
strategy:
fail-fast: false
matrix:
protobuf: [protobuf@21, protobuf@29]
steps:
- uses: actions/checkout@v7.0.1
- name: Install codegen dependencies
run: |
brew install cmake openssl gflags leveldb snappy gperftools
if [ "${{ matrix.protobuf }}" = protobuf@21 ]; then
# Same local tap as compile-with-make-cmake-protobuf21.
brew tap-new --no-git local/pins
cp homebrew-formula/protobuf@21.rb "$(brew --repository local/pins)/Formula/"
brew install local/pins/protobuf@21
else
brew install ${{ matrix.protobuf }}
fi
- name: Verify FlatBuffers generation and runtime
env:
CXX: clang++
JOBS: 2
run: |
export CMAKE_PREFIX_PATH="$(brew --prefix ${{ matrix.protobuf }});$(brew --prefix)"
export OPENSSL_ROOT_DIR="$(brew --prefix openssl)"
bash tools/flatbuffers/verify.sh "$RUNNER_TEMP/flatbuffers-codegen"

compile-with-make-cmake-protobuf21:
runs-on: macos-latest # https://github.com/actions/runner-images

Expand Down
27 changes: 26 additions & 1 deletion BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@ licenses(["notice"]) # Apache v2

exports_files(["LICENSE"])

config_setting(
name = "brpc_with_flatbuffers",
define_values = {"BRPC_WITH_FLATBUFFERS": "true"},
visibility = ["//visibility:public"],
)

COPTS = [
"-fno-omit-frame-pointer",
] + select({
Expand All @@ -45,6 +51,9 @@ DEFINES = [
}) + select({
"//bazel/config:brpc_with_thrift": ["ENABLE_THRIFT_FRAMED_PROTOCOL=1"],
"//conditions:default": [],
}) + select({
":brpc_with_flatbuffers": ["BRPC_WITH_FLATBUFFERS=1"],
"//conditions:default": ["BRPC_WITH_FLATBUFFERS=0"],
}) + select({
"//bazel/config:brpc_with_thrift_legacy_version": [],
"//conditions:default": ["THRIFT_STDCXX=std"],
Expand Down Expand Up @@ -125,6 +134,14 @@ genrule(
"//conditions:default": "0",
}) +
"""
#ifdef BRPC_WITH_FLATBUFFERS
#undef BRPC_WITH_FLATBUFFERS
#endif
#define BRPC_WITH_FLATBUFFERS """ + select({
":brpc_with_flatbuffers": "1",
"//conditions:default": "0",
}) +
"""
#ifdef BUTIL_USE_CPU_FREQUENCY
#undef BUTIL_USE_CPU_FREQUENCY
#endif
Expand Down Expand Up @@ -542,6 +559,8 @@ brpc_proto_library(
visibility = ["//visibility:public"],
)

FLATBUFFERS_SRC_PATTERNS = ["src/brpc/flatbuffers/*.cpp"]

URMA_SRC_PATTERNS = [
"src/brpc/urma/*.cpp",
"src/brpc/urma/**/*.cpp",
Expand All @@ -565,7 +584,7 @@ BRPC_BASE_SRCS = glob(
"src/brpc/policy/thrift_protocol.cpp",
"src/brpc/event_dispatcher_epoll.cpp",
"src/brpc/event_dispatcher_kqueue.cpp",
] + URMA_SRC_PATTERNS,
] + URMA_SRC_PATTERNS + FLATBUFFERS_SRC_PATTERNS,
)

cc_library(
Expand All @@ -576,6 +595,9 @@ cc_library(
"src/brpc/**/thrift*.cpp",
]),
"//conditions:default": [],
}) + select({
":brpc_with_flatbuffers": glob(FLATBUFFERS_SRC_PATTERNS),
"//conditions:default": [],
}) + select({
"//bazel/config:brpc_with_urma_use_real": URMA_SRCS,
"//bazel/config:brpc_with_urma": URMA_SRCS + [
Expand Down Expand Up @@ -615,6 +637,9 @@ cc_library(
"@org_apache_thrift//:thrift",
],
"//conditions:default": [],
}) + select({
":brpc_with_flatbuffers": ["@com_github_google_flatbuffers//:runtime_cc"],
"//conditions:default": [],
}),
)

Expand Down
1 change: 1 addition & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ option(WITH_MESALINK "With MesaLink" OFF)
option(WITH_BORINGSSL "With BoringSSL" OFF)
option(WITH_DEBUG_SYMBOLS "With debug symbols" ON)
option(WITH_THRIFT "With thrift framed protocol supported" OFF)
option(WITH_FLATBUFFERS "With FlatBuffers message support (headers only)" OFF)
option(WITH_BTHREAD_TRACER "With bthread tracer supported" OFF)
option(WITH_SNAPPY "With snappy" OFF)
option(WITH_RDMA "With RDMA" OFF)
Expand Down
16 changes: 16 additions & 0 deletions MODULE.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,3 +79,19 @@ git_repository(
remote = 'https://atomgit.com/openeuler/umdk.git',
commit = '564ee727a55523d4351a8fb3c94292b388ebb924', # v26.06.0_CAM
)

# Do not use `bazel_dep(name = 'flatbuffers', version = '25.2.10')` here.
# The BCR module pulls gRPC and JS/Go/Swift rule dependencies for FlatBuffers'
# full upstream build. bRPC only needs runtime_cc and flatc, and gRPC would
# otherwise introduce another BoringSSL version even when BRPC_WITH_FLATBUFFERS
# is false. Keep this archive and checksum in sync with WORKSPACE.
flatbuffers_http_archive = use_repo_rule(
'@bazel_tools//tools/build_defs/repo:http.bzl',
'http_archive',
)
flatbuffers_http_archive(
name = 'com_github_google_flatbuffers',
sha256 = 'b9c2df49707c57a48fc0923d52b8c73beb72d675f9d44b2211e4569be40a7421',
strip_prefix = 'flatbuffers-25.2.10',
urls = ['https://github.com/google/flatbuffers/archive/refs/tags/v25.2.10.tar.gz'],
)
3 changes: 3 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,9 @@ JSON2PB_SOURCES = $(foreach d,$(JSON2PB_DIRS),$(wildcard $(addprefix $(d)/*,$(SR
JSON2PB_OBJS = $(addsuffix .o, $(basename $(JSON2PB_SOURCES)))

BRPC_DIRS = src/brpc src/brpc/details src/brpc/builtin src/brpc/policy src/brpc/policy/mysql src/brpc/rdma
ifeq ($(WITH_FLATBUFFERS),1)
BRPC_DIRS += src/brpc/flatbuffers
endif
ifeq ($(WITH_URMA),1)
BRPC_DIRS += src/brpc/urma
endif
Expand Down
8 changes: 8 additions & 0 deletions WORKSPACE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,14 @@ http_archive(
urls = ["https://github.com/google/crc32c/archive/1.1.2.tar.gz"],
)

# Optional FlatBuffers support uses runtime_cc; keep this version in sync with MODULE.bazel.
http_archive(
name = "com_github_google_flatbuffers",
integrity = "sha256-ucLfSXB8V6SPwJI9UrjHO+ty1nX51EsiEeRWm+QKdCE=",
strip_prefix = "flatbuffers-25.2.10",
urls = ["https://github.com/google/flatbuffers/archive/refs/tags/v25.2.10.tar.gz"],
)

http_archive(
name = "com_github_google_glog", # 2021-05-07T23:06:39Z
patch_args = ["-p1"],
Expand Down
19 changes: 17 additions & 2 deletions config_brpc.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,10 @@ else
LDD=ldd
fi

TEMP=`getopt -o v: --long headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-rdma,with-urma,with-urma-mock,without-urma-mock,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,with-riscv-zvbc,with-riscv-zbc,with-cpu-frequency,nodebugsymbols,werror -n 'config_brpc' -- "$@"`
TEMP=`getopt -o v: --long headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-flatbuffers,with-rdma,with-urma,with-urma-mock,without-urma-mock,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,with-riscv-zvbc,with-riscv-zbc,with-cpu-frequency,nodebugsymbols,werror -n 'config_brpc' -- "$@"`
WITH_GLOG=0
WITH_THRIFT=0
WITH_FLATBUFFERS=0
WITH_RDMA=0
WITH_URMA=0
URMA_MOCK_MODE=auto
Expand Down Expand Up @@ -91,6 +92,7 @@ while true; do
--cxx ) CXX=$2; shift 2 ;;
--with-glog ) WITH_GLOG=1; shift 1 ;;
--with-thrift) WITH_THRIFT=1; shift 1 ;;
--with-flatbuffers) WITH_FLATBUFFERS=1; shift 1 ;;
--with-rdma) WITH_RDMA=1; shift 1 ;;
--with-urma) WITH_URMA=1; shift 1 ;;
--with-urma-mock) URMA_MOCK_MODE=on; shift 1 ;;
Expand Down Expand Up @@ -479,14 +481,15 @@ append_to_output "HDRS=$($ECHO $HDRS)"
append_to_output "LIBS=$($ECHO $LIBS)"
append_to_output "PROTOC=$PROTOC"
append_to_output "PROTOBUF_HDR=$PROTOBUF_HDR"
append_to_output "WITH_FLATBUFFERS=$WITH_FLATBUFFERS"
append_to_output "CC=$CC"
append_to_output "CXX=$CXX"
append_to_output "GCC_VERSION=$GCC_VERSION"
append_to_output "STATIC_LINKINGS=$STATIC_LINKINGS"
append_to_output "DYNAMIC_LINKINGS=$DYNAMIC_LINKINGS"

# CPP means C PreProcessing, not C PlusPlus
CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_GLOG=$WITH_GLOG -DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=$BRPC_DEBUG_BTHREAD_SCHE_SAFETY -DBRPC_DEBUG_LOCK=$BRPC_DEBUG_LOCK -DBUTIL_USE_CPU_FREQUENCY=$WITH_CPU_FREQUENCY"
CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_GLOG=$WITH_GLOG -DBRPC_WITH_FLATBUFFERS=$WITH_FLATBUFFERS -DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=$BRPC_DEBUG_BTHREAD_SCHE_SAFETY -DBRPC_DEBUG_LOCK=$BRPC_DEBUG_LOCK -DBUTIL_USE_CPU_FREQUENCY=$WITH_CPU_FREQUENCY"

# Avoid over-optimizations of TLS variables by GCC>=4.8
# See: https://github.com/apache/brpc/issues/1693
Expand All @@ -506,6 +509,12 @@ if [ "$SYSTEM" = "Darwin" ]; then
fi
fi

if [ $WITH_FLATBUFFERS != 0 ]; then
FLATBUFFERS_HDR=$(find_dir_of_header_or_die flatbuffers/flatbuffers.h) || exit 1
append_to_output_headers "$FLATBUFFERS_HDR"
print_success "Found FlatBuffers headers: $FLATBUFFERS_HDR"
fi

if [ $WITH_THRIFT != 0 ]; then
THRIFT_LIB=$(find_dir_of_lib_or_die thriftnb)
THRIFT_HDR=$(find_dir_of_header_or_die thrift/Thrift.h)
Expand Down Expand Up @@ -692,6 +701,11 @@ cat << EOF > src/butil/config.h
#endif
#define BRPC_WITH_GLOG $WITH_GLOG

#ifdef BRPC_WITH_FLATBUFFERS
#undef BRPC_WITH_FLATBUFFERS
#endif
#define BRPC_WITH_FLATBUFFERS $WITH_FLATBUFFERS

#ifdef BUTIL_USE_CPU_FREQUENCY
#undef BUTIL_USE_CPU_FREQUENCY
#endif
Expand All @@ -714,6 +728,7 @@ print_info "C++ std: $CXXFLAGS"
print_info "System: $SYSTEM"
if [ $WITH_GLOG -ne 0 ]; then print_info "With glog: yes"; fi
if [ $WITH_THRIFT -ne 0 ]; then print_info "With thrift: yes"; fi
if [ $WITH_FLATBUFFERS -ne 0 ]; then print_info "With FlatBuffers: yes (headers only)"; fi
if [ $WITH_RDMA -ne 0 ]; then print_info "With RDMA: yes"; fi
if [ $WITH_URMA -ne 0 ]; then print_info "With URMA: yes"; fi
if [ $WITH_MESALINK -ne 0 ]; then print_info "With MesaLink: yes"; fi
Expand Down
116 changes: 116 additions & 0 deletions docs/cn/flatbuffers.md
View file
Open in desktop
Loading
Loading