protobuf: Update to the latest version

[tiagogaspar8]

Maintainer: Ken Keys kkeys@caida.org @kenkeys
Environment: All

Description:

The protobuf package is extremely out of date. I have tried to update it but unfortunately, I don't have enough knowledge to do it myself, and as I'm trying to update the netdata package, it needs the latest version.

[Ra2-IFV]

It introduced a new dependency, abseil, openwrt doen't have this

[tiagogaspar8]

I managed to compile protobuf, I will submit a pull request as soon as I can

[Ra2-IFV]

Yeah I also did it, just need to get a copy abseil and place it under third_party/abseil-cpp

index 7e60e8498..9cb94b726 100644
--- a/libs/protobuf/Makefile
+++ b/libs/protobuf/Makefile
@@ -8,19 +8,19 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=protobuf
-PKG_VERSION:=3.17.3
-PKG_RELEASE:=2
+PKG_VERSION:=28.3
+PKG_RELEASE:=1
 
-PKG_SOURCE:=$(PKG_NAME)-cpp-$(PKG_VERSION).tar.gz
-PKG_SOURCE_URL:=https://github.com/google/protobuf/releases/download/v$(PKG_VERSION)
-PKG_HASH:=51cec99f108b83422b7af1170afd7aeb2dd77d2bcbb7b6bad1f92509e9ccf8cb
+PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz
+PKG_SOURCE_URL:=https://github.com/protocolbuffers/protobuf/releases/download/v$(PKG_VERSION)
+PKG_HASH:=7c3ebd7aaedd86fa5dc479a0fda803f602caaf78d8aff7ce83b89e1b8ae7442a
 
 PKG_MAINTAINER:=Ken Keys <kkeys@caida.org>
 PKG_LICENSE:=BSD-3-Clause
 PKG_LICENSE_FILES:=LICENSE
 PKG_CPE_ID:=cpe:/a:google:protobuf
 
-CMAKE_SOURCE_SUBDIR:=cmake
+# CMAKE_SOURCE_SUBDIR:=cmake
 
 include $(INCLUDE_DIR)/package.mk
 include $(INCLUDE_DIR)/host-build.mk

[tiagogaspar8]

You don't need that, protobuff automatically pulls, compiles and installs abseil, I can share the makefile in a pull request later today, I'd appreciate your input 😁

[vortexilation]

How about separating abseil-cpp dependency from protobuf ? , this will make version pinning and updating easier without dependent to each other :

# SPDX-License-Identifier: GPL-2.0-only
#
# Copyright (C) 2021 Martin Schneider <martschneider@google.com>

include $(TOPDIR)/rules.mk

PKG_NAME:=abseil-cpp
PKG_VERSION:=20240116.1
PKG_RELEASE:=1

PKG_SOURCE_PROTO:=git
PKG_SOURCE_URL:=https://github.com/abseil/abseil-cpp.git
PKG_MIRROR_HASH:=1afa569daf7268a724924ab98b54af8cdbbefa33dc8e044ce5df9bd5e708f640
PKG_SOURCE_DATE:=2024-11-08
PKG_SOURCE_VERSION:=2f9e432cce407ce0ae50676696666f33a77d42ac

PKG_LICENSE:=MIT
PKG_LICENSE_FILES:=LICENSE

include $(INCLUDE_DIR)/package.mk
include $(INCLUDE_DIR)/cmake.mk

define Package/abseil-cpp
	SECTION:=libs
	CATEGORY:=Libraries
	TITLE:=Abseil Common Libraries
	URL:=https://github.com/abseil/abseil-cpp
	DEPENDS:=+libstdcpp
endef

define Package/abseil-cpp/description
	The repository contains the Abseil C++ library code. 
	Abseil is an open-source collection of C++ code (compliant to C++14) designed to augment the C++ standard library.
endef

CMAKE_OPTIONS+=-DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON

define Package/abseil-cpp/install
	$(INSTALL_DIR) $(1)/usr/lib
	$(CP) $(PKG_INSTALL_DIR)/usr/lib/* $(1)/usr/lib/
endef

define Build/InstallDev
	$(INSTALL_DIR) $(1)/usr/lib
	$(CP) $(PKG_INSTALL_DIR)/usr/lib/* $(1)/usr/lib
	$(INSTALL_DIR) $(1)/usr/include
	$(CP) $(PKG_INSTALL_DIR)/usr/include/* $(1)/usr/include
endef

$(eval $(call BuildPackage,abseil-cpp))

diff -Naur a/libs/protobuf/Makefile b/libs/protobuf/Makefile
--- a/libs/protobuf/Makefile
+++ b/libs/protobuf/Makefile
@@ -8,19 +8,21 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=protobuf
-PKG_VERSION:=3.17.3
-PKG_RELEASE:=2
+PKG_VERSION:=25.3
+PKG_RELEASE:=1
 
-PKG_SOURCE:=$(PKG_NAME)-cpp-$(PKG_VERSION).tar.gz
-PKG_SOURCE_URL:=https://github.com/google/protobuf/releases/download/v$(PKG_VERSION)
-PKG_HASH:=51cec99f108b83422b7af1170afd7aeb2dd77d2bcbb7b6bad1f92509e9ccf8cb
+PKG_SOURCE_PROTO:=git
+PKG_SOURCE_URL:=https://github.com/protocolbuffers/protobuf.git
+PKG_MIRROR_HASH:=63ddc2978bc7cdd357f66ed70f273a787960e990a328887b7f65d119094c7353
+PKG_SOURCE_DATE:=2024-11-08
+PKG_SOURCE_VERSION:=4a2aef570deb2bfb8927426558701e8bfc26f2a4
 
 PKG_MAINTAINER:=Ken Keys <kkeys@caida.org>
 PKG_LICENSE:=BSD-3-Clause
 PKG_LICENSE_FILES:=LICENSE
 PKG_CPE_ID:=cpe:/a:google:protobuf
 
-CMAKE_SOURCE_SUBDIR:=cmake
+CMAKE_BINARY_SUBDIR:=build
 
 include $(INCLUDE_DIR)/package.mk
 include $(INCLUDE_DIR)/host-build.mk
@@ -31,7 +33,7 @@
   CATEGORY:=Libraries
   TITLE:=A structured data encoding library
   URL:=https://github.com/google/protobuf
-  DEPENDS:=+zlib +libpthread +libatomic +libstdcpp
+  DEPENDS:=+zlib +libpthread +libatomic +libstdcpp +abseil-cpp
 endef
 
 define Package/protobuf
@@ -74,9 +76,12 @@
 	-Dprotobuf_BUILD_PROTOC_BINARIES=ON \
 	-Dprotobuf_BUILD_TESTS=OFF \
 	-Dprotobuf_WITH_ZLIB=ON \
-	-DBUILD_SHARED_LIBS=ON
+	-DBUILD_SHARED_LIBS=ON \
+	-Dprotobuf_ABSL_PROVIDER=package \
+	-Dabsl_DIR=$(STAGING_DIR)/usr/lib/cmake/absl
 
 TARGET_LDFLAGS += -latomic
+TARGET_CFLAGS += -fPIC
 
 define Build/InstallDev
 	$(call Build/InstallDev/cmake,$(1))
@@ -84,28 +89,23 @@
 	$(SED) 's,/usr/lib,$$$${exec_prefix}/lib,g' $(1)/usr/lib/pkgconfig/protobuf.pc
 	$(SED) 's,/usr/include,$$$${prefix}/include,g' $(1)/usr/lib/pkgconfig/protobuf-lite.pc
 	$(SED) 's,/usr/lib,$$$${exec_prefix}/lib,g' $(1)/usr/lib/pkgconfig/protobuf-lite.pc
+	$(INSTALL_DIR) $(1)/usr/lib $(1)/usr/include
+	$(CP) $(PKG_INSTALL_DIR)/usr/include/* $(1)/usr/include/
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/* $(1)/usr/lib/
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/cmake $(1)/usr/lib/cmake
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/pkgconfig $(1)/usr/lib/pkgconfig
 endef
 
 define Package/protobuf-lite/install
-	$(INSTALL_DIR) \
-		$(1)/usr/lib
-
-	$(CP) \
-		$(PKG_INSTALL_DIR)/usr/lib/libprotobuf-lite.so* \
-		$(1)/usr/lib/
+	$(INSTALL_DIR) $(1)/usr/lib
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/libutf8_* $(1)/usr/lib/
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/libprotobuf-lite.so* $(1)/usr/lib/
 endef
 
 define Package/protobuf/install
-	$(INSTALL_DIR) \
-		$(1)/usr/lib
-
-	$(CP) \
-		$(PKG_INSTALL_DIR)/usr/lib/libprotoc.so*  \
-		$(1)/usr/lib/
-
-	$(CP) \
-		$(PKG_INSTALL_DIR)/usr/lib/libprotobuf.so* \
-		$(1)/usr/lib/
+	$(INSTALL_DIR) $(1)/usr/lib
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/libprotoc.so* $(1)/usr/lib/
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/libprotobuf.so* $(1)/usr/lib/
 endef
 
 $(eval $(call BuildPackage,protobuf))

#
# Copyright (C) 2011 OpenWrt.org
#
# This is free software, licensed under the GNU General Public License v2.
# See /LICENSE for more information.
#

include $(TOPDIR)/rules.mk

PKG_NAME:=libprotobuf-c
PKG_VERSION:=1.5.0
PKG_RELEASE:=1

PKG_SOURCE_PROTO:=git
PKG_SOURCE_URL:=https://github.com/protobuf-c/protobuf-c.git
PKG_MIRROR_HASH:=09f22b5b5ef76c51969eefea9dd4d3e8cc5064b5c64de996d0596887c8cda34b
PKG_SOURCE_DATE:=2024-11-10
PKG_SOURCE_VERSION:=2480f4d9d2fa97e5511ed0914ee529a344e969a7
#PKG_BUILD_DIR:=$(BUILD_DIR)/protobuf-c-$(PKG_VERSION)
#HOST_BUILD_DIR:=$(BUILD_DIR_HOST)/protobuf-c-$(PKG_VERSION)

PKG_MAINTAINER:=
PKG_LICENSE:=BSD-2-Clause
PKG_LICENSE_FILES:=LICENSE
PKG_CPE_ID:=cpe:/a:protobuf-c_project:protobuf-c

HOST_BUILD_DEPENDS:=protobuf/host
PKG_BUILD_DEPENDS:=protobuf

CMAKE_INSTALL:=1
CMAKE_SOURCE_SUBDIR:=build-cmake

include $(INCLUDE_DIR)/package.mk
include $(INCLUDE_DIR)/host-build.mk
include $(INCLUDE_DIR)/cmake.mk
include $(INCLUDE_DIR)/nls.mk

define Package/libprotobuf-c
  TITLE:=Protocol Buffers library
  SECTION:=libs
  CATEGORY:=Libraries
  DEPENDS:=+libpthread +libatomic +libstdcpp +protobuf
  URL:=https://github.com/protobuf-c/protobuf-c
endef

define Package/libprotobuf-c/description
  Runtime library to use Google Protocol Buffers from C applications.
  Protocol Buffers are a way of encoding structured data in an efficient yet
  extensible format. Google uses Protocol Buffers for almost all of its
  internal RPC protocols and file formats.
endef

##For protoc & git runtime at configure
ifeq ($(CONFIG_BUILD_NLS),y)
	export LD_LIBRARY_PATH=$(STAGING_DIR)/usr/lib:$(ICONV_PREFIX)/lib:$(INTL_PREFIX)/lib
endif

CMAKE_HOST_OPTIONS += \
	-DBUILD_SHARED_LIBS=ON \
	-DCMAKE_CXX_STANDARD=11 \
	-DCMAKE_SKIP_RPATH=OFF \
	-DCMAKE_INSTALL_RPATH="${STAGING_DIR_HOSTPKG}/lib"

CMAKE_OPTIONS += \
	-DBUILD_SHARED_LIBS=ON \
	-DBUILD_PROTOC=ON \
	-DCMAKE_SKIP_RPATH=OFF \
	-DCMAKE_INSTALL_RPATH="${STAGING_DIR}/usr/lib"

define Build/InstallDev
	$(INSTALL_DIR) $(1)/usr/include
	$(CP) $(PKG_INSTALL_DIR)/usr/include/* $(1)/usr/include
	$(INSTALL_DIR) $(1)/usr/lib
	$(CP) $(PKG_INSTALL_DIR)/usr/lib/* $(1)/usr/lib/
	$(INSTALL_DIR) $(1)/usr/bin
	$(CP) $(PKG_INSTALL_DIR)/usr/bin/protoc-gen-c $(1)/usr/bin
	$(LN) protoc-gen-c $(1)/usr/bin/protoc-c
endef

define Package/libprotobuf-c/install
	$(INSTALL_DIR) $(1)/usr/lib
	$(CP) $(PKG_INSTALL_DIR)/usr/lib/libprotobuf-c.so* $(1)/usr/lib/
	
endef

$(eval $(call BuildPackage,libprotobuf-c))
$(eval $(call HostBuild))

I suggest to use protobuf version 25.3 or 26.0 as version above it is not supported by protobuf-c

Above Makefile changes has been tested by compiling frr and netdata v2.0.0.

[tiagogaspar8]

You managed to compile netdata 2.0? 😲
I have a pull request open here for aged and I was looking for help on how to get it to build...
#22677
Also, do you want to open a pull request for Protobuf?

[Ra2-IFV]

I fixed the patch for protobuf 28.3, if anyone needs it

I suggest to use protobuf version 25.3 or 26.0 as version above it is not supported by protobuf-c

Might need refreshing if you are updating to a different version

--- a/cmake/install.cmake
+++ b/cmake/install.cmake
@@ -35,8 +35,8 @@ endif ()
 
 foreach(_library ${_protobuf_libraries})
   if (UNIX AND NOT APPLE)
-    set_property(TARGET ${_library}
-      PROPERTY INSTALL_RPATH "$ORIGIN")
+    # set_property(TARGET ${_library}
+    #   PROPERTY INSTALL_RPATH "$ORIGIN")
   elseif (APPLE)
     set_property(TARGET ${_library}
       PROPERTY INSTALL_RPATH "@loader_path")
@@ -62,8 +62,8 @@ if (protobuf_BUILD_PROTOC_BINARIES)
   endif ()
   foreach (binary IN LISTS _protobuf_binaries)
     if (UNIX AND NOT APPLE)
-      set_property(TARGET ${binary}
-        PROPERTY INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}")
+      # set_property(TARGET ${binary}
+      #   PROPERTY INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}")
     elseif (APPLE)
       set_property(TARGET ${binary}
         PROPERTY INSTALL_RPATH "@loader_path/../lib")

[vortexilation]

@tiagogaspar8
Yes, but forgot to mention it's all on glibc, not musl, i will try musl now.
Probably harder on musl.

About your post on that PR, i am not encountering the same thing on glibc.

You could open up protobuf PR based on my finding if you want.

Here is the rough draft for netdata 2.0 package and libs package :
netdata.zip

The netdata 2.0 package from me is different, it using cmake build system instead.

[vortexilation]

@tiagogaspar8
Your previous issue on netdata repo, if I am not mistaken I have encountered it before, try to not using bundled abseil-cpp/protobuf from netdata.
Use it's own external abseil-cpp and protobuf OpenWrt package like above.

[EDIT]
On musl, it's totally different world, lot's of new errors.

Stuck with :

Error loading shared library libatomic.so.1: No such file or directory (needed by /home/user/works/openwrt/staging_dir/target-x86_64_musl/usr/bin/protoc)
Error loading shared library libprotoc.so.25.3.0: No such file or directory (needed by /home/user/works/openwrt/staging_dir/target-x86_64_musl/usr/bin/protoc)

Even tough I have set LD_LIBRARY_PATH

[EDIT#2]
Managed to compiled netdata 2.0 for musl targeting x86/64.

Need to add the following to protobuf Makefile :

TARGET_LDFLAGS += \
	-Wl,-rpath $(STAGING_DIR_ROOT)/lib:$(STAGING_DIR_ROOT)/usr/lib

[tiagogaspar8]

Hey @vortexilation, I was trying your suggestion but it's not working, and now I noticed you mentioned it's working for you when targeting x86/64, could you try your setup targeting another arch besides x86?
For reference, the latest error I'm seeing is:

[422/872] cd /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0 && /mnt/openwrt/ss_mod/staging_dir/host/bin/cmake -E make_directory /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/CMakeFiles && /mnt/openwrt/ss_mod/staging_dir/host/bin/cmake -E touch /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/CMakeFiles/ebpf-co-re-complete && /mnt/openwrt/ss_mod/staging_dir/host/bin/cmake -E touch /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/ebpf-co-re-prefix/src/ebpf-co-re-stamp/ebpf-co-re-done
[423/872] cd /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/ebpf-code-legacy-prefix/src/ebpf-code-legacy-build && sh -c "mkdir -p /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/ebpf-legacy-build/ebpf.d && mv /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/ebpf-legacy/*netdata_ebpf_*.o /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/ebpf-legacy-build/ebpf.d" && /mnt/openwrt/ss_mod/staging_dir/host/bin/cmake -E touch /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/ebpf-code-legacy-prefix/src/ebpf-code-legacy-stamp/ebpf-code-legacy-build
[424/872] cd /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0 && /mnt/openwrt/ss_mod/staging_dir/target-aarch64_cortex-a53_musl/usr/bin/protoc-28.3.0 -I/mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/src/aclk/aclk-schemas --cpp_out=/mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/src/aclk/aclk-schemas /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/src/aclk/aclk-schemas/proto/aclk/v1/lib.proto
FAILED: src/aclk/aclk-schemas/proto/aclk/v1/lib.pb.cc src/aclk/aclk-schemas/proto/aclk/v1/lib.pb.h /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/src/aclk/aclk-schemas/proto/aclk/v1/lib.pb.cc /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/src/aclk/aclk-schemas/proto/aclk/v1/lib.pb.h 
cd /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0 && /mnt/openwrt/ss_mod/staging_dir/target-aarch64_cortex-a53_musl/usr/bin/protoc-28.3.0 -I/mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/src/aclk/aclk-schemas --cpp_out=/mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/src/aclk/aclk-schemas /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/src/aclk/aclk-schemas/proto/aclk/v1/lib.proto
qemu-aarch64-static: Could not open '/lib/ld-musl-aarch64.so.1': No such file or directory
ninja: build stopped: subcommand failed.
make[2]: *** [Makefile:106: /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/.built] Error 1

Thanks!

[vortexilation]

@tiagogaspar8
Trying to build for aarch64_cortex-a53_musl now, but my build environment quite messy as currently testing for kernel 6.12.

Could you try to locate "ld-musl-aarch64.so.1" in your staging_dir directory?. I assume it's on $(STAGING_DIR_ROOT)/lib ?.

[EDIT]
Try to add this in your netdata Makefile :

export LD_LIBRARY_PATH=$(STAGING_DIR_ROOT)/lib

[tiagogaspar8]

Tried it but no luck unfortunately 😢
I think the issue stems from this line here:
https://github.com/netdata/netdata/blob/v2.0.0/packaging/cmake/Modules/NetdataProtobuf.cmake#L174
Netdata is trying to compile the files for aclk-schemas but it doesn't pass the LD flags to protobuf as it should, not sure how to fix it though

[vortexilation]

/mnt/openwrt/ss_mod/staging_dir/target-aarch64_cortex-a53_musl/usr/bin/protoc-28.3.0 -I/mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/src/aclk/aclk-schemas --cpp_out=/mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/src/aclk/aclk-schemas /mnt/openwrt/ss_mod/build_dir/target-aarch64_cortex-a53_musl/netdata-v2.0.0/src/aclk/aclk-schemas/proto/aclk/v1/lib.proto

I think it failed to run protoc successfully because of could not found ld-musl-aarch64.so.1 dependency.

Where do you put the line ?

export LD_LIBRARY_PATH=$(STAGING_DIR_ROOT)/lib

[tiagogaspar8]

BTW, forgot to say I found the mentioned lib in staging_dir/target-aarch64_cortex-a53_musl/root-mediatek/lib

I think it failed to run protoc successfully because of could not found ld-musl-aarch64.so.1 dependency.

Yes, I just thought it was looking for it in the wrong place because the error states it's looking for it in the /lib, which for me means root filesystem of the building host, am I wrong?

Where do you put the line?

I put it above the CMAKE_OPTIONS as in your makefile

[vortexilation]

BTW, forgot to say I found the mentioned lib in staging_dir/target-aarch64_cortex-a53_musl/root-mediatek/lib

Yes that's exactly what $(STAGING_DIR_ROOT)/lib is.

Yes, I just thought it was looking for it in the wrong place because the error states it's looking for it in the /lib, which for me means root filesystem of the building host, am I wrong?

Yes, you are correct, the netdata build system is not also looking at the OpenWrt toolchain, $(STAGING_DIR_ROOT)/lib , which is ld-musl-aarch64.so.1 located.

Btw what is your target device?