protobuf: Update to the latest version #25194

tiagogaspar8 · 2024-10-24T11:19:01Z

Maintainer: Ken Keys [email protected] @kenkeys
Environment: All

Description:

The protobuf package is extremely out of date. I have tried to update it but unfortunately, I don't have enough knowledge to do it myself, and as I'm trying to update the netdata package, it needs the latest version.

Ra2-IFV · 2024-11-13T14:01:38Z

It introduced a new dependency, abseil, openwrt doen't have this

tiagogaspar8 · 2024-11-13T14:19:23Z

I managed to compile protobuf, I will submit a pull request as soon as I can

Ra2-IFV · 2024-11-13T14:29:31Z

Yeah I also did it, just need to get a copy abseil and place it under third_party/abseil-cpp

index 7e60e8498..9cb94b726 100644
--- a/libs/protobuf/Makefile
+++ b/libs/protobuf/Makefile
@@ -8,19 +8,19 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=protobuf
-PKG_VERSION:=3.17.3
-PKG_RELEASE:=2
+PKG_VERSION:=28.3
+PKG_RELEASE:=1
 
-PKG_SOURCE:=$(PKG_NAME)-cpp-$(PKG_VERSION).tar.gz
-PKG_SOURCE_URL:=https://github.com/google/protobuf/releases/download/v$(PKG_VERSION)
-PKG_HASH:=51cec99f108b83422b7af1170afd7aeb2dd77d2bcbb7b6bad1f92509e9ccf8cb
+PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz
+PKG_SOURCE_URL:=https://github.com/protocolbuffers/protobuf/releases/download/v$(PKG_VERSION)
+PKG_HASH:=7c3ebd7aaedd86fa5dc479a0fda803f602caaf78d8aff7ce83b89e1b8ae7442a
 
 PKG_MAINTAINER:=Ken Keys <[email protected]>
 PKG_LICENSE:=BSD-3-Clause
 PKG_LICENSE_FILES:=LICENSE
 PKG_CPE_ID:=cpe:/a:google:protobuf
 
-CMAKE_SOURCE_SUBDIR:=cmake
+# CMAKE_SOURCE_SUBDIR:=cmake
 
 include $(INCLUDE_DIR)/package.mk
 include $(INCLUDE_DIR)/host-build.mk

tiagogaspar8 · 2024-11-13T14:31:03Z

You don't need that, protobuff automatically pulls, compiles and installs abseil, I can share the makefile in a pull request later today, I'd appreciate your input 😁

vortexilation · 2024-11-13T14:54:39Z

How about separating abseil-cpp dependency from protobuf ? , this will make version pinning and updating easier without dependent to each other :

# SPDX-License-Identifier: GPL-2.0-only
#
# Copyright (C) 2021 Martin Schneider <[email protected]>

include $(TOPDIR)/rules.mk

PKG_NAME:=abseil-cpp
PKG_VERSION:=20240116.1
PKG_RELEASE:=1

PKG_SOURCE_PROTO:=git
PKG_SOURCE_URL:=https://github.com/abseil/abseil-cpp.git
PKG_MIRROR_HASH:=1afa569daf7268a724924ab98b54af8cdbbefa33dc8e044ce5df9bd5e708f640
PKG_SOURCE_DATE:=2024-11-08
PKG_SOURCE_VERSION:=2f9e432cce407ce0ae50676696666f33a77d42ac

PKG_LICENSE:=MIT
PKG_LICENSE_FILES:=LICENSE

include $(INCLUDE_DIR)/package.mk
include $(INCLUDE_DIR)/cmake.mk

define Package/abseil-cpp
	SECTION:=libs
	CATEGORY:=Libraries
	TITLE:=Abseil Common Libraries
	URL:=https://github.com/abseil/abseil-cpp
	DEPENDS:=+libstdcpp
endef

define Package/abseil-cpp/description
	The repository contains the Abseil C++ library code. 
	Abseil is an open-source collection of C++ code (compliant to C++14) designed to augment the C++ standard library.
endef

CMAKE_OPTIONS+=-DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON

define Package/abseil-cpp/install
	$(INSTALL_DIR) $(1)/usr/lib
	$(CP) $(PKG_INSTALL_DIR)/usr/lib/* $(1)/usr/lib/
endef

define Build/InstallDev
	$(INSTALL_DIR) $(1)/usr/lib
	$(CP) $(PKG_INSTALL_DIR)/usr/lib/* $(1)/usr/lib
	$(INSTALL_DIR) $(1)/usr/include
	$(CP) $(PKG_INSTALL_DIR)/usr/include/* $(1)/usr/include
endef

$(eval $(call BuildPackage,abseil-cpp))

diff -Naur a/libs/protobuf/Makefile b/libs/protobuf/Makefile
--- a/libs/protobuf/Makefile
+++ b/libs/protobuf/Makefile
@@ -8,19 +8,21 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=protobuf
-PKG_VERSION:=3.17.3
-PKG_RELEASE:=2
+PKG_VERSION:=25.3
+PKG_RELEASE:=1
 
-PKG_SOURCE:=$(PKG_NAME)-cpp-$(PKG_VERSION).tar.gz
-PKG_SOURCE_URL:=https://github.com/google/protobuf/releases/download/v$(PKG_VERSION)
-PKG_HASH:=51cec99f108b83422b7af1170afd7aeb2dd77d2bcbb7b6bad1f92509e9ccf8cb
+PKG_SOURCE_PROTO:=git
+PKG_SOURCE_URL:=https://github.com/protocolbuffers/protobuf.git
+PKG_MIRROR_HASH:=63ddc2978bc7cdd357f66ed70f273a787960e990a328887b7f65d119094c7353
+PKG_SOURCE_DATE:=2024-11-08
+PKG_SOURCE_VERSION:=4a2aef570deb2bfb8927426558701e8bfc26f2a4
 
 PKG_MAINTAINER:=Ken Keys <[email protected]>
 PKG_LICENSE:=BSD-3-Clause
 PKG_LICENSE_FILES:=LICENSE
 PKG_CPE_ID:=cpe:/a:google:protobuf
 
-CMAKE_SOURCE_SUBDIR:=cmake
+CMAKE_BINARY_SUBDIR:=build
 
 include $(INCLUDE_DIR)/package.mk
 include $(INCLUDE_DIR)/host-build.mk
@@ -31,7 +33,7 @@
   CATEGORY:=Libraries
   TITLE:=A structured data encoding library
   URL:=https://github.com/google/protobuf
-  DEPENDS:=+zlib +libpthread +libatomic +libstdcpp
+  DEPENDS:=+zlib +libpthread +libatomic +libstdcpp +abseil-cpp
 endef
 
 define Package/protobuf
@@ -74,9 +76,12 @@
 	-Dprotobuf_BUILD_PROTOC_BINARIES=ON \
 	-Dprotobuf_BUILD_TESTS=OFF \
 	-Dprotobuf_WITH_ZLIB=ON \
-	-DBUILD_SHARED_LIBS=ON
+	-DBUILD_SHARED_LIBS=ON \
+	-Dprotobuf_ABSL_PROVIDER=package \
+	-Dabsl_DIR=$(STAGING_DIR)/usr/lib/cmake/absl
 
 TARGET_LDFLAGS += -latomic
+TARGET_CFLAGS += -fPIC
 
 define Build/InstallDev
 	$(call Build/InstallDev/cmake,$(1))
@@ -84,28 +89,23 @@
 	$(SED) 's,/usr/lib,$$$${exec_prefix}/lib,g' $(1)/usr/lib/pkgconfig/protobuf.pc
 	$(SED) 's,/usr/include,$$$${prefix}/include,g' $(1)/usr/lib/pkgconfig/protobuf-lite.pc
 	$(SED) 's,/usr/lib,$$$${exec_prefix}/lib,g' $(1)/usr/lib/pkgconfig/protobuf-lite.pc
+	$(INSTALL_DIR) $(1)/usr/lib $(1)/usr/include
+	$(CP) $(PKG_INSTALL_DIR)/usr/include/* $(1)/usr/include/
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/* $(1)/usr/lib/
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/cmake $(1)/usr/lib/cmake
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/pkgconfig $(1)/usr/lib/pkgconfig
 endef
 
 define Package/protobuf-lite/install
-	$(INSTALL_DIR) \
-		$(1)/usr/lib
-
-	$(CP) \
-		$(PKG_INSTALL_DIR)/usr/lib/libprotobuf-lite.so* \
-		$(1)/usr/lib/
+	$(INSTALL_DIR) $(1)/usr/lib
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/libutf8_* $(1)/usr/lib/
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/libprotobuf-lite.so* $(1)/usr/lib/
 endef
 
 define Package/protobuf/install
-	$(INSTALL_DIR) \
-		$(1)/usr/lib
-
-	$(CP) \
-		$(PKG_INSTALL_DIR)/usr/lib/libprotoc.so*  \
-		$(1)/usr/lib/
-
-	$(CP) \
-		$(PKG_INSTALL_DIR)/usr/lib/libprotobuf.so* \
-		$(1)/usr/lib/
+	$(INSTALL_DIR) $(1)/usr/lib
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/libprotoc.so* $(1)/usr/lib/
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/libprotobuf.so* $(1)/usr/lib/
 endef
 
 $(eval $(call BuildPackage,protobuf))

#
# Copyright (C) 2011 OpenWrt.org
#
# This is free software, licensed under the GNU General Public License v2.
# See /LICENSE for more information.
#

include $(TOPDIR)/rules.mk

PKG_NAME:=libprotobuf-c
PKG_VERSION:=1.5.0
PKG_RELEASE:=1

PKG_SOURCE_PROTO:=git
PKG_SOURCE_URL:=https://github.com/protobuf-c/protobuf-c.git
PKG_MIRROR_HASH:=09f22b5b5ef76c51969eefea9dd4d3e8cc5064b5c64de996d0596887c8cda34b
PKG_SOURCE_DATE:=2024-11-10
PKG_SOURCE_VERSION:=2480f4d9d2fa97e5511ed0914ee529a344e969a7
#PKG_BUILD_DIR:=$(BUILD_DIR)/protobuf-c-$(PKG_VERSION)
#HOST_BUILD_DIR:=$(BUILD_DIR_HOST)/protobuf-c-$(PKG_VERSION)

PKG_MAINTAINER:=
PKG_LICENSE:=BSD-2-Clause
PKG_LICENSE_FILES:=LICENSE
PKG_CPE_ID:=cpe:/a:protobuf-c_project:protobuf-c

HOST_BUILD_DEPENDS:=protobuf/host
PKG_BUILD_DEPENDS:=protobuf

CMAKE_INSTALL:=1
CMAKE_SOURCE_SUBDIR:=build-cmake

include $(INCLUDE_DIR)/package.mk
include $(INCLUDE_DIR)/host-build.mk
include $(INCLUDE_DIR)/cmake.mk
include $(INCLUDE_DIR)/nls.mk

define Package/libprotobuf-c
  TITLE:=Protocol Buffers library
  SECTION:=libs
  CATEGORY:=Libraries
  DEPENDS:=+libpthread +libatomic +libstdcpp +protobuf
  URL:=https://github.com/protobuf-c/protobuf-c
endef

define Package/libprotobuf-c/description
  Runtime library to use Google Protocol Buffers from C applications.
  Protocol Buffers are a way of encoding structured data in an efficient yet
  extensible format. Google uses Protocol Buffers for almost all of its
  internal RPC protocols and file formats.
endef

##For protoc & git runtime at configure
ifeq ($(CONFIG_BUILD_NLS),y)
	export LD_LIBRARY_PATH=$(STAGING_DIR)/usr/lib:$(ICONV_PREFIX)/lib:$(INTL_PREFIX)/lib
endif

CMAKE_HOST_OPTIONS += \
	-DBUILD_SHARED_LIBS=ON \
	-DCMAKE_CXX_STANDARD=11 \
	-DCMAKE_SKIP_RPATH=OFF \
	-DCMAKE_INSTALL_RPATH="${STAGING_DIR_HOSTPKG}/lib"

CMAKE_OPTIONS += \
	-DBUILD_SHARED_LIBS=ON \
	-DBUILD_PROTOC=ON \
	-DCMAKE_SKIP_RPATH=OFF \
	-DCMAKE_INSTALL_RPATH="${STAGING_DIR}/usr/lib"

define Build/InstallDev
	$(INSTALL_DIR) $(1)/usr/include
	$(CP) $(PKG_INSTALL_DIR)/usr/include/* $(1)/usr/include
	$(INSTALL_DIR) $(1)/usr/lib
	$(CP) $(PKG_INSTALL_DIR)/usr/lib/* $(1)/usr/lib/
	$(INSTALL_DIR) $(1)/usr/bin
	$(CP) $(PKG_INSTALL_DIR)/usr/bin/protoc-gen-c $(1)/usr/bin
	$(LN) protoc-gen-c $(1)/usr/bin/protoc-c
endef

define Package/libprotobuf-c/install
	$(INSTALL_DIR) $(1)/usr/lib
	$(CP) $(PKG_INSTALL_DIR)/usr/lib/libprotobuf-c.so* $(1)/usr/lib/
	
endef

$(eval $(call BuildPackage,libprotobuf-c))
$(eval $(call HostBuild))

I suggest to use protobuf version 25.3 or 26.0 as version above it is not supported by protobuf-c

Above Makefile changes has been tested by compiling frr and netdata v2.0.0.

tiagogaspar8 · 2024-11-13T16:57:15Z

You managed to compile netdata 2.0? 😲
I have a pull request open here for aged and I was looking for help on how to get it to build...
#22677
Also, do you want to open a pull request for Protobuf?

Ra2-IFV · 2024-11-13T17:08:30Z

I fixed the patch for protobuf 28.3, if anyone needs it

I suggest to use protobuf version 25.3 or 26.0 as version above it is not supported by protobuf-c

Might need refreshing if you are updating to a different version

--- a/cmake/install.cmake
+++ b/cmake/install.cmake
@@ -35,8 +35,8 @@ endif ()
 
 foreach(_library ${_protobuf_libraries})
   if (UNIX AND NOT APPLE)
-    set_property(TARGET ${_library}
-      PROPERTY INSTALL_RPATH "$ORIGIN")
+    # set_property(TARGET ${_library}
+    #   PROPERTY INSTALL_RPATH "$ORIGIN")
   elseif (APPLE)
     set_property(TARGET ${_library}
       PROPERTY INSTALL_RPATH "@loader_path")
@@ -62,8 +62,8 @@ if (protobuf_BUILD_PROTOC_BINARIES)
   endif ()
   foreach (binary IN LISTS _protobuf_binaries)
     if (UNIX AND NOT APPLE)
-      set_property(TARGET ${binary}
-        PROPERTY INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}")
+      # set_property(TARGET ${binary}
+      #   PROPERTY INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}")
     elseif (APPLE)
       set_property(TARGET ${binary}
         PROPERTY INSTALL_RPATH "@loader_path/../lib")

vortexilation · 2024-11-13T17:42:42Z

@tiagogaspar8
Yes, but forgot to mention it's all on glibc, not musl, i will try musl now.
Probably harder on musl.

About your post on that PR, i am not encountering the same thing on glibc.

You could open up protobuf PR based on my finding if you want.

Here is the rough draft for netdata 2.0 package and libs package :
netdata.zip

The netdata 2.0 package from me is different, it using cmake build system instead.

vortexilation · 2024-11-13T18:21:55Z

@tiagogaspar8
Your previous issue on netdata repo, if I am not mistaken I have encountered it before, try to not using bundled abseil-cpp/protobuf from netdata.
Use it's own external abseil-cpp and protobuf OpenWrt package like above.

[EDIT]
On musl, it's totally different world, lot's of new errors.

Stuck with :

Error loading shared library libatomic.so.1: No such file or directory (needed by /home/user/works/openwrt/staging_dir/target-x86_64_musl/usr/bin/protoc)
Error loading shared library libprotoc.so.25.3.0: No such file or directory (needed by /home/user/works/openwrt/staging_dir/target-x86_64_musl/usr/bin/protoc)

Even tough I have set LD_LIBRARY_PATH

[EDIT#2]
Managed to compiled netdata 2.0 for musl targeting x86/64.

Need to add the following to protobuf Makefile :

TARGET_LDFLAGS += \
	-Wl,-rpath $(STAGING_DIR_ROOT)/lib:$(STAGING_DIR_ROOT)/usr/lib

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

protobuf: Update to the latest version #25194

protobuf: Update to the latest version #25194

tiagogaspar8 commented Oct 24, 2024

Ra2-IFV commented Nov 13, 2024

tiagogaspar8 commented Nov 13, 2024

Ra2-IFV commented Nov 13, 2024

tiagogaspar8 commented Nov 13, 2024

vortexilation commented Nov 13, 2024 •

edited

Loading

tiagogaspar8 commented Nov 13, 2024 •

edited

Loading

Ra2-IFV commented Nov 13, 2024

vortexilation commented Nov 13, 2024

vortexilation commented Nov 13, 2024 •

edited

Loading

protobuf: Update to the latest version #25194

protobuf: Update to the latest version #25194

Comments

tiagogaspar8 commented Oct 24, 2024

Ra2-IFV commented Nov 13, 2024

tiagogaspar8 commented Nov 13, 2024

Ra2-IFV commented Nov 13, 2024

tiagogaspar8 commented Nov 13, 2024

vortexilation commented Nov 13, 2024 • edited Loading

tiagogaspar8 commented Nov 13, 2024 • edited Loading

Ra2-IFV commented Nov 13, 2024

vortexilation commented Nov 13, 2024

vortexilation commented Nov 13, 2024 • edited Loading

vortexilation commented Nov 13, 2024 •

edited

Loading

tiagogaspar8 commented Nov 13, 2024 •

edited

Loading

vortexilation commented Nov 13, 2024 •

edited

Loading