protobuf upgrade

README.md

@@ -52,7 +52,7 @@ Detailed instructions are provided below.
 ```
 python >= 3.8
 gcc >= 6.4
-protobuf >= 4.21.12
+protobuf >= 4.25.1
 cmake >= 3.13.4
 make >= 4.2.1 or ninja >= 1.10.2
 java >= 1.11 (optional)

docker/Dockerfile.llvm-project

@@ -33,7 +33,7 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"]
                   libtool make maven ninja-build openjdk-11-jdk-headless \
                   python3 python3-dev python3-distutils python3-numpy \
                   python3-pip python3-pytest-xdist python3-setuptools \
-                  python3-typing-extensions zlib1g-dev && \
+                  python3-typing-extensions zlib1g-dev wget tar && \
           rm -rf /var/lib/apt/lists/* && \
           ln -sf /usr/bin/pytest-3 /usr/bin/pytest; \
        elif [ "${distro}" = "rhel" ] || [ "${distro}" = "fedora" ]; then \
@@ -48,7 +48,7 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"]
               file java-11-openjdk-devel java-11-openjdk-headless \
               gcc gcc-c++ git libtool make ncurses-devel ninja-build \
               python39 python39-devel python39-pip python39-setuptools \
-              python39-wheel tzdata-java zlib-devel && \
+              python39-wheel tzdata-java zlib-devel wget tar && \
 	  # Workaround broken ubi8 amd64 image installing python3.12 as
 	  # dependency of clang, which also breaks the /usr/bin/pip3
 	  # symlink creation
@@ -64,13 +64,32 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"]
     && pip3 install --upgrade pip
 
 # Install protobuf
-ARG PROTOBUF_VERSION=21.12
-RUN git clone -b v${PROTOBUF_VERSION} --recursive https://github.com/protocolbuffers/protobuf.git \
-    && cd protobuf && ./autogen.sh \
-    && ./configure --enable-static=no \
-    && make -j${NPROC} install && ldconfig \
-    && cd python && python3 setup.py install --cpp_implementation \
-    && cd ../.. && rm -rf protobuf
+ARG PROTOBUF_VERSION=25.1
+ARG INSTALL_PROTOBUF_PATH=/usr/local
+ARG BUILD_TYPE=Release
+ARG CORE_NUMBER=2
+ARG ORIGINAL_PATH=${pwd}
+RUN cd .. && \
+    wget https://github.com/abseil/abseil-cpp/releases/download/20230802.2/abseil-cpp-20230802.2.tar.gz && \
+    tar -xvf abseil-cpp-20230802.2.tar.gz && \
+    wget https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protobuf-25.1.tar.gz && \
+    tar -xvf protobuf-25.1.tar.gz
+RUN cd /protobuf-25.1 && \
+    mkdir build_source && cd build_source && \
+    cmake -G Ninja ../ \
+          -DBUILD_SHARED_LIBS=OFF \
+          -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH \
+          -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+          -Dprotobuf_BUILD_TESTS=OFF \
+          -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
+          -DABSL_ROOT_DIR="${ORIGINAL_PATH}/abseil-cpp-20230802.2"\
+          -DCMAKE_CXX_STANDARD=17 \
+          -DABSL_PROPAGATE_CXX_STD=on \
+    && cmake --build . --target install --parallel $CORE_NUMBER
+
+RUN export PATH=$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$INSTALL_PROTOBUF_PATH/bin:$PATH && \
+    export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" && \
+    export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include"
 
 # Install jsoniter
 ARG JSONITER_VERSION=0.9.23

docs/BuildOnWindows.md

@@ -15,8 +15,8 @@ Build protobuf as a static library.
 
 [same-as-file]: <> (utils/install-protobuf.cmd)
 ```shell
-REM Check out protobuf v21.12
-set protobuf_version=21.12
+REM Check out protobuf v4.25.1
+set protobuf_version=4.25.1
 git clone -b v%protobuf_version% --recursive https://github.com/protocolbuffers/protobuf.git
 
 set root_dir=%cd%
@@ -42,7 +42,7 @@ set PATH=%root_dir%\protobuf_install\bin;%PATH%
 
 If you wish to be able to run all the ONNX-MLIR tests, you will also need to install the matching version of protobuf through pip. Note that this is included in the requirements.txt file at the root of onnx-mlir, so if you plan on using it, you won't need to explicitly install protobuf.
 ```shell
-python3 -m pip install protobuf==4.21.12
+python3 -m pip install protobuf==4.25.1
 ```
 
 #### MLIR

docs/Prerequisite.md

@@ -6,7 +6,7 @@
 ```
 python >= 3.8
 gcc >= 6.4
-protobuf >= 4.21.12
+protobuf >= 4.25.1
 cmake >= 3.13.4
 make >= 4.2.1 or ninja >= 1.10.2
 java >= 1.11 (optional)

requirements.txt

@@ -3,6 +3,6 @@ lit~=15.0
 # and np.int which are used heavily in onnx-mlir.
 numpy==2.0.1
 onnx==1.17.0
-protobuf==4.21.12
+protobuf==4.25.1
 pytest==8.3.2
 pytest-xdist==3.6.1

utils/cppcheck_scan/cppcheck_suppressions.txt

@@ -15,3 +15,10 @@ preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-o
 #            <location file="/workdir/llvm-project/llvm/include/llvm-c/DataTypes.h" line="38" column="0"/>
 #        </error>
 preprocessorErrorDirective:/workdir/llvm-project/llvm/include/llvm-c/DataTypes.h:38
+# Build image has a newer protobuf than onnx-mlir source. This causes preprocessor issues when scanning with cppcheck.
+#        <error id="preprocessorErrorDirective" severity="error" msg="#error &quot;This file was generated by a newer version of protoc which is&quot;" verbose="#error &quot;This file was generated by a newer version of protoc which is&quot;">
+#            <location file="/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-data.pb.h" line="15" column="0"/>
+#        </error>
+preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-data.pb.h:15
+preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-ml.pb.h:15
+preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-operators-ml.pb.h:15

utils/install-protobuf.cmd

@@ -1,8 +1,9 @@
-REM Check out protobuf v21.12
-set protobuf_version=21.12
+REM Check out protobuf v4.25.1
+set protobuf_version=4.25.1
 git clone -b v%protobuf_version% --recursive https://github.com/protocolbuffers/protobuf.git
 
 set root_dir=%cd%
+git submodule update --init --recursive
 md protobuf_build
 cd protobuf_build
 call cmake %root_dir%\protobuf\cmake -G "Ninja" ^

utils/install-protobuf.sh

@@ -1,10 +1,34 @@
-# Check out protobuf
-PROTOBUF_VERSION=21.12
-git clone -b v${PROTOBUF_VERSION} --depth 1 --recursive https://github.com/protocolbuffers/protobuf.git
+# Exit immediately if a command exits with a non-zero status.
+set -e
 
-cd protobuf
-./autogen.sh
-./configure --enable-static=no
-make -j$(sysctl -n hw.logicalcpu) install
-cd python
-python3 setup.py install --cpp_implementation
+export INSTALL_PROTOBUF_PATH=~/work/protobuf_install # Changed to a dedicated install directory
+export BUILD_TYPE=Release
+export CORE_NUMBER=1
+
+# Build protobuf from source with -fPIC on Unix-like system
+ORIGINAL_PATH=$(pwd)
+cd ..
+wget https://github.com/abseil/abseil-cpp/releases/download/20230802.2/abseil-cpp-20230802.2.tar.gz
+tar -xvf abseil-cpp-20230802.2.tar.gz
+
+wget https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protobuf-25.1.tar.gz
+tar -xvf protobuf-25.1.tar.gz
+cd protobuf-25.1
+mkdir build_source && cd build_source
+cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DABSL_ROOT_DIR="${ORIGINAL_PATH}/../abseil-cpp-20230802.2" -DCMAKE_CXX_STANDARD=17 -DABSL_PROPAGATE_CXX_STD=on ..
+if [ "$INSTALL_PROTOBUF_PATH" == "/usr" ]; then
+    # Don't use sudo for root
+    if [[ "$(id -u)" == "0" ]]; then
+      cmake --build . --target install --parallel $CORE_NUMBER
+    else
+      # install Protobuf on default system path so it needs sudo permission
+      sudo cmake --build . --target install --parallel $CORE_NUMBER
+    fi
+else
+    cmake --build . --target install --parallel $CORE_NUMBER
+    export PATH=$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$INSTALL_PROTOBUF_PATH/bin:$PATH
+    export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib"
+    export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include"
+fi
+protoc --version
+cd $ORIGINAL_PATH