From 24625746945001d4cd4f1ba44084e989874aadb7 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Wed, 19 Jan 2022 17:20:27 -0800 Subject: [PATCH 01/10] Change the way how docker installs py38. --- tensorflow/tools/ci_build/Dockerfile.cpu-py38 | 7 ++++-- .../ci_build/install/install_python38.sh | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100755 tensorflow/tools/ci_build/install/install_python38.sh diff --git a/tensorflow/tools/ci_build/Dockerfile.cpu-py38 b/tensorflow/tools/ci_build/Dockerfile.cpu-py38 index 3d401f56647b64..6672423935cc57 100644 --- a/tensorflow/tools/ci_build/Dockerfile.cpu-py38 +++ b/tensorflow/tools/ci_build/Dockerfile.cpu-py38 @@ -16,10 +16,13 @@ RUN apt -y install gcc-7 g++-7 RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 700 \ --slave /usr/bin/g++ g++ /usr/bin/g++-7 -# The following line installs the Python 3.8 cross-compilation toolchain. -RUN /install/install_pi_python3x_toolchain.sh "3.8" +# Install Python 3.8 +# libffi-dev is needed for ctypes module of python3.8 +RUN apt install libffi-dev +RUN /install/install_python38.sh RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 0 RUN python3 -V +RUN /install/install_pip_packages_by_version.sh pip3 RUN /install/install_bazel.sh RUN /install/install_proto3.sh diff --git a/tensorflow/tools/ci_build/install/install_python38.sh b/tensorflow/tools/ci_build/install/install_python38.sh new file mode 100755 index 00000000000000..2428cacb3d24c0 --- /dev/null +++ b/tensorflow/tools/ci_build/install/install_python38.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +cd /usr/src +wget https://www.python.org/ftp/python/3.8.2/Python-3.8.2.tgz +tar xzf Python-3.8.2.tgz +cd Python-3.8.2 +./configure --enable-optimizations --with-ensurepip=install --prefix=/usr/ +make altinstall +rm /usr/src/Python-3.8.2.tgz +rm -rf /usr/src/Python-3.8.2 + From 82fc37d08ea9accccacc6bf3ac1b8f7d8fea453a Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Thu, 20 Jan 2022 09:38:27 -0800 Subject: [PATCH 02/10] Fix permission. --- tensorflow/tools/ci_build/Dockerfile.cpu-py38 | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/tools/ci_build/Dockerfile.cpu-py38 b/tensorflow/tools/ci_build/Dockerfile.cpu-py38 index 6672423935cc57..9a59570d498b80 100644 --- a/tensorflow/tools/ci_build/Dockerfile.cpu-py38 +++ b/tensorflow/tools/ci_build/Dockerfile.cpu-py38 @@ -19,6 +19,7 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 700 \ # Install Python 3.8 # libffi-dev is needed for ctypes module of python3.8 RUN apt install libffi-dev +RUN chmod 777 /install/install_python38.sh RUN /install/install_python38.sh RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 0 RUN python3 -V From fc09c62095a01c17241f1a9d3a2998038fb78319 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Thu, 20 Jan 2022 12:01:30 -0800 Subject: [PATCH 03/10] Fix sqlite module --- tensorflow/tools/ci_build/Dockerfile.cpu-py38 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/tools/ci_build/Dockerfile.cpu-py38 b/tensorflow/tools/ci_build/Dockerfile.cpu-py38 index 9a59570d498b80..8904e59d1f3607 100644 --- a/tensorflow/tools/ci_build/Dockerfile.cpu-py38 +++ b/tensorflow/tools/ci_build/Dockerfile.cpu-py38 @@ -19,6 +19,8 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 700 \ # Install Python 3.8 # libffi-dev is needed for ctypes module of python3.8 RUN apt install libffi-dev +# libsqlite3-dev is needed for module named '_sqlite3' +RUN apt install libsqlite3-dev RUN chmod 777 /install/install_python38.sh RUN /install/install_python38.sh RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 0 From 7abbddf35cd97f2631ba00731839e76fe060e422 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Fri, 21 Jan 2022 13:48:34 -0800 Subject: [PATCH 04/10] Update to 3.8.10 to be consistent with tensorflow:devel --- tensorflow/tools/ci_build/install/install_python38.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/tools/ci_build/install/install_python38.sh b/tensorflow/tools/ci_build/install/install_python38.sh index 2428cacb3d24c0..c3c8a8f7aa311e 100755 --- a/tensorflow/tools/ci_build/install/install_python38.sh +++ b/tensorflow/tools/ci_build/install/install_python38.sh @@ -15,11 +15,11 @@ # ============================================================================== cd /usr/src -wget https://www.python.org/ftp/python/3.8.2/Python-3.8.2.tgz -tar xzf Python-3.8.2.tgz -cd Python-3.8.2 +wget https://www.python.org/ftp/python/3.8.10/Python-3.8.10.tgz +tar xzf Python-3.8.10.tgz +cd Python-3.8.10 ./configure --enable-optimizations --with-ensurepip=install --prefix=/usr/ make altinstall -rm /usr/src/Python-3.8.2.tgz -rm -rf /usr/src/Python-3.8.2 +rm /usr/src/Python-3.8.10.tgz +rm -rf /usr/src/Python-3.8.10 From 8e194ff823a8a401e9e0b4c306397defed60114b Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Wed, 9 Feb 2022 23:53:10 -0800 Subject: [PATCH 05/10] Add missing "packaging" dependency --- .../tools/ci_build/install/install_pip_packages_by_version.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/tools/ci_build/install/install_pip_packages_by_version.sh b/tensorflow/tools/ci_build/install/install_pip_packages_by_version.sh index 1847a025722997..14ab1b8072cf6b 100755 --- a/tensorflow/tools/ci_build/install/install_pip_packages_by_version.sh +++ b/tensorflow/tools/ci_build/install/install_pip_packages_by_version.sh @@ -43,6 +43,7 @@ PACKAGES=( "protobuf" "scipy" "scikit-learn" + "packaging" "pandas" "psutil" "py-cpuinfo" From 3b0c4ccab302a544c805229ec75067f45880c875 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Thu, 10 Feb 2022 10:37:36 -0800 Subject: [PATCH 06/10] Enforcing tb-nightly version to be >= 2.9.0a20220210 --- .../tools/ci_build/install/install_pip_packages_by_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/install/install_pip_packages_by_version.sh b/tensorflow/tools/ci_build/install/install_pip_packages_by_version.sh index 14ab1b8072cf6b..2c54fac6776f7a 100755 --- a/tensorflow/tools/ci_build/install/install_pip_packages_by_version.sh +++ b/tensorflow/tools/ci_build/install/install_pip_packages_by_version.sh @@ -58,7 +58,7 @@ PACKAGES=( "keras_preprocessing" "h5py" "tf-estimator-nightly" - "tb-nightly" + "tb-nightly>=2.9.0a20220210" "argparse" "dm-tree" "dill" From 94c43d5915402553acba726c4d2606b65493c259 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Fri, 1 Apr 2022 10:18:20 -0700 Subject: [PATCH 07/10] Update install_bazel.sh Bump up bazel version to 5.1.0 --- tensorflow/tools/ci_build/install/install_bazel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/install/install_bazel.sh b/tensorflow/tools/ci_build/install/install_bazel.sh index 063e1f97bace86..f2b731a3f67584 100755 --- a/tensorflow/tools/ci_build/install/install_bazel.sh +++ b/tensorflow/tools/ci_build/install/install_bazel.sh @@ -15,7 +15,7 @@ # ============================================================================== # Select bazel version. -BAZEL_VERSION="3.7.2" +BAZEL_VERSION="5.1.0" set +e local_bazel_ver=$(bazel version 2>&1 | grep -i label | awk '{print $3}') From 1a37d7b0fbee7e1004b4485a06e98f44fe0229b8 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Tue, 12 Apr 2022 10:51:59 -0700 Subject: [PATCH 08/10] Update install_bazel.sh Update bazel to 5.1.1 --- tensorflow/tools/ci_build/install/install_bazel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/install/install_bazel.sh b/tensorflow/tools/ci_build/install/install_bazel.sh index f2b731a3f67584..593838ec65e7ca 100755 --- a/tensorflow/tools/ci_build/install/install_bazel.sh +++ b/tensorflow/tools/ci_build/install/install_bazel.sh @@ -15,7 +15,7 @@ # ============================================================================== # Select bazel version. -BAZEL_VERSION="5.1.0" +BAZEL_VERSION="5.1.1" set +e local_bazel_ver=$(bazel version 2>&1 | grep -i label | awk '{print $3}') From ea897418430dd4a2082e0a3c9285ae00e0d14052 Mon Sep 17 00:00:00 2001 From: intel-ravig <104024558+intel-ravig@users.noreply.github.com> Date: Wed, 27 Apr 2022 18:13:40 -0700 Subject: [PATCH 09/10] Build from Ubuntu 18.04 and cmake fix Changed Dockerfile to pull from Ubuntu 18.04 and removed third party cmake repo. --- tensorflow/tools/ci_build/Dockerfile.cpu-py38 | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/tensorflow/tools/ci_build/Dockerfile.cpu-py38 b/tensorflow/tools/ci_build/Dockerfile.cpu-py38 index 8904e59d1f3607..f576846351c9f2 100644 --- a/tensorflow/tools/ci_build/Dockerfile.cpu-py38 +++ b/tensorflow/tools/ci_build/Dockerfile.cpu-py38 @@ -1,13 +1,16 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 -LABEL maintainer="Gauri Deshpande " +LABEL maintainer="Ravi Gupta " # Copy and run the install scripts. +RUN apt-get update +RUN apt-get upgrade -y +RUN apt-get install cmake -y + COPY install/*.sh /install/ -RUN /install/install_bootstrap_deb_packages.sh -RUN add-apt-repository -y ppa:openjdk-r/ppa && \ - add-apt-repository -y ppa:george-edison55/cmake-3.x -RUN /install/install_deb_packages.sh +RUN bash -x /install/install_bootstrap_deb_packages.sh +RUN add-apt-repository -y ppa:openjdk-r/ppa +RUN bash -x /install/install_deb_packages.sh # Install gcc 7 RUN add-apt-repository ppa:ubuntu-toolchain-r/test @@ -27,10 +30,10 @@ RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 0 RUN python3 -V RUN /install/install_pip_packages_by_version.sh pip3 -RUN /install/install_bazel.sh -RUN /install/install_proto3.sh -RUN /install/install_buildifier.sh -RUN /install/install_auditwheel.sh -RUN /install/install_golang.sh +RUN bash -x /install/install_bazel.sh +RUN bash -x /install/install_proto3.sh +RUN bash -x /install/install_buildifier.sh +RUN bash -x /install/install_auditwheel.sh +RUN bash -x /install/install_golang.sh # Set up the master bazelrc configuration file. COPY install/.bazelrc /etc/bazel.bazelrc From 40116420d4096d303a2bffdf0f9c6dba7967a7e7 Mon Sep 17 00:00:00 2001 From: Ravi Gupta Date: Wed, 27 Apr 2022 18:25:56 -0700 Subject: [PATCH 10/10] Build from Ubuntu 18.04 and cmake fix Changed Dockerfile to pull from Ubuntu 18.04 and removed third party cmake repo. --- tensorflow/tools/ci_build/Dockerfile.cpu-py38 | 25 +++++++++++-------- .../ci_build/install/install_deb_packages.sh | 1 - 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/tensorflow/tools/ci_build/Dockerfile.cpu-py38 b/tensorflow/tools/ci_build/Dockerfile.cpu-py38 index 8904e59d1f3607..f576846351c9f2 100644 --- a/tensorflow/tools/ci_build/Dockerfile.cpu-py38 +++ b/tensorflow/tools/ci_build/Dockerfile.cpu-py38 @@ -1,13 +1,16 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 -LABEL maintainer="Gauri Deshpande " +LABEL maintainer="Ravi Gupta " # Copy and run the install scripts. +RUN apt-get update +RUN apt-get upgrade -y +RUN apt-get install cmake -y + COPY install/*.sh /install/ -RUN /install/install_bootstrap_deb_packages.sh -RUN add-apt-repository -y ppa:openjdk-r/ppa && \ - add-apt-repository -y ppa:george-edison55/cmake-3.x -RUN /install/install_deb_packages.sh +RUN bash -x /install/install_bootstrap_deb_packages.sh +RUN add-apt-repository -y ppa:openjdk-r/ppa +RUN bash -x /install/install_deb_packages.sh # Install gcc 7 RUN add-apt-repository ppa:ubuntu-toolchain-r/test @@ -27,10 +30,10 @@ RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 0 RUN python3 -V RUN /install/install_pip_packages_by_version.sh pip3 -RUN /install/install_bazel.sh -RUN /install/install_proto3.sh -RUN /install/install_buildifier.sh -RUN /install/install_auditwheel.sh -RUN /install/install_golang.sh +RUN bash -x /install/install_bazel.sh +RUN bash -x /install/install_proto3.sh +RUN bash -x /install/install_buildifier.sh +RUN bash -x /install/install_auditwheel.sh +RUN bash -x /install/install_golang.sh # Set up the master bazelrc configuration file. COPY install/.bazelrc /etc/bazel.bazelrc diff --git a/tensorflow/tools/ci_build/install/install_deb_packages.sh b/tensorflow/tools/ci_build/install/install_deb_packages.sh index 7eaadb52c5e3fa..d83d37a1491579 100755 --- a/tensorflow/tools/ci_build/install/install_deb_packages.sh +++ b/tensorflow/tools/ci_build/install/install_deb_packages.sh @@ -28,7 +28,6 @@ if [[ "$1" != "" ]] && [[ "$1" != "--without_cmake" ]]; then fi # Install dependencies from ubuntu deb repository. -apt-key adv --keyserver keyserver.ubuntu.com --recv 084ECFC5828AB726 apt-get update if [[ "$ubuntu_version" == "14" ]]; then