Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix issue related to the installation of Python #236

Merged
merged 13 commits into from
Feb 27, 2025
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/coverage/cpp.develop.coverage_report.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ Directory: ..
------------------------------------------------------------------------------
File Lines Exec Cover Missing
------------------------------------------------------------------------------
client/cpp/CSVParserUtil.cpp 345 291 84% 37-45,48,50,239,241,264-265,269-270,286,292,304,313-314,317,323,331-332,335,345,351,363,368,373,379-387,389,425,435-437,474-476,478,503-506
client/cpp/CSVParserUtil.cpp 345 290 84% 37-45,48,50,239,241,264-265,269-270,286,292,304,313-314,317,323,331-332,335,345,351,363,368,373,379-387,389,425,435-437,450,474-476,478,503-506
client/cpp/VDMSClient.cc 20 20 100%
src/AutoDeleteNode.cc 9 8 88% 40
src/BackendNeo4j.cc 121 0 0% 4,6-17,20,24,29-41,46-47,52,55-58,61-62,64-70,73,78,82-83,85-86,89,92,95-96,98,102,104,106-109,111,114-116,118,122,131-132,138,140,142-144,147,150-152,155-159,161-175,178,182,184,186,195,197-200,204-205,207-208,211-215,220,224-226,228
Expand Down Expand Up @@ -57,5 +57,5 @@ utils/src/comm/Exception.cc 6 0 0% 35-40
utils/src/stats/SystemStats.cc 250 249 99% 453
utils/src/timers/TimerMap.cc 82 75 91% 126,151,153,155-158
------------------------------------------------------------------------------
TOTAL 10238 6537 63%
TOTAL 10238 6536 63%
------------------------------------------------------------------------------
2 changes: 1 addition & 1 deletion .github/coverage/cpp.develop.coverage_value.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
63.8504
63.8406
4 changes: 2 additions & 2 deletions .github/scripts/Dockerfile.checkin
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ RUN git clone https://github.com/opencv/opencv.git /dependencies/opencv && \

# VALIJSON
# hadolint ignore=DL3003
RUN python -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" && \
RUN python3 -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" && \
git clone --branch ${VALIJSON_VERSION} https://github.com/tristanpenman/valijson.git /dependencies/valijson && \
cd /dependencies/valijson && cp -r include/* /usr/local/include/ && \
mkdir -p /opt/dist/usr/local/include/ && cp -r include/* /opt/dist/usr/local/include/
Expand All @@ -122,7 +122,7 @@ RUN python -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" &&
# hadolint ignore=DL3003,SC2086
RUN git clone --branch ${FAISS_VERSION} https://github.com/facebookresearch/faiss.git /dependencies/faiss && \
cd /dependencies/faiss && mkdir build && cd build && \
cmake -DFAISS_ENABLE_GPU=OFF -DPython_EXECUTABLE=$(which python) \
cmake -DFAISS_ENABLE_GPU=OFF -DPython_EXECUTABLE=$(which python3) \
-DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release .. && \
make ${BUILD_THREADS} && make install DESTDIR=/opt/dist && make install && \
git clone https://github.com/tonyzhang617/FLINNG.git /dependencies/FLINNG && \
Expand Down
4 changes: 2 additions & 2 deletions .github/scripts/run_coverage_py.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ cd /vdms/tests/python

./run_python_tests.sh
./run_python_aws_tests.sh -u ${AWS_ACCESS_KEY_ID} -p ${AWS_SECRET_ACCESS_KEY}
python -m coverage report -m 2>&1 | tee /vdms/tests/coverage_report/python.new.coverage_report.txt
python -m coverage xml -o /vdms/tests/coverage_report/python.new.coverage_report.xml
python3 -m coverage report -m 2>&1 | tee /vdms/tests/coverage_report/python.new.coverage_report.txt
python3 -m coverage xml -o /vdms/tests/coverage_report/python.new.coverage_report.xml

echo "DONE"

Expand Down
25 changes: 19 additions & 6 deletions .github/scripts/setup_vdms.sh
Original file line number Diff line number Diff line change
Expand Up @@ -136,8 +136,9 @@ fi


# SETUP PYTHON VERSION
version_exists=$(echo "$(python${PYTHON_BASE} --version | cut -d ' ' -f 2)" || echo false)
if [ "${version_exists}" != "${PYTHON_VERSION}" ]
version_exists=$(echo "$(python3 --version | cut -d ' ' -f 2)" || echo false)
# Compare the current version of Python with the $PYTHON_VERSION
if $(dpkg --compare-versions "${version_exists}" "lt" "${PYTHON_VERSION}")
then
echo "Installing python ${PYTHON_VERSION}..."
apt update -y
Expand All @@ -147,10 +148,22 @@ then
tar -xzf Python-${PYTHON_VERSION}.tgz
cd Python-${PYTHON_VERSION}
./configure --enable-optimizations && make -j && make altinstall

# Update the path to where the new Python version is installed

# if the new version of Python is installed in /usr/local/bin
if [ -f "/usr/local/bin/python${PYTHON_BASE}" ]; then
# and there is already a symbolic link pointing to /usr/bin/python3
if [ -f "/usr/bin/python3" ]; then
# Remove that outdated path to Python3
rm /usr/bin/python3
# and update the new path to Python3
ln -s /usr/local/bin/python${PYTHON_BASE} /usr/bin/python3
fi
fi
else
echo "python ${PYTHON_VERSION} already installed"
fi
alias python=$(which python${PYTHON_BASE})
alias python3=$(which python${PYTHON_BASE})

python${PYTHON_BASE} -m venv ${VIRTUAL_ENV}
Expand All @@ -159,7 +172,7 @@ export PATH="$VIRTUAL_ENV/bin:$PATH"

if [ "${BUILD_COVERAGE}" = "on" ]; then
apt-get install -y --no-install-suggests --no-install-recommends gdb
python -m pip install --no-cache-dir "gcovr>=7.0"
python3 -m pip install --no-cache-dir "gcovr>=7.0"
curl -L -o ${WORKSPACE}/minio https://dl.min.io/server/minio/release/linux-amd64/minio
chmod +x ${WORKSPACE}/minio
mkdir -p ${WORKSPACE}/minio_files/minio-bucket
Expand Down Expand Up @@ -237,7 +250,7 @@ make install


# INSTALL PYTHON PACKAGES
python -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" "coverage>=7.3.1" \
python3 -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" "coverage>=7.3.1" \
"protobuf==4.${PROTOBUF_VERSION}" "cryptography>=42.0.7"


Expand All @@ -251,7 +264,7 @@ cp -r include/* /usr/local/include/
git clone --branch ${FAISS_VERSION} https://github.com/facebookresearch/faiss.git $VDMS_DEP_DIR/faiss
cd $VDMS_DEP_DIR/faiss
mkdir build && cd build
cmake -DFAISS_ENABLE_GPU=OFF -DPython_EXECUTABLE=$(which python) \
cmake -DFAISS_ENABLE_GPU=OFF -DPython_EXECUTABLE=$(which python3) \
-DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release ..
make ${BUILD_THREADS}
make install
Expand Down
8 changes: 4 additions & 4 deletions INSTALL.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
Here is the detailed process of installation of VDMS dependencies.

## Dependencies
To install VDMS, we must install the necessary dependencies via apt, github, and pip (Python 3.9+).
To install VDMS, we must install the necessary dependencies via apt, github, and pip (Python 3.12+).

### Install Debian/Ubuntu Packages
Here we will install the Debian/Ubuntu packages.
Expand Down Expand Up @@ -58,8 +58,8 @@ mkdir -p $VDMS_DEP_DIR


#### Python3 Packages
It is expected that you have Python3.9 or higher installed on your system.
All python calls will use Python3.9+; therefore you may find it convenient to set alias for python.
It is expected that you have Python3.12 or higher installed on your system.
All python calls will use Python3.12+; therefore you may find it convenient to set alias for python.
Here we will install Python 3.12.3 from the Python website.
```bash
PYTHON_VERSION=3.12.3
Expand All @@ -75,7 +75,7 @@ If you prefer, you can install the the Python 3 version available on the OS plat
sudo apt-get install -y python3-dev python3-pip
```

***NOTE:*** If multiple versions of Python 3 are present on your system, verify you are using Python3.9 or higher. You can specify the specific verison and set an alias for `python` and/or `python3` to easily use the desired python version. This can be done using the following:
***NOTE:*** If multiple versions of Python 3 are present on your system, verify you are using Python3.12 or higher. You can specify the specific verison and set an alias for `python` and/or `python3` to easily use the desired python version. This can be done using the following:
```bash
alias python=/usr/bin/python3.x
alias python3=/usr/bin/python3.x
Expand Down
4 changes: 2 additions & 2 deletions docker/base/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ RUN git clone https://github.com/opencv/opencv.git /dependencies/opencv && \

# VALIJSON
# hadolint ignore=DL3003
RUN python -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" && \
RUN python3 -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" && \
git clone --branch ${VALIJSON_VERSION} https://github.com/tristanpenman/valijson.git /dependencies/valijson && \
cd /dependencies/valijson && cp -r include/* /usr/local/include/ && \
mkdir -p /opt/dist/usr/local/include/ && cp -r include/* /opt/dist/usr/local/include/
Expand All @@ -122,7 +122,7 @@ RUN python -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" &&
# hadolint ignore=DL3003,SC2086
RUN git clone --branch ${FAISS_VERSION} https://github.com/facebookresearch/faiss.git /dependencies/faiss && \
cd /dependencies/faiss && mkdir build && cd build && \
cmake -DFAISS_ENABLE_GPU=OFF -DPython_EXECUTABLE=$(which python) \
cmake -DFAISS_ENABLE_GPU=OFF -DPython_EXECUTABLE=$(which python3) \
-DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release .. && \
make ${BUILD_THREADS} && make install DESTDIR=/opt/dist && make install && \
git clone https://github.com/tonyzhang617/FLINNG.git /dependencies/FLINNG && \
Expand Down