Merge branch 'main' into win_llm_demo

Author: mzegla

Makefile

@@ -636,9 +636,9 @@ test_python_clients:
 	@echo "Download models"
 	@if [ ! -d "tests/python/models" ]; then cd tests/python && \
 		mkdir models && \
-		docker run -u $(id -u):$(id -g) -v ${PWD}/tests/python/models:/models openvino/ubuntu20_dev:latest omz_downloader --name resnet-50-tf --output_dir /models && \
-		docker run -u $(id -u):$(id -g) -v ${PWD}/tests/python/models:/models:rw openvino/ubuntu20_dev:latest omz_converter --name resnet-50-tf --download_dir /models --output_dir /models --precisions FP32 && \
-		docker run -u $(id -u):$(id -g) -v ${PWD}/tests/python/models:/models:rw openvino/ubuntu20_dev:latest mv /models/public/resnet-50-tf/FP32 /models/public/resnet-50-tf/1; fi
+		docker run -u $(id -u):$(id -g) -v ${PWD}/tests/python/models:/models openvino/ubuntu20_dev:2024.6.0 omz_downloader --name resnet-50-tf --output_dir /models && \
+		docker run -u $(id -u):$(id -g) -v ${PWD}/tests/python/models:/models:rw openvino/ubuntu20_dev:2024.6.0 omz_converter --name resnet-50-tf --download_dir /models --output_dir /models --precisions FP32 && \
+		docker run -u $(id -u):$(id -g) -v ${PWD}/tests/python/models:/models:rw openvino/ubuntu20_dev:2024.6.0 mv /models/public/resnet-50-tf/FP32 /models/public/resnet-50-tf/1; fi
 	@echo "Start test container"
 	@docker run -d --rm --name $(PYTHON_CLIENT_TEST_CONTAINER_NAME) -v ${PWD}/tests/python/models/public/resnet-50-tf:/models/public/resnet-50-tf -p $(PYTHON_CLIENT_TEST_REST_PORT):8000 -p $(PYTHON_CLIENT_TEST_GRPC_PORT):9000 openvino/model_server:latest --model_name resnet --model_path /models/public/resnet-50-tf --port 9000 --rest_port 8000 && \
 		sleep 10

ci/build_test_OnCommit.groovy

@@ -97,6 +97,7 @@ pipeline {
                       def windows = load 'ci/loadWin.groovy'
                       if (windows != null) {
                         try {
+                          windows.setup_bazel_remote_cache()
                           windows.install_dependencies()
                           windows.clean()
                           windows.build_and_test()

ci/build_test_OnCommitWin.groovy

@@ -12,6 +12,7 @@ pipeline {
                     def windows = load 'ci/loadWin.groovy'
                     if (windows != null) {
                         try {
+                          windows.setup_bazel_remote_cache()
                           windows.cleanup_directories()
                           windows.install_dependencies()
                           windows.clean()

ci/loadWin.groovy

@@ -128,4 +128,22 @@ def archive_artifacts(){
     archiveArtifacts allowEmptyArchive: true, artifacts: "win_test.log"
 }
 
-return this
+def setup_bazel_remote_cache(){
+    def bazel_remote_cache_url = env.OVMS_BAZEL_REMOTE_CACHE_URL
+    def content = "build --remote_cache=\"${bazel_remote_cache_url}\""
+    def filePath = '.user.bazelrc'
+    def command = "echo ${content} > ${filePath}"
+    status = bat(returnStatus: true, script: command)
+    if ( status != 0) {
+        println "Failed to set up bazel remote cache for Windows"
+        return
+    }
+    command = "cat ${filePath}"
+    status = bat(returnStatus: true, script: command)
+    if ( status != 0) {
+        println "Failed to read file"
+        return
+    }
+}
+
+return this

client/python/ovmsclient/samples/README.md

@@ -34,8 +34,8 @@ pip3 install -r requirements.txt
 Download [Resnet50-tf Model](https://github.com/openvinotoolkit/open_model_zoo/blob/master/models/public/resnet-50-tf/README.md) and convert it into Intermediate Representation format:
 ```bash
 mkdir models
-docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models openvino/ubuntu20_dev:latest omz_downloader --name resnet-50-tf --output_dir /models
-docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models:rw openvino/ubuntu20_dev:latest omz_converter --name resnet-50-tf --download_dir /models --output_dir /models --precisions FP32
+docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models openvino/ubuntu20_dev:2024.6.0 omz_downloader --name resnet-50-tf --output_dir /models
+docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models:rw openvino/ubuntu20_dev:2024.6.0 omz_converter --name resnet-50-tf --download_dir /models --output_dir /models --precisions FP32
 mv ${PWD}/models/public/resnet-50-tf/FP32 ${PWD}/models/public/resnet-50-tf/1
 ```
 

demos/image_classification/go/README.md

@@ -20,8 +20,8 @@ Where `PATH_TO_MODELS` is the path to the directory with models on the host file
 For example:
 ```bash
 mkdir models
-docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models openvino/ubuntu20_dev:latest omz_downloader --name resnet-50-tf --output_dir /models
-docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models:rw openvino/ubuntu20_dev:latest omz_converter --name resnet-50-tf --download_dir /models --output_dir /models --precisions FP32
+docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models openvino/ubuntu20_dev:2024.6.0 omz_downloader --name resnet-50-tf --output_dir /models
+docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models:rw openvino/ubuntu20_dev:2024.6.0 omz_converter --name resnet-50-tf --download_dir /models --output_dir /models --precisions FP32
 mv ${PWD}/models/public/resnet-50-tf/FP32 ${PWD}/models/public/resnet-50-tf/1
 
 tree models/public/resnet-50-tf

demos/model_ensemble/python/Makefile

@@ -27,14 +27,14 @@ venv:
 
 download_models: venv
 	mkdir -p workspace/models/argmax
-	docker run -u $(id -u):$(id -g) -e http_proxy=${http_proxy} -e https_proxy=${https_proxy} -e no_proxy=${no_proxy} -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:latest omz_downloader --name googlenet-v2-tf --output_dir /models
-	docker run -u $(id -u):$(id -g) -e http_proxy=${http_proxy} -e https_proxy=${https_proxy} -e no_proxy=${no_proxy} -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:latest omz_downloader --name resnet-50-tf --output_dir /models
+	docker run -u $(id -u):$(id -g) -e http_proxy=${http_proxy} -e https_proxy=${https_proxy} -e no_proxy=${no_proxy} -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:2024.6.0 omz_downloader --name googlenet-v2-tf --output_dir /models
+	docker run -u $(id -u):$(id -g) -e http_proxy=${http_proxy} -e https_proxy=${https_proxy} -e no_proxy=${no_proxy} -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:2024.6.0 omz_downloader --name resnet-50-tf --output_dir /models
 	python3 ../../../tests/models/argmax_sum.py --input_size 1001 --export_dir $(THIS_DIR)/workspace/models/argmax
 
 convert_models: download_models
-	docker run -u $(id -u):$(id -g) -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:latest omz_converter --name googlenet-v2-tf --download_dir /models --output_dir /models --precisions FP32
-	docker run -u $(id -u):$(id -g) -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:latest omz_converter --name resnet-50-tf --download_dir /models --output_dir /models --precisions FP32
-	docker run -u $(id -u):$(id -g) -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:latest mo --input input1,input2 --input_shape '[1,1001],[1,1001]' --saved_model_dir /models/argmax/ --output_dir /models/argmax/1
+	docker run -u $(id -u):$(id -g) -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:2024.6.0 omz_converter --name googlenet-v2-tf --download_dir /models --output_dir /models --precisions FP32
+	docker run -u $(id -u):$(id -g) -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:2024.6.0 omz_converter --name resnet-50-tf --download_dir /models --output_dir /models --precisions FP32
+	docker run -u $(id -u):$(id -g) -v $(THIS_DIR)/workspace/models:/models:rw openvino/ubuntu18_dev:2024.6.0 mo --input input1,input2 --input_shape '[1,1001],[1,1001]' --saved_model_dir /models/argmax/ --output_dir /models/argmax/1
 
 prepare_repository: convert_models
 	mkdir -p models/googlenet-v2-tf/1

demos/universal-sentence-encoder/README.md

@@ -31,7 +31,7 @@ The image `openvino/model_server:2023.3` and newer includes ready to use OpenVIN
 ## Start the model server in a container
 You can start the service with a command:
 ```bash
-docker run -d --name ovms -p 9000:9000 -p 8000:8000 -v $(pwd)/universal-sentence-encoder-multilingual:/model openvino/model_server:latest --model_name usem --model_path /model --cpu_extension /ovms/lib/libopenvino_tokenizers.so --plugin_config '{"NUM_STREAMS": 1}' --port 9000 --rest_port 8000
+docker run -d --name ovms -p 9000:9000 -p 8000:8000 -v $(pwd)/universal-sentence-encoder-multilingual:/model openvino/model_server:latest --model_name usem --model_path /model --cpu_extension /ovms/lib/libopenvino_tokenizers.so --plugin_config "{\"NUM_STREAMS\": 1}" --port 9000 --rest_port 8000
 ```
 
 Check the container logs to confirm successful start:

docs/accelerators.md

@@ -6,8 +6,8 @@ Download ResNet50 model
 
 ```bash
 mkdir models
-docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models openvino/ubuntu20_dev:latest omz_downloader --name resnet-50-tf --output_dir /models
-docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models:rw openvino/ubuntu20_dev:latest omz_converter --name resnet-50-tf --download_dir /models --output_dir /models --precisions FP32
+docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models openvino/ubuntu20_dev:2024.6.0 omz_downloader --name resnet-50-tf --output_dir /models
+docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models:rw openvino/ubuntu20_dev:2024.6.0 omz_converter --name resnet-50-tf --download_dir /models --output_dir /models --precisions FP32
 mv ${PWD}/models/public/resnet-50-tf/FP32 ${PWD}/models/public/resnet-50-tf/1
 ```
 
@@ -146,7 +146,7 @@ LATENCY
 docker run --rm -d --device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \
 -v ${PWD}/models/public/resnet-50-tf:/opt/model -p 9001:9001 openvino/model_server:latest-gpu \
 --model_path /opt/model --model_name resnet --port 9001 \
---plugin_config '{"PERFORMANCE_HINT": "LATENCY"}' \
+--plugin_config "{\"PERFORMANCE_HINT\": \"LATENCY\"}" \
 --target_device AUTO
 ```
 
@@ -156,7 +156,7 @@ THROUGHPUT
 docker run --rm -d --device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \
 -v ${PWD}/models/public/resnet-50-tf:/opt/model -p 9001:9001 openvino/model_server:latest-gpu \
 --model_path /opt/model --model_name resnet --port 9001 \
---plugin_config '{"PERFORMANCE_HINT": "THROUGHPUT"}' \
+--plugin_config "{\"PERFORMANCE_HINT\": \"THROUGHPUT\"}" \
 --target_device AUTO
 ```
 
@@ -208,4 +208,4 @@ Example command to run container with NPU:
 docker run --device /dev/accel -p 9000:9000 --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \
 -v ${PWD}/models/public/resnet-50-tf:/opt/model openvino/model_server:latest --model_path /opt/model --model_name resnet --port 9000 --target_device NPU
 ```
-Check more info about the [NPU driver for Linux](https://github.com/intel/linux-npu-driver).
+Check more info about the [NPU driver for Linux](https://github.com/intel/linux-npu-driver).

docs/llm/reference.md

@@ -118,8 +118,6 @@ for chunk in stream:
         break
 ```
 
-> NOTE: To leverage LLM graph cancellation upon client disconnection, use `stream=True` parameter.
-
 ## Models Directory
 
 In node configuration we set `models_path` indicating location of the directory with files loaded by LLM engine. It loads following files: