diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_basic/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_basic/spec.yaml index 9b216aa4f3..8a0648d582 100644 --- a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_basic/spec.yaml +++ b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_basic/spec.yaml @@ -4,7 +4,7 @@ type: pipeline tags: Preview: "" -version: 0.0.83 +version: 0.0.85 name: llm_ingest_dataset_to_acs_basic display_name: LLM - Dataset to ACS Pipeline is_deterministic: false @@ -110,7 +110,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_validate_deployments:0.0.77' + component: 'azureml:llm_rag_validate_deployments:0.0.80' identity: type: user_identity inputs: @@ -136,7 +136,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_crack_and_chunk:0.0.74' + component: 'azureml:llm_rag_crack_and_chunk:0.0.77' inputs: input_data: ${{parent.inputs.input_data}} input_glob: ${{parent.inputs.input_glob}} @@ -159,7 +159,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_generate_embeddings:0.0.67' + component: 'azureml:llm_rag_generate_embeddings:0.0.70' inputs: chunks_source: type: uri_folder @@ -209,7 +209,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_update_acs_index:0.0.71' + component: 'azureml:llm_rag_update_acs_index:0.0.74' inputs: embeddings: type: uri_folder @@ -229,7 +229,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_register_mlindex_asset:0.0.71' + component: 'azureml:llm_rag_register_mlindex_asset:0.0.74' inputs: storage_uri: ${{parent.jobs.create_acs_index_job.outputs.index}} asset_name: ${{parent.inputs.embeddings_dataset_name}} diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_user_id/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_user_id/spec.yaml index fea24ea9c4..f27eb35120 100644 --- a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_user_id/spec.yaml +++ b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_acs_user_id/spec.yaml @@ -4,7 +4,7 @@ type: pipeline tags: Preview: "" -version: 0.0.81 +version: 0.0.83 name: llm_ingest_dataset_to_acs_user_id display_name: LLM - Dataset to ACS Pipeline is_deterministic: false @@ -110,7 +110,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_validate_deployments:0.0.77' + component: 'azureml:llm_rag_validate_deployments:0.0.80' identity: type: user_identity inputs: @@ -136,7 +136,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_crack_and_chunk:0.0.74' + component: 'azureml:llm_rag_crack_and_chunk:0.0.77' identity: type: user_identity inputs: @@ -161,7 +161,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_generate_embeddings:0.0.67' + component: 'azureml:llm_rag_generate_embeddings:0.0.70' identity: type: user_identity inputs: @@ -215,7 +215,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_update_acs_index:0.0.71' + component: 'azureml:llm_rag_update_acs_index:0.0.74' identity: type: user_identity inputs: @@ -237,7 +237,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_register_mlindex_asset:0.0.71' + component: 'azureml:llm_rag_register_mlindex_asset:0.0.74' identity: type: user_identity inputs: diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_basic/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_basic/spec.yaml index 85360d05d4..f8e4e9e253 100644 --- a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_basic/spec.yaml +++ b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_basic/spec.yaml @@ -4,7 +4,7 @@ type: pipeline tags: Preview: "" -version: 0.0.82 +version: 0.0.84 name: llm_ingest_dataset_to_faiss_basic display_name: LLM - Dataset to FAISS Pipeline is_deterministic: false @@ -102,7 +102,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_validate_deployments:0.0.77' + component: 'azureml:llm_rag_validate_deployments:0.0.80' identity: type: user_identity inputs: @@ -125,7 +125,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_crack_and_chunk:0.0.74' + component: 'azureml:llm_rag_crack_and_chunk:0.0.77' inputs: input_data: ${{parent.inputs.input_data}} input_glob: ${{parent.inputs.input_glob}} @@ -148,7 +148,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_generate_embeddings:0.0.67' + component: 'azureml:llm_rag_generate_embeddings:0.0.70' inputs: chunks_source: type: uri_folder @@ -198,7 +198,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_create_faiss_index:0.0.72' + component: 'azureml:llm_rag_create_faiss_index:0.0.75' inputs: embeddings: type: uri_folder @@ -216,7 +216,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_register_mlindex_asset:0.0.71' + component: 'azureml:llm_rag_register_mlindex_asset:0.0.74' inputs: storage_uri: ${{parent.jobs.create_faiss_index_job.outputs.index}} asset_name: ${{parent.inputs.embeddings_dataset_name}} diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_user_id/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_user_id/spec.yaml index 30f46119bf..7b16babc18 100644 --- a/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_user_id/spec.yaml +++ b/assets/large_language_models/components_pipelines/data_ingestion_dataset_to_faiss_user_id/spec.yaml @@ -4,7 +4,7 @@ type: pipeline tags: Preview: "" -version: 0.0.81 +version: 0.0.83 name: llm_ingest_dataset_to_faiss_user_id display_name: LLM - Dataset to FAISS Pipeline is_deterministic: false @@ -102,7 +102,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_validate_deployments:0.0.77' + component: 'azureml:llm_rag_validate_deployments:0.0.80' identity: type: user_identity inputs: @@ -125,7 +125,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_crack_and_chunk:0.0.74' + component: 'azureml:llm_rag_crack_and_chunk:0.0.77' identity: type: user_identity inputs: @@ -150,7 +150,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_generate_embeddings:0.0.67' + component: 'azureml:llm_rag_generate_embeddings:0.0.70' identity: type: user_identity inputs: @@ -204,7 +204,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_create_faiss_index:0.0.72' + component: 'azureml:llm_rag_create_faiss_index:0.0.75' identity: type: user_identity inputs: @@ -224,7 +224,7 @@ jobs: properties: compute_specification: automatic: true - component: 'azureml:llm_rag_register_mlindex_asset:0.0.71' + component: 'azureml:llm_rag_register_mlindex_asset:0.0.74' identity: type: user_identity inputs: diff --git a/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml b/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml index 8363a3c15e..8c36b2fd89 100644 --- a/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml +++ b/assets/large_language_models/rag/components/crack_and_chunk/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.75 +version: 0.0.77 name: llm_rag_crack_and_chunk display_name: LLM - Crack and Chunk Data is_deterministic: true diff --git a/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml b/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml index a71b648760..7619981361 100644 --- a/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml +++ b/assets/large_language_models/rag/components/crack_and_chunk_and_embed/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.46 +version: 0.0.48 name: llm_rag_crack_and_chunk_and_embed display_name: LLM - Crack, Chunk and Embed Data is_deterministic: true diff --git a/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml b/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml index 7c022851a0..62dad5ba9a 100644 --- a/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml +++ b/assets/large_language_models/rag/components/crack_chunk_embed_index_and_register/spec.yaml @@ -1,5 +1,5 @@ name: llm_rag_crack_chunk_embed_index_and_register -version: 0.0.34 +version: 0.0.36 tags: Preview: "" diff --git a/assets/large_language_models/rag/components/crawl_url/spec.yaml b/assets/large_language_models/rag/components/crawl_url/spec.yaml index f67e8b3d57..94397c7d8e 100644 --- a/assets/large_language_models/rag/components/crawl_url/spec.yaml +++ b/assets/large_language_models/rag/components/crawl_url/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.33 +version: 0.0.35 name: llm_rag_crawl_url display_name: LLM - Crawl URL to Retrieve Data is_deterministic: true diff --git a/assets/large_language_models/rag/components/create_faiss_index/spec.yaml b/assets/large_language_models/rag/components/create_faiss_index/spec.yaml index 2b6f1a2aaa..c7f70d7b68 100644 --- a/assets/large_language_models/rag/components/create_faiss_index/spec.yaml +++ b/assets/large_language_models/rag/components/create_faiss_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.73 +version: 0.0.75 name: llm_rag_create_faiss_index display_name: LLM - Create FAISS Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/create_promptflow/spec.yaml b/assets/large_language_models/rag/components/create_promptflow/spec.yaml index 3b21a7850f..9f0fffe781 100644 --- a/assets/large_language_models/rag/components/create_promptflow/spec.yaml +++ b/assets/large_language_models/rag/components/create_promptflow/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.84 +version: 0.0.86 name: llm_rag_create_promptflow display_name: LLM - Create Prompt Flow is_deterministic: true diff --git a/assets/large_language_models/rag/components/data_import_acs/spec.yaml b/assets/large_language_models/rag/components/data_import_acs/spec.yaml index 021e1d1e45..76a7146194 100644 --- a/assets/large_language_models/rag/components/data_import_acs/spec.yaml +++ b/assets/large_language_models/rag/components/data_import_acs/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.69 +version: 0.0.71 name: llm_rag_data_import_acs display_name: LLM - Import Data from ACS is_deterministic: false diff --git a/assets/large_language_models/rag/components/generate_embeddings/spec.yaml b/assets/large_language_models/rag/components/generate_embeddings/spec.yaml index 481223f8c0..c0df214689 100644 --- a/assets/large_language_models/rag/components/generate_embeddings/spec.yaml +++ b/assets/large_language_models/rag/components/generate_embeddings/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.68 +version: 0.0.70 name: llm_rag_generate_embeddings display_name: LLM - Generate Embeddings is_deterministic: true diff --git a/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml b/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml index 0cd2b85726..974cc515f3 100644 --- a/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml +++ b/assets/large_language_models/rag/components/generate_embeddings_parallel/spec.yaml @@ -4,7 +4,7 @@ type: parallel tags: Preview: "" -version: 0.0.74 +version: 0.0.76 name: llm_rag_generate_embeddings_parallel display_name: LLM - Generate Embeddings Parallel is_deterministic: true diff --git a/assets/large_language_models/rag/components/git_clone/spec.yaml b/assets/large_language_models/rag/components/git_clone/spec.yaml index 23cf96d082..a8252799e9 100644 --- a/assets/large_language_models/rag/components/git_clone/spec.yaml +++ b/assets/large_language_models/rag/components/git_clone/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.72 +version: 0.0.74 name: llm_rag_git_clone display_name: LLM - Clone Git Repo is_deterministic: true diff --git a/assets/large_language_models/rag/components/image_embed_index/spec.yaml b/assets/large_language_models/rag/components/image_embed_index/spec.yaml index 58f3803063..ab756969d0 100644 --- a/assets/large_language_models/rag/components/image_embed_index/spec.yaml +++ b/assets/large_language_models/rag/components/image_embed_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.26 +version: 0.0.28 name: llm_rag_image_embed_index display_name: LLM - Embedding images with Florence is_deterministic: true diff --git a/assets/large_language_models/rag/components/qa_data_generation/spec.yaml b/assets/large_language_models/rag/components/qa_data_generation/spec.yaml index 68f80d54fe..758ae9f732 100644 --- a/assets/large_language_models/rag/components/qa_data_generation/spec.yaml +++ b/assets/large_language_models/rag/components/qa_data_generation/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.72 +version: 0.0.74 name: llm_rag_qa_data_generation display_name: LLM - Generate QnA Test Data is_deterministic: true diff --git a/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml b/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml index a0facd583f..43e412b009 100644 --- a/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml +++ b/assets/large_language_models/rag/components/register_mlindex_asset/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.72 +version: 0.0.74 name: llm_rag_register_mlindex_asset display_name: LLM - Register Vector Index Asset is_deterministic: true diff --git a/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml b/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml index d78951ee6e..e50d949643 100644 --- a/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml +++ b/assets/large_language_models/rag/components/register_qa_data_asset/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.65 +version: 0.0.67 name: llm_rag_register_qa_data_asset display_name: LLM - Register QA Generation Data Asset is_deterministic: true diff --git a/assets/large_language_models/rag/components/update_acs_index/spec.yaml b/assets/large_language_models/rag/components/update_acs_index/spec.yaml index a884bd2b74..70266a8c8b 100644 --- a/assets/large_language_models/rag/components/update_acs_index/spec.yaml +++ b/assets/large_language_models/rag/components/update_acs_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.72 +version: 0.0.74 name: llm_rag_update_acs_index display_name: LLM - Update ACS Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml b/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml index 7085f52acf..0cae00fc15 100644 --- a/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml +++ b/assets/large_language_models/rag/components/update_azure_cosmos_mongo_vcore_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.28 +version: 0.0.30 name: llm_rag_update_cosmos_mongo_vcore_index display_name: LLM - Update Azure Cosmos Mongo vCore Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/update_milvus_index/spec.yaml b/assets/large_language_models/rag/components/update_milvus_index/spec.yaml index 7faaa4a1db..16393cc576 100644 --- a/assets/large_language_models/rag/components/update_milvus_index/spec.yaml +++ b/assets/large_language_models/rag/components/update_milvus_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.28 +version: 0.0.30 name: llm_rag_update_milvus_index display_name: LLM - Update Milvus Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml b/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml index dfd99646fb..d08890ae36 100644 --- a/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml +++ b/assets/large_language_models/rag/components/update_pinecone_index/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.40 +version: 0.0.42 name: llm_rag_update_pinecone_index display_name: LLM - Update Pinecone Index is_deterministic: true diff --git a/assets/large_language_models/rag/components/validate_deployments/spec.yaml b/assets/large_language_models/rag/components/validate_deployments/spec.yaml index 70e84e3408..869173d33d 100644 --- a/assets/large_language_models/rag/components/validate_deployments/spec.yaml +++ b/assets/large_language_models/rag/components/validate_deployments/spec.yaml @@ -4,7 +4,7 @@ type: command tags: Preview: "" -version: 0.0.78 +version: 0.0.80 name: llm_rag_validate_deployments display_name: LLM - Validate Deployments is_deterministic: false diff --git a/assets/large_language_models/rag/environments/rag/context/conda_dependencies.yaml b/assets/large_language_models/rag/environments/rag/context/conda_dependencies.yaml index f51d292bab..21cd740566 100644 --- a/assets/large_language_models/rag/environments/rag/context/conda_dependencies.yaml +++ b/assets/large_language_models/rag/environments/rag/context/conda_dependencies.yaml @@ -8,7 +8,7 @@ dependencies: - scikit-learn - pip: # RAG package - - azureml-rag[azure,langchain,cognitive_search,data_generation,pinecone,milvus,azure_cosmos_mongo_vcore,azure_cosmos_nosql]==0.2.37 + - azureml-rag[azure,langchain,cognitive_search,data_generation,milvus,azure_cosmos_mongo_vcore,azure_cosmos_nosql]==0.2.37.1 # Azure - azure-ai-formrecognizer==3.3.1 - azure-identity=={{latest-pypi-version}} diff --git a/assets/large_language_models/rag/environments/rag_embeddings/context/conda_dependencies.yaml b/assets/large_language_models/rag/environments/rag_embeddings/context/conda_dependencies.yaml index ea1f444160..e29184bf72 100644 --- a/assets/large_language_models/rag/environments/rag_embeddings/context/conda_dependencies.yaml +++ b/assets/large_language_models/rag/environments/rag_embeddings/context/conda_dependencies.yaml @@ -7,7 +7,7 @@ dependencies: - pip=24.0 - pip: # RAG package - - azureml-rag[azure,faiss,langchain,cognitive_search,document_parsing,data_generation,pinecone,milvus,azure_cosmos_mongo_vcore,azure_cosmos_nosql]==0.2.37 + - azureml-rag[azure,faiss,langchain,cognitive_search,document_parsing,data_generation,milvus,azure_cosmos_mongo_vcore,azure_cosmos_nosql]==0.2.37.1 # Azure AI - azure-ai-formrecognizer==3.3.1 # Azure diff --git a/assets/training/automl/environments/ai-ml-automl-dnn-text-gpu-ptca/context/Dockerfile b/assets/training/automl/environments/ai-ml-automl-dnn-text-gpu-ptca/context/Dockerfile index faf565fa14..688b5a646c 100644 --- a/assets/training/automl/environments/ai-ml-automl-dnn-text-gpu-ptca/context/Dockerfile +++ b/assets/training/automl/environments/ai-ml-automl-dnn-text-gpu-ptca/context/Dockerfile @@ -7,7 +7,7 @@ RUN pip install 'azureml-defaults=={{latest-pypi-version}}' RUN pip uninstall -y onnxruntime -RUN pip install onnx==1.16.2 +RUN pip install onnx==1.17.0 RUN pip uninstall -y onnxruntime-training RUN pip install -i https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/ onnxruntime-training==1.18.0 RUN TORCH_CUDA_ARCH_LIST="5.2;6.0;7.0;8.0;8.6;9.0" python -m onnxruntime.training.ortmodule.torch_cpp_extensions.install diff --git a/assets/training/automl/environments/ai-ml-automl-dnn-text-gpu/context/Dockerfile b/assets/training/automl/environments/ai-ml-automl-dnn-text-gpu/context/Dockerfile index 9cf5f650ad..f964de0d4e 100644 --- a/assets/training/automl/environments/ai-ml-automl-dnn-text-gpu/context/Dockerfile +++ b/assets/training/automl/environments/ai-ml-automl-dnn-text-gpu/context/Dockerfile @@ -62,7 +62,7 @@ RUN HOROVOD_WITH_PYTORCH=1 pip install --no-cache-dir git+https://github.com/hor # by fixing dependencies in the base packages RUN pip list && \ pip install pyarrow==14.0.2 \ - 'transformers[sentencepiece,torch]==4.36.2' \ + 'transformers[sentencepiece,torch]==4.37.2' \ aiohttp==3.10.2 # end pip install diff --git a/assets/training/automl/environments/ai-ml-automl/context/Dockerfile b/assets/training/automl/environments/ai-ml-automl/context/Dockerfile index e4495427c0..4a25c5454e 100644 --- a/assets/training/automl/environments/ai-ml-automl/context/Dockerfile +++ b/assets/training/automl/environments/ai-ml-automl/context/Dockerfile @@ -47,7 +47,7 @@ RUN pip install \ azureml-dataset-runtime=={{latest-pypi-version}} \ azureml-defaults=={{latest-pypi-version}} \ # TODO: replace the hard coded above by a referenceto azureml-train-automl[tabular] - 'mlflow-skinny==2.9.2' \ + 'mlflow-skinny==2.15.1' \ 'xgboost==1.5.2' \ 'prophet==1.1.4' \ 'inference-schema' \ diff --git a/assets/training/finetune_acft_common/components/validation/spec.yaml b/assets/training/finetune_acft_common/components/validation/spec.yaml index 5b9bf725e3..de667aab36 100644 --- a/assets/training/finetune_acft_common/components/validation/spec.yaml +++ b/assets/training/finetune_acft_common/components/validation/spec.yaml @@ -1,14 +1,14 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json type: command -version: 0.0.6 +version: 0.0.7 name: finetune_common_validation display_name: Common Validation Component description: Component to validate the finetune job against Validation Service is_deterministic: True -environment: azureml://registries/azureml/environments/acpt-pytorch-2.2-cuda12.1/versions/18 +environment: azureml://registries/azureml/environments/acpt-pytorch-2.2-cuda12.1/versions/23 code: ../../src/validation diff --git a/assets/training/finetune_acft_hf_nlp/components/finetune/chat_completion/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/finetune/chat_completion/spec.yaml index 07a67b3fdf..38f70381bc 100644 --- a/assets/training/finetune_acft_hf_nlp/components/finetune/chat_completion/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/finetune/chat_completion/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: chat_completion_finetune -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Chat Completion Finetune description: Component to finetune Hugging Face pretrained models for chat completion task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/chat_completion_finetune) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/finetune diff --git a/assets/training/finetune_acft_hf_nlp/components/finetune/question_answering/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/finetune/question_answering/spec.yaml index a07f61f967..75b5b86377 100644 --- a/assets/training/finetune_acft_hf_nlp/components/finetune/question_answering/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/finetune/question_answering/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: question_answering_finetune -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Question Answering Finetune description: Component to finetune Hugging Face pretrained models for extractive question answering task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/question_answering_finetune) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/finetune diff --git a/assets/training/finetune_acft_hf_nlp/components/finetune/summarization/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/finetune/summarization/spec.yaml index 069e3e931d..a3a471ed08 100644 --- a/assets/training/finetune_acft_hf_nlp/components/finetune/summarization/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/finetune/summarization/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: summarization_finetune -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Summarization Finetune description: Component to finetune Hugging Face pretrained models for summarization task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/summarization_finetune) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/finetune diff --git a/assets/training/finetune_acft_hf_nlp/components/finetune/text_classification/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/finetune/text_classification/spec.yaml index 76b2cd04f7..c4f9e79bef 100644 --- a/assets/training/finetune_acft_hf_nlp/components/finetune/text_classification/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/finetune/text_classification/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: text_classification_finetune -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: false @@ -8,7 +8,7 @@ is_deterministic: false display_name: Text Classification Finetune description: Component to finetune Hugging Face pretrained models for text classification task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/text_classification_finetune) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/finetune diff --git a/assets/training/finetune_acft_hf_nlp/components/finetune/text_generation/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/finetune/text_generation/spec.yaml index a5103acfb3..82f88816bd 100644 --- a/assets/training/finetune_acft_hf_nlp/components/finetune/text_generation/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/finetune/text_generation/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: text_generation_finetune -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Text Generation Finetune description: Component to finetune model for Text Generation task -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/finetune diff --git a/assets/training/finetune_acft_hf_nlp/components/finetune/token_classification/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/finetune/token_classification/spec.yaml index a8515c1efd..7ea0f1ecd0 100644 --- a/assets/training/finetune_acft_hf_nlp/components/finetune/token_classification/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/finetune/token_classification/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: token_classification_finetune -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: false @@ -8,7 +8,7 @@ is_deterministic: false display_name: Token Classification Finetune description: Component to finetune Hugging Face pretrained models for token classification task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/token_classification_finetune) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/finetune diff --git a/assets/training/finetune_acft_hf_nlp/components/finetune/translation/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/finetune/translation/spec.yaml index dcc0890d91..4ec03205e7 100644 --- a/assets/training/finetune_acft_hf_nlp/components/finetune/translation/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/finetune/translation/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: translation_finetune -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Translation Finetune description: Component to finetune Hugging Face pretrained models for translation task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/translation_finetune) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/finetune diff --git a/assets/training/finetune_acft_hf_nlp/components/model_converter/common/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/model_converter/common/spec.yaml index a2ac35e56e..7d9b49cf5e 100644 --- a/assets/training/finetune_acft_hf_nlp/components/model_converter/common/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/model_converter/common/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: ft_nlp_model_converter -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Common Model Converter description: Component to convert the finetune job output to pytorch and mlflow model -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/model_converter diff --git a/assets/training/finetune_acft_hf_nlp/components/model_import/chat_completion/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/model_import/chat_completion/spec.yaml index 0c19a0ff68..1fd29aabee 100644 --- a/assets/training/finetune_acft_hf_nlp/components/model_import/chat_completion/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/model_import/chat_completion/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: chat_completion_model_import -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Chat Completion Model Import description: Component to import PyTorch / MLFlow model. See [docs](https://aka.ms/azureml/components/chat_completion_model_import) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/model_selector diff --git a/assets/training/finetune_acft_hf_nlp/components/model_import/question_answering/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/model_import/question_answering/spec.yaml index 803a96418f..3c9b6f12a5 100644 --- a/assets/training/finetune_acft_hf_nlp/components/model_import/question_answering/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/model_import/question_answering/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: question_answering_model_import -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Question Answering Model Import description: Component to import PyTorch / MLFlow model. See [docs](https://aka.ms/azureml/components/question_answering_model_import) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/model_selector diff --git a/assets/training/finetune_acft_hf_nlp/components/model_import/summarization/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/model_import/summarization/spec.yaml index 14c6641d7e..bf31286433 100644 --- a/assets/training/finetune_acft_hf_nlp/components/model_import/summarization/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/model_import/summarization/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: summarization_model_import -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Summarization Model Import description: Component to import PyTorch / MLFlow model. See [docs](https://aka.ms/azureml/components/summarization_model_import) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/model_selector diff --git a/assets/training/finetune_acft_hf_nlp/components/model_import/text_classification/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/model_import/text_classification/spec.yaml index 03ea9ad8a6..fcec786fff 100644 --- a/assets/training/finetune_acft_hf_nlp/components/model_import/text_classification/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/model_import/text_classification/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: text_classification_model_import -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Text Classification Model Import description: Component to import PyTorch / MLFlow model. See [docs](https://aka.ms/azureml/components/text_classification_model_import) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/model_selector diff --git a/assets/training/finetune_acft_hf_nlp/components/model_import/text_generation/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/model_import/text_generation/spec.yaml index 2b8ef96dca..e7c88b299d 100644 --- a/assets/training/finetune_acft_hf_nlp/components/model_import/text_generation/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/model_import/text_generation/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: text_generation_model_import -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Text Generation Model Import description: Import PyTorch / MLFlow model -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/model_selector diff --git a/assets/training/finetune_acft_hf_nlp/components/model_import/token_classification/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/model_import/token_classification/spec.yaml index f2545e8cae..95ae17fcac 100644 --- a/assets/training/finetune_acft_hf_nlp/components/model_import/token_classification/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/model_import/token_classification/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: token_classification_model_import -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Token Classification Model Import description: Component to import PyTorch / MLFlow model. See [docs](https://aka.ms/azureml/components/token_classification_model_import) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/model_selector diff --git a/assets/training/finetune_acft_hf_nlp/components/model_import/translation/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/model_import/translation/spec.yaml index da48dd8eda..b7f4c39347 100644 --- a/assets/training/finetune_acft_hf_nlp/components/model_import/translation/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/model_import/translation/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: translation_model_import -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Translation Model Import description: Component to import PyTorch / MLFlow model. See [docs](https://aka.ms/azureml/components/translation_model_import) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/model_selector diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/chat_completion/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/chat_completion/spec.yaml index 2d5dec123f..dd8e79c417 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/chat_completion/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/chat_completion/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: chat_completion_pipeline -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Chat Completion Pipeline description: Pipeline Component to finetune Hugging Face pretrained models for chat completion task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/chat_completion_pipeline) to learn more. @@ -495,7 +495,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -532,7 +532,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' chat_completion_model_import: type: command - component: azureml:chat_completion_model_import:0.0.63 + component: azureml:chat_completion_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -543,7 +543,7 @@ jobs: validation_output: '${{parent.jobs.ft_nlp_common_validation.outputs.validation_info}}' chat_completion_datapreprocess: type: command - component: azureml:chat_completion_datapreprocess:0.0.63 + component: azureml:chat_completion_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -560,7 +560,7 @@ jobs: model_selector_output: '${{parent.jobs.chat_completion_model_import.outputs.output_dir}}' chat_completion_finetune: type: command - component: azureml:chat_completion_finetune:0.0.63 + component: azureml:chat_completion_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -618,7 +618,7 @@ jobs: # mlflow_model_folder: '${{parent.outputs.mlflow_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/question_answering/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/question_answering/spec.yaml index cd9e192ebc..42a7d2a335 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/question_answering/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/question_answering/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: question_answering_pipeline -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Question Answering Pipeline description: Pipeline Component to finetune Hugging Face pretrained models for extractive question answering task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/question_answering_pipeline) to learn more. @@ -541,7 +541,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -578,7 +578,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' question_answering_model_import: type: command - component: azureml:question_answering_model_import:0.0.63 + component: azureml:question_answering_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -589,7 +589,7 @@ jobs: validation_output: '${{parent.jobs.ft_nlp_common_validation.outputs.validation_info}}' question_answering_datapreprocess: type: command - component: azureml:question_answering_datapreprocess:0.0.63 + component: azureml:question_answering_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -614,7 +614,7 @@ jobs: model_selector_output: '${{parent.jobs.question_answering_model_import.outputs.output_dir}}' question_answering_finetune: type: command - component: azureml:question_answering_finetune:0.0.63 + component: azureml:question_answering_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -671,7 +671,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' @@ -681,7 +681,7 @@ jobs: mlflow_model_folder: '${{parent.outputs.mlflow_model_folder}}' model_prediction: type: command - component: azureml:model_prediction:0.0.33 + component: azureml:model_prediction:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' resources: instance_type: '${{parent.inputs.instance_type_model_evaluation}}' @@ -697,7 +697,7 @@ jobs: evaluation_config_params: '${{parent.inputs.evaluation_config_params}}' compute_metrics: type: command - component: azureml:compute_metrics:0.0.33 + component: azureml:compute_metrics:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' resources: instance_type: '${{parent.inputs.instance_type_model_evaluation}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/summarization/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/summarization/spec.yaml index f85f4473dd..0a1920e036 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/summarization/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/summarization/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: summarization_pipeline -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Summarization Pipeline description: Pipeline Component to finetune Hugging Face pretrained models for summarization task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/summarization_pipeline) to learn more. @@ -512,7 +512,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -549,7 +549,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' summarization_model_import: type: command - component: azureml:summarization_model_import:0.0.63 + component: azureml:summarization_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -560,7 +560,7 @@ jobs: validation_output: '${{parent.jobs.ft_nlp_common_validation.outputs.validation_info}}' summarization_datapreprocess: type: command - component: azureml:summarization_datapreprocess:0.0.63 + component: azureml:summarization_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -580,7 +580,7 @@ jobs: model_selector_output: '${{parent.jobs.summarization_model_import.outputs.output_dir}}' summarization_finetune: type: command - component: azureml:summarization_finetune:0.0.63 + component: azureml:summarization_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -637,7 +637,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' @@ -647,7 +647,7 @@ jobs: mlflow_model_folder: '${{parent.outputs.mlflow_model_folder}}' model_prediction: type: command - component: azureml:model_prediction:0.0.33 + component: azureml:model_prediction:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' resources: instance_type: '${{parent.inputs.instance_type_model_evaluation}}' @@ -663,7 +663,7 @@ jobs: evaluation_config_params: '${{parent.inputs.evaluation_config_params}}' compute_metrics: type: command - component: azureml:compute_metrics:0.0.33 + component: azureml:compute_metrics:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' resources: instance_type: '${{parent.inputs.instance_type_model_evaluation}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_classification/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_classification/spec.yaml index 239859e107..cb08be0299 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_classification/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_classification/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: text_classification_pipeline -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Text Classification Pipeline description: Pipeline component to finetune Hugging Face pretrained models for text classification task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/text_classification_pipeline) to learn more. @@ -514,7 +514,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -551,7 +551,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' text_classification_model_import: type: command - component: azureml:text_classification_model_import:0.0.63 + component: azureml:text_classification_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -562,7 +562,7 @@ jobs: validation_output: '${{parent.jobs.ft_nlp_common_validation.outputs.validation_info}}' text_classification_datapreprocess: type: command - component: azureml:text_classification_datapreprocess:0.0.63 + component: azureml:text_classification_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -583,7 +583,7 @@ jobs: model_selector_output: '${{parent.jobs.text_classification_model_import.outputs.output_dir}}' text_classification_finetune: type: command - component: azureml:text_classification_finetune:0.0.63 + component: azureml:text_classification_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -640,7 +640,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' @@ -650,7 +650,7 @@ jobs: mlflow_model_folder: '${{parent.outputs.mlflow_model_folder}}' model_prediction: type: command - component: azureml:model_prediction:0.0.33 + component: azureml:model_prediction:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' resources: instance_type: '${{parent.inputs.instance_type_model_evaluation}}' @@ -666,7 +666,7 @@ jobs: evaluation_config_params: '${{parent.inputs.evaluation_config_params}}' compute_metrics: type: command - component: azureml:compute_metrics:0.0.33 + component: azureml:compute_metrics:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' resources: instance_type: '${{parent.inputs.instance_type_model_evaluation}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation/spec.yaml index 61e85a2e9f..ef984737c0 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: text_generation_pipeline -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Text Generation Pipeline description: Pipeline component for text generation @@ -528,7 +528,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -566,7 +566,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' text_generation_model_import: type: command - component: azureml:text_generation_model_import:0.0.63 + component: azureml:text_generation_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -578,7 +578,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_datapreprocess: type: command - component: azureml:text_generation_datapreprocess:0.0.63 + component: azureml:text_generation_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -598,7 +598,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_finetune: type: command - component: azureml:text_generation_finetune:0.0.63 + component: azureml:text_generation_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -656,7 +656,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' @@ -667,7 +667,7 @@ jobs: mlflow_model_folder: '${{parent.outputs.mlflow_model_folder}}' model_prediction: type: command - component: azureml:model_prediction:0.0.33 + component: azureml:model_prediction:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' resources: instance_type: '${{parent.inputs.instance_type_model_evaluation}}' @@ -683,7 +683,7 @@ jobs: evaluation_config_params: '${{parent.inputs.evaluation_config_params}}' compute_metrics: type: command - component: azureml:compute_metrics:0.0.33 + component: azureml:compute_metrics:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' resources: instance_type: '${{parent.inputs.instance_type_model_evaluation}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_basic_high/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_basic_high/spec.yaml index 0a0909bdbc..82d87fb2e2 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_basic_high/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_basic_high/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: text_generation_pipeline_singularity_basic_high -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Text Generation Pipeline Singularity Basic High description: Pipeline component for text generation @@ -520,7 +520,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -563,7 +563,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' text_generation_model_import: type: command - component: azureml:text_generation_model_import:0.0.63 + component: azureml:text_generation_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -580,7 +580,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_datapreprocess: type: command - component: azureml:text_generation_datapreprocess:0.0.63 + component: azureml:text_generation_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -605,7 +605,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_finetune: type: command - component: azureml:text_generation_finetune:0.0.63 + component: azureml:text_generation_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -667,7 +667,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_basic_low/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_basic_low/spec.yaml index 628e4f6879..8755e6ec26 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_basic_low/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_basic_low/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: text_generation_pipeline_singularity_basic_low -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Text Generation Pipeline Singularity Basic Low description: Pipeline component for text generation @@ -520,7 +520,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -563,7 +563,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' text_generation_model_import: type: command - component: azureml:text_generation_model_import:0.0.63 + component: azureml:text_generation_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -580,7 +580,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_datapreprocess: type: command - component: azureml:text_generation_datapreprocess:0.0.63 + component: azureml:text_generation_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -605,7 +605,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_finetune: type: command - component: azureml:text_generation_finetune:0.0.63 + component: azureml:text_generation_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -667,7 +667,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_basic_medium/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_basic_medium/spec.yaml index 91726724c4..78279c5358 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_basic_medium/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_basic_medium/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: text_generation_pipeline_singularity_basic_medium -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Text Generation Pipeline Singularity Basic Medium description: Pipeline component for text generation @@ -520,7 +520,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -563,7 +563,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' text_generation_model_import: type: command - component: azureml:text_generation_model_import:0.0.63 + component: azureml:text_generation_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -580,7 +580,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_datapreprocess: type: command - component: azureml:text_generation_datapreprocess:0.0.63 + component: azureml:text_generation_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -605,7 +605,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_finetune: type: command - component: azureml:text_generation_finetune:0.0.63 + component: azureml:text_generation_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -667,7 +667,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_premium_high/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_premium_high/spec.yaml index e1eabca2b9..8dfe00ca85 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_premium_high/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_premium_high/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: text_generation_pipeline_singularity_premium_high -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Text Generation Pipeline Singularity Premium High description: Pipeline component for text generation @@ -520,7 +520,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -563,7 +563,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' text_generation_model_import: type: command - component: azureml:text_generation_model_import:0.0.63 + component: azureml:text_generation_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -580,7 +580,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_datapreprocess: type: command - component: azureml:text_generation_datapreprocess:0.0.63 + component: azureml:text_generation_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -605,7 +605,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_finetune: type: command - component: azureml:text_generation_finetune:0.0.63 + component: azureml:text_generation_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -667,7 +667,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_premium_low/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_premium_low/spec.yaml index bb45f1374e..a4285475a0 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_premium_low/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_premium_low/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: text_generation_pipeline_singularity_premium_low -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Text Generation Pipeline Singularity Premium Low description: Pipeline component for text generation @@ -520,7 +520,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -563,7 +563,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' text_generation_model_import: type: command - component: azureml:text_generation_model_import:0.0.63 + component: azureml:text_generation_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -580,7 +580,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_datapreprocess: type: command - component: azureml:text_generation_datapreprocess:0.0.63 + component: azureml:text_generation_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -605,7 +605,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_finetune: type: command - component: azureml:text_generation_finetune:0.0.63 + component: azureml:text_generation_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -667,7 +667,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_premium_medium/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_premium_medium/spec.yaml index 79f7735b49..4af207340e 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_premium_medium/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_premium_medium/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: text_generation_pipeline_singularity_premium_medium -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Text Generation Pipeline Singularity Premium Medium description: Pipeline component for text generation @@ -520,7 +520,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -563,7 +563,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' text_generation_model_import: type: command - component: azureml:text_generation_model_import:0.0.63 + component: azureml:text_generation_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -580,7 +580,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_datapreprocess: type: command - component: azureml:text_generation_datapreprocess:0.0.63 + component: azureml:text_generation_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -605,7 +605,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_finetune: type: command - component: azureml:text_generation_finetune:0.0.63 + component: azureml:text_generation_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -667,7 +667,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_standard_high/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_standard_high/spec.yaml index 5cd0b38cfd..4f7cb9eb53 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_standard_high/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_standard_high/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: text_generation_pipeline_singularity_standard_high -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Text Generation Pipeline Singularity Standard High description: Pipeline component for text generation @@ -520,7 +520,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -563,7 +563,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' text_generation_model_import: type: command - component: azureml:text_generation_model_import:0.0.63 + component: azureml:text_generation_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -580,7 +580,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_datapreprocess: type: command - component: azureml:text_generation_datapreprocess:0.0.63 + component: azureml:text_generation_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -605,7 +605,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_finetune: type: command - component: azureml:text_generation_finetune:0.0.63 + component: azureml:text_generation_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -667,7 +667,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_standard_low/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_standard_low/spec.yaml index 3f22614f04..6e98780d8c 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_standard_low/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_standard_low/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: text_generation_pipeline_singularity_standard_low -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Text Generation Pipeline Singularity Standard Low description: Pipeline component for text generation @@ -520,7 +520,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -563,7 +563,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' text_generation_model_import: type: command - component: azureml:text_generation_model_import:0.0.63 + component: azureml:text_generation_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -580,7 +580,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_datapreprocess: type: command - component: azureml:text_generation_datapreprocess:0.0.63 + component: azureml:text_generation_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -605,7 +605,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_finetune: type: command - component: azureml:text_generation_finetune:0.0.63 + component: azureml:text_generation_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -667,7 +667,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_standard_medium/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_standard_medium/spec.yaml index f551fb3069..97ec3207d0 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_standard_medium/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/text_generation_singularity_standard_medium/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: text_generation_pipeline_singularity_standard_medium -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Text Generation Pipeline Singularity Standard Medium description: Pipeline component for text generation @@ -520,7 +520,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -563,7 +563,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' text_generation_model_import: type: command - component: azureml:text_generation_model_import:0.0.63 + component: azureml:text_generation_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -580,7 +580,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_datapreprocess: type: command - component: azureml:text_generation_datapreprocess:0.0.63 + component: azureml:text_generation_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -605,7 +605,7 @@ jobs: system_properties: '${{parent.inputs.system_properties}}' text_generation_finetune: type: command - component: azureml:text_generation_finetune:0.0.63 + component: azureml:text_generation_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -667,7 +667,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/token_classification/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/token_classification/spec.yaml index 9ca78c9155..e1346bc0f3 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/token_classification/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/token_classification/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: token_classification_pipeline -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Token Classification Pipeline description: Pipeline component to finetune Hugging Face pretrained models for token classification task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/token_classification_pipeline) to learn more. @@ -507,7 +507,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -544,7 +544,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' token_classification_model_import: type: command - component: azureml:token_classification_model_import:0.0.63 + component: azureml:token_classification_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -555,7 +555,7 @@ jobs: validation_output: '${{parent.jobs.ft_nlp_common_validation.outputs.validation_info}}' token_classification_datapreprocess: type: command - component: azureml:token_classification_datapreprocess:0.0.63 + component: azureml:token_classification_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -574,7 +574,7 @@ jobs: model_selector_output: '${{parent.jobs.token_classification_model_import.outputs.output_dir}}' token_classification_finetune: type: command - component: azureml:token_classification_finetune:0.0.63 + component: azureml:token_classification_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -631,7 +631,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' @@ -641,7 +641,7 @@ jobs: mlflow_model_folder: '${{parent.outputs.mlflow_model_folder}}' model_prediction: type: command - component: azureml:model_prediction:0.0.33 + component: azureml:model_prediction:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' resources: instance_type: '${{parent.inputs.instance_type_model_evaluation}}' @@ -657,7 +657,7 @@ jobs: evaluation_config_params: '${{parent.inputs.evaluation_config_params}}' compute_metrics: type: command - component: azureml:compute_metrics:0.0.33 + component: azureml:compute_metrics:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' resources: instance_type: '${{parent.inputs.instance_type_model_evaluation}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/translation/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/translation/spec.yaml index e24285371b..2af8b75c81 100644 --- a/assets/training/finetune_acft_hf_nlp/components/pipeline_components/translation/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/pipeline_components/translation/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: translation_pipeline -version: 0.0.65 +version: 0.0.66 type: pipeline display_name: Translation Pipeline description: Pipeline component to finetune Hugging Face pretrained models for translation task. The component supports optimizations such as LoRA, Deepspeed and ONNXRuntime for performance enhancement. See [docs](https://aka.ms/azureml/components/translation_pipeline) to learn more. @@ -504,7 +504,7 @@ outputs: jobs: ft_nlp_common_validation: type: command - component: azureml:ft_nlp_common_validation:0.0.63 + component: azureml:ft_nlp_common_validation:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -541,7 +541,7 @@ jobs: auto_find_batch_size: '${{parent.inputs.auto_find_batch_size}}' translation_model_import: type: command - component: azureml:translation_model_import:0.0.63 + component: azureml:translation_model_import:0.0.64 compute: '${{parent.inputs.compute_model_import}}' resources: instance_type: '${{parent.inputs.instance_type_model_import}}' @@ -552,7 +552,7 @@ jobs: validation_output: '${{parent.jobs.ft_nlp_common_validation.outputs.validation_info}}' translation_datapreprocess: type: command - component: azureml:translation_datapreprocess:0.0.63 + component: azureml:translation_datapreprocess:0.0.64 compute: '${{parent.inputs.compute_preprocess}}' resources: instance_type: '${{parent.inputs.instance_type_preprocess}}' @@ -571,7 +571,7 @@ jobs: model_selector_output: '${{parent.jobs.translation_model_import.outputs.output_dir}}' translation_finetune: type: command - component: azureml:translation_finetune:0.0.63 + component: azureml:translation_finetune:0.0.64 compute: '${{parent.inputs.compute_finetune}}' distribution: type: pytorch @@ -628,7 +628,7 @@ jobs: pytorch_model_folder: '${{parent.outputs.pytorch_model_folder}}' ft_nlp_model_converter: type: command - component: azureml:ft_nlp_model_converter:0.0.63 + component: azureml:ft_nlp_model_converter:0.0.64 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' @@ -638,7 +638,7 @@ jobs: mlflow_model_folder: '${{parent.outputs.mlflow_model_folder}}' model_prediction: type: command - component: azureml:model_prediction:0.0.33 + component: azureml:model_prediction:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' resources: instance_type: '${{parent.inputs.instance_type_model_evaluation}}' @@ -654,7 +654,7 @@ jobs: evaluation_config_params: '${{parent.inputs.evaluation_config_params}}' compute_metrics: type: command - component: azureml:compute_metrics:0.0.33 + component: azureml:compute_metrics:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' resources: instance_type: '${{parent.inputs.instance_type_model_evaluation}}' diff --git a/assets/training/finetune_acft_hf_nlp/components/preprocess/chat_completion/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/preprocess/chat_completion/spec.yaml index 0d32b55e76..c7010123f7 100644 --- a/assets/training/finetune_acft_hf_nlp/components/preprocess/chat_completion/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/preprocess/chat_completion/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: chat_completion_datapreprocess -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Chat Completion DataPreProcess description: Component to preprocess data for chat completion task. See [docs](https://aka.ms/azureml/components/chat_completion_datapreprocess) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/preprocess diff --git a/assets/training/finetune_acft_hf_nlp/components/preprocess/question_answering/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/preprocess/question_answering/spec.yaml index f37be970dc..7a6615e5da 100644 --- a/assets/training/finetune_acft_hf_nlp/components/preprocess/question_answering/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/preprocess/question_answering/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: question_answering_datapreprocess -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Question Answering DataPreProcess description: Component to preprocess data for question answering task. See [docs](https://aka.ms/azureml/components/question_answering_datapreprocess) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/preprocess diff --git a/assets/training/finetune_acft_hf_nlp/components/preprocess/summarization/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/preprocess/summarization/spec.yaml index d0e6a382c5..5ac24b5d66 100644 --- a/assets/training/finetune_acft_hf_nlp/components/preprocess/summarization/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/preprocess/summarization/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: summarization_datapreprocess -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Summarization DataPreProcess description: Component to preprocess data for summarization task. See [docs](https://aka.ms/azureml/components/summarization_datapreprocess) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/preprocess diff --git a/assets/training/finetune_acft_hf_nlp/components/preprocess/text_classification/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/preprocess/text_classification/spec.yaml index 35710a2033..c6e7aca760 100644 --- a/assets/training/finetune_acft_hf_nlp/components/preprocess/text_classification/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/preprocess/text_classification/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: text_classification_datapreprocess -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Text Classification DataPreProcess description: Component to preprocess data for single label classification task. See [docs](https://aka.ms/azureml/components/text_classification_datapreprocess) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/preprocess diff --git a/assets/training/finetune_acft_hf_nlp/components/preprocess/text_generation/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/preprocess/text_generation/spec.yaml index b32fa8b8e6..00a7b0299b 100644 --- a/assets/training/finetune_acft_hf_nlp/components/preprocess/text_generation/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/preprocess/text_generation/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: text_generation_datapreprocess -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Text Generation DataPreProcess description: Component to preprocess data for text generation task -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/preprocess diff --git a/assets/training/finetune_acft_hf_nlp/components/preprocess/token_classification/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/preprocess/token_classification/spec.yaml index 861921b8dd..ee38f272b1 100644 --- a/assets/training/finetune_acft_hf_nlp/components/preprocess/token_classification/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/preprocess/token_classification/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: token_classification_datapreprocess -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Token Classification DataPreProcess description: Component to preprocess data for token classification task. See [docs](https://aka.ms/azureml/components/token_classification_datapreprocess) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/preprocess diff --git a/assets/training/finetune_acft_hf_nlp/components/preprocess/translation/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/preprocess/translation/spec.yaml index 02f29397ac..b9499cd8a0 100644 --- a/assets/training/finetune_acft_hf_nlp/components/preprocess/translation/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/preprocess/translation/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: translation_datapreprocess -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Translation DataPreProcess description: Component to preprocess data for translation task. See [docs](https://aka.ms/azureml/components/translation_datapreprocess) to learn more. -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/preprocess diff --git a/assets/training/finetune_acft_hf_nlp/components/validation/common/spec.yaml b/assets/training/finetune_acft_hf_nlp/components/validation/common/spec.yaml index 187d575be1..e7b78fadda 100644 --- a/assets/training/finetune_acft_hf_nlp/components/validation/common/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components/validation/common/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: ft_nlp_common_validation -version: 0.0.63 +version: 0.0.64 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: Common Validation Component description: Component to validate the finetune job against Validation Service -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 code: ../../../src/validation diff --git a/assets/training/finetune_acft_hf_nlp/components_v2/data_import/common/spec.yaml b/assets/training/finetune_acft_hf_nlp/components_v2/data_import/common/spec.yaml index c3f2b8765f..a0a9ca7f88 100644 --- a/assets/training/finetune_acft_hf_nlp/components_v2/data_import/common/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components_v2/data_import/common/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: oss_text_generation_data_import -version: 0.0.25 +version: 0.0.26 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: OSS Text Generation Data Import description: FTaaS component to copy user training data to output -environment: azureml://registries/azureml/environments/acft-hf-nlp-data-import/versions/4 +environment: azureml://registries/azureml/environments/acft-hf-nlp-data-import/versions/7 inputs: task_name: diff --git a/assets/training/finetune_acft_hf_nlp/components_v2/finetune/chat_completion/spec.yaml b/assets/training/finetune_acft_hf_nlp/components_v2/finetune/chat_completion/spec.yaml index a3a66eee58..6c3c1b44c5 100644 --- a/assets/training/finetune_acft_hf_nlp/components_v2/finetune/chat_completion/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components_v2/finetune/chat_completion/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: oss_chat_completion_finetune -version: 0.0.25 +version: 0.0.26 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: OSS Chat Completion Finetune description: FTaaS component to finetune model for Chat Completion task -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 inputs: task_name: diff --git a/assets/training/finetune_acft_hf_nlp/components_v2/finetune/text_generation/spec.yaml b/assets/training/finetune_acft_hf_nlp/components_v2/finetune/text_generation/spec.yaml index f96b591587..11cceb143b 100644 --- a/assets/training/finetune_acft_hf_nlp/components_v2/finetune/text_generation/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components_v2/finetune/text_generation/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: oss_text_generation_finetune -version: 0.0.25 +version: 0.0.26 type: command is_deterministic: True @@ -8,7 +8,7 @@ is_deterministic: True display_name: OSS Text Generation Finetune description: FTaaS component to finetune model for Text Generation task -environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/77 +environment: azureml://registries/azureml/environments/acft-hf-nlp-gpu/versions/78 inputs: task_name: diff --git a/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/chat_completion/spec.yaml b/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/chat_completion/spec.yaml index d5e1d24959..31b448c9c2 100644 --- a/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/chat_completion/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/chat_completion/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: oss_chat_completion_pipeline -version: 0.0.25 +version: 0.0.26 type: pipeline display_name: OSS Chat Completion Pipeline description: FTaaS Pipeline component for chat completion @@ -123,7 +123,7 @@ outputs: jobs: oss_text_generation_data_import: type: command - component: azureml:oss_text_generation_data_import:0.0.25 + component: azureml:oss_text_generation_data_import:0.0.26 compute: '${{parent.inputs.compute_data_import}}' resources: instance_type: '${{parent.inputs.instance_type_data_import}}' @@ -142,7 +142,7 @@ jobs: user_column_names: 'messages' oss_chat_completion_finetune: type: command - component: azureml:oss_chat_completion_finetune:0.0.25 + component: azureml:oss_chat_completion_finetune:0.0.26 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' diff --git a/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/text_generation/spec.yaml b/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/text_generation/spec.yaml index 893aa018c7..540a6bdb85 100644 --- a/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/text_generation/spec.yaml +++ b/assets/training/finetune_acft_hf_nlp/components_v2/pipeline_components/text_generation/spec.yaml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json name: oss_text_generation_pipeline -version: 0.0.25 +version: 0.0.26 type: pipeline display_name: OSS Text Generation Pipeline description: FTaaS Pipeline component for text generation @@ -138,7 +138,7 @@ outputs: jobs: oss_text_generation_data_import: type: command - component: azureml:oss_text_generation_data_import:0.0.25 + component: azureml:oss_text_generation_data_import:0.0.26 compute: '${{parent.inputs.compute_data_import}}' resources: instance_type: '${{parent.inputs.instance_type_data_import}}' @@ -157,7 +157,7 @@ jobs: user_column_names: '${{parent.inputs.text_key}},${{parent.inputs.ground_truth_key}}' oss_text_generation_finetune: type: command - component: azureml:oss_text_generation_finetune:0.0.25 + component: azureml:oss_text_generation_finetune:0.0.26 compute: '${{parent.inputs.compute_finetune}}' resources: instance_type: '${{parent.inputs.instance_type_finetune}}' diff --git a/assets/training/finetune_acft_image/components/framework_selector/spec.yaml b/assets/training/finetune_acft_image/components/framework_selector/spec.yaml index 7ea22b667c..e6c05f4797 100644 --- a/assets/training/finetune_acft_image/components/framework_selector/spec.yaml +++ b/assets/training/finetune_acft_image/components/framework_selector/spec.yaml @@ -1,14 +1,14 @@ $schema: https://azuremlschemas.azureedge.net/development/commandComponent.schema.json type: command -version: 0.0.18 +version: 0.0.19 name: image_framework_selector display_name: Framework Selector for Image Tasks description: Framework selector control flow component for image tasks is_deterministic: true -environment: azureml://registries/azureml/environments/acpt-automl-image-framework-selector-gpu/versions/43 +environment: azureml://registries/azureml/environments/acpt-automl-image-framework-selector-gpu/versions/44 code: ../../src/framework_selector diff --git a/assets/training/finetune_acft_image/components/model_output_selector/spec.yaml b/assets/training/finetune_acft_image/components/model_output_selector/spec.yaml index 844ab4bc79..1a93806d58 100644 --- a/assets/training/finetune_acft_image/components/model_output_selector/spec.yaml +++ b/assets/training/finetune_acft_image/components/model_output_selector/spec.yaml @@ -1,14 +1,14 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json type: command -version: 0.0.17 +version: 0.0.18 name: image_model_output_selector display_name: Model output selector for image components description: Model output selector control flow component for image tasks is_deterministic: true -environment: azureml://registries/azureml/environments/acpt-automl-image-framework-selector-gpu/versions/43 +environment: azureml://registries/azureml/environments/acpt-automl-image-framework-selector-gpu/versions/44 code: ../../src/model_output_selector diff --git a/assets/training/finetune_acft_image/components/pipeline_components/classification/spec.yaml b/assets/training/finetune_acft_image/components/pipeline_components/classification/spec.yaml index 93ab63d6da..c57a0b9627 100644 --- a/assets/training/finetune_acft_image/components/pipeline_components/classification/spec.yaml +++ b/assets/training/finetune_acft_image/components/pipeline_components/classification/spec.yaml @@ -1,7 +1,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json type: pipeline -version: 0.0.21 +version: 0.0.22 name: image_classification_pipeline display_name: Image Classification Pipeline description: Pipeline component for image classification. @@ -255,7 +255,7 @@ jobs: finetune_common_validation: type: command - component: azureml:finetune_common_validation:0.0.6 + component: azureml:finetune_common_validation:0.0.7 compute: ${{parent.inputs.compute_model_import}} inputs: train_mltable_path: ${{parent.inputs.training_data}} @@ -269,7 +269,7 @@ jobs: framework_selector: type: command - component: azureml:image_framework_selector:0.0.18 + component: azureml:image_framework_selector:0.0.19 compute: ${{parent.inputs.compute_model_import}} inputs: task_type: 'image-classification' @@ -372,7 +372,7 @@ jobs: output_selector: type: command - component: azureml:image_model_output_selector:0.0.17 + component: azureml:image_model_output_selector:0.0.18 compute: ${{parent.inputs.compute_model_import}} inputs: mlflow_model_t: ${{parent.jobs.image_classification_runtime_component.outputs.mlflow_model_folder}} diff --git a/assets/training/finetune_acft_image/components/pipeline_components/hf_classification/spec.yaml b/assets/training/finetune_acft_image/components/pipeline_components/hf_classification/spec.yaml index 4e1eb7f5a4..c38b4eee8d 100644 --- a/assets/training/finetune_acft_image/components/pipeline_components/hf_classification/spec.yaml +++ b/assets/training/finetune_acft_image/components/pipeline_components/hf_classification/spec.yaml @@ -1,7 +1,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json type: pipeline -version: 0.0.21 +version: 0.0.22 name: transformers_image_classification_pipeline display_name: Image Classification HuggingFace Transformers Pipeline description: Pipeline component for image classification using HuggingFace transformers models. @@ -393,7 +393,7 @@ outputs: jobs: finetune_common_validation: type: command - component: azureml:finetune_common_validation:0.0.6 + component: azureml:finetune_common_validation:0.0.7 compute: ${{parent.inputs.compute_model_import}} inputs: mlflow_model_path: ${{parent.inputs.mlflow_model}} @@ -483,7 +483,7 @@ jobs: model_prediction: type: command - component: azureml:model_prediction:0.0.34 + component: azureml:model_prediction:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' inputs: task: '${{parent.inputs.task_name}}' @@ -498,7 +498,7 @@ jobs: compute_metrics: type: command - component: azureml:compute_metrics:0.0.33 + component: azureml:compute_metrics:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' inputs: task: '${{parent.inputs.task_name}}' diff --git a/assets/training/finetune_acft_image/components/pipeline_components/instance_segmentation/spec.yaml b/assets/training/finetune_acft_image/components/pipeline_components/instance_segmentation/spec.yaml index 289c2fe004..8cdfd297a7 100644 --- a/assets/training/finetune_acft_image/components/pipeline_components/instance_segmentation/spec.yaml +++ b/assets/training/finetune_acft_image/components/pipeline_components/instance_segmentation/spec.yaml @@ -1,7 +1,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json type: pipeline -version: 0.0.23 +version: 0.0.24 name: image_instance_segmentation_pipeline display_name: Image Instance Segmentation Pipeline description: Pipeline component for image instance segmentation. @@ -300,7 +300,7 @@ jobs: finetune_common_validation: type: command - component: azureml:finetune_common_validation:0.0.6 + component: azureml:finetune_common_validation:0.0.7 compute: ${{parent.inputs.compute_model_import}} inputs: train_mltable_path: ${{parent.inputs.training_data}} @@ -314,7 +314,7 @@ jobs: framework_selector: type: command - component: azureml:image_framework_selector:0.0.18 + component: azureml:image_framework_selector:0.0.19 compute: ${{parent.inputs.compute_model_import}} inputs: task_type: ${{parent.inputs.task_type}} @@ -424,7 +424,7 @@ jobs: output_selector: type: command - component: azureml:image_model_output_selector:0.0.17 + component: azureml:image_model_output_selector:0.0.18 compute: ${{parent.inputs.compute_model_import}} inputs: mlflow_model_t: ${{parent.jobs.image_instance_segmentation_runtime_component.outputs.mlflow_model_folder}} diff --git a/assets/training/finetune_acft_image/components/pipeline_components/mmd_od_is/spec.yaml b/assets/training/finetune_acft_image/components/pipeline_components/mmd_od_is/spec.yaml index d8a457ec67..ea1303a2e8 100644 --- a/assets/training/finetune_acft_image/components/pipeline_components/mmd_od_is/spec.yaml +++ b/assets/training/finetune_acft_image/components/pipeline_components/mmd_od_is/spec.yaml @@ -1,7 +1,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json type: pipeline -version: 0.0.23 +version: 0.0.24 name: mmdetection_image_objectdetection_instancesegmentation_pipeline display_name: Image Object Detection and Instance Segmentation MMDetection Pipeline description: Pipeline component for image object detection and instance segmentation using MMDetection models. @@ -391,7 +391,7 @@ outputs: jobs: finetune_common_validation: type: command - component: azureml:finetune_common_validation:0.0.6 + component: azureml:finetune_common_validation:0.0.7 compute: ${{parent.inputs.compute_model_import}} inputs: mlflow_model_path: ${{parent.inputs.mlflow_model}} @@ -482,7 +482,7 @@ jobs: model_prediction: type: command - component: azureml:model_prediction:0.0.34 + component: azureml:model_prediction:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' inputs: task: '${{parent.inputs.task_name}}' @@ -497,7 +497,7 @@ jobs: compute_metrics: type: command - component: azureml:compute_metrics:0.0.33 + component: azureml:compute_metrics:0.0.35 compute: '${{parent.inputs.compute_model_evaluation}}' inputs: task: '${{parent.inputs.task_name}}' diff --git a/assets/training/finetune_acft_image/components/pipeline_components/object_detection/spec.yaml b/assets/training/finetune_acft_image/components/pipeline_components/object_detection/spec.yaml index 3d32d63123..7399c3f987 100644 --- a/assets/training/finetune_acft_image/components/pipeline_components/object_detection/spec.yaml +++ b/assets/training/finetune_acft_image/components/pipeline_components/object_detection/spec.yaml @@ -1,7 +1,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.json type: pipeline -version: 0.0.23 +version: 0.0.24 name: image_object_detection_pipeline display_name: Image Object Detection Pipeline description: Pipeline component for image object detection. @@ -325,7 +325,7 @@ jobs: finetune_common_validation: type: command - component: azureml:finetune_common_validation:0.0.6 + component: azureml:finetune_common_validation:0.0.7 compute: ${{parent.inputs.compute_model_import}} inputs: train_mltable_path: ${{parent.inputs.training_data}} @@ -339,7 +339,7 @@ jobs: framework_selector: type: command - component: azureml:image_framework_selector:0.0.18 + component: azureml:image_framework_selector:0.0.19 compute: ${{parent.inputs.compute_model_import}} inputs: task_type: ${{parent.inputs.task_type}} @@ -451,7 +451,7 @@ jobs: output_selector: type: command - component: azureml:image_model_output_selector:0.0.17 + component: azureml:image_model_output_selector:0.0.18 compute: ${{parent.inputs.compute_model_import}} inputs: mlflow_model_t: ${{parent.jobs.image_object_detection_runtime_component.outputs.mlflow_model_folder}} diff --git a/assets/training/finetune_acft_image/environments/acft_image_huggingface/context/Dockerfile b/assets/training/finetune_acft_image/environments/acft_image_huggingface/context/Dockerfile index e6429dce98..9f8f74d74f 100644 --- a/assets/training/finetune_acft_image/environments/acft_image_huggingface/context/Dockerfile +++ b/assets/training/finetune_acft_image/environments/acft_image_huggingface/context/Dockerfile @@ -1,5 +1,5 @@ # PTCA image -FROM mcr.microsoft.com/aifx/acpt/stable-ubuntu2004-cu118-py310-torch222:{{latest-image-tag:biweekly\.\d{6}\.\d{1}.*}} +FROM mcr.microsoft.com/aifx/acpt/stable-ubuntu2204-cu118-py310-torch222:{{latest-image-tag:biweekly\.\d{6}\.\d{1}.*}} USER root RUN apt-get -y update diff --git a/assets/training/finetune_acft_image/environments/acft_image_huggingface/context/requirements.txt b/assets/training/finetune_acft_image/environments/acft_image_huggingface/context/requirements.txt index a042652a14..97824fed8c 100644 --- a/assets/training/finetune_acft_image/environments/acft_image_huggingface/context/requirements.txt +++ b/assets/training/finetune_acft_image/environments/acft_image_huggingface/context/requirements.txt @@ -12,4 +12,4 @@ optimum==1.19.0 diffusers==0.27.2 huggingface-hub==0.25.2 setuptools>=71.1.0 -mlflow==2.12.1 \ No newline at end of file +mlflow==2.16.0 \ No newline at end of file diff --git a/assets/training/finetune_acft_image/environments/acft_image_mmdetection/context/Dockerfile b/assets/training/finetune_acft_image/environments/acft_image_mmdetection/context/Dockerfile index 26039f2e6b..6ec8b3c43f 100644 --- a/assets/training/finetune_acft_image/environments/acft_image_mmdetection/context/Dockerfile +++ b/assets/training/finetune_acft_image/environments/acft_image_mmdetection/context/Dockerfile @@ -1,5 +1,5 @@ # PTCA image -FROM mcr.microsoft.com/aifx/acpt/stable-ubuntu2004-cu118-py310-torch222:{{latest-image-tag:biweekly\.\d{6}\.\d{1}.*}} +FROM mcr.microsoft.com/aifx/acpt/stable-ubuntu2204-cu118-py310-torch222:{{latest-image-tag:biweekly\.\d{6}\.\d{1}.*}} USER root RUN apt-get -y update @@ -21,4 +21,4 @@ RUN sed -i 's/2.2.0/2.3.0/' /opt/conda/envs/ptca/lib/python3.10/site-packages/mm # Vulnerability Fix RUN pip install gunicorn==22.0.0 -# dummy number to change when needing to force rebuild without changing the definition: 1 \ No newline at end of file +# dummy number to change when needing to force rebuild without changing the definition: 2 \ No newline at end of file diff --git a/assets/training/finetune_acft_image/environments/acft_video_mmtracking/context/Dockerfile b/assets/training/finetune_acft_image/environments/acft_video_mmtracking/context/Dockerfile index e7cbae41cd..87d88dec2f 100644 --- a/assets/training/finetune_acft_image/environments/acft_video_mmtracking/context/Dockerfile +++ b/assets/training/finetune_acft_image/environments/acft_video_mmtracking/context/Dockerfile @@ -1,5 +1,5 @@ # PTCA image -FROM mcr.microsoft.com/aifx/acpt/stable-ubuntu2004-cu118-py310-torch222:{{latest-image-tag:biweekly\.\d{6}\.\d{1}.*}} +FROM mcr.microsoft.com/aifx/acpt/stable-ubuntu2204-cu118-py310-torch222:{{latest-image-tag:biweekly\.\d{6}\.\d{1}.*}} USER root RUN apt-get -y update diff --git a/assets/training/finetune_acft_image/environments/acpt_image_framework_selector/context/Dockerfile b/assets/training/finetune_acft_image/environments/acpt_image_framework_selector/context/Dockerfile index e75290d8c2..9c5e376f89 100644 --- a/assets/training/finetune_acft_image/environments/acpt_image_framework_selector/context/Dockerfile +++ b/assets/training/finetune_acft_image/environments/acpt_image_framework_selector/context/Dockerfile @@ -1,5 +1,5 @@ # PTCA image -FROM mcr.microsoft.com/aifx/acpt/stable-ubuntu2004-cu118-py310-torch222:{{latest-image-tag:biweekly\.\d{6}\.\d{1}.*}} +FROM mcr.microsoft.com/aifx/acpt/stable-ubuntu2204-cu118-py310-torch222:{{latest-image-tag:biweekly\.\d{6}\.\d{1}.*}} RUN apt-get -y update diff --git a/assets/training/model_management/components/convert_model_to_mlflow/spec.yaml b/assets/training/model_management/components/convert_model_to_mlflow/spec.yaml index 3b9885517b..fc8e168813 100644 --- a/assets/training/model_management/components/convert_model_to_mlflow/spec.yaml +++ b/assets/training/model_management/components/convert_model_to_mlflow/spec.yaml @@ -1,7 +1,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json name: convert_model_to_mlflow -version: 0.0.34 +version: 0.0.35 type: command is_deterministic: True @@ -19,7 +19,7 @@ command: | pip_pkg_str="${pip_pkgs[*]}" if [[ -n "$pip_pkg_str" ]]; then echo "Installing $pip_pkg_str"; pip install $pip_pkg_str; echo "pip installation completed. For any installation error please check above logs"; fi; echo "Running model conversion ... " - python -u run_model_preprocess.py $[[--model-id ${{inputs.model_id}}]] $[[--task-name ${{inputs.task_name}}]] $[[--model-download-metadata ${{inputs.model_download_metadata}}]] $[[--license-file-path ${{inputs.license_file_path}}]] $[[--hf-config-args "${{inputs.hf_config_args}}"]] $[[--hf-tokenizer-args "${{inputs.hf_tokenizer_args}}"]] $[[--hf-model-args "${{inputs.hf_model_args}}"]] $[[--hf-pipeline-args "${{inputs.hf_pipeline_args}}"]] $[[--hf-config-class ${{inputs.hf_config_class}}]] $[[--hf-model-class ${{inputs.hf_model_class}}]] $[[--hf-tokenizer-class ${{inputs.hf_tokenizer_class}}]] $[[--hf-use-experimental-features ${{inputs.hf_use_experimental_features}}]] $[[--extra-pip-requirements "${{inputs.extra_pip_requirements}}"]] $[[--inference-base-image "${{inputs.inference_base_image}}"]] --vllm-enabled ${{inputs.vllm_enabled}} --model-framework ${{inputs.model_framework}} --model-path ${{inputs.model_path}} --mlflow-model-output-dir ${{outputs.mlflow_model_folder}} --model-flavor ${{inputs.model_flavor}} + python -u run_model_preprocess.py $[[--model-id ${{inputs.model_id}}]] $[[--task-name ${{inputs.task_name}}]] $[[--model-download-metadata ${{inputs.model_download_metadata}}]] $[[--license-file-path ${{inputs.license_file_path}}]] $[[--hf-config-args "${{inputs.hf_config_args}}"]] $[[--hf-tokenizer-args "${{inputs.hf_tokenizer_args}}"]] $[[--hf-model-args "${{inputs.hf_model_args}}"]] $[[--hf-pipeline-args "${{inputs.hf_pipeline_args}}"]] $[[--hf-config-class ${{inputs.hf_config_class}}]] $[[--hf-model-class ${{inputs.hf_model_class}}]] $[[--hf-tokenizer-class ${{inputs.hf_tokenizer_class}}]] $[[--hf-use-experimental-features ${{inputs.hf_use_experimental_features}}]] $[[--extra-pip-requirements "${{inputs.extra_pip_requirements}}"]] $[[--inference-base-image "${{inputs.inference_base_image}}"]] --vllm-enabled ${{inputs.vllm_enabled}} --model-framework ${{inputs.model_framework}} $[[--model-path "${{inputs.model_path}}"]] $[[--model-path-mmd "${{inputs.model_path_mmd}}"]] --mlflow-model-output-dir ${{outputs.mlflow_model_folder}} --model-flavor ${{inputs.model_flavor}} echo "Completed model conversion ... " inputs: @@ -79,6 +79,7 @@ inputs: - mask-generation - video-multi-object-tracking - visual-question-answering + - image-feature-extraction description: A Hugging face task on which model was trained on. A required parameter for transformers MLflow flavor. Can be provided as input here or in model_download_metadata JSON file. optional: true @@ -156,7 +157,13 @@ inputs: type: uri_folder description: Path to the model. mode: ro_mount - optional: false + optional: true + + model_path_mmd: + type: uri_folder + description: Path to the MMD model. + mode: ro_mount + optional: true license_file_path: type: uri_file diff --git a/assets/training/model_management/components/import_model/spec.yaml b/assets/training/model_management/components/import_model/spec.yaml index 709105add4..2dd61c9b80 100644 --- a/assets/training/model_management/components/import_model/spec.yaml +++ b/assets/training/model_management/components/import_model/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: import_model display_name: Import model description: Import a model into a workspace or a registry -version: 0.0.40 +version: 0.0.41 # Pipeline inputs inputs: @@ -100,6 +100,7 @@ inputs: - mask-generation - video-multi-object-tracking - visual-question-answering + - image-feature-extraction optional: true type: string @@ -218,7 +219,7 @@ outputs: jobs: validation_trigger_import: - component: azureml:validation_trigger_import:0.0.13 + component: azureml:validation_trigger_import:0.0.14 compute: ${{parent.inputs.compute}} resources: instance_type: '${{parent.inputs.instance_type}}' @@ -271,7 +272,7 @@ jobs: type: uri_folder convert_model_to_mlflow: - component: azureml:convert_model_to_mlflow:0.0.34 + component: azureml:convert_model_to_mlflow:0.0.35 compute: ${{parent.inputs.compute}} resources: instance_type: '${{parent.inputs.instance_type}}' diff --git a/assets/training/model_management/components/validation_trigger_import/spec.yaml b/assets/training/model_management/components/validation_trigger_import/spec.yaml index 94b87f080b..afa560c6e0 100644 --- a/assets/training/model_management/components/validation_trigger_import/spec.yaml +++ b/assets/training/model_management/components/validation_trigger_import/spec.yaml @@ -4,7 +4,7 @@ type: command name: validation_trigger_import display_name: Validation Trigger Import model description: Component for enabling validation of import pipeline. -version: 0.0.13 +version: 0.0.14 # Pipeline inputs inputs: @@ -67,6 +67,7 @@ inputs: - image-classification - text-to-image - chat-completion + - image-feature-extraction optional: true type: string diff --git a/assets/training/model_management/src/azureml/model/mgmt/processors/factory.py b/assets/training/model_management/src/azureml/model/mgmt/processors/factory.py index bb57653ebf..cf41cd448c 100644 --- a/assets/training/model_management/src/azureml/model/mgmt/processors/factory.py +++ b/assets/training/model_management/src/azureml/model/mgmt/processors/factory.py @@ -36,6 +36,7 @@ DinoV2MLFlowConvertor, LLaVAMLFlowConvertor, SegmentAnythingMLFlowConvertor, + VirchowMLFlowConvertor ) @@ -84,6 +85,10 @@ def get_mlflow_convertor(model_framework, model_dir, output_dir, temp_dir, trans return SegmentAnythingMLflowConvertorFactory.create_mlflow_convertor( model_dir, output_dir, temp_dir, translate_params ) + elif task == PyFuncSupportedTasks.IMAGE_FEATURE_EXTRACTION.value: + return VirchowMLflowConvertorFactory.create_mlflow_convertor( + model_dir, output_dir, temp_dir, translate_params + ) else: raise Exception( f"Models from {model_framework} for task {task} and model {model_id} " @@ -299,6 +304,19 @@ def create_mlflow_convertor(model_dir, output_dir, temp_dir, translate_params): ) +class VirchowMLflowConvertorFactory(MLflowConvertorFactoryInterface): + """Factory class for segment anything Virchow model.""" + + def create_mlflow_convertor(model_dir, output_dir, temp_dir, translate_params): + """Create MLflow convertor for segment anything Virchow model.""" + return VirchowMLFlowConvertor( + model_dir=model_dir, + output_dir=output_dir, + temp_dir=temp_dir, + translate_params=translate_params, + ) + + class MMLabTrackingMLflowConvertorFactory(MLflowConvertorFactoryInterface): """Factory class for MMTrack video model family.""" diff --git a/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/config.py b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/config.py index 6637ff1204..fe941a6fcd 100644 --- a/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/config.py +++ b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/config.py @@ -56,6 +56,9 @@ class SupportedTasks(_CustomEnum): IMAGE_OBJECT_DETECTION = "image-object-detection" IMAGE_INSTANCE_SEGMENTATION = "image-instance-segmentation" + # Virchow + IMAGE_FEATURE_EXTRACTION = "image-feature-extraction" + class ModelFamilyPrefixes(_CustomEnum): """Prefixes for some of the models converted to PyFunc MLflow.""" @@ -65,3 +68,6 @@ class ModelFamilyPrefixes(_CustomEnum): # DinoV2 model family. DINOV2 = "facebook/dinov2" + + # Virchow model family. + VIRCHOW = "paige-ai/Virchow" diff --git a/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/convertors.py b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/convertors.py index e20fc66285..f4c45a3393 100644 --- a/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/convertors.py +++ b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/convertors.py @@ -41,6 +41,8 @@ MLflowSchemaLiterals as SegmentAnythingMLFlowSchemaLiterals, MLflowLiterals as SegmentAnythingMLflowLiterals from azureml.model.mgmt.processors.pyfunc.vision.config import \ MLflowSchemaLiterals as VisionMLFlowSchemaLiterals, MMDetLiterals +from azureml.model.mgmt.processors.pyfunc.virchow.config import \ + MLflowSchemaLiterals as VirchowMLFlowSchemaLiterals, MLflowLiterals as VirchowMLflowLiterals logger = get_logger(__name__) @@ -1136,3 +1138,101 @@ def save_as_mlflow(self): conda_env=conda_env_file, code_path=None, ) + + +class VirchowMLFlowConvertor(PyFuncMLFLowConvertor): + """PyFunc MLfLow convertor for Virchow models.""" + + MODEL_DIR = os.path.join(os.path.dirname(__file__), "virchow") + COMMON_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "common") + + def __init__(self, **kwargs): + """Initialize MLflow convertor for Virchow models.""" + super().__init__(**kwargs) + if self._task not in \ + [SupportedTasks.IMAGE_FEATURE_EXTRACTION.value]: + raise Exception("Unsupported task") + + def get_model_signature(self) -> ModelSignature: + """Return MLflow model signature with input and output schema for the given input task. + + :return: MLflow model signature. + :rtype: mlflow.models.signature.ModelSignature + """ + input_schema = Schema( + [ + ColSpec(VirchowMLFlowSchemaLiterals.INPUT_COLUMN_IMAGE_DATA_TYPE, + VirchowMLFlowSchemaLiterals.INPUT_COLUMN_IMAGE), + ColSpec(VirchowMLFlowSchemaLiterals.INPUT_COLUMN_TEXT_DATA_TYPE, + VirchowMLFlowSchemaLiterals.INPUT_COLUMN_TEXT), + ] + ) + params = ParamSchema( + [ + ParamSpec(VirchowMLflowLiterals.DEVICE_TYPE, + DataType.string, "cuda"), + ParamSpec(VirchowMLflowLiterals.TO_HALF_PRECISION, + DataType.boolean, False), + ] + ) + if self._task == SupportedTasks.IMAGE_FEATURE_EXTRACTION.value: + output_schema = Schema( + [ + ColSpec(VirchowMLFlowSchemaLiterals.OUTPUT_COLUMN_DATA_TYPE, + VirchowMLFlowSchemaLiterals.OUTPUT_COLUMN_PROBS), + ColSpec(VirchowMLFlowSchemaLiterals.OUTPUT_COLUMN_DATA_TYPE, + VirchowMLFlowSchemaLiterals.OUTPUT_COLUMN_LABELS), + ColSpec(VirchowMLFlowSchemaLiterals.OUTPUT_COLUMN_DATA_TYPE, + VirchowMLFlowSchemaLiterals.OUTPUT_COLUMN_IMAGE_FEATURES), + ColSpec(VirchowMLFlowSchemaLiterals.OUTPUT_COLUMN_DATA_TYPE, + VirchowMLFlowSchemaLiterals.OUTPUT_COLUMN_TEXT_FEATURES), + ] + ) + else: + raise Exception("Unsupported task") + + return ModelSignature(inputs=input_schema, outputs=output_schema, params=params) + + def save_as_mlflow(self): + """Prepare model for save to MLflow.""" + sys.path.append(self.MODEL_DIR) + + from virchow_mlflow_model_wrapper import VirchowModelWrapper + mlflow_model_wrapper = VirchowModelWrapper() + + artifacts_dict = self._prepare_artifacts_dict() + conda_env_file = os.path.join(self.MODEL_DIR, "conda.yaml") + code_path = self._get_code_path() + + super()._save( + mlflow_model_wrapper=mlflow_model_wrapper, + artifacts_dict=artifacts_dict, + conda_env=conda_env_file, + code_path=code_path, + ) + + def _get_code_path(self): + """Return code path for saving mlflow model depending on task type. + + :return: code path + :rtype: List[str] + """ + code_path = [ + os.path.join(self.MODEL_DIR, "virchow_mlflow_model_wrapper.py"), + os.path.join(self.MODEL_DIR, "config.py"), + os.path.join(self.COMMON_DIR, "vision_utils.py") + ] + + return code_path + + def _prepare_artifacts_dict(self) -> Dict: + """Prepare artifacts dict for MLflow model. + + :return: artifacts dict + :rtype: Dict + """ + artifacts_dict = { + VirchowMLflowLiterals.CHECKPOINT_PATH: self._model_dir+"/pytorch_model.bin", + VirchowMLflowLiterals.CONFIG_PATH: self._model_dir+"/config.json" + } + return artifacts_dict diff --git a/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/virchow/conda.yaml b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/virchow/conda.yaml new file mode 100644 index 0000000000..43b178be2a --- /dev/null +++ b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/virchow/conda.yaml @@ -0,0 +1,17 @@ +channels: +- conda-forge +dependencies: +- python=3.10.14 +- pip<=24.0 +- pip: + - mlflow==2.13.2 + - cffi==1.16.0 + - cloudpickle==2.2.1 + - numpy==1.23.5 + - pandas==2.2.2 + - pyyaml==6.0.1 + - requests==2.32.3 + - timm==1.0.9,>=0.9.11 + - torch>2 + - pillow>=10 +name: mlflow-env diff --git a/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/virchow/config.py b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/virchow/config.py new file mode 100644 index 0000000000..dd8dd49302 --- /dev/null +++ b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/virchow/config.py @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Common Config.""" + +from enum import Enum + +from mlflow.types import DataType + + +class _CustomEnum(Enum): + @classmethod + def has_value(cls, value): + return value in cls._value2member_map_ + + +class MLflowSchemaLiterals: + """MLflow model signature related schema.""" + + INPUT_COLUMN_IMAGE_DATA_TYPE = DataType.binary + INPUT_COLUMN_IMAGE = "image" + INPUT_COLUMN_TEXT_DATA_TYPE = DataType.string + INPUT_COLUMN_TEXT = "text" + OUTPUT_COLUMN_DATA_TYPE = DataType.string + OUTPUT_COLUMN_PROBS = "probs" + OUTPUT_COLUMN_LABELS = "labels" + OUTPUT_COLUMN_IMAGE_FEATURES = "image_features" + OUTPUT_COLUMN_TEXT_FEATURES = "text_features" + + +class MLflowLiterals: + """MLflow export related literals.""" + + CHECKPOINT_PATH = "checkpoint_path" + CONFIG_PATH = "config_path" + DEVICE_TYPE = "device_type" + TO_HALF_PRECISION = "to_half_precision" diff --git a/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/virchow/virchow_mlflow_model_wrapper.py b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/virchow/virchow_mlflow_model_wrapper.py new file mode 100644 index 0000000000..8e2188138e --- /dev/null +++ b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/virchow/virchow_mlflow_model_wrapper.py @@ -0,0 +1,97 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""MLFlow pyfunc wrapper for Virchow models.""" + +import timm +import json + +import mlflow.pyfunc +import torch +import pandas as pd +import io +from PIL import Image +from timm.data import resolve_data_config +from timm.data.transforms_factory import create_transform +from timm.layers import SwiGLUPacked +from config import MLflowSchemaLiterals +import logging +logger = logging.getLogger("mlflow") # Set log level to debugging +logger.setLevel(logging.DEBUG) + + +class VirchowModelWrapper(mlflow.pyfunc.PythonModel): + """MLFlow pyfunc wrapper for Virchow models.""" + + def load_context(self, context): + """Load a MLflow model with pyfunc.load_model(). + + :param context: MLflow context containing artifacts that the model can use for inference + :type context: mlflow.pyfunc.PythonModelContext + """ + config_path = context.artifacts["config_path"] + checkpoint_path = context.artifacts["checkpoint_path"] + # config = json.loads(config_path.read_text()) + with open(config_path) as f: + config = json.load(f) + self.model = timm.create_model( + model_name="vit_huge_patch14_224", + checkpoint_path=checkpoint_path, + mlp_layer=SwiGLUPacked, + act_layer=torch.nn.SiLU, + pretrained_cfg=config["pretrained_cfg"], + **config["model_args"] + ) + self.model.eval() + self.transforms = create_transform( + **resolve_data_config(self.model.pretrained_cfg, model=self.model) + ) + + # def predict(self, image_input_path: str, params: dict = None): + def predict( + self, + context: mlflow.pyfunc.PythonModelContext, + input_data: pd.DataFrame, + params: pd.DataFrame) -> pd.DataFrame: + """Perform inference on the input data. + + :param context: MLflow context containing artifacts that the model can use for inference + :type context: mlflow.pyfunc.PythonModelContext + :param input_data: Pandas DataFrame with columns ["image"], ["prompt"] and ["direct_question"], where + the image is either a url or a base64 string, the prompt is the dialog so far between the + user and the model and the direct question is a prompt with a single question from the user. + :type input_data: pd.DataFrame + :return: Pandas dataframe with column ["response"] containing the model's response to the dialog so far. + """ + from vision_utils import process_image + pil_images = [ + Image.open(io.BytesIO(process_image(image))) + for image in input_data[MLflowSchemaLiterals.INPUT_COLUMN_IMAGE] + ] + # image = input_data["image"] + # pil_image = Image.open(io.BytesIO(process_image(pil_images[0]))) + pil_image = self.transforms(pil_images[0]).unsqueeze(0) # size: 1 x 3 x 224 x 224 + + device_type = params.get("device_type", "cuda") + to_half_precision = params.get("to_half_precision", False) + + with torch.inference_mode(), torch.autocast( + device_type=device_type, dtype=torch.float16 + ): + output = self.model(pil_image) # size: 1 x 257 x 1280 + + class_token = output[:, 0] # size: 1 x 1280 + # patch_tokens = output[:, 1:] # size: 1 x 256 x 1280 + + # use the class token only as the embedding + # size: 1 x 1280 + embedding = class_token + + # the model output will be fp32 because the final operation is a LayerNorm that is ran in mixed precision + # optionally, you can convert the embedding to fp16 for efficiency in downstream use + if to_half_precision: + embedding = embedding.to(torch.float16) + + df_result = pd.DataFrame() + df_result['output'] = embedding.tolist() + return df_result diff --git a/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/virchow/vision_utils.py b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/virchow/vision_utils.py new file mode 100644 index 0000000000..a0af9f0d17 --- /dev/null +++ b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/virchow/vision_utils.py @@ -0,0 +1,234 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Helper utils for vision Mlflow models.""" + +import base64 +import io +import logging +import os +import re +import requests +import uuid + +import PIL +import pandas as pd +import numpy as np +import torch + +from ast import literal_eval +from PIL import Image, UnidentifiedImageError +from typing import Union + + +logger = logging.getLogger(__name__) + +# Uncomment the following line for mlflow debug mode +# logging.getLogger("mlflow").setLevel(logging.DEBUG) + + +def save_image(output_folder: str, img: PIL.Image.Image, format: str) -> str: + """ + Save image in a folder designated for batch output and return image file path. + + :param output_folder: directory path where we need to save files + :type output_folder: str + :param img: image object + :type img: PIL.Image.Image + :param format: format to save image + :type format: str + :return: file name of image. + :rtype: str + """ + filename = f"image_{uuid.uuid4()}.{format.lower()}" + img.save(os.path.join(output_folder, filename), format=format) + return filename + + +def get_pil_image(image: bytes) -> PIL.Image.Image: + """ + Convert image bytes to PIL image. + + :param image: image bytes + :type image: bytes + :return: PIL image object + :rtype: PIL.Image.Image + """ + try: + return Image.open(io.BytesIO(image)) + except UnidentifiedImageError as e: + logger.error("Invalid image format. Please use base64 encoding for input images.") + raise e + + +def image_to_base64(img: PIL.Image.Image, format: str) -> str: + """ + Convert image into Base64 encoded string. + + :param img: image object + :type img: PIL.Image.Image + :param format: image format + :type format: str + :return: base64 encoded string + :rtype: str + """ + buffered = io.BytesIO() + img.save(buffered, format=format) + img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") + return img_str + + +def process_image(image: Union[str, bytes]) -> bytes: + """Process image. + + If input image is in bytes format, return it as it is. + If input image is in base64 string format, decode it to bytes. + If input image is in url format, download it and return bytes. + https://github.com/mlflow/mlflow/blob/master/examples/flower_classifier/image_pyfunc.py + + :param image: image in base64 string format or url or bytes. + :type image: string or bytes + :return: decoded image. + :rtype: bytes + """ + if isinstance(image, bytes): + return image + elif isinstance(image, str): + if _is_valid_url(image): + try: + response = requests.get(image) + response.raise_for_status() # Raise exception in case of unsuccessful response code. + image = response.content + return image + except requests.exceptions.RequestException as ex: + raise ValueError(f"Unable to retrieve image from url string due to exception: {ex}") + else: + try: + return base64.b64decode(image) + except ValueError: + raise ValueError( + "The provided image string cannot be decoded. " "Expected format is base64 string or url string." + ) + else: + raise ValueError( + f"Image received in {type(image)} format which is not supported. " + "Expected format is bytes, base64 string or url string." + ) + + +def process_image_pandas_series(image_pandas_series: pd.Series) -> pd.Series: + """Process image in Pandas series form. + + If input image is in bytes format, return it as it is. + If input image is in base64 string format, decode it to bytes. + If input image is in url format, download it and return bytes. + https://github.com/mlflow/mlflow/blob/master/examples/flower_classifier/image_pyfunc.py + + :param img: pandas series with image in base64 string format or url or bytes. + :type img: pd.Series + :return: decoded image in pandas series format. + :rtype: Pandas Series + """ + image = image_pandas_series[0] + return pd.Series(process_image(image)) + + +def _is_valid_url(text: str) -> bool: + """Check if text is url or base64 string. + + :param text: text to validate + :type text: str + :return: True if url else false + :rtype: bool + """ + regex = ( + "((http|https)://)(www.)?" + + "[a-zA-Z0-9@:%._\\+~#?&//=\\-]" + + "{2,256}\\.[a-z]" + + "{2,6}\\b([-a-zA-Z0-9@:%" + + "._\\+~#?&//=]*)" + ) + p = re.compile(regex) + + # If the string is empty + # return false + if str is None: + return False + + # Return if the string + # matched the ReGex + if re.search(p, text): + return True + else: + return False + + +def get_current_device() -> torch.device: + """Get current cuda device. + + :return: current device + :rtype: torch.device + """ + # check if GPU is available + if torch.cuda.is_available(): + try: + # get the current device index + device_idx = torch.distributed.get_rank() + except RuntimeError as ex: + if "Default process group has not been initialized".lower() in str(ex).lower(): + device_idx = 0 + else: + logger.error(str(ex)) + raise ex + return torch.device(type="cuda", index=device_idx) + else: + return torch.device(type="cpu") + + +def string_to_nested_float_list(input_str: str) -> list: + """Convert string to nested list of floats. + + :return: string converted to nested list of floats + :rtype: list + """ + if input_str in ["null", "None", "", "nan", "NoneType", np.nan, None]: + return None + try: + # Use ast.literal_eval to safely evaluate the string into a list + nested_list = literal_eval(input_str) + + # Recursive function to convert all numbers in the nested list to floats + def to_floats(lst) -> list: + """ + Recursively convert all numbers in a nested list to floats. + + :param lst: nested list + :type lst: list + :return: nested list of floats + :rtype: list + """ + return [to_floats(item) if isinstance(item, list) else float(item) for item in lst] + + # Use the recursive function to process the nested list + return to_floats(nested_list) + except (ValueError, SyntaxError) as e: + # In case of an error during conversion, print an error message + print(f"Invalid input {input_str}: {e}, ignoring.") + return None + + +def bool_array_to_pil_image(bool_array: np.ndarray) -> PIL.Image.Image: + """Convert boolean array to PIL Image. + + :param bool_array: boolean array + :type bool_array: np.array + :return: PIL Image + :rtype: PIL.Image.Image + """ + # Convert boolean array to uint8 + uint8_array = bool_array.astype(np.uint8) * 255 + + # Create a PIL Image + pil_image = Image.fromarray(uint8_array) + + return pil_image