Merge branch 'main' into ijen/curated_env

.github/workflows/check-new-assets.yaml

@@ -47,5 +47,5 @@ jobs:
       - name: Check PR labels for "safe to publish" label
         if: fromJSON(env.continue) && !contains(github.event.pull_request.labels.*.name, 'safe to publish')
         run: |
-          echo "::error::This PR contains a brand new asset and requires the 'safe to publish' label to run this check. Assets must be properly tested before being added Github and the public registry."
-          exit 1
+          echo "::error::This PR contains a brand new asset and requires the 'safe to publish' label to run this check. Assets must be properly evaluated before being added Github and the public registry. Please reach out to System Registry Content team to review this PR."
+          exit 1

.github/workflows/environments-ci.yaml

@@ -84,6 +84,10 @@ jobs:
         id: build-changed-images
         run: python -u $scripts_environment_dir/build.py -i "${{ github.event.inputs.asset_dirs || env.default_asset_dirs }}" -a $asset_config_filename -o '${{ runner.temp }}'/$built_dir -l '${{ runner.temp }}'/$build_logs_artifact_name -P -g $resource_group -r $container_registry -T 'python -V' -c "${{ needs.check-directory-file-changes.outputs.files-changed }}" -U 'https://github.com/aquasecurity/trivy/releases/download/v0.42.1/trivy_0.42.1_Linux-64bit.deb'
 
+      - name: Validate build logs for additional deprecated dependencies
+        id: validate-build-logs
+        run: python -u $scripts_environment_dir/validate_build_logs.py -l '${{ runner.temp }}'/$build_logs_artifact_name
+
       - name: Upload build logs
         uses: actions/upload-artifact@v3
         if: always()

assets/aistudio/environments/ai_studio_dev/context/requirements.txt

@@ -1,18 +1,13 @@
 # Install Python SDK
-ipykernel
+azure-ai-ml
 openai>1.0
-# Promptflow
-promptflow[azure]>=1.11.0
-promptflow-evals
-promptflow-rag
-keyrings.alt
 # hardcoded the version of azureml-mlflow here for faster Docker image building speed
 azureml-mlflow==1.57.0
 # copilot dependencies
-langchain==0.1.17
-langchain-openai==0.1.6
+keyrings.alt
+ipykernel
+langchain
+langchain-openai
 semantic-kernel
 pytest
 keyring
-# # inference server dependencies (moved to Docker file)
-# azureml-inference-server-http

assets/aml-benchmark/components/batch_benchmark_config_generator/spec.yaml

@@ -4,7 +4,7 @@ type: command
 name: batch_benchmark_config_generator
 display_name: Batch Benchmark Config Generator
 description: Generates the config for the batch score component.
-version: 0.0.6
+version: 0.0.8
 is_deterministic: true
 
 inputs:
@@ -86,7 +86,7 @@ outputs:
     description: The config json file for the batch score component.
 
 code: ../src
-environment: azureml://registries/azureml/environments/model-evaluation/versions/34
+environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
   python -m aml_benchmark.batch_config_generator.main 
   --scoring_url '${{inputs.scoring_url}}'

assets/aml-benchmark/components/batch_benchmark_inference/spec.yaml

@@ -4,7 +4,7 @@ type: pipeline
 name: batch_benchmark_inference
 display_name: Batch Benchmark Inference
 description: Components for batch endpoint inference
-version: 0.0.11
+version: 0.0.12
 
 inputs:
   input_dataset:
@@ -183,7 +183,7 @@ jobs:
   # Preparer
   batch_inference_preparer: 
     type: command
-    component: azureml:batch_inference_preparer:0.0.12
+    component: azureml:batch_inference_preparer:0.0.13
     inputs:
       input_dataset: ${{parent.inputs.input_dataset}}
       model_type: ${{parent.inputs.model_type}}
@@ -201,7 +201,7 @@ jobs:
   # Config generation
   config_generation:
     type: command
-    component: azureml:batch_benchmark_config_generator:0.0.6
+    component: azureml:batch_benchmark_config_generator:0.0.7
     inputs:
       scoring_url: ${{parent.inputs.endpoint_url}}
       connection_name: ${{parent.inputs.connections_name}}
@@ -248,7 +248,7 @@ jobs:
   # Reformat
   batch_output_formatter: 
     type: command
-    component: azureml:batch_output_formatter:0.0.12
+    component: azureml:batch_output_formatter:0.0.13
     inputs:
       model_type: ${{parent.inputs.model_type}}
       batch_inference_output: ${{parent.jobs.endpoint_batch_score.outputs.mini_batch_results_output_directory}}

assets/aml-benchmark/components/batch_benchmark_inference_with_inference_compute/spec.yaml

@@ -4,7 +4,7 @@ type: pipeline
 name: batch_benchmark_inference_with_inference_compute
 display_name: Batch Benchmark Inference (With Inference compute Support)
 description: Components for batch endpoint inference with inference compute support.
-version: 0.0.6
+version: 0.0.7
 
 inputs:
   input_dataset:
@@ -187,7 +187,7 @@ jobs:
   # Preparer
   batch_inference_preparer: 
     type: command
-    component: azureml:batch_inference_preparer:0.0.12
+    component: azureml:batch_inference_preparer:0.0.13
     inputs:
       input_dataset: ${{parent.inputs.input_dataset}}
       model_type: ${{parent.inputs.model_type}}
@@ -205,7 +205,7 @@ jobs:
   # Config generation
   config_generation:
     type: command
-    component: azureml:batch_benchmark_config_generator:0.0.6
+    component: azureml:batch_benchmark_config_generator:0.0.7
     inputs:
       scoring_url: ${{parent.inputs.endpoint_url}}
       connection_name: ${{parent.inputs.connections_name}}
@@ -253,7 +253,7 @@ jobs:
   # Reformat
   batch_output_formatter: 
     type: command
-    component: azureml:batch_output_formatter:0.0.12
+    component: azureml:batch_output_formatter:0.0.13
     inputs:
       model_type: ${{parent.inputs.model_type}}
       batch_inference_output: ${{parent.jobs.endpoint_batch_score.outputs.mini_batch_results_output_directory}}

assets/aml-benchmark/components/batch_inference_preparer/spec.yaml

@@ -4,7 +4,7 @@ type: command
 name: batch_inference_preparer
 display_name: Batch Inference Preparer
 description: Prepare the jsonl file and endpoint for batch inference component.
-version: 0.0.12
+version: 0.0.13
 is_deterministic: true
 
 inputs:
@@ -65,7 +65,7 @@ outputs:
     description: Path to the folder where the ground truth metadata will be stored.
 
 code: ../src
-environment: azureml://registries/azureml/environments/model-evaluation/versions/34
+environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
   python -m aml_benchmark.batch_inference_preparer.main
   --batch_input_pattern '${{inputs.batch_input_pattern}}'

assets/aml-benchmark/components/batch_output_formatter/spec.yaml

@@ -1,5 +1,5 @@
 name: batch_output_formatter
-version: 0.0.12
+version: 0.0.13
 display_name: Batch Output Formatter
 is_deterministic: True
 type: command
@@ -71,7 +71,7 @@ outputs:
   unsafe_content_blocked_requests:
     type: uri_file
 code: ../src
-environment: azureml://registries/azureml/environments/model-evaluation/versions/34
+environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 
 resources:
   instance_count: 1

assets/aml-benchmark/components/benchmark_embedding_model/spec.yaml

@@ -4,7 +4,7 @@ type: command
 name: benchmark_embedding_model
 display_name: Benchmark Embedding Model
 description: Component for benchmarking an embedding model via MTEB.
-version: 0.0.1
+version: 0.0.2
 
 inputs:
   endpoint_url:
@@ -57,7 +57,7 @@ outputs:
     description: Directory where the benchmark metrics will be saved.
 
 code: ../src
-environment: azureml://registries/azureml/environments/model-evaluation/versions/34
+environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
   python -m aml_benchmark.benchmark_embedding_model.main
   $[[--endpoint_url ${{inputs.endpoint_url}}]]

assets/aml-benchmark/components/benchmark_result_aggregator/spec.yaml

@@ -4,7 +4,7 @@ type: command
 name: benchmark_result_aggregator
 display_name: Benchmark result aggregator
 description: Aggregate quality metrics, performance metrics and all of the metadata from the pipeline. Also add them to the root run.
-version: 0.0.11
+version: 0.0.12
 is_deterministic: false
 
 inputs:
@@ -23,7 +23,7 @@ outputs:
     description: The json file with all of the aggregated results.
 
 code: ../src
-environment: azureml://registries/azureml/environments/model-evaluation/versions/34
+environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
   python -m aml_benchmark.benchmark_result_aggregator.main 
   $[[--quality_metrics_path ${{inputs.quality_metrics}}]] 

assets/aml-benchmark/components/compute_performance_metrics/spec.yaml

@@ -4,7 +4,7 @@ type: command
 name: compute_performance_metrics
 display_name: Compute Performance Metrics
 description: Performs performance metric post processing using data from a model inference run.
-version: 0.0.8
+version: 0.0.9
 is_deterministic: true
 
 inputs:
@@ -57,7 +57,7 @@ outputs:
     description: Path to the file where the calculated performance metric results will be stored.
 
 code: ../src
-environment: azureml://registries/azureml/environments/model-evaluation/versions/34
+environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
   python -m aml_benchmark.compute_performance_metrics.main
   --performance_data ${{inputs.performance_data}}

assets/aml-benchmark/components/dataset_downloader/spec.yaml

@@ -4,7 +4,7 @@ type: command
 name: dataset_downloader
 display_name: Dataset Downloader
 description: Downloads the dataset onto blob store.
-version: 0.0.8
+version: 0.0.9
 
 inputs:
   dataset_name: 
@@ -34,7 +34,7 @@ outputs:
     description: Path to the directory where the dataset will be downloaded.
 
 code: ../src
-environment: azureml://registries/azureml/environments/model-evaluation/versions/34
+environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
   python -m aml_benchmark.dataset_downloader.main
   $[[--dataset_name ${{inputs.dataset_name}}]] 

assets/aml-benchmark/components/dataset_preprocessor/spec.yaml

@@ -4,7 +4,7 @@ type: command
 name:  dataset_preprocessor 
 display_name: Dataset Preprocessor
 description: Dataset Preprocessor
-version: 0.0.8
+version: 0.0.9
 is_deterministic: true
 
 inputs:
@@ -50,7 +50,7 @@ outputs:
 
 code: ../src
 
-environment: azureml://registries/azureml/environments/model-evaluation/versions/34
+environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 
 command: >-
   python -m aml_benchmark.dataset_preprocessor.main

assets/aml-benchmark/components/dataset_sampler/spec.yaml

@@ -4,7 +4,7 @@ type: command
 name: dataset_sampler
 display_name: Dataset Sampler
 description: Samples a dataset containing JSONL file(s).
-version: 0.0.8
+version: 0.0.9
 
 inputs:
   dataset: 
@@ -47,7 +47,7 @@ outputs:
     description: Path to the jsonl file where the sampled dataset will be saved.
 
 code: ../src
-environment: azureml://registries/azureml/environments/model-evaluation/versions/34
+environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
   python -m aml_benchmark.dataset_sampler.main 
   --dataset ${{inputs.dataset}} 

assets/aml-benchmark/components/inference_postprocessor/spec.yaml

@@ -4,7 +4,7 @@ type: command
 name: inference_postprocessor
 display_name: Inference Postprocessor
 description: Inference Postprocessor
-version: 0.0.9
+version: 0.0.10
 is_deterministic: true
 
 inputs:
@@ -130,7 +130,7 @@ outputs:
 
 code: ../src
 
-environment: azureml://registries/azureml/environments/model-evaluation/versions/34
+environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 
 command: >-
   python -m aml_benchmark.inference_postprocessor.main

assets/aml-benchmark/components/prompt_crafter/spec.yaml

@@ -6,7 +6,7 @@ display_name: Prompt Crafter
 description: This component is used to create prompts from a given dataset. From a 
   given jinja prompt template, it will generate prompts. It can also create 
   few-shot prompts given a few-shot dataset and the number of shots.
-version: 0.0.11
+version: 0.0.12
 is_deterministic: true
 
 inputs:
@@ -134,7 +134,7 @@ outputs:
     description: Output file path where few_shot_prompt data will be written.
 
 code: ../src
-environment: azureml://registries/azureml/environments/model-evaluation/versions/34
+environment: azureml://registries/azureml/environments/model-evaluation/labels/latest
 command: >-
   python -m aml_benchmark.prompt_crafter.main 
   --test_data ${{inputs.test_data}}

assets/aml-benchmark/components/src/aml_benchmark/batch_config_generator/main.py

@@ -13,7 +13,7 @@
 from aml_benchmark.utils.exceptions import swallow_all_exceptions
 from aml_benchmark.utils.aml_run_utils import str2bool
 from aml_benchmark.utils.exceptions import BenchmarkUserException
-from aml_benchmark.utils.constants import AuthenticationType, get_api_type, get_endpoint_type
+from aml_benchmark.utils.constants import AuthenticationType, ApiType, get_api_type, get_endpoint_type
 from aml_benchmark.utils.error_definitions import BenchmarkUserError
 from azureml._common._error_definition.azureml_error import AzureMLError
 
@@ -305,11 +305,13 @@ def main(
     )
     request_settings_dict = _get_request_settings(additional_headers_dict, max_retry_time_interval)
 
+    api_type = get_api_type(merged_scoring_url)
+    api_dict = {"type": api_type}
+    if api_type == ApiType.Completion:
+        api_dict["response_segment_size"] = response_segment_size
+
     config_dict = {
-        "api": {
-            "type": get_api_type(merged_scoring_url),
-            "response_segment_size": response_segment_size
-        },
+        "api": api_dict,
         "authentication": authentication_dict,
         "concurrency_settings": {
             "initial_worker_count": initial_worker_count,

assets/aml-benchmark/components/src/setup.py

@@ -7,7 +7,7 @@
 
 setup(
     name="aml-benchmark",
-    version="0.7",
+    version="0.8",
     description="AzureML Benchmarking SDK",
     author="Microsoft Corp",
     entry_points={},

assets/common/components/mlflow_model_local_validation/spec.yaml

@@ -1,15 +1,15 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 
 name: mlflow_model_local_validation
-version: 0.0.15
+version: 0.0.16
 type: command
 
 is_deterministic: True
 
 display_name: MLFlow model local validation
 description: Validates if a MLFLow model can be loaded on a compute and is usable for inferencing.
 
-environment: azureml://registries/azureml/environments/python-sdk-v2/versions/22
+environment: azureml://registries/azureml/environments/python-sdk-v2/versions/23
 
 code: ../../src/
 command: >-

assets/common/components/register_model/spec.yaml

@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: register_model
-version: 0.0.18
+version: 0.0.19
 type: command
 
 is_deterministic: True
@@ -9,7 +9,7 @@ display_name: Register model
 description:
   Register a model to a workspace or a registry. The component works on compute with [MSI](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-create-manage-compute-instance?tabs=python) attached.
 
-environment: azureml://registries/azureml/environments/python-sdk-v2/versions/22
+environment: azureml://registries/azureml/environments/python-sdk-v2/versions/23
 
 code: ../../src
 command: >-

assets/designer/environments/designer-cv-transform/asset.yaml

@@ -0,0 +1,6 @@
+name: designer-cv-transform
+version: 82
+type: environment
+spec: spec.yaml
+extra_config: environment.yaml
+categories: ["Designer", "Python"]