Skip to content

Commit

Permalink
Merge branch 'main' into dev/aisgupta/upgrade_ubuntu_base_image
Browse files Browse the repository at this point in the history
  • Loading branch information
ashg1910 authored Nov 7, 2024
2 parents 48dd4f5 + 82ec219 commit e0d9f29
Show file tree
Hide file tree
Showing 143 changed files with 4,169 additions and 23 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ tags:
Preview: ""
name: llm_ingest_db_to_acs
display_name: LLM - SQL Datastore to ACS Pipeline
version: 0.0.96
version: 0.0.97
description: Single job pipeline to chunk data from AzureML sql data store, and create ACS embeddings index
settings:
default_compute: serverless
Expand Down Expand Up @@ -164,7 +164,7 @@ jobs:
properties:
compute_specification:
automatic: true
component: "azureml:llm_rag_generate_embeddings:0.0.64"
component: "azureml:llm_rag_generate_embeddings:0.0.66"
inputs:
chunks_source:
type: uri_folder
Expand Down Expand Up @@ -216,7 +216,7 @@ jobs:
path: ${{parent.inputs.acs_config}}
outputs:
index: ${{parent.outputs.grounding_index}}
component: "azureml:llm_rag_update_acs_index:0.0.68"
component: "azureml:llm_rag_update_acs_index:0.0.70"
type: command
#########################################
db_sample_loading_generator:
Expand All @@ -241,7 +241,7 @@ jobs:
#########################################
generate_sample_embeddings:
type: command
component: "azureml:llm_rag_generate_embeddings:0.0.64"
component: "azureml:llm_rag_generate_embeddings:0.0.66"
resources:
instance_count: ${{parent.inputs.serverless_instance_count}}
instance_type: ${{parent.inputs.serverless_instance_type}}
Expand All @@ -262,7 +262,7 @@ jobs:
#########################################
create_sample_acs_index_job:
type: command
component: "azureml:llm_rag_update_acs_index:0.0.68"
component: "azureml:llm_rag_update_acs_index:0.0.70"
resources:
instance_count: ${{parent.inputs.serverless_instance_count}}
instance_type: ${{parent.inputs.serverless_instance_type}}
Expand Down Expand Up @@ -296,7 +296,7 @@ jobs:
outputs:
asset_id:
type: uri_file
component: "azureml:llm_rag_register_mlindex_asset:0.0.68"
component: "azureml:llm_rag_register_mlindex_asset:0.0.70"
type: command
create_prompt_flow:
environment_variables:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ tags:
Preview: ""
name: llm_ingest_db_to_faiss
display_name: LLM - SQL Datastore to FAISS Pipeline
version: 0.0.96
version: 0.0.97
description: Single job pipeline to chunk data from AzureML sql data store, and create FAISS embeddings index
settings:
default_compute: serverless
Expand Down Expand Up @@ -154,7 +154,7 @@ jobs:
properties:
compute_specification:
automatic: true
component: "azureml:llm_rag_generate_embeddings:0.0.64"
component: "azureml:llm_rag_generate_embeddings:0.0.66"
inputs:
chunks_source:
type: uri_folder
Expand Down Expand Up @@ -203,7 +203,7 @@ jobs:
path: ${{parent.jobs.generate_meta_embeddings.outputs.embeddings}}
outputs:
index: ${{parent.outputs.grounding_index}}
component: "azureml:llm_rag_create_faiss_index:0.0.69"
component: "azureml:llm_rag_create_faiss_index:0.0.71"
type: command

#########################################
Expand All @@ -229,7 +229,7 @@ jobs:
#########################################
generate_sample_embeddings:
type: command
component: "azureml:llm_rag_generate_embeddings:0.0.64"
component: "azureml:llm_rag_generate_embeddings:0.0.66"
resources:
instance_count: ${{parent.inputs.serverless_instance_count}}
instance_type: ${{parent.inputs.serverless_instance_type}}
Expand All @@ -250,7 +250,7 @@ jobs:
#########################################
create_sample_faiss_index_job:
type: command
component: "azureml:llm_rag_create_faiss_index:0.0.69"
component: "azureml:llm_rag_create_faiss_index:0.0.71"
resources:
instance_count: ${{parent.inputs.serverless_instance_count}}
instance_type: ${{parent.inputs.serverless_instance_type}}
Expand Down Expand Up @@ -280,7 +280,7 @@ jobs:
outputs:
asset_id:
type: uri_file
component: "azureml:llm_rag_register_mlindex_asset:0.0.68"
component: "azureml:llm_rag_register_mlindex_asset:0.0.70"
type: command
create_prompt_flow:
environment_variables:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.js
type: pipeline

name: llm_ingest_dbcopilot_acs_e2e
version: 0.0.65
version: 0.0.66
display_name: Data Ingestion for DB Data Output to ACS E2E Deployment
description: Single job pipeline to chunk data from AzureML DB Datastore and create acs embeddings index

Expand Down Expand Up @@ -170,7 +170,7 @@ jobs:
#########################################
generate_meta_embeddings:
type: command
component: "azureml:llm_rag_generate_embeddings:0.0.64"
component: "azureml:llm_rag_generate_embeddings:0.0.66"
resources:
instance_count: ${{parent.inputs.serverless_instance_count}}
instance_type: ${{parent.inputs.serverless_instance_type}}
Expand All @@ -191,7 +191,7 @@ jobs:
#########################################
create_meta_acs_index_job:
type: command
component: "azureml:llm_rag_update_acs_index:0.0.68"
component: "azureml:llm_rag_update_acs_index:0.0.70"
resources:
instance_count: ${{parent.inputs.serverless_instance_count}}
instance_type: ${{parent.inputs.serverless_instance_type}}
Expand Down Expand Up @@ -233,7 +233,7 @@ jobs:
#########################################
generate_sample_embeddings:
type: command
component: "azureml:llm_rag_generate_embeddings:0.0.64"
component: "azureml:llm_rag_generate_embeddings:0.0.66"
resources:
instance_count: ${{parent.inputs.serverless_instance_count}}
instance_type: ${{parent.inputs.serverless_instance_type}}
Expand All @@ -254,7 +254,7 @@ jobs:
#########################################
create_sample_acs_index_job:
type: command
component: "azureml:llm_rag_update_acs_index:0.0.68"
component: "azureml:llm_rag_update_acs_index:0.0.70"
resources:
instance_count: ${{parent.inputs.serverless_instance_count}}
instance_type: ${{parent.inputs.serverless_instance_type}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.js
type: pipeline

name: llm_ingest_dbcopilot_faiss_e2e
version: 0.0.65
version: 0.0.66
display_name: Data Ingestion for DB Data Output to FAISS E2E Deployment
description: Single job pipeline to chunk data from AzureML DB Datastore and create faiss embeddings index

Expand Down Expand Up @@ -160,7 +160,7 @@ jobs:
#########################################
generate_meta_embeddings:
type: command
component: "azureml:llm_rag_generate_embeddings:0.0.64"
component: "azureml:llm_rag_generate_embeddings:0.0.66"
resources:
instance_count: ${{parent.inputs.serverless_instance_count}}
instance_type: ${{parent.inputs.serverless_instance_type}}
Expand All @@ -181,7 +181,7 @@ jobs:
#########################################
create_meta_faiss_index_job:
type: command
component: "azureml:llm_rag_create_faiss_index:0.0.69"
component: "azureml:llm_rag_create_faiss_index:0.0.71"
resources:
instance_count: ${{parent.inputs.serverless_instance_count}}
instance_type: ${{parent.inputs.serverless_instance_type}}
Expand Down Expand Up @@ -221,7 +221,7 @@ jobs:
#########################################
generate_sample_embeddings:
type: command
component: "azureml:llm_rag_generate_embeddings:0.0.64"
component: "azureml:llm_rag_generate_embeddings:0.0.66"
resources:
instance_count: ${{parent.inputs.serverless_instance_count}}
instance_type: ${{parent.inputs.serverless_instance_type}}
Expand All @@ -242,7 +242,7 @@ jobs:
#########################################
create_sample_faiss_index_job:
type: command
component: "azureml:llm_rag_create_faiss_index:0.0.69"
component: "azureml:llm_rag_create_faiss_index:0.0.71"
resources:
instance_count: ${{parent.inputs.serverless_instance_count}}
instance_type: ${{parent.inputs.serverless_instance_type}}
Expand Down
24 changes: 24 additions & 0 deletions assets/models/system/ALLaM-2-7b-instruct/MLmodel
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
flavors:
hftransformersv2:
code: null
hf_config_class: AutoConfig
hf_pretrained_class: AutoModelForCausalLM
hf_tokenizer_class: AutoTokenizer
model_data: data
pytorch_version: 1.13.1
task_type: chat-completion
transformers_version: 4.42.4
python_function:
data: data
env: conda.yaml
loader_module: azureml.evaluate.mlflow.hftransformers
python_version: 3.9.19
metadata:
azureml.base_image: mcr.microsoft.com/azureml/curated/foundation-model-inference:49
base_model_name: ALLaM-2-7b-instruct
base_model_task: chat-completion
is_acft_model: true
is_finetuned_model: false
mlflow_version: 2.12.2
model_uuid: 3573e6b5dc7649c79c0be21b48cbe095
utc_time_created: '2024-08-31 13:39:08.777155'
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
flavors:
python_function:
artifacts:
model_dir:
path: artifacts/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224
uri: ../../BiomedCLIP-PubMedBERT_256-vit_base_patch16_224/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224
cloudpickle_version: 2.2.1
code: code
env:
conda: conda.yaml
virtualenv: python_env.yaml
loader_module: mlflow.pyfunc.model
python_model: python_model.pkl
python_version: 3.9.19
streamable: false
metadata:
base_model_name: null
base_model_task: zero-shot-image-classification
is_acft_model: true
is_finetuned_model: false
mlflow_version: 2.15.1
model_size_bytes: 789393877
model_uuid: 87e0b5b21dc34fad8381248be8107f2b
signature:
inputs: '[{"type": "binary", "name": "image", "required": true}, {"type": "string",
"name": "text", "required": true}]'
outputs: '[{"type": "string", "name": "probs", "required": true}, {"type": "string",
"name": "labels", "required": true}]'
params: null
utc_time_created: '2024-09-19 19:59:15.785066'
30 changes: 30 additions & 0 deletions assets/models/system/Deci-DeciCoder-1b/MLmodel
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
flavors:
hftransformersv2:
code: null
config_hf_load_kwargs:
trust_remote_code: true
hf_config_class: AutoConfig
hf_pretrained_class: AutoModelForCausalLM
hf_tokenizer_class: AutoTokenizer
huggingface_id: Deci/DeciCoder-1b
model_data: data
model_hf_load_args:
trust_remote_code: true
pytorch_version: 1.13.1
task_type: text-generation
tokenizer_hf_load_kwargs:
trust_remote_code: true
transformers_version: 4.34.0
python_function:
data: data
env: conda.yaml
loader_module: azureml.evaluate.mlflow.hftransformers
python_version: 3.8.18
metadata:
base_model_name: Deci/DeciCoder-1b
base_model_task: text-generation
is_acft_model: true
is_finetuned_model: false
mlflow_version: 2.6.0
model_uuid: 045f50822e264144b88dc8fc9884de99
utc_time_created: '2023-12-12 10:24:21.482548'
30 changes: 30 additions & 0 deletions assets/models/system/Deci-DeciLM-7B-instruct/MLmodel
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
flavors:
hftransformersv2:
code: null
config_hf_load_kwargs:
trust_remote_code: true
hf_config_class: AutoConfig
hf_pretrained_class: AutoModelForCausalLM
hf_tokenizer_class: AutoTokenizer
huggingface_id: Deci/DeciLM-7B-instruct
model_data: data
model_hf_load_args:
trust_remote_code: true
pytorch_version: 2.0.0+cu117
task_type: text-generation
tokenizer_hf_load_kwargs:
trust_remote_code: true
transformers_version: 4.36.0
python_function:
data: data
env: conda.yaml
loader_module: azureml.evaluate.mlflow.hftransformers
python_version: 3.8.18
metadata:
base_model_name: Deci/DeciLM-7B-instruct
base_model_task: text-generation
is_acft_model: true
is_finetuned_model: false
mlflow_version: 2.6.0
model_uuid: cacc364686f743da842ec07d0af5ff54
utc_time_created: '2023-12-13 11:49:35.201762'
30 changes: 30 additions & 0 deletions assets/models/system/Deci-DeciLM-7B/MLmodel
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
flavors:
hftransformersv2:
code: null
config_hf_load_kwargs:
trust_remote_code: true
hf_config_class: AutoConfig
hf_pretrained_class: AutoModelForCausalLM
hf_tokenizer_class: AutoTokenizer
huggingface_id: Deci/DeciLM-7B
model_data: data
model_hf_load_args:
trust_remote_code: true
pytorch_version: 2.0.0+cu117
task_type: text-generation
tokenizer_hf_load_kwargs:
trust_remote_code: true
transformers_version: 4.36.0
python_function:
data: data
env: conda.yaml
loader_module: azureml.evaluate.mlflow.hftransformers
python_version: 3.8.18
metadata:
base_model_name: Deci/DeciLM-7B
base_model_task: text-generation
is_acft_model: true
is_finetuned_model: false
mlflow_version: 2.6.0
model_uuid: 209d8ac9ffdb4c5881a34143dac3217b
utc_time_created: '2023-12-13 11:38:47.400961'
28 changes: 28 additions & 0 deletions assets/models/system/Llama-2-13b-chat/MLmodel
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
flavors:
hftransformersv2:
code: code
hf_config_class: LlamaConfig
hf_predict_module: llama_predict_meta
hf_pretrained_class: LlamaForCausalLM
hf_tokenizer_class: LlamaTokenizer
model_data: data
pytorch_version: 2.0.1+cu117
task_type: chat-completion
transformers_version: 4.31.0
model_hf_load_kwargs:
torch_dtype: torch.float16
device_map: auto
python_function:
code: code
data: data
env: conda.yaml
loader_module: azureml.evaluate.mlflow.hftransformers
python_version: 3.8.16
metadata:
azureml.base_image: mcr.microsoft.com/azureml/curated/foundation-model-inference:45
base_model_name: Llama-2-13b-chat
base_model_task: chat-completion
model_provider_name: meta
mlflow_version: 2.3.1
model_uuid: 192e572d7e464297a5b7b3916aaafab5
utc_time_created: '2023-07-24 06:22:15.253950'
Loading

0 comments on commit e0d9f29

Please sign in to comment.