From 1f2de23199eadc504f717cd13cef54061c322240 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Thu, 9 Oct 2025 13:38:40 +0200 Subject: [PATCH] Add prompt tuning experiment with sample vocab A new initialization method was added to prompt tuning in #2815. This PR adds an experiment config for this method to the MetaMathQA benchmark. Testing locally, this got a test accuracy of 36%, compared to 25% with random initialization. --- .../adapter_config.json | 17 +++++++++++++++++ .../training_params.json | 6 ++++++ 2 files changed, 23 insertions(+) create mode 100644 method_comparison/MetaMathQA/experiments/prompt_tuning/llama-3.2-3B-sample_vocab-lr_0.001/adapter_config.json create mode 100644 method_comparison/MetaMathQA/experiments/prompt_tuning/llama-3.2-3B-sample_vocab-lr_0.001/training_params.json diff --git a/method_comparison/MetaMathQA/experiments/prompt_tuning/llama-3.2-3B-sample_vocab-lr_0.001/adapter_config.json b/method_comparison/MetaMathQA/experiments/prompt_tuning/llama-3.2-3B-sample_vocab-lr_0.001/adapter_config.json new file mode 100644 index 0000000000..8b9f142797 --- /dev/null +++ b/method_comparison/MetaMathQA/experiments/prompt_tuning/llama-3.2-3B-sample_vocab-lr_0.001/adapter_config.json @@ -0,0 +1,17 @@ +{ + "auto_mapping": null, + "base_model_name_or_path": null, + "inference_mode": false, + "num_attention_heads": 24, + "num_layers": 28, + "num_transformer_submodules": 1, + "num_virtual_tokens": 200, + "peft_type": "PROMPT_TUNING", + "prompt_tuning_init": "SAMPLE_VOCAB", + "prompt_tuning_init_text": null, + "revision": null, + "task_type": "CAUSAL_LM", + "token_dim": 3072, + "tokenizer_kwargs": null, + "tokenizer_name_or_path": null +} diff --git a/method_comparison/MetaMathQA/experiments/prompt_tuning/llama-3.2-3B-sample_vocab-lr_0.001/training_params.json b/method_comparison/MetaMathQA/experiments/prompt_tuning/llama-3.2-3B-sample_vocab-lr_0.001/training_params.json new file mode 100644 index 0000000000..8a120ad9a8 --- /dev/null +++ b/method_comparison/MetaMathQA/experiments/prompt_tuning/llama-3.2-3B-sample_vocab-lr_0.001/training_params.json @@ -0,0 +1,6 @@ +{ + "optimizer_kwargs": { + "lr": 1e-3 + } +} +