processing-pubmed-context.yaml

name: pubmed_rag
cache: true
output_path: .
steps:
    - _target_: ragfoundry.processing.dataset_loaders.loaders.HFLoader
      inputs: train
      dataset_config:
            path: bigbio/pubmed_qa
            split: train

    - _target_: ragfoundry.processing.dataset_loaders.loaders.HFLoader
      inputs: test
      dataset_config:
            path: bigbio/pubmed_qa
            name: pubmed_qa_labeled_fold0_source
            split: test

    - _target_: ragfoundry.processing.global_steps.sampling.ShuffleSelect
      inputs: train
      limit: 50000

    - _target_: ragfoundry.processing.local_steps.common_datasets.PubMed
      inputs: [train, test]

    - _target_: ragfoundry.processing.local_steps.context.DocumentsJoiner
      inputs: [train, test]
      docs_key: positive_passages
      k: 5

    - _target_: ragfoundry.processing.local_steps.prompter.TextPrompter
      inputs: [train, test]
      prompt_file: ragfoundry/processing/prompts/qa.txt
      output_key: prompt
      mapping:
            question: query
            context: positive_passages

    - _target_: ragfoundry.processing.global_steps.output.OutputData
      inputs: [train, test]
      prefix: pubmed-rag