WikidataTextEmbedding/docker-compose.yml at main · philippesaade-wmde/WikidataTextEmbedding · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
services:
  data_processing_save_labels_descriptions:
    build:
      context: .
      dockerfile: ./docker/1_Data_Processing_save_labels_descriptions/Dockerfile
    volumes:
      - ./data:/data  # Mount the ./data folder from the host to /data in the container
    tty: true
    container_name: data_processing_save_ids
    environment:
      PYTHONUNBUFFERED: 1
      LANGUAGE: "de"
      OFFSET: 0

  data_processing_save_items_per_lang:
    build:
      context: .
      dockerfile: ./docker/2_Data_Processing_save_items_per_lang/Dockerfile
    volumes:
      - ./data:/data  # Mount the ./data folder from the host to /data in the container
    tty: true
    container_name: data_processing_save_entities
    environment:
      PYTHONUNBUFFERED: 1

  add_wikidata_to_db:
    build:
      context: .
      dockerfile: ./docker/3_Add_Wikidata_to_DB/Dockerfile
    volumes:
      - ./data:/data
      - ~/.cache/huggingface:/root/.cache/huggingface
    tty: true
    container_name: add_wikidata_to_astra
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    environment:
      PYTHONUNBUFFERED: 1
      MODEL: "jina"
      SAMPLE: "true"
      API_KEY: "datastax_wikidata.json"
      EMBED_BATCH_SIZE: 8
      QUERY_BATCH_SIZE: 100
      OFFSET: 0
      COLLECTION_NAME: "wikidatav1"
      LANGUAGE: 'en'
      TEXTIFIER_LANGUAGE: 'en'
      DB_PATH: 'sqlite_enwiki.db'
    network_mode: "host"

  run_retrieval:
    build:
      context: .
      dockerfile: ./docker/4_Run_Retrieval/Dockerfile
    volumes:
      - ./data:/data
      - ~/.cache/huggingface:/root/.cache/huggingface
    tty: true
    container_name: run_retrieval
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]

    environment:
      PYTHONUNBUFFERED: 1
      MODEL: "jina"
      API_KEY: "datastax_wikidata.json"
      COLLECTION_NAME: "wikidatav1"
      BATCH_SIZE: 1
      EVALUATION_PATH: "Mintaka/processed_dataframe.pkl"
      # COMPARATIVE: "true"
      # COMPARATIVE_COLS: "Correct QID,Wrong QID"
      QUERY_COL: "Question"
      QUERY_LANGUAGE: "en"
      DB_LANGUAGE: "en"
      PREFIX: ""
    network_mode: "host"

  run_rerank:
    build:
      context: .
      dockerfile: ./docker/5_Run_Rerank/Dockerfile
    volumes:
      - ./data:/data
      - ~/.cache/huggingface:/root/.cache/huggingface
    tty: true
    container_name: run_rank
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]

    environment:
      PYTHONUNBUFFERED: 1
      MODEL: "jina"
      RETRIEVAL_FILENAME: "retrieval_results_Mintaka-wikidata_test_v1-en"
      BATCH_SIZE: 1
      QUERY_COL: "Question"
      LANGUAGE: "de"
    network_mode: "host"

  push_huggingface:
    build:
      context: .
      dockerfile: ./docker/6_Push_Huggingface/Dockerfile
    volumes:
      - ./data:/data
    tty: true
    container_name: push_huggingface
    environment:
      PYTHONUNBUFFERED: 1
      QUEUE_SIZE: 5000
      NUM_PROCESSES: 4
      SKIPLINES: 0
      ITERATION: 36

  create_prototype:
      build:
        context: .
        dockerfile: ./docker/7_Create_Prototype/Dockerfile
      volumes:
        - ./data:/data
        - ~/.cache/huggingface:/root/.cache/huggingface
      tty: true
      container_name: create_prototype
      environment:
        PYTHONUNBUFFERED: 1
        MODEL: "jinaapi"
        API_KEY: "datastax_wikidata.json"
        EMBED_BATCH_SIZE: 10
        NUM_PROCESSES: 23
        OFFSET: 0
        COLLECTION_NAME: "wikidata_prototype"
        LANGUAGE: 'en'
        TEXTIFIER_LANGUAGE: 'en'
      network_mode: "host"