ENH: Add gguf models of llama-2-chat (#981)

xorbitsai · Feb 4, 2024 · 1b9b8c8 · 1b9b8c8
1 parent 9a8221c
commit 1b9b8c8
Showing 1 changed file with 60 additions and 0 deletions.
diff --git a/xinference/model/llm/llm_family.json b/xinference/model/llm/llm_family.json
@@ -827,6 +827,66 @@
         ],
         "model_id": "meta-llama/Llama-2-70b-chat-hf",
         "model_revision": "36d9a7388cc80e5f4b3e9701ca2f250d21a96c30"
+      },
+      {
+        "model_format": "ggufv2",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "Q2_K",
+          "Q3_K_S",
+          "Q3_K_M",
+          "Q3_K_L",
+          "Q4_0",
+          "Q4_K_S",
+          "Q4_K_M",
+          "Q5_0",
+          "Q5_K_S",
+          "Q5_K_M",
+          "Q6_K",
+          "Q8_0"
+        ],
+        "model_id": "TheBloke/Llama-2-7B-Chat-GGUF",
+        "model_file_name_template": "llama-2-7b-chat.{quantization}.gguf"
+      },
+      {
+        "model_format": "ggufv2",
+        "model_size_in_billions": 13,
+        "quantizations": [
+          "Q2_K",
+          "Q3_K_S",
+          "Q3_K_M",
+          "Q3_K_L",
+          "Q4_0",
+          "Q4_K_S",
+          "Q4_K_M",
+          "Q5_0",
+          "Q5_K_S",
+          "Q5_K_M",
+          "Q6_K",
+          "Q8_0"
+        ],
+        "model_id": "TheBloke/Llama-2-13B-chat-GGUF",
+        "model_file_name_template": "llama-2-13b-chat.{quantization}.gguf"
+      },
+      {
+        "model_format": "ggufv2",
+        "model_size_in_billions": 70,
+        "quantizations": [
+          "Q2_K",
+          "Q3_K_S",
+          "Q3_K_M",
+          "Q3_K_L",
+          "Q4_0",
+          "Q4_K_S",
+          "Q4_K_M",
+          "Q5_0",
+          "Q5_K_S",
+          "Q5_K_M",
+          "Q6_K",
+          "Q8_0"
+        ],
+        "model_id": "TheBloke/Llama-2-70B-Chat-GGUF",
+        "model_file_name_template": "llama-2-70b-chat.{quantization}.gguf"
       }
     ],
     "prompt_style": {