dpsalvatierra
diff --git a/‎.circleci/continue_config.yml
+3-1 b/‎.circleci/continue_config.yml
+3-1
diff --git a/‎.gitignore
+4-1 b/‎.gitignore
+4-1
diff --git a/‎gpt4all-api/docker-compose.gpu.yaml
+3-1 b/‎gpt4all-api/docker-compose.gpu.yaml
+3-1
diff --git a/‎gpt4all-api/docker-compose.yaml
+4-1 b/‎gpt4all-api/docker-compose.yaml
+4-1
diff --git a/‎gpt4all-api/gpt4all.code-workspace
+7 b/‎gpt4all-api/gpt4all.code-workspace
+7
diff --git a/‎gpt4all-api/gpt4all_api/Dockerfile.buildkit
-2 b/‎gpt4all-api/gpt4all_api/Dockerfile.buildkit
-2
diff --git a/‎gpt4all-api/gpt4all_api/app/__init__.py b/‎gpt4all-api/gpt4all_api/app/__init__.py
diff --git a/‎gpt4all-api/gpt4all_api/app/api_v1/__init__.py b/‎gpt4all-api/gpt4all_api/app/api_v1/__init__.py
diff --git a/‎gpt4all-api/gpt4all_api/app/api_v1/routes/__init__.py b/‎gpt4all-api/gpt4all_api/app/api_v1/routes/__init__.py
diff --git a/‎gpt4all-api/gpt4all_api/app/tests/test_endpoints.py
+16-6 b/‎gpt4all-api/gpt4all_api/app/tests/test_endpoints.py
+16-6
diff --git a/‎gpt4all-api/gpt4all_api/models/README.md
+1 b/‎gpt4all-api/gpt4all_api/models/README.md
+1
diff --git a/‎gpt4all-api/makefile
+5-5 b/‎gpt4all-api/makefile
+5-5
diff --git a/‎gpt4all-backend/llama.cpp-mainline b/‎gpt4all-backend/llama.cpp-mainline
diff --git a/‎gpt4all-backend/llamamodel.cpp
+18-5 b/‎gpt4all-backend/llamamodel.cpp
+18-5
diff --git a/‎gpt4all-backend/llmodel.cpp
+7 b/‎gpt4all-backend/llmodel.cpp
+7
diff --git a/‎gpt4all-backend/llmodel_c.cpp
+8-20 b/‎gpt4all-backend/llmodel_c.cpp
+8-20
diff --git a/‎gpt4all-backend/llmodel_c.h
+2-13 b/‎gpt4all-backend/llmodel_c.h
+2-13
diff --git a/‎gpt4all-bindings/golang/Makefile
+2-2 b/‎gpt4all-bindings/golang/Makefile
+2-2
diff --git a/‎gpt4all-bindings/golang/binding.cpp
+3-4 b/‎gpt4all-bindings/golang/binding.cpp
+3-4
diff --git a/‎gpt4all-bindings/golang/placeholder b/‎gpt4all-bindings/golang/placeholder
diff --git a/‎gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModel.java
+4-3 b/‎gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModel.java
+4-3
@@ -856,6 +856,7 @@ jobs:
       - node/install-packages:
           app-dir: gpt4all-bindings/typescript
           pkg-manager: yarn
+          override-ci-command: yarn install
       - run:
           command: | 
             cd gpt4all-bindings/typescript
@@ -885,6 +886,7 @@ jobs:
       - node/install-packages:
           app-dir: gpt4all-bindings/typescript
           pkg-manager: yarn
+          override-ci-command: yarn install
       - run:
           command: | 
             cd gpt4all-bindings/typescript
@@ -994,7 +996,7 @@ jobs:
           command: |
             cd gpt4all-bindings/typescript
             npm set //registry.npmjs.org/:_authToken=$NPM_TOKEN
-            npm publish --access public --tag alpha
+            npm publish
 
 workflows:
   version: 2
 
@@ -183,4 +183,7 @@ build_*
 build-*
 
 # IntelliJ
-.idea/
+.idea/
+
+# gguf files
+*.gguf
@@ -8,8 +8,10 @@ services:
     environment:
       - HUGGING_FACE_HUB_TOKEN=token
       - USE_FLASH_ATTENTION=false
-      - MODEL_ID=''
+      - MODEL_ID=${EMBEDDING}
       - NUM_SHARD=1
+    env_file:
+      - ./gpt4all_api/.env
     command: --model-id $MODEL_ID --num-shard $NUM_SHARD
     volumes:
       - ./:/data
 
@@ -7,13 +7,16 @@ services:
     restart: always #restart on error (usually code compilation from save during bad state)
     ports:
       - "4891:4891"
+    env_file:
+      - './gpt4all_api/.env'
     environment:
       - APP_ENVIRONMENT=dev
       - WEB_CONCURRENCY=2
       - LOGLEVEL=debug
       - PORT=4891
-      - model=ggml-mpt-7b-chat.bin
+      - model=${MODEL_BIN}
       - inference_mode=cpu
     volumes:
       - './gpt4all_api/app:/app'
+      - './gpt4all_api/models:/models'
     command: ["/start-reload.sh"]
@@ -0,0 +1,7 @@
+{
+	"folders": [
+		{
+			"path": ".."
+		}
+	]
+}
@@ -1,8 +1,6 @@
 # syntax=docker/dockerfile:1.0.0-experimental
 FROM tiangolo/uvicorn-gunicorn:python3.11
 
-ARG MODEL_BIN=ggml-mpt-7b-chat.bin
-
 # Put first so anytime this file changes other cached layers are invalidated.
 COPY gpt4all_api/requirements.txt /requirements.txt
 
 
@@ -2,24 +2,34 @@
 Use the OpenAI python API to test gpt4all models.
 """
 from typing import List, get_args
+import os
+from dotenv import load_dotenv
 
 import openai
 
 openai.api_base = "http://localhost:4891/v1"
-
 openai.api_key = "not needed for a local LLM"
 
+# Load the .env file
+env_path = 'gpt4all-api/gpt4all_api/.env'
+load_dotenv(dotenv_path=env_path)
+
+# Fetch MODEL_ID from .env file
+model_id = os.getenv('MODEL_BIN', 'default_model_id')
+embedding = os.getenv('EMBEDDING', 'default_embedding_model_id')
+print (model_id)
+print (embedding)
 
 def test_completion():
-    model = "ggml-mpt-7b-chat.bin"
+    model = model_id
     prompt = "Who is Michael Jordan?"
     response = openai.Completion.create(
         model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
     )
     assert len(response['choices'][0]['text']) > len(prompt)
 
 def test_streaming_completion():
-    model = "ggml-mpt-7b-chat.bin"
+    model = model_id
     prompt = "Who is Michael Jordan?"
     tokens = []
     for resp in openai.Completion.create(
@@ -38,7 +48,7 @@ def test_streaming_completion():
 
 
 def test_batched_completion():
-    model = "ggml-mpt-7b-chat.bin"
+    model = model_id
     prompt = "Who is Michael Jordan?"
     response = openai.Completion.create(
         model=model, prompt=[prompt] * 3, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
@@ -48,12 +58,12 @@ def test_batched_completion():
 
 
 def test_embedding():
-    model = "ggml-all-MiniLM-L6-v2-f16.bin"
+    model = embedding
     prompt = "Who is Michael Jordan?"
     response = openai.Embedding.create(model=model, input=prompt)
     output = response["data"][0]["embedding"]
     args = get_args(List[float])
 
     assert response["model"] == model
     assert isinstance(output, list)
-    assert all(isinstance(x, args) for x in output)
+    assert all(isinstance(x, args) for x in output)
@@ -0,0 +1 @@
+## Place your gguf models and embeddings on this folder
@@ -28,19 +28,19 @@ clean_testenv:
 fresh_testenv: clean_testenv testenv
 
 venv:
-	if [ ! -d $(ROOT_DIR)/env ]; then $(PYTHON) -m venv $(ROOT_DIR)/env; fi
+	if [ ! -d $(ROOT_DIR)/venv ]; then $(PYTHON) -m venv $(ROOT_DIR)/venv; fi
 
 dependencies: venv
-	source $(ROOT_DIR)/env/bin/activate; $(PYTHON) -m pip install -r $(ROOT_DIR)/$(APP_NAME)/requirements.txt
+	source $(ROOT_DIR)/venv/bin/activate; $(PYTHON) -m pip install -r $(ROOT_DIR)/$(APP_NAME)/requirements.txt
 
 clean: clean_testenv
 	# Remove existing environment
-	rm -rf $(ROOT_DIR)/env;
+	rm -rf $(ROOT_DIR)/venv;
 	rm -rf $(ROOT_DIR)/$(APP_NAME)/*.pyc;
 
 
 black:
-	source $(ROOT_DIR)/env/bin/activate; black -l 120 -S --target-version py38 $(APP_NAME)
+	source $(ROOT_DIR)/venv/bin/activate; black -l 120 -S --target-version py38 $(APP_NAME)
 
 isort:
-	source $(ROOT_DIR)/env/bin/activate; isort  --ignore-whitespace --atomic -w 120 $(APP_NAME)
+	source $(ROOT_DIR)/venv/bin/activate; isort  --ignore-whitespace --atomic -w 120 $(APP_NAME)
@@ -385,22 +385,35 @@ DLL_EXPORT const char *get_build_variant() {
 }
 
 DLL_EXPORT bool magic_match(const char * fname) {
-
     struct ggml_context * ctx_meta = NULL;
     struct gguf_init_params params = {
         /*.no_alloc = */ true,
         /*.ctx      = */ &ctx_meta,
     };
     gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
+    if (!ctx_gguf) {
+        std::cerr << __func__ << ": gguf_init_from_file failed\n";
         return false;
+    }
+
+    bool valid = true;
+
+    int gguf_ver = gguf_get_version(ctx_gguf);
+    if (valid && gguf_ver > 3) {
+        std::cerr << __func__ << ": unsupported gguf version: " << gguf_ver << "\n";
+        valid = false;
+    }
 
-    bool isValid = gguf_get_version(ctx_gguf) <= 3;
     auto arch = get_arch_name(ctx_gguf);
-    isValid = isValid && (arch == "llama" || arch == "starcoder" || arch == "falcon" || arch == "mpt");
+    if (valid && !(arch == "llama" || arch == "starcoder" || arch == "falcon" || arch == "mpt")) {
+        if (!(arch == "gptj" || arch == "bert")) { // we support these via other modules
+            std::cerr << __func__ << ": unsupported model architecture: " << arch << "\n";
+        }
+        valid = false;
+    }
 
     gguf_free(ctx_gguf);
-    return isValid;
+    return valid;
 }
 
 DLL_EXPORT LLModel *construct() {
 
@@ -123,11 +123,18 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
 }
 
 const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
+    bool buildVariantMatched = false;
     for (const auto& i : implementationList()) {
         if (buildVariant != i.m_buildVariant) continue;
+        buildVariantMatched = true;
+
         if (!i.m_magicMatch(fname)) continue;
         return &i;
     }
+
+    if (!buildVariantMatched) {
+        std::cerr << "LLModel ERROR: Could not find any implementations for build variant: " << buildVariant << "\n";
+    }
     return nullptr;
 }
 
 
@@ -11,45 +11,33 @@ struct LLModelWrapper {
     ~LLModelWrapper() { delete llModel; }
 };
 
-
 thread_local static std::string last_error_message;
 
-
 llmodel_model llmodel_model_create(const char *model_path) {
-    auto fres = llmodel_model_create2(model_path, "auto", nullptr);
+    const char *error;
+    auto fres = llmodel_model_create2(model_path, "auto", &error);
     if (!fres) {
-        fprintf(stderr, "Invalid model file\n");
+        fprintf(stderr, "Unable to instantiate model: %s\n", error);
     }
     return fres;
 }
 
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error) {
+llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error) {
     auto wrapper = new LLModelWrapper;
-    int error_code = 0;
 
     try {
         wrapper->llModel = LLModel::Implementation::construct(model_path, build_variant);
+        if (!wrapper->llModel) {
+            last_error_message = "Model format not supported (no matching implementation found)";
+        }
     } catch (const std::exception& e) {
-        error_code = EINVAL;
         last_error_message = e.what();
     }
 
     if (!wrapper->llModel) {
         delete std::exchange(wrapper, nullptr);
-        // Get errno and error message if none
-        if (error_code == 0) {
-            if (errno != 0) {
-                error_code = errno;
-                last_error_message = std::strerror(error_code);
-            } else {
-                error_code = ENOTSUP;
-                last_error_message = "Model format not supported (no matching implementation found)";
-            }
-        }
-        // Set error argument
         if (error) {
-            error->message = last_error_message.c_str();
-            error->code = error_code;
+            *error = last_error_message.c_str();
         }
     }
     return reinterpret_cast<llmodel_model*>(wrapper);
 
@@ -23,17 +23,6 @@ extern "C" {
  */
 typedef void *llmodel_model;
 
-/**
- * Structure containing any errors that may eventually occur
- */
-struct llmodel_error {
-    const char *message;  // Human readable error description; Thread-local; guaranteed to survive until next llmodel C API call
-    int code;             // errno; 0 if none
-};
-#ifndef __cplusplus
-typedef struct llmodel_error llmodel_error;
-#endif
-
 /**
  * llmodel_prompt_context structure for holding the prompt context.
  * NOTE: The implementation takes care of all the memory handling of the raw logits pointer and the
@@ -105,10 +94,10 @@ DEPRECATED llmodel_model llmodel_model_create(const char *model_path);
  * Recognises correct model type from file at model_path
  * @param model_path A string representing the path to the model file; will only be used to detect model type.
  * @param build_variant A string representing the implementation to use (auto, default, avxonly, ...),
- * @param error A pointer to a llmodel_error; will only be set on error.
+ * @param error A pointer to a string; will only be set on error.
  * @return A pointer to the llmodel_model instance; NULL on error.
  */
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error);
+llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error);
 
 /**
  * Destroy a llmodel instance.
 
@@ -139,7 +139,7 @@ $(info I CXX:      $(CXXV))
 $(info )
 
 llmodel.o:
-	mkdir buildllm
+	[ -e buildllm ] || mkdir buildllm
 	cd buildllm && cmake ../../../gpt4all-backend/ $(CMAKEFLAGS) && make
 	cd buildllm && cp -rf CMakeFiles/llmodel.dir/llmodel_c.cpp.o ../llmodel_c.o
 	cd buildllm && cp -rf CMakeFiles/llmodel.dir/llmodel.cpp.o ../llmodel.o
@@ -150,7 +150,7 @@ clean:
 	rm -rf buildllm
 	rm -rf example/main
 
-binding.o: 
+binding.o: binding.cpp binding.h
 	$(CXX) $(CXXFLAGS) binding.cpp -o binding.o -c $(LDFLAGS)
 
 libgpt4all.a: binding.o llmodel.o
 
@@ -17,11 +17,10 @@
 
 void* load_model(const char *fname, int n_threads) {
     // load the model
-    llmodel_error new_error{};
+    const char *new_error;
     auto model = llmodel_model_create2(fname, "auto", &new_error);
-    if (model == nullptr ){
-        fprintf(stderr, "%s: error '%s'\n",
-                __func__, new_error.message);
+    if (model == nullptr) {
+        fprintf(stderr, "%s: error '%s'\n", __func__, new_error);
         return nullptr;
     }
     if (!llmodel_loadModel(model, fname)) {
 
@@ -1,6 +1,7 @@
 package com.hexadevlabs.gpt4all;
 
 import jnr.ffi.Pointer;
+import jnr.ffi.byref.PointerByReference;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -176,7 +177,7 @@ public LLModel(Path modelPath) {
         modelName = modelPath.getFileName().toString();
         String modelPathAbs = modelPath.toAbsolutePath().toString();
 
-        LLModelLibrary.LLModelError error = new LLModelLibrary.LLModelError(jnr.ffi.Runtime.getSystemRuntime());
+        PointerByReference error = new PointerByReference();
 
         // Check if model file exists
         if(!Files.exists(modelPath)){
@@ -192,7 +193,7 @@ public LLModel(Path modelPath) {
         model = library.llmodel_model_create2(modelPathAbs, "auto", error);
 
         if(model == null) {
-            throw new IllegalStateException("Could not load, gpt4all backend returned error: " + error.message);
+            throw new IllegalStateException("Could not load, gpt4all backend returned error: " + error.getValue().getString(0));
         }
         library.llmodel_loadModel(model, modelPathAbs);
 
@@ -631,4 +632,4 @@ public void close() throws Exception {
         library.llmodel_model_destroy(model);
     }
 
-}
+}
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+## Place your gguf models and embeddings on this folder`
Original file line number	Diff line number	Diff line change
`@@ -123,11 +123,18 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat`
`123`	`123`	`}`
`124`	`124`
`125`	`125`	`const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {`
	`126`	`+ bool buildVariantMatched = false;`
`126`	`127`	`for (const auto& i : implementationList()) {`
`127`	`128`	`if (buildVariant != i.m_buildVariant) continue;`
	`129`	`+ buildVariantMatched = true;`
	`130`	`+`
`128`	`131`	`if (!i.m_magicMatch(fname)) continue;`
`129`	`132`	`return &i;`
`130`	`133`	`}`
	`134`	`+`
	`135`	`+ if (!buildVariantMatched) {`
	`136`	`+ std::cerr << "LLModel ERROR: Could not find any implementations for build variant: " << buildVariant << "\n";`
	`137`	`+ }`
`131`	`138`	`return nullptr;`
`132`	`139`	`}`
`133`	`140`