pinecone deployment

Author: teknetik

.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "envFile": "${workspaceFolder}/.env"
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,16 @@
+{
+    "files.exclude": {
+        "**/.git": true,
+        "**/.svn": true,
+        "**/.hg": true,
+        "**/CVS": true,
+        "**/.DS_Store": true,
+        "**/*.pyc": true,
+        "**/__pycache__": true
+    },
+    "python.pythonPath": " /home/teknetik/.local/share/virtualenvs/documentation-helper-TKW46dO5/bin/python",
+    "python.linting.pylintArgs": [
+        "--load-plugins",
+        "pylint_flask"
+    ]
+}

Pipfile

@@ -23,4 +23,3 @@ tqdm = "*"
 
 [requires]
 python_version = "3.11"
-python_full_version = "3.11.0"

Pipfile.lock

@@ -13,7 +13,7 @@
     environment=os.environ["PINECONE_ENVIRONMENT_REGION"],
 )
 
-INDEX_NAME = "langchain-doc-index"
+INDEX_NAME = "langchain-doc"
 
 
 def run_llm(query: str, chat_history: List[Dict[str, Any]] = []):

backend/core.py

@@ -0,0 +1,16 @@
+/home/teknetik/websites/docs.kong/docs.konghq.com/gateway/3.3.x/
+/home/teknetik/websites/docs.kong/docs.konghq.com/konnect/
+/home/teknetik/websites/docs.kong/docs.konghq.com/api/
+/home/teknetik/websites/docs.kong/docs.konghq.com/getting-started-guide/
+/home/teknetik/websites/docs.kong/docs.konghq.com/mesh/latest/
+
+
+#Not done
+
+/home/teknetik/websites/docs.kong/docs.konghq.com/mesh/changelog/
+/home/teknetik/websites/docs.kong/docs.konghq.com/plugins/
+/home/teknetik/websites/docs.kong/docs.konghq.com/studio/
+/home/teknetik/websites/docs.kong/docs.konghq.com/kubernetes-ingress-controller/
+/home/teknetik/websites/docs.kong/docs.konghq.com/install/
+/home/teknetik/websites/docs.kong/docs.konghq.com/konnect-platform/
+/home/teknetik/websites/docs.kong/docs.konghq.com/deck/

dirs_ingested.txt

@@ -4,21 +4,37 @@
 from langchain.embeddings import OpenAIEmbeddings
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores import Pinecone
+from langchain.document_loaders import UnstructuredHTMLLoader
 import pinecone
 
 pinecone.init(
     api_key=os.environ["PINECONE_API_KEY"],
     environment=os.environ["PINECONE_ENVIRONMENT_REGION"],
 )
-INDEX_NAME = "langchain-doc-index"
+INDEX_NAME = "langchain-doc"
+
+import os
+
+def get_files_in_dir(directory):
+    file_list = []
+
+    # Walk through directory
+    for dirpath, dirnames, filenames in os.walk(directory):
+        for file in filenames:
+            # Check if the file is .html
+            if file.endswith('.html'):
+                file_list.append(os.path.join(dirpath, file))
+            #file_list.append(os.path.join(dirpath, file))
+    return file_list
 
 
-def ingest_docs():
-    loader = ReadTheDocsLoader("python.langchain.com/en/latest/index.html")
+def ingest_docs(file):
+    #loader = ReadTheDocsLoader(path="/home/teknetik/websites/docs.kong/docs.konghq.com/index.html")
+    loader =  UnstructuredHTMLLoader(file)
     raw_documents = loader.load()
     print(f"loaded {len(raw_documents)} documents")
     text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=400, chunk_overlap=50, separators=["\n\n", "\n", " ", ""]
+        chunk_size=1000, chunk_overlap=200, separators=["\n\n", "\n", " ", ""]
     )
     documents = text_splitter.split_documents(raw_documents)
     for doc in documents:
@@ -33,4 +49,17 @@ def ingest_docs():
 
 
 if __name__ == "__main__":
-    ingest_docs()
+    ###
+    # Change the directory, only uploading one version at a time
+    #
+    # Ensure to update the meta data with the product and version or other applicable metadata
+    #
+    ###
+    directory_to_scan = "/home/teknetik/websites/docs.kong/docs.konghq.com/mesh/latest/"  # Change this to your target directory
+    file_list = get_files_in_dir(directory_to_scan)
+    file_num = len(file_list)
+    i =1
+    for file in file_list:
+        print(file + " " + str(i) + " of " + str(file_num))
+        ingest_docs(file)
+        i += 1