[add] main_llm_tokenizer experiment

Author: dianjixz

projects/llm_framework/main_llm_tokenizer/Kconfig

@@ -0,0 +1,34 @@
+import os
+
+Import('env')
+with open(env['PROJECT_TOOL_S']) as f:
+    exec(f.read())
+
+SRCS = Glob('src/*.c*')
+INCLUDE = []
+PRIVATE_INCLUDE = []
+REQUIREMENTS = ['pthread']
+STATIC_LIB = []
+DYNAMIC_LIB = []
+DEFINITIONS = []
+DEFINITIONS_PRIVATE = []
+LDFLAGS = []
+LINK_SEARCH_PATH = []
+STATIC_FILES = []
+
+STATIC_FILES += [AFile('_tokenizer.py')]
+
+env['COMPONENTS'].append({'target':'llm_tokenizer',
+                          'SRCS':SRCS,
+                          'INCLUDE':INCLUDE,
+                          'PRIVATE_INCLUDE':PRIVATE_INCLUDE,
+                          'REQUIREMENTS':REQUIREMENTS,
+                          'STATIC_LIB':STATIC_LIB,
+                          'DYNAMIC_LIB':DYNAMIC_LIB,
+                          'DEFINITIONS':DEFINITIONS,
+                          'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE,
+                          'LDFLAGS':LDFLAGS,
+                          'LINK_SEARCH_PATH':LINK_SEARCH_PATH,
+                          'STATIC_FILES':STATIC_FILES,
+                          'REGISTER':'project'
+                          })

projects/llm_framework/main_llm_tokenizer/SConstruct

@@ -0,0 +1,132 @@
+#!/bin/env python3
+import zmq
+from tokenizers import Tokenizer
+import http.server
+import socketserver
+import threading
+import socket
+import json
+from http.server import HTTPServer, BaseHTTPRequestHandler
+tokenizers_obj = {}
+tokenizers_content = {}
+server_obj = {}
+url_path_map = {}
+
+context = zmq.Context()
+zmq_socket = context.socket(zmq.REP)
+zmq_socket.bind("ipc:///tmp/rpc.tokenizer")
+
+
+class Request(BaseHTTPRequestHandler):
+    timeout = 5
+    server_version = "Apache"
+    def do_GET(self):
+        server_ip, server_port = self.server.server_address
+        val = 'http://localhost:{}'.format(server_port)
+        token_obj = tokenizers_obj[url_path_map[val]]
+
+        print(self.path)
+        self.send_response(200)
+        self.send_header("type", "get")
+        self.end_headers()
+        if self.path == "/bos_id":
+            bos_id = token_obj.bos_id
+            # print(bos_id)
+            # to json
+            if bos_id is None:
+                msg = json.dumps({"bos_id": -1})
+            else:
+                msg = json.dumps({"bos_id": bos_id})
+        elif self.path == "/eos_id":
+            eos_id = token_obj.eos_id
+            if eos_id is None:
+                msg = json.dumps({"eos_id": -1})
+            else:
+                msg = json.dumps({"eos_id": eos_id})
+        else:
+            msg = "error"
+        print(msg)
+        msg = str(msg).encode()
+        self.wfile.write(msg)
+
+    def do_POST(self):
+        server_ip, server_port = self.server.server_address
+        val = 'http://localhost:{}'.format(server_port)
+        token_obj = tokenizers_obj[url_path_map[val]]
+        data = self.rfile.read(
+            int(self.headers["content-length"])
+        )
+        data = data.decode()
+        self.send_response(200)
+        self.send_header("type", "post")
+        self.end_headers()
+        if self.path == "/encode":
+            req = json.loads(data)
+            prompt = req['text']
+            token_ids = token_obj.encode(prompt, tokenizers_content[url_path_map[val]])
+            if token_ids is None:
+                msg = json.dumps({"token_ids": -1})
+            else:
+                msg = json.dumps({"token_ids": token_ids})
+        elif self.path == "/decode":
+            req = json.loads(data)
+            token_ids = req["token_ids"]
+            text = token_obj.decode(token_ids)
+            if text is None:
+                msg = json.dumps({"text": ""})
+            else:
+                msg = json.dumps({"text": text})
+        else:
+            msg = "error"
+        print(msg)
+        msg = str(msg).encode()
+        self.wfile.write(msg)
+
+
+def start_server(httpd):
+    httpd.serve_forever()
+
+def find_free_port():
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(('', 0))
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        return s.getsockname()[1]
+
+def rpc_forever():
+    while True:
+        try:
+            message_parts = socket.recv_multipart()
+            action = message_parts[0].decode('utf-8')
+            rawmsg = message_parts[1].decode('utf-8')
+            
+            val = 'None'
+            if action == 'creat_tokenizer':
+                json_args = json.loads(rawmsg)
+                tokenizer_path = json_args['path']
+                tokenizer_content = json_args['content']
+                tokenizers_content[tokenizer_path] = tokenizer_content
+                tokenizers_obj[tokenizer_path] = Tokenizer.from_file(tokenizer_path)
+                server_obj[tokenizer_path] = socketserver.TCPServer(("", 0), Request)
+                server_ip, server_port = server_obj[tokenizer_path].server_address
+                val = 'http://localhost:{}'.format(server_port)
+                url_path_map[val] = tokenizer_path
+                thread = threading.Thread(target=start_server, args=(server_obj[tokenizer_path]))
+                thread.daemon = True
+                thread.start()
+                
+            if action == 'close_tokenizer':
+                tokenizer_path = rawmsg.decode('utf-8')
+                server_obj[tokenizer_path].shutdown()
+                server_obj[tokenizer_path].server_close()
+                del server_obj[tokenizer_path]
+                del tokenizers_obj[tokenizer_path]
+                del tokenizers_content[tokenizer_path]
+                for key, value in list(url_path_map.items()):
+                    if value == tokenizer_path:
+                        del url_path_map[key]
+            zmq_socket.send(val.encode('utf-8'))
+        except:
+            pass
+
+if __name__ == "__main__":
+    rpc_forever()

projects/llm_framework/main_llm_tokenizer/_tokenizer.py

@@ -0,0 +1,25 @@
+/*
+ * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#include <stdio.h>
+#include <unistd.h>
+
+int main() {
+    if (access("./_tokenizer.py", F_OK) == 0) {
+        char *args[] = {"python3", "./_tokenizer.py", NULL};
+        if (execvp("python3", args) == -1) {
+            perror("execvp");
+            return 1;
+        }
+    } else if (access("/opt/m5stack/share/_tokenizer.py", F_OK) == 0) {
+        char *args[] = {"python3", "/opt/m5stack/share/_tokenizer.py", NULL};
+        if (execvp("python3", args) == -1) {
+            perror("execvp");
+            return 1;
+        }
+    }
+    perror("_tokenizer.py miss");
+    return 0;
+}