Add llamacpp python scripts

.gitignore

@@ -38,3 +38,4 @@ vite.config.js.timestamp-*
 vite.config.ts.timestamp-*
 raise.code-workspace
 recc.md
+__pycache__/

src-python/llamacpp.py

@@ -0,0 +1,31 @@
+from llama_cpp import Llama
+from pydantic import BaseModel
+
+class LlamaItem(BaseModel):
+    prompt: str
+    model_path: str
+    temperature: float = 0.2,
+    top_p: float = 0.95,
+    top_k: int = 40,
+    max_tokens: int = 256,
+    presence_penalty: float = 0,
+    frequency_penalty: float = 0,
+    repeat_penalty: float = 1.1,
+    n_ctx: int = 2000
+
+def stream_chat_llamacpp(item:LlamaItem):
+    if last_model_path != item.model_path or llm is None or n_ctx != item.n_ctx:
+        llm = Llama(model_path=item.model_path, n_ctx=n_ctx)
+        last_model_path = item.model_path
+        n_ctx = item.n_ctx
+    chunks = llm.create_completion(
+        prompt = item.prompt,
+    )
+    for chunk in chunks:
+        cont = chunk
+        print(cont, end="")
+        yield cont.encode()
+
+n_ctx = 2000
+last_model_path = ""
+llm:Llama

src-python/main.py

@@ -0,0 +1,25 @@
+from fastapi import FastAPI, Header
+from fastapi.responses import StreamingResponse
+from llamacpp import LlamaItem, stream_chat_llamacpp
+from typing import Annotated, Union
+import os
+
+app = FastAPI()
+key_dir = os.path.join(os.getcwd(), "key.txt")
+f = open(key_dir, 'r')
+key = f.read()
+f.close()
+
+@app.post("/llamacpp")
+async def llamacpp(item:LlamaItem, x_risu_auth: Annotated[Union[str, None], Header()] = None):
+    if key != x_risu_auth:
+        return {"error": "Invalid key"}
+    return StreamingResponse(stream_chat_llamacpp(item))
+
+@app.get("/")
+async def autha():
+    return {"dir": key_dir}
+
+@app.get("/auth")
+async def auth():
+    return {"dir": key_dir}

src-python/run.py

@@ -0,0 +1,13 @@
+import uvicorn
+import os
+import uuid
+import subprocess
+import sys
+
+if __name__ == "__main__":
+    key_dir = os.path.join(os.getcwd(), "key.txt")
+    with open(key_dir, "w") as f:
+        f.write(uuid.uuid4().hex)
+    subprocess.check_call([sys.executable, "-m", "pip", "install", "pydantic"])
+    subprocess.check_call([sys.executable, "-m", "pip", "install", "llama-cpp-python"])
+    uvicorn.run("main:app", host="0.0.0.0", port=8912)