From dcd8af2895c721192cdeb8829a16e41e3a6a3380 Mon Sep 17 00:00:00 2001 From: Tanner Collin Date: Mon, 20 Jan 2025 18:24:59 +0000 Subject: [PATCH] Keep model loaded in memory, saves 5 seconds --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 77e695b..5b52a38 100644 --- a/main.py +++ b/main.py @@ -23,7 +23,7 @@ def controller_message(message): return False def llama(prompt): - data = dict(model='llama3.1', prompt=prompt, stream=False) + data = dict(model='llama3.1', prompt=prompt, stream=False, keep_alive=-1) try: r = requests.post(LLAMA_URL, json=data, timeout=20) r.raise_for_status()