Keep model loaded in memory, saves 5 seconds
This commit is contained in:
2
main.py
2
main.py
@@ -23,7 +23,7 @@ def controller_message(message):
|
||||
return False
|
||||
|
||||
def llama(prompt):
|
||||
data = dict(model='llama3.1', prompt=prompt, stream=False)
|
||||
data = dict(model='llama3.1', prompt=prompt, stream=False, keep_alive=-1)
|
||||
try:
|
||||
r = requests.post(LLAMA_URL, json=data, timeout=20)
|
||||
r.raise_for_status()
|
||||
|
Reference in New Issue
Block a user