Upload 3 files

app.py

@@ -14,7 +14,7 @@ from dc import (infer, _infer, pass_result, get_diffusers_model_list, get_sample
     update_textual_inversion, set_textual_inversion_prompt, create_mask_now)
 # Translator
 from llmdolphin import (dolphin_respond_auto, dolphin_parse_simple,
-    get_llm_formats, get_dolphin_model_format, get_dolphin_models,
+    get_llm_formats, get_dolphin_model_format, get_dolphin_models, get_dolphin_loras, select_dolphin_lora, add_dolphin_loras,
     get_dolphin_model_info, select_dolphin_model, select_dolphin_format, get_dolphin_sysprompt)
 # Tagger
 from tagger.v2 import v2_upsampling_prompt, V2_ALL_MODELS
@@ -352,9 +352,9 @@ with gr.Blocks(fill_width=True, elem_id="container", css=css, delete_cache=(60,
 
                 with gr.Tab("Translation Settings"):
                     chatbot = gr.Chatbot(render_markdown=False, visible=False) # component for auto-translation
-                    chat_model = gr.Dropdown(choices=get_dolphin_models(), value=get_dolphin_models()[0][1], allow_custom_value=True, label="Model")
-                    chat_model_info = gr.Markdown(value=get_dolphin_model_info(get_dolphin_models()[0][1]), label="Model info")
-                    chat_format = gr.Dropdown(choices=get_llm_formats(), value=get_dolphin_model_format(get_dolphin_models()[0][1]), label="Message format")
+                    chat_model = gr.Dropdown(choices=get_dolphin_models(), value=get_dolphin_models()[0], allow_custom_value=True, label="Model")
+                    chat_model_info = gr.Markdown(value=get_dolphin_model_info(get_dolphin_models()[0]), label="Model info")
+                    chat_format = gr.Dropdown(choices=get_llm_formats(), value=get_dolphin_model_format(get_dolphin_models()[0]), label="Message format")
                     with gr.Row():
                         chat_tokens = gr.Slider(minimum=1, maximum=4096, value=512, step=1, label="Max tokens")
                         chat_temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
@@ -362,6 +362,9 @@ with gr.Blocks(fill_width=True, elem_id="container", css=css, delete_cache=(60,
                         chat_topk = gr.Slider(minimum=0, maximum=100, value=40, step=1, label="Top-k")
                         chat_rp = gr.Slider(minimum=0.0, maximum=2.0, value=1.1, step=0.1, label="Repetition penalty")
                     chat_sysmsg = gr.Textbox(value=get_dolphin_sysprompt(), label="System message")
+                    with gr.Accordion("Loras", open=True, visible=False):
+                        chat_lora = gr.Dropdown(choices=get_dolphin_loras(), value=get_dolphin_loras()[0], allow_custom_value=True, label="Lora")
+                        chat_lora_scale = gr.Slider(minimum=0.0, maximum=1.0, value=1.0, step=0.01, label="Lora scale")
 
         examples = gr.Examples(
             examples = [
@@ -444,7 +447,7 @@ with gr.Blocks(fill_width=True, elem_id="container", css=css, delete_cache=(60,
         api_name="infer_translate",
     ).success(
         fn=dolphin_respond_auto,
-        inputs=[prompt, chatbot, chat_model, chat_sysmsg, chat_tokens, chat_temperature, chat_topp, chat_topk, chat_rp, state],
+        inputs=[prompt, chatbot, chat_model, chat_sysmsg, chat_tokens, chat_temperature, chat_topp, chat_topk, chat_rp, chat_lora, chat_lora_scale, state],
         outputs=[chatbot, result, prompt],
         queue=True,
         show_progress="full",
@@ -546,6 +549,8 @@ with gr.Blocks(fill_width=True, elem_id="container", css=css, delete_cache=(60,
     .success(lambda: None, None, chatbot, queue=False, show_api=False)
     chat_format.change(select_dolphin_format, [chat_format, state], [chat_format, state], queue=False, show_api=False)\
     .success(lambda: None, None, chatbot, queue=False, show_api=False)
+    chat_lora.change(select_dolphin_lora, [chat_lora, state], [chat_lora, state], queue=True, show_progress="full")\
+    .success(lambda: None, None, chatbot, queue=False, show_api=False)
 
     # Tagger
     with gr.Tab("Tags Transformer with Tagger"):

llmdolphin.py

@@ -4,6 +4,8 @@ from pathlib import Path
 import re
 import torch
 import gc
+import os
+import urllib
 from typing import Any
 from huggingface_hub import hf_hub_download, HfApi
 from llama_cpp import Llama
@@ -15,14 +17,17 @@ from ja_to_danbooru.ja_to_danbooru import jatags_to_danbooru_tags
 import wrapt_timeout_decorator
 from llama_cpp_agent.messages_formatter import MessagesFormatter
 from formatter import mistral_v1_formatter, mistral_v2_formatter, mistral_v3_tekken_formatter
-from llmenv import llm_models, llm_models_dir, llm_formats, llm_languages, dolphin_system_prompt
+from llmenv import llm_models, llm_models_dir, llm_loras, llm_loras_dir, llm_formats, llm_languages, dolphin_system_prompt
 import subprocess
 subprocess.run("rm -rf /data-nvme/zerogpu-offload/*", env={}, shell=True)
 
 
-llm_models_tupled_list = []
+llm_models_list = []
+llm_loras_list = []
 default_llm_model_filename = list(llm_models.keys())[0]
+default_llm_lora_filename = list(llm_loras.keys())[0]
 device = "cuda" if torch.cuda.is_available() else "cpu"
+HF_TOKEN = os.getenv("HF_TOKEN", False)
 
 
 def to_list(s: str):
@@ -68,43 +73,170 @@ def is_japanese(s: str):
     return False
 
 
-def update_llm_model_tupled_list():
-    global llm_models_tupled_list
-    llm_models_tupled_list = []
-    for k, v in llm_models.items():
-        name = k
-        value = k
-        llm_models_tupled_list.append((name, value))
+def get_dir_size(path: str):
+    total = 0
+    with os.scandir(path) as it:
+        for entry in it:
+            if entry.is_file():
+                total += entry.stat().st_size
+            elif entry.is_dir():
+                total += get_dir_size(entry.path)
+    return total
+
+
+def get_dir_size_gb(path: str):
+    try:
+        size_gb = get_dir_size(path) / (1024 ** 3)
+        print(f"Dir size: {size_gb:.2f} GB ({path})")
+    except Exception as e:
+        size_gb = 999
+        print(f"Error while retrieving the used storage: {e}.")
+    finally:
+        return size_gb
+
+
+def clean_dir(path: str, size_gb: float, limit_gb: float):
+    try:
+        files = os.listdir(path)
+        files = [os.path.join(path, f) for f in files if f.endswith(".gguf") and default_llm_model_filename not in f and default_llm_lora_filename not in f]
+        files.sort(key=os.path.getatime, reverse=False)
+        req_bytes = int((size_gb - limit_gb) * (1024 ** 3))
+        for file in files:
+            if req_bytes < 0: break
+            size = os.path.getsize(file)
+            Path(file).unlink()
+            req_bytes -= size
+            print(f"Deleted: {file}")
+    except Exception as e:
+        print(e)
+
+
+def update_storage(path: str, limit_gb: float=50.0):
+    size_gb = get_dir_size_gb(path)
+    if size_gb > limit_gb:
+        print("Cleaning storage...")
+        clean_dir(path, size_gb, limit_gb)
+        #get_dir_size_gb(path)
+
+
+def split_hf_url(url: str):
+    try:
+        s = list(re.findall(r'^(?:https?://huggingface.co/)(?:(datasets|spaces)/)?(.+?/.+?)/\w+?/.+?/(?:(.+)/)?(.+?.\w+)(?:\?download=true)?$', url)[0])
+        if len(s) < 4: return "", "", "", ""
+        repo_id = s[1]
+        if s[0] == "datasets": repo_type = "dataset"
+        elif s[0] == "spaces": repo_type = "space"
+        else: repo_type = "model"
+        subfolder = urllib.parse.unquote(s[2]) if s[2] else None
+        filename = urllib.parse.unquote(s[3])
+        return repo_id, filename, subfolder, repo_type
+    except Exception as e:
+        print(e)
+
+
+def hf_url_exists(url: str):
+    hf_token = HF_TOKEN
+    repo_id, filename, subfolder, repo_type = split_hf_url(url)
+    api = HfApi(token=hf_token)
+    return api.file_exists(repo_id=repo_id, filename=filename, repo_type=repo_type, token=hf_token)
+
+
+def get_repo_type(repo_id: str):
+    try:
+        api = HfApi(token=HF_TOKEN)
+        if api.repo_exists(repo_id=repo_id, repo_type="dataset", token=HF_TOKEN): return "dataset"
+        elif api.repo_exists(repo_id=repo_id, repo_type="space", token=HF_TOKEN): return "space"
+        elif api.repo_exists(repo_id=repo_id, token=HF_TOKEN): return "model"
+        else: return None
+    except Exception as e:
+        print(e)
+        raise Exception(f"Repo not found: {repo_id} {e}")
+
+
+def get_hf_blob_url(repo_id: str, repo_type: str, path: str):
+    if repo_type == "model": return f"https://huggingface.co/{repo_id}/blob/main/{path}"
+    elif repo_type == "dataset": return f"https://huggingface.co/datasets/{repo_id}/blob/main/{path}"
+    elif repo_type == "space": return f"https://huggingface.co/spaces/{repo_id}/blob/main/{path}"
+
+
+def get_gguf_url(s: str):
+    def find_gguf(d: dict, keys: dict):
+        paths = []
+        for key, size in keys.items():
+            if size != 0: l = [p for p, s in d.items() if key.lower() in p.lower() and s < size]
+            else: l = [p for p in d.keys() if key.lower() in p.lower()]
+            if len(l) > 0: paths.append(l[0])
+        if len(paths) > 0: return paths[0]
+        return list(d.keys())[0]
+
+    try:
+        if s.lower().endswith(".gguf"): return s
+        repo_type = get_repo_type(s)
+        if repo_type is None: return s
+        repo_id = s
+        api = HfApi(token=HF_TOKEN)
+        gguf_dict = {i.path: i.size for i in api.list_repo_tree(repo_id=repo_id, repo_type=repo_type, recursive=True, token=HF_TOKEN) if i.path.endswith(".gguf")}
+        if len(gguf_dict) == 0: return s
+        return get_hf_blob_url(repo_id, repo_type, find_gguf(gguf_dict, {"Q5_K_M": 6000000000, "Q4_K_M": 0, "Q4": 0}))
+    except Exception as e:
+        print(e)
+        return s
+
+
+def download_hf_file(directory, url, progress=gr.Progress(track_tqdm=True)):
+    hf_token = HF_TOKEN
+    repo_id, filename, subfolder, repo_type = split_hf_url(url)
+    try:
+        print(f"Downloading {url} to {directory}")
+        if subfolder is not None: path = hf_hub_download(repo_id=repo_id, filename=filename, subfolder=subfolder, repo_type=repo_type, local_dir=directory, token=hf_token)
+        else: path = hf_hub_download(repo_id=repo_id, filename=filename, repo_type=repo_type, local_dir=directory, token=hf_token)
+        return path
+    except Exception as e:
+        print(f"Failed to download: {e}")
+        return None
+
+
+def update_llm_model_list():
+    global llm_models_list
+    llm_models_list = []
+    for k in llm_models.keys():
+        llm_models_list.append(k)
     model_files = Path(llm_models_dir).glob('*.gguf')
     for path in model_files:
-        name = path.name
-        value = path.name
-        llm_models_tupled_list.append((name, value))
-    llm_models_tupled_list = list_uniq(llm_models_tupled_list)
-    return llm_models_tupled_list
-
-
-def download_llm_models():
-    global llm_models_tupled_list
-    llm_models_tupled_list = []
-    for k, v in llm_models.items():
-        try:
-            hf_hub_download(repo_id = v[0], filename = k, local_dir = llm_models_dir)
-        except Exception:
-            continue
-        name = k
-        value = k
-        llm_models_tupled_list.append((name, value))
+        llm_models_list.append(path.name)
+    llm_models_list = list_uniq(llm_models_list)
+    return llm_models_list
 
 
 def download_llm_model(filename: str):
-    if not filename in llm_models.keys(): return default_llm_model_filename
+    if filename not in llm_models.keys(): return default_llm_model_filename
     try:
-        hf_hub_download(repo_id = llm_models[filename][0], filename = filename, local_dir = llm_models_dir)
+        hf_hub_download(repo_id=llm_models[filename][0], filename=filename, local_dir=llm_models_dir, token=HF_TOKEN)
     except Exception as e:
         print(e)
         return default_llm_model_filename
-    update_llm_model_tupled_list()
+    update_llm_model_list()
+    return filename
+
+
+def update_llm_lora_list():
+    global llm_loras_list
+    llm_loras_list = list(llm_loras.keys()).copy()
+    model_files = Path(llm_loras_dir).glob('*.gguf')
+    for path in model_files:
+        llm_loras_list.append(path.name)
+    llm_loras_list = list_uniq([""] + llm_loras_list)
+    return llm_loras_list
+
+
+def download_llm_lora(filename: str):
+    if not filename in llm_loras.keys(): return ""
+    try:
+        download_hf_file(llm_loras_dir, llm_loras[filename])
+    except Exception as e:
+        print(e)
+        return ""
+    update_llm_lora_list()
     return filename
 
 
@@ -122,9 +254,18 @@ def select_dolphin_model(filename: str, state: dict, progress=gr.Progress(track_
     value = download_llm_model(filename)
     progress(1, desc="Model loaded.")
     md = get_dolphin_model_info(filename)
+    update_storage(llm_models_dir)
     return gr.update(value=value, choices=get_dolphin_models()), gr.update(value=get_dolphin_model_format(value)), gr.update(value=md), state
 
 
+def select_dolphin_lora(filename: str, state: dict, progress=gr.Progress(track_tqdm=True)):
+    progress(0, desc="Loading lora...")
+    value = download_llm_lora(filename)
+    progress(1, desc="Lora loaded.")
+    update_storage(llm_loras_dir)
+    return gr.update(value=value, choices=get_dolphin_loras()), state
+
+
 def select_dolphin_format(format_name: str, state: dict):
     set_state(state, "override_llm_format", llm_formats[format_name])
     return gr.update(value=format_name), state
@@ -134,7 +275,11 @@ download_llm_model(default_llm_model_filename)
 
 
 def get_dolphin_models():
-    return update_llm_model_tupled_list()
+    return update_llm_model_list()
+
+
+def get_dolphin_loras():
+    return update_llm_lora_list()
 
 
 def get_llm_formats():
@@ -157,33 +302,41 @@ def get_dolphin_model_format(filename: str):
 
 def add_dolphin_models(query: str, format_name: str):
     global llm_models
-    api = HfApi()
-    add_models = {}
-    format = llm_formats[format_name]
-    filename = ""
-    repo = ""
     try:
-        s = list(re.findall(r'^(?:https?://huggingface.co/)?(.+?/.+?)(?:/.*/(.+?.gguf).*?)?$', query)[0])
-        if s and  "" in s: s.remove("")
-        if len(s) == 1:
-            repo = s[0]
-            if not api.repo_exists(repo_id = repo): return gr.update()
-            files = api.list_repo_files(repo_id = repo)
-            for file in files:
-                if str(file).endswith(".gguf"): add_models[filename] = [repo, format]
-        elif len(s) >= 2:
-            repo = s[0]
-            filename = s[1]
-            if not api.repo_exists(repo_id = repo) or not api.file_exists(repo_id = repo, filename = filename): return gr.update()
-            add_models[filename] = [repo, format]
+        add_models = {}
+        format = llm_formats[format_name]
+        filename = ""
+        repo = ""
+        query = get_gguf_url(query)
+        if hf_url_exists(query):
+            s = list(re.findall(r'^https?://huggingface.co/(.+?/.+?)/(?:blob|resolve)/main/(.+.gguf)(?:\?download=true)?$', query)[0])
+            if len(s) == 2:
+                repo = s[0]
+                filename = s[1]
+                add_models[filename] = [repo, format]
         else: return gr.update()
     except Exception as e:
         print(e)
         return gr.update()
     llm_models = (llm_models | add_models).copy()
-    update_llm_model_tupled_list()
+    update_llm_model_list()
     choices = get_dolphin_models()
-    return gr.update(choices=choices, value=choices[-1][1])
+    return gr.update(choices=choices, value=choices[-1])
+
+
+def add_dolphin_loras(query: str):
+    global llm_loras
+    try:
+        add_loras = {}
+        query = get_gguf_url(query)
+        if hf_url_exists(query): add_loras[Path(query).name] = query
+    except Exception as e:
+        print(e)
+        return gr.update()
+    llm_loras = (llm_loras | add_loras).copy()
+    update_llm_lora_list()
+    choices = get_dolphin_loras()
+    return gr.update(choices=choices, value=choices[-1])
 
 
 def get_dolphin_sysprompt(state: dict={}):
@@ -221,6 +374,7 @@ def get_raw_prompt(msg: str):
     return re.sub(r'[*/:_"#\n]', ' ', ", ".join(m)).lower() if m else ""
 
 
+# https://llama-cpp-python.readthedocs.io/en/latest/api-reference/
 @torch.inference_mode()
 @spaces.GPU(duration=59)
 def dolphin_respond(
@@ -233,6 +387,8 @@ def dolphin_respond(
     top_p: float = 0.95,
     top_k: int = 40,
     repeat_penalty: float = 1.1,
+    lora: str = "",
+    lora_scale: float = 1.0,
     state: dict = {},
     progress=gr.Progress(track_tqdm=True),
 ):
@@ -244,12 +400,18 @@ def dolphin_respond(
         if override_llm_format: chat_template = override_llm_format
         else: chat_template = llm_models[model][1]
 
+        kwargs = {}
+        if lora:
+            kwargs["lora_path"] = str(Path(f"{llm_loras_dir}/{lora}"))
+            kwargs["lora_scale"] = lora_scale
+        else:
+            kwargs["flash_attn"] = True
         llm = Llama(
             model_path=str(model_path),
-            flash_attn=True,
             n_gpu_layers=81, # 81
             n_batch=1024,
             n_ctx=8192, #8192
+            **kwargs,
         )
         provider = LlamaCppPythonProvider(llm)
 
@@ -339,6 +501,8 @@ def dolphin_respond_auto(
     top_p: float = 0.95,
     top_k: int = 40,
     repeat_penalty: float = 1.1,
+    lora: str = "",
+    lora_scale: float = 1.0,
     state: dict = {},
     progress=gr.Progress(track_tqdm=True),
 ):
@@ -351,12 +515,18 @@ def dolphin_respond_auto(
         if override_llm_format: chat_template = override_llm_format
         else: chat_template = llm_models[model][1]
 
+        kwargs = {}
+        if lora:
+            kwargs["lora_path"] = str(Path(f"{llm_loras_dir}/{lora}"))
+            kwargs["lora_scale"] = lora_scale
+        else:
+            kwargs["flash_attn"] = True
         llm = Llama(
             model_path=str(model_path),
-            flash_attn=True,
             n_gpu_layers=81, # 81
             n_batch=1024,
             n_ctx=8192, #8192
+            **kwargs,
         )
         provider = LlamaCppPythonProvider(llm)
 
@@ -452,6 +622,8 @@ def respond_playground(
     top_p: float = 0.95,
     top_k: int = 40,
     repeat_penalty: float = 1.1,
+    lora: str = "",
+    lora_scale: float = 1.0,
     state: dict = {},
     progress=gr.Progress(track_tqdm=True),
 ):
@@ -462,12 +634,18 @@ def respond_playground(
         if override_llm_format: chat_template = override_llm_format
         else: chat_template = llm_models[model][1]
 
+        kwargs = {}
+        if lora:
+            kwargs["lora_path"] = str(Path(f"{llm_loras_dir}/{lora}"))
+            kwargs["lora_scale"] = lora_scale
+        else:
+            kwargs["flash_attn"] = True
         llm = Llama(
             model_path=str(model_path),
-            flash_attn=True,
             n_gpu_layers=81, # 81
             n_batch=1024,
             n_ctx=8192, #8192
+            **kwargs,
         )
         provider = LlamaCppPythonProvider(llm)
 

llmenv.py

@@ -1,5 +1,7 @@
 from llama_cpp_agent import MessagesFormatterType
 from formatter import mistral_v1_formatter, mistral_v2_formatter, mistral_v3_tekken_formatter
+from pathlib import Path
+
 
 llm_models = {
     #"": ["", MessagesFormatterType.LLAMA_3],
@@ -16,10 +18,12 @@ llm_models = {
     "Captain-Eris-Diogenes_Twilight-V0.420-12B.i1-Q4_K_M.gguf": ["mradermacher/Captain-Eris-Diogenes_Twilight-V0.420-12B-i1-GGUF", MessagesFormatterType.MISTRAL],
     "Captain_Eris_Noctis-12B-v0.420.Q4_K_M.gguf": ["mradermacher/Captain_Eris_Noctis-12B-v0.420-GGUF", MessagesFormatterType.MISTRAL],
     "Capt_Eris_Noctis-V0.420-Dark-Science-12B.i1-Q4_K_M.gguf": ["mradermacher/Capt_Eris_Noctis-V0.420-Dark-Science-12B-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Capt_Eris_Noctis-Dark-Science-12B-v0.420.Q4_K_M.gguf": ["mradermacher/Capt_Eris_Noctis-Dark-Science-12B-v0.420-GGUF", MessagesFormatterType.MISTRAL],
     "Mistral-Nemo-Prism-12B-v2.i1-Q4_K_M.gguf": ["mradermacher/Mistral-Nemo-Prism-12B-v2-i1-GGUF", MessagesFormatterType.MISTRAL],
     "Mahou-1.5-mistral-nemo-12B.i1-Q4_K_M.gguf": ["mradermacher/Mahou-1.5-mistral-nemo-12B-i1-GGUF", MessagesFormatterType.MISTRAL],
     "Nemo-Merge-DPO-v1v5.Q4_K_M.gguf": ["mradermacher/Nemo-Merge-DPO-v1v5-GGUF", MessagesFormatterType.MISTRAL],
     "Nemo-Merge-DPO-v1ori.Q4_K_M.gguf": ["mradermacher/Nemo-Merge-DPO-v1ori-GGUF", MessagesFormatterType.MISTRAL],
+    "H-Mistral-Nemo-00-fp16.Q4_K_M.gguf": ["mradermacher/H-Mistral-Nemo-00-fp16-GGUF", MessagesFormatterType.MISTRAL],
     "Mistralnemo-dpo-v7-rp-pantsftv1.Q4_K_M.gguf": ["mradermacher/Mistralnemo-dpo-v7-rp-pantsftv1-GGUF", MessagesFormatterType.MISTRAL],
     "Nemo12B-Merge-DPOv1-sftv1.Q4_K_M.gguf": ["mradermacher/Nemo12B-Merge-DPOv1-sftv1-GGUF", MessagesFormatterType.MISTRAL],
     "MN-12B-Mag-Mell-Q4_K_M.gguf": ["inflatebot/MN-12B-Mag-Mell-R1-GGUF", MessagesFormatterType.MISTRAL],
@@ -84,6 +88,8 @@ llm_models = {
     "Brapnemo.Q4_K_M.gguf": ["mradermacher/Brapnemo-GGUF", MessagesFormatterType.MISTRAL],
     "J.O.S.I.E.3-Beta11-7B-slerp.Q5_K_M.gguf": ["mradermacher/J.O.S.I.E.3-Beta11-7B-slerp-GGUF", MessagesFormatterType.MISTRAL],
     "Chono-Noctis_12B.i1-Q4_K_M.gguf": ["mradermacher/Chono-Noctis_12B-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Valkyyrie-14b-v1.Q4_K_M.gguf": ["mradermacher/Valkyyrie-14b-v1-GGUF", MessagesFormatterType.LLAMA_3],
+    "MN-Slush-GGLD-Wayfarer.i1-Q4_K_M.gguf": ["mradermacher/MN-Slush-GGLD-Wayfarer-i1-GGUF", MessagesFormatterType.MISTRAL],
     "Trinas_Nectar-8B-model_stock.i1-Q4_K_M.gguf": ["mradermacher/Trinas_Nectar-8B-model_stock-i1-GGUF", MessagesFormatterType.MISTRAL],
     "ChatWaifu_Magnum_V0.2.Q4_K_M.gguf": ["mradermacher/ChatWaifu_Magnum_V0.2-GGUF", MessagesFormatterType.MISTRAL],
     "ChatWaifu_12B_v2.0.Q5_K_M.gguf": ["mradermacher/ChatWaifu_12B_v2.0-GGUF", MessagesFormatterType.MISTRAL],
@@ -96,6 +102,72 @@ llm_models = {
     #"": ["", MessagesFormatterType.OPEN_CHAT],
     #"": ["", MessagesFormatterType.CHATML],
     #"": ["", MessagesFormatterType.PHI_3],
+    "qwen2.5-7b-upscaled.i1-Q4_K_M.gguf": ["mradermacher/qwen2.5-7b-upscaled-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "MaidenlessNoMore-7B.i1-Q4_K_M.gguf": ["mradermacher/MaidenlessNoMore-7B-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Poppy_Porpoise-v0.4-L3-8B.i1-Q5_K_M.gguf": ["mradermacher/Poppy_Porpoise-v0.4-L3-8B-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "KoDolph-2x8b-Instruct.Q5_K_M.gguf": ["mradermacher/KoDolph-2x8b-Instruct-GGUF", MessagesFormatterType.LLAMA_3],
+    "llama-3-wissenschaft-8B.i1-Q5_K_M.gguf": ["mradermacher/llama-3-wissenschaft-8B-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "WestOrcaDPO-7B-GTA.i1-Q5_K_M.gguf": ["mradermacher/WestOrcaDPO-7B-GTA-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "llama3-8b-spaetzle-v20.Q5_K_M.gguf": ["mradermacher/llama3-8b-spaetzle-v20-GGUF", MessagesFormatterType.LLAMA_3],
+    "TQ2.5-14B-Neon-v1.i1-Q4_K_M.gguf": ["mradermacher/TQ2.5-14B-Neon-v1-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Qwenvergence-14B-v9.Q4_K_M.gguf": ["mradermacher/Qwenvergence-14B-v9-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "EdgeRunner-Command-AgentF-llama31-2.Q5_K_M.gguf": ["mradermacher/EdgeRunner-Command-AgentF-llama31-2-GGUF", MessagesFormatterType.LLAMA_3],
+    "MN-Slush-GGLD.i1-Q4_K_M.gguf": ["mradermacher/MN-Slush-GGLD-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Dark-Science-12B-v0.420.i1-Q4_K_M.gguf": ["mradermacher/Dark-Science-12B-v0.420-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Undi95-LewdStorytellerMix-8b-64k.Q5_K_M.gguf": ["mradermacher/Undi95-LewdStorytellerMix-8b-64k-GGUF", MessagesFormatterType.LLAMA_3],
+    "Qwen2.5-14B-Emergedv2.Q4_K_M.gguf": ["mradermacher/Qwen2.5-14B-Emergedv2-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "music_generation_model.Q5_K_M.gguf": ["mradermacher/music_generation_model-GGUF", MessagesFormatterType.MISTRAL],
+    "Fu_sion_HA-8B-SLERP.Q5_K_M.gguf": ["mradermacher/Fu_sion_HA-8B-SLERP-GGUF", MessagesFormatterType.LLAMA_3],
+    "Eunoia-Gemma-9B-o1-Indo.i1-Q4_K_M.gguf": ["mradermacher/Eunoia-Gemma-9B-o1-Indo-i1-GGUF", MessagesFormatterType.ALPACA],
+    "educa-ai-nemo-sft.Q4_K_M.gguf": ["mradermacher/educa-ai-nemo-sft-GGUF", MessagesFormatterType.MISTRAL],
+    "hitchens.Q5_K_M.gguf": ["mradermacher/hitchens-GGUF", MessagesFormatterType.LLAMA_3],
+    "fq2.5-7b-it-normalize_false.Q5_K_M.gguf": ["mradermacher/fq2.5-7b-it-normalize_false-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "llama-3-open-hermes-disco.Q5_K_M.gguf": ["mradermacher/llama-3-open-hermes-disco-GGUF", MessagesFormatterType.LLAMA_3],
+    "Qwen2.5-7B-olm-v1.3.Q5_K_M.gguf": ["mradermacher/Qwen2.5-7B-olm-v1.3-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Ice0.52.1-16.01-RP.Q5_K_M.gguf": ["mradermacher/Ice0.52.1-16.01-RP-GGUF", MessagesFormatterType.MISTRAL],
+    "ultiima-14B.Q4_K_M.gguf": ["mradermacher/ultiima-14B-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Qwen2-7B-Instruct-Response-Exp.i1-Q5_K_M.gguf": ["mradermacher/Qwen2-7B-Instruct-Response-Exp-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Ice0.50.1-16.01-RP.Q5_K_M.gguf": ["mradermacher/Ice0.50.1-16.01-RP-GGUF", MessagesFormatterType.MISTRAL],
+    "StrangeMerges_17-7B-dare_ties.Q5_K_M.gguf": ["mradermacher/StrangeMerges_17-7B-dare_ties-GGUF", MessagesFormatterType.MISTRAL],
+    "mistral-nemo-kartoffel-12B.i1-Q4_K_M.gguf": ["mradermacher/mistral-nemo-kartoffel-12B-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Euphrates-14B.Q4_K_M.gguf": ["mradermacher/Euphrates-14B-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Daughter-of-Rhodia-12B.Q4_K_M.gguf": ["mradermacher/Daughter-of-Rhodia-12B-GGUF", MessagesFormatterType.MISTRAL],
+    "mpn_mistral7bv3_sft.i1-Q5_K_M.gguf": ["mradermacher/mpn_mistral7bv3_sft-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Instrumentality-RP-12B-RU-2.Q4_K_M.gguf": ["mradermacher/Instrumentality-RP-12B-RU-2-GGUF", MessagesFormatterType.MISTRAL],
+    "1852-dark-9b-q4_k_m.gguf": ["ClaudioItaly/1852-Dark-9B-Q4_K_M-GGUF", MessagesFormatterType.ALPACA],
+    "Gemma-Radiation-RP-9B-lorablated.Q4_K_M.gguf": ["mradermacher/Gemma-Radiation-RP-9B-lorablated-GGUF", MessagesFormatterType.ALPACA],
+    "What_A_Thrill-8B-Model_Stock.Q5_K_M.gguf": ["mradermacher/What_A_Thrill-8B-Model_Stock-GGUF", MessagesFormatterType.LLAMA_3],
+    "14b-Qwen2.5-Infermatic-Crea-v2.i1-Q4_K_M.gguf": ["mradermacher/14b-Qwen2.5-Infermatic-Crea-v2-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Ice0.51-16.01-RP.Q5_K_M.gguf": ["mradermacher/Ice0.51-16.01-RP-GGUF", MessagesFormatterType.MISTRAL],
+    "Qwen2.5-14B-Emerged.i1-Q4_K_M.gguf": ["mradermacher/Qwen2.5-14B-Emerged-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Howdy-8B-LINEAR.Q5_K_M.gguf": ["mradermacher/Howdy-8B-LINEAR-GGUF", MessagesFormatterType.LLAMA_3],
+    "Qwen2.5-3B-Model-Stock-v2.Q5_K_M.gguf": ["mradermacher/Qwen2.5-3B-Model-Stock-v2-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Qwen2.5-3B-Model-Stock.Q5_K_M.gguf": ["mradermacher/Qwen2.5-3B-Model-Stock-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Eunoia-GwQ-Gemma-9B.Q4_K_M.gguf": ["mradermacher/Eunoia-GwQ-Gemma-9B-GGUF", MessagesFormatterType.ALPACA],
+    "oh-dcft-v3.1-claude-3-5-haiku-20241022-qwen.i1-Q5_K_M.gguf": ["mradermacher/oh-dcft-v3.1-claude-3-5-haiku-20241022-qwen-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "XortronUncensored2025.1.Q4_K_M.gguf": ["mradermacher/XortronUncensored2025.1-GGUF", MessagesFormatterType.MISTRAL],
+    "Llama-3.1-8B-Instruct-travelplanner-SFT.Q5_K_M.gguf": ["mradermacher/Llama-3.1-8B-Instruct-travelplanner-SFT-GGUF", MessagesFormatterType.LLAMA_3],
+    "Saba2-14B-Preview.Q4_K_M.gguf": ["mradermacher/Saba2-14B-Preview-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "qwen2.5-3b-model-stock-v2-q5_k_m.gguf": ["bunnycore/Qwen2.5-3B-Model-Stock-v2-Q5_K_M-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "MISCHIEVOUS-12B-Mix_0.3v.Q4_K_M.gguf": ["mradermacher/MISCHIEVOUS-12B-Mix_0.3v-GGUF", MessagesFormatterType.MISTRAL],
+    "Zelus-8B-Model_Stock.Q5_K_M.gguf": ["mradermacher/Zelus-8B-Model_Stock-GGUF", MessagesFormatterType.LLAMA_3],
+    "llama3.1-8b-cpt-sea-lionv3-instruct.i1-Q5_K_M.gguf": ["mradermacher/llama3.1-8b-cpt-sea-lionv3-instruct-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "LongRAG-Qwen2.5-7B-Instruct.Q4_K_S.gguf": ["mradermacher/LongRAG-Qwen2.5-7B-Instruct-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Tsunami-Instruct-14B.Q4_K_M.gguf": ["mradermacher/Tsunami-Instruct-14B-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "MN-12B-solracht-EXPERIMENTAL-011425.Q4_K_M.gguf": ["mradermacher/MN-12B-solracht-EXPERIMENTAL-011425-GGUF", MessagesFormatterType.MISTRAL],
+    "Llamaverse-3.1-8B-Instruct.Q5_K_M.gguf": ["mradermacher/Llamaverse-3.1-8B-Instruct-GGUF", MessagesFormatterType.LLAMA_3],
+    "Morphing-8B-Model_Stock.Q5_K_M.gguf": ["mradermacher/Morphing-8B-Model_Stock-GGUF", MessagesFormatterType.LLAMA_3],
+    "Not_Even_My_Final_Form-8B-Model_Stock.Q5_K_M.gguf": ["mradermacher/Not_Even_My_Final_Form-8B-Model_Stock-GGUF", MessagesFormatterType.LLAMA_3],
+    "Qwen2.5-7B-sft-ultrachat.Q5_K_M.gguf": ["mradermacher/Qwen2.5-7B-sft-ultrachat-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Kosmos-EVAA-Franken-Immersive-v40-8B.Q5_K_M.gguf": ["mradermacher/Kosmos-EVAA-Franken-Immersive-v40-8B-GGUF", MessagesFormatterType.LLAMA_3],
+    "light-7b-beta.Q5_K_M.gguf": ["mradermacher/light-7b-beta-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "light-3B-beta.Q5_K_M.gguf": ["mradermacher/light-3B-beta-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Magnolia-v4-12B.Q4_K_M.gguf": ["mradermacher/Magnolia-v4-12B-GGUF", MessagesFormatterType.MISTRAL],
+    "Darkest-muse-v1-lorablated-v2.i1-Q4_K_M.gguf": ["mradermacher/Darkest-muse-v1-lorablated-v2-i1-GGUF", MessagesFormatterType.ALPACA],
+    "Eunoia-Gemma-9B-o1-Indo.Q4_K_M.gguf": ["mradermacher/Eunoia-Gemma-9B-o1-Indo-GGUF", MessagesFormatterType.ALPACA],
+    "VISION-1.Q5_K_M.gguf": ["mradermacher/VISION-1-GGUF", MessagesFormatterType.LLAMA_3],
+    "RigoChat-7b-v2.i1-Q5_K_M.gguf": ["mradermacher/RigoChat-7b-v2-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "italy-10b-q5_k_m.gguf": ["ClaudioItaly/Italy-10B-Q5_K_M-GGUF", MessagesFormatterType.ALPACA],
     "SJT-14B.Q4_K_M.gguf": ["mradermacher/SJT-14B-GGUF", MessagesFormatterType.OPEN_CHAT],
     "Hermes-Llama-3.2-CoT-Summary.Q5_K_M.gguf": ["mradermacher/Hermes-Llama-3.2-CoT-Summary-GGUF", MessagesFormatterType.LLAMA_3],
     "Rombo-LLM-V2.5-Qwen-7b.Q5_K_M.gguf": ["mradermacher/Rombo-LLM-V2.5-Qwen-7b-GGUF", MessagesFormatterType.OPEN_CHAT],
@@ -2057,6 +2129,7 @@ llm_models = {
     "tifa-7b-qwen2-v0.1.q4_k_m.gguf": ["Tifa-RP/Tifa-7B-Qwen2-v0.1-GGUF", MessagesFormatterType.OPEN_CHAT],
     "Holland-Magnum-Merge-R2.i1-Q5_K_M.gguf": ["mradermacher/Holland-Magnum-Merge-R2-i1-GGUF", MessagesFormatterType.LLAMA_3],
     "SEA-E.i1-Q5_K_M.gguf": ["mradermacher/SEA-E-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Wayfarer-12B-Q4_K_M.gguf": ["bartowski/Wayfarer-12B-GGUF", MessagesFormatterType.CHATML],
     "Oumuamua-7b-RP_Q5_K_M.gguf": ["Aratako/Oumuamua-7b-RP-GGUF", MessagesFormatterType.MISTRAL],
     "MasherAI-7B-v6.1.Q5_K_M.gguf": ["mradermacher/MasherAI-7B-v6.1-GGUF", MessagesFormatterType.MISTRAL],
     "ContextualKunoichi_KTO-7B.Q5_K_M.gguf": ["mradermacher/ContextualKunoichi_KTO-7B-GGUF", MessagesFormatterType.MISTRAL],
@@ -2080,7 +2153,14 @@ llm_models = {
 }
 
 
+llm_loras_urls = [
+    "https://huggingface.co/ggml-org/LoRA-Qwen2.5-32B-Instruct-abliterated-F16-GGUF/blob/main/LoRA-Qwen2.5-32B-Instruct-abliterated-f16.gguf",
+]
+llm_loras = {str(Path(u).name): u for u in llm_loras_urls}
+
+
 llm_models_dir = "./llm_models"
+llm_loras_dir = "./llm_loras"
 
 
 llm_formats = {