Fix inference code in readme.

README.md

@@ -34,55 +34,46 @@ This repo contains a low-rank adapter for LLaMA-7b fit on the Stanford Alpaca da
 
 Model can be easily loaded with AutoModelForCausalLM.
 ``` python
-
 import torch
 from peft import PeftModel
 import transformers
-
-assert (
-    "LlamaTokenizer" in transformers._import_structure["models.llama"]
-), "LLaMA is now in HuggingFace's main branch.\nPlease reinstall it: pip uninstall transformers && pip install git+https://github.com/huggingface/transformers.git"
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from peft import PeftModel, PeftConfig
 from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig
 
-tokenizer = LlamaTokenizer.from_pretrained("decapoda-research/llama-7b-hf")
+base_model_path = "meta-llama/Llama-2-7b-hf"
+adapter_path = "OdiaGenAI/odiagenAI-model-v0"
 
-BASE_MODEL = "decapoda-research/llama-7b-hf"
-LORA_WEIGHTS = "OdiaGenAI/odiagenAI-model-v0"
+tokenizer = AutoTokenizer.from_pretrained(base_model_path, trust_remote_code=True)
+tokenizer.pad_token = tokenizer.eos_token
 
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_compute_dtype=torch.float16,
+)
 
-model = LlamaForCausalLM.from_pretrained(
-        BASE_MODEL,
-        load_in_8bit=False,
-        torch_dtype=torch.float16,
-        device_map="auto",
-    )
-model = PeftModel.from_pretrained(
-        model, LORA_WEIGHTS, torch_dtype=torch.float16, force_download=True
-    )
+base_model = AutoModelForCausalLM.from_pretrained(
+    base_model_path,
+    quantization_config=bnb_config,
+    device_map="auto",
+    trust_remote_code=True
+)
 
-def generate_prompt(instruction, input=None):
-    if input:
-        return f"""ନିମ୍ନରେ ଏକ ନିର୍ଦ୍ଦେଶନାମା ଯାହାକି ଏକ କାର୍ଯ୍ୟକୁ ବର୍ଣ୍ଣନା କରେ, ଏକ ଇନପୁଟ୍ ସହିତ ଯୋଡି ଯାହା ପରବର୍ତ୍ତୀ ପ୍ରସଙ୍ଗ ପ୍ରଦାନ କରେ | ଏକ ପ୍ରତିକ୍ରିୟା ଲେଖନ୍ତୁ ଯାହା ଅନୁରୋଧକୁ ସଠିକ୍ ଭାବରେ ସମାପ୍ତ କରେ |
-### ନିର୍ଦ୍ଦେଶ:
-{instruction}
-### ଇନପୁଟ୍:
-{input}
-### ପ୍ରତିକ୍ରିୟା:"""
-    else:
-        return f"""ନିମ୍ନରେ ଏକ ନିର୍ଦ୍ଦେଶ ଯାହାକି ଏକ କାର୍ଯ୍ୟକୁ ବର୍ଣ୍ଣନା କରେ | ଏକ ପ୍ରତିକ୍ରିୟା ଲେଖନ୍ତୁ ଯାହା ଅନୁରୋଧକୁ ସଠିକ୍ ଭାବରେ ସମାପ୍ତ କରେ |
-### ନିର୍ଦ୍ଦେଶ:
-{instruction}
-### ପ୍ରତିକ୍ରିୟା:"""
-
-prompt = generate_prompt(instruction, input)
-inputs = tokenizer(prompt, return_tensors="pt")
+model = PeftModel.from_pretrained(base_model, adapter_path)
+
+instruction = "ଭାରତ ବିଷୟରେ କିଛି କୁହନ୍ତୁ"
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+inputs = tokenizer(instruction, return_tensors="pt").to(device)
 input_ids = inputs["input_ids"].to(device)
 generation_config = GenerationConfig(
     temperature=0.1,
     top_p=0.75,
     top_k=40,
     num_beams=4,
-    **kwargs,
 )
 with torch.no_grad():
     generation_output = model.generate(
@@ -94,9 +85,7 @@ with torch.no_grad():
     )
 s = generation_output.sequences[0]
 output = tokenizer.decode(s)
-print(output.split("### Response:")[1].strip())
-
-
+print(output)
 ```