add mistral and zephir runs

RESULTS

@@ -19,6 +19,9 @@ galactica-1.2b             0.0192
 galactica-6.7b             0.0352
 mpt-instruct-7b            0.0641
 pmc-llama-7b               0.0224
+---
+mistral-7b-instruct        0.1858
+zephir-7b                  0.2019
 
 # trad automatique anglais (pas l'air de marcher)
 en/bloomz-3b               0.1153    ??

run_mistral.py

+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+
+def main(result_path: str, corpus_path: str, model: str = 'HuggingFaceH4/zephyr-7b-beta', template_id: str = '0'):
+    checkpoint = model 
+
+    tokenizer = AutoTokenizer.from_pretrained(checkpoint)
+    tokenizer.pad_token = tokenizer.eos_token
+    tokenizer.padding_side = "right"
+
+    quant_config=BitsAndBytesConfig(
+        #load_in_8bit=True,
+        # llm_int8_threshold=6.0,
+        load_in_4bit=True,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_compute_type=torch.bfloat16,
+        #llm_int8_enable_fp32_cpu_offload=True, 
+    )
+    device_map = {
+        "": 0
+    }
+    llm = AutoModelForCausalLM.from_pretrained(checkpoint, device_map=device_map, torch_dtype=torch.float16)#, load_in_8bit=True) #quantization_config=quant_config)#, load_in_8bit=True) 
+
+    def generate(input_string):
+        messages = [
+            #{
+            #    "role": "system",
+            #    "content": "You are a friendly chatbot who responds accuractly to the user without explaining the answer.",
+            #},
+            {"role": "user", "content": input_string},
+        ]
+
+        encodeds = tokenizer.apply_chat_template(messages, return_tensors="pt")
+
+        #inputs = tokenizer(input_string, return_tensors="pt")
+        outputs = llm.generate(encodeds.to('cuda'), max_new_tokens=32)#, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)#, pad_token_id=tokenizer.eos_token_id)
+
+        generated = tokenizer.decode(outputs[0], skip_special_tokens=True)
+        generated = generated[len(input_string):].split('[/INST]')[1]
+        return generated
+        #return generated.split('<|assistant|>')[-1].split('\n')[1]
+        #return generated[len(input_string):]
+
+    import deft
+    results = deft.run_inference(generate, corpus_path, deft.template_from_id(template_id))
+    deft.write_results(results, result_path)
+
+if __name__ == '__main__':
+    import fire
+    fire.Fire(main)

run_zephir.py

+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+
+def main(result_path: str, corpus_path: str, model: str = 'HuggingFaceH4/zephyr-7b-beta', template_id: str = '0'):
+    checkpoint = model 
+
+    tokenizer = AutoTokenizer.from_pretrained(checkpoint)
+    tokenizer.pad_token = tokenizer.eos_token
+    tokenizer.padding_side = "right"
+
+    quant_config=BitsAndBytesConfig(
+        #load_in_8bit=True,
+        # llm_int8_threshold=6.0,
+        load_in_4bit=True,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_compute_type=torch.bfloat16,
+        #llm_int8_enable_fp32_cpu_offload=True, 
+    )
+    device_map = {
+        "": 0
+    }
+    llm = AutoModelForCausalLM.from_pretrained(checkpoint, device_map=device_map, torch_dtype=torch.float16)#, load_in_8bit=True) #quantization_config=quant_config)#, load_in_8bit=True) 
+
+    def generate(input_string):
+        messages = [
+            {
+                "role": "system",
+                "content": "You are a friendly chatbot who responds accuractly to the user without explaining the answer.",
+            },
+            {"role": "user", "content": input_string},
+        ]
+
+        encodeds = tokenizer.apply_chat_template(messages, return_tensors="pt")
+
+        #inputs = tokenizer(input_string, return_tensors="pt")
+        outputs = llm.generate(encodeds.to('cuda'), max_new_tokens=32)#, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)#, pad_token_id=tokenizer.eos_token_id)
+
+        generated = tokenizer.decode(outputs[0], skip_special_tokens=True)
+        return generated.split('<|assistant|>')[-1].split('\n')[1]
+        #return generated[len(input_string):]
+
+    import deft
+    results = deft.run_inference(generate, corpus_path, deft.template_from_id(template_id))
+    deft.write_results(results, result_path)
+
+if __name__ == '__main__':
+    import fire
+    fire.Fire(main)

runs

@@ -33,3 +33,7 @@ python run_llama2_finetuned.py output/llama2-13b-deft_prompt0.txt data/dev.json
 python run_llama2_finetuned.py output/llama2-13b-deft-comp_prompt0.txt data/dev.json llama-2-13b-hf models/llama-2-13b-deft-comp | tee logs/llama2-13b-deft-comp_prompt0.txt
 python run_llama2_finetuned.py output/llama2-70b-deft_prompt0.txt data/dev.json llama-2-70b-hf models/llama-2-70b-deft | tee logs/llama2-70b-deft_prompt0.txt
 python run_llama2_finetuned.py output/llama2-70b-deft-comp_prompt0.txt data/dev.json llama-2-70b-hf models/llama-2-70b-deft-comp | tee logs/llama2-70b-deft-comp_prompt0.txt
+
+# mistral
+python run_zephir.py output/zephir-7b_prompt0.txt clean/data/dev.json HuggingFaceH4/zephyr-7b-beta | tee logs/zerphir-7b_prompt0.txt
+python run_mistral.py output/mistral-7b-instruct-0.1_prompt0.txt clean/data/dev.json mistralai/Mistral-7B-Instruct-v0.1 | tee logs/mistral-7b-instruct-0.1_prompt0.txt