add vicuna results

RESULTS

@@ -14,6 +14,9 @@ tk-instruct-11b-def        0.1826
 oasst-sft-1-pythia-12b     0.0705
 opt-iml-1.3b               0.0673    
 opt-iml-30b                0.1442    
+galactica-125m             0.0128
+galactica-1.2b             0.0192
+galactica-6.7b             0.0352
 
 # trad automatique anglais (pas l'air de marcher)
 en/bloomz-3b               0.1153    ??
@@ -30,6 +33,7 @@ llama_7B+alpaca-native     0.1153
 llama_7B+deft              0.1378    
 llama_13B                  0.0769    
 llama_13B+alpaca           0.1474
+llama_13B+vicuna           0.1538
 llama_13B+deft             0.1730    
 llama_30B                  0.1442    
 llama_30B+alpaca           0.1923

run_galactica.py

+#https://huggingface.co/facebook/opt-iml-30b
+from transformers import AutoTokenizer, OPTForCausalLM
+import torch
+
+def main(result_path: str, corpus_path: str, model: str = 'galactica-30b', load_in_8bit=False, template_num: int = 0):
+    checkpoint = 'facebook/' + model
+    llm = OPTForCausalLM.from_pretrained(checkpoint, device_map="auto", load_in_8bit=load_in_8bit)
+    tokenizer = AutoTokenizer.from_pretrained(checkpoint)
+
+    def generate(input_string):
+        with torch.no_grad():
+            inputs = tokenizer(input_string, return_tensors="pt").input_ids.to("cuda")
+            outputs = llm.generate(inputs, max_new_tokens=32)
+
+            generated = tokenizer.decode(outputs[0], skip_special_tokens=True)
+        return generated[len(input_string):]
+
+    import deft
+    results = deft.run_inference(generate, corpus_path, deft.lm_templates[template_num], add_left_parenthesis=False)
+    deft.write_results(results, result_path)
+
+if __name__ == '__main__':
+    import fire
+    fire.Fire(main)

runs

@@ -5,6 +5,7 @@ python run_bloomz.py output/bloomz-560m_prompt0.txt ../json/dev.json bloomz-560m
 python run_tkinstruct.py output/tk-instruct-3b-def_prompt0.txt ../json/dev.json tk-instruct-3b-def | tee logs/tk-instruct-3b-def_prompt0.txt
 python run_tkinstruct.py output/tk-instruct-11b-def_prompt0.txt ../json/dev.json tk-instruct-11b-def | tee logs/tk-instruct-11b-def_prompt0.txt
 python run_opt-iml.py output/opt-iml-30b_prompt0.txt ../json/dev.json opt-iml-30b | tee logs/opt-iml-30b_prompt0.txt
+python run_galactica.py output/galactica-30b_prompt0.txt ../json/dev.json galactica-30b | tee logs/galactica-30b_prompt0.txt
 
 python run_api.py output/code-cushman-001_prompt0.txt ../json/dev.json openai/code-cushman-001 | tee logs/code-cushman-001_prompt0.txt
 python run_api.py output/code-davinci-002_prompt0.txt ../json/dev.json openai/code-davinci-002 | tee logs/code-davinci-002_prompt0.txt