add latest runs for submission

RESULTS

@@ -17,6 +17,8 @@ opt-iml-30b                0.1442
 galactica-125m             0.0128
 galactica-1.2b             0.0192
 galactica-6.7b             0.0352
+mpt-instruct-7b            0.0641
+pmc-llama-7b               0.0224
 
 # trad automatique anglais (pas l'air de marcher)
 en/bloomz-3b               0.1153    ??

compare_outputs.py

+import sys
+
+instances = {}
+with open(sys.argv[1]) as fp:
+    for line in fp:
+        id, labels = line.strip().split(';')
+        instances[id] = ''.join(sorted(labels.split('|'))).lower()
+
+instances2 = {}
+with open(sys.argv[2]) as fp:
+    for line in fp:
+        id, labels = line.strip().split(';')
+        instances2[id] = ''.join(sorted(labels.split('|'))).lower()
+
+assert set(instances.keys()) == set(instances2.keys())
+
+identical = 0
+for id in instances.keys():
+    if instances[id] == instances2[id]:
+        identical += 1
+
+print(identical /  len(list(instances.keys())))

deft.py

@@ -41,7 +41,10 @@ def extract_answer(answer, num_answers=5):
         selected = re.findall(r'(\b[a-%s]\b)' % letters[num_answers - 1], answer)
     else:
         selected = [x.replace(')', '').replace('(', '') for x in selected]
-    return list(sorted(set([letter.lower() for letter in selected])))
+    result = list(sorted(set([letter.lower() for letter in selected])))
+    if len(result) == 0:
+        result = ['a']
+    return result
 
 #def hamming(a, b, num):
 #    A = [c.upper() if c in a else c for c in letters[:num]]

requirements.txt

@@ -17,3 +17,6 @@ loralib
 sentencepiece
 git+https://github.com/huggingface/transformers.git
 git+https://github.com/huggingface/peft.git
+
+#for mpt
+einops