add vicuna results

RESULTS

@@ -14,6 +14,9 @@ tk-instruct-11b-def        0.1826
 oasst-sft-1-pythia-12b     0.0705
 opt-iml-1.3b               0.0673    
 opt-iml-30b                0.1442    
+galactica-125m             0.0128
+galactica-1.2b             0.0192
+galactica-6.7b             0.0352
 
 # trad automatique anglais (pas l'air de marcher)
 en/bloomz-3b               0.1153    ??
@@ -30,6 +33,7 @@ llama_7B+alpaca-native     0.1153
 llama_7B+deft              0.1378    
 llama_13B                  0.0769    
 llama_13B+alpaca           0.1474
+llama_13B+vicuna           0.1538
 llama_13B+deft             0.1730    
 llama_30B                  0.1442    
 llama_30B+alpaca           0.1923

output/llama_vicuna-13b_prompt0.txt

+d0f974df4406bf2f933962bf0d0c0447a80d66719031e3b4a35c2ff3b3ccc594;c
+26006d212c5ac91ace2cedb8ba7be0493cdbdab0fa3f7225b587cf5245e6c30e;b
+2532bce14adf9a1eb8c15b26e8141101fcf85ea15a5012fa6890b68370be537f;a
+03a1eb7ea99f0f2cc40f9c8869b612db51c29fa06ec4593ff81132bd5ac59812;e
+593f437c4abfe68a134c11579f955f5312a5c2a10f225ab2a97296c34131c60a;d
+0cc8d2cf6a48f16cccb62669baf999554c9bbdad37336eadfbf739c1626bf5f6;d
+9c59bf7ccf6fd142cec0ef8f2f8ad5e6d07465b861862c7b4af725a45e6d77f2;b
+1f3b737b3ec97b278f300c898845e2356ccac519a84a2de909f45cdb5e79198d;e
+5bc9fcc436747c4910cc03cb6dd75b898ae6ad9d7f718b99dab59cfad43ecf8e;e
+0774ce9d3eec365156f7ef48c440c8ce484a2c2014ce4d42994bdd4b323568eb;c
+133c33270d44fc1d2a085f6a6ba0b64bcfe7a7f43fcc740d07acc11bf924dcb5;c
+d2be657ca4b06a953849c5645923876e19abb74aef63ce46ac9867904b40ba5e;b
+3c707b53a2189df15f9ee227a63498e690724b89c9c7f3fdf88350319c540ce7;e
+6d4c03d77b6077f33f53d7d0968a11e3f94f41bcc5df244a91f88a9d09a3372f;c
+e9508d865fd3b3f0e7aa65a1b9613a6b2461dc4f30306f51da8fc7647f57b28b;a
+55d436f919192ef9c2faf616a1155abd783a96f0b1c747900c4274d5b6c22a05;a
+0acb5c0af2a51add0461cb0a1301310e4b6cf14709b28fab6482b6100be51241;d
+362902d44341292ea9064ec4436f379f269cf58df4fc5e5b8a883bc8ec943300;a
+f20ef5ca3714c515637ff2b733854c0990aabe8133d08d269074392b24a108f8;e
+19f087607388d9a9017fc5456bb39125f30f998929b47d51c384ff5b9dc98215;b
+7751960e61f58777325bbab2e60a54055e88be9665ed919b29fc2b6bcf6cc27a;d
+838e8c06fe1651cffca55a6816752fc2cf482a580f3fcc7411c06a2c92e5344e;b
+6821e68094b1a7cd4e3660642fc0b7d9dfa8606ec57a13b93e459e2d835e0152;c
+dcdc1cb6a938eb57545d24b397463e0d8b7e389f6008761015cb53aef6569751;a
+1a6d11bdfd3e33c57892cec5f7eaecd9301dd6b163e5b90c3b562ba3eeba5152;a
+b1f9f1de8c497ab6dc989227ebaa01dfa1b2643209bf6b16bc0fe08d6ab55214;d
+e8e226e29459a265175f5ee193d4f7b4b5e700ea8147971e02626a7fee4a623d;d
+0b6e8e761c8886bf00f10b16128a88aa1a19eee29c38f2bd49959e65b266fd97;c
+db9061516f62d6a9a163414cfe530cfe25bae10bb64c9a990351bbac9f83583d;c
+b36af8422b00feefb35b23b95e7c9045b2db89b8b893b30c8703d2caa4bf48c5;e
+3ea2ef1e40057e5750e8ea3bd7c5de22cbb25f5971a7156cfd5b90a2eb13e00f;b
+d092a603e897ff884b46f7d04e18ce4ddefe98d5fd278c8f69117fdb35e13737;c
+6c3bc681064a2742f2306c79f7c42a2f8848291f95df46b1673c674ea760dbab;a|b|d
+0fea3be17abc222e1557da232b24a7d210768c710f69324454f37cd5bbed5cc2;c
+bd7d6613391216ff43b7d7dc1b4ce0b0938c9e37d8bce3404251a3599d0c2631;b
+93fc74a8bfa6796824731c249eec3eedaed80a67a555794c7b011e4a3a7bd156;c
+4c21cb628285a86bbe87ef5f7329812e507f3357d7324a1918746427cf302d00;c
+cdfa80c398e3178e0e414cc7ac89720c37a1c5785ab7be59468e50b3c6973263;c
+ba127dc297feb9c2f8b1915635ac795f97bf93a92ab7262e5e38101ceea51b78;d
+ea971cc6487a08debaf8f052a76460450488e742dde86e63082951ee37ff07d2;b
+acb4378a630b038c6a00a1b8b674a683e371a44da6af983e3f526c49e1d9e35c;c
+373f823dbf212abeb4dffec4875caeadfc60fa5d6f6d7d5e59af9bb031c9c6b8;b
+5f5c715865c2dea673db2f385b7784e6e0959a02311a6d3be351e07442b6c7e3;a
+62c17bb5f178c3ee366147a3fb8e3e08ded8f8889f12b5afbeadc0b48ad85ed2;d
+5ec85e18c4dc220cab969b7109c0650cb052b0f2c2f4aaaf0285279640f17e77;b
+bdc11129e137656f83452df10907a0a4c6c008d5c7e11443893bec9e6e9f0f52;d
+bb9b898b7379885c70ae3dde3690f3d169acc77aa25ba5280459ee812adc2e67;b
+3b4ba5a011722b6993babdcace4ab64af6249a317bb599e5cf9cf9b69b721d2e;a
+6157730b95869c9a5cb7a153add870081786090d76704da08beb7416cd60a302;c
+1125ebfc90138b066ce7b116ec3daaddb2646c36ffa7d544641ff42c8a484479;c
+6598ec9c10584dc0af4a357889466146d3919553c9574d04cbe32f565e889c21;e
+d30405fdda77805e9fbe4a35f9f66ee631f01a1abd28522e8f131221049847f1;e
+160ae8f17ccc5649945fbbc43d1f79be406603a1d2f48cb9778aea6f40618e0a;b
+39eb2d4f462134f9e89b43331cc2a46f10ae4a21e4e41b1515996f11cf3bc182;a
+e63b02a359038b9ca6c8a0547ea076cbe90b457baa4332232697279db1a2124b;d
+a04be3f9e3caa4cdbf9b43265ca3e9ddaa051713b0a21bf31f28f008f0717d01;c
+307d1070bf63a5d52b79569bf235d2205555980e9ec83500a0b119dc19e5bf57;e
+1a3148d2012b2352e3a15fb41a3a2efa154573c3223ee602e4ff49459e52aeb5;a
+778548a68b9532a310fc6134fd70804e3e34cc6e1d8623b5afa113e8d551574d;a
+29b21fc980d660f3b33bfed6c59a269d6cb4c9512764eccd16718c813a77f823;d
+4daef687028d4d6b0d6748100af096dfec71b4db3764ece0b0d42afe247b46f8;b
+8966e53e83d6b79fc979dece1e99298ba8973e3f37a4d8b49746036a90a3374f;c
+0666f011eef0fa8608a09a17694972205b090faca5c9b236aa34ea21fcfd8644;d
+f18070c719bb51f25f4cb5e726154effbef865b8ddbd4e465accd5515216205b;b
+6c5106434785aa94aaf1b1cf7732fedc86038668351004d42004119afdf440ab;b
+03eb4b11edc583bd3b3bb0d8ba39895522fae0f9f4630574b86c0edba5892988;e
+3b754ca26ee8012a0621b8a3f225616addcd92b862eb5b44fbafc879c774120e;d
+89df1517155161cf49f4a71b50c66efb56b49d6a0cfbaa1ed1c01242064249f1;b
+df92a6c481de8c4eb9a3f0250dbcc670b9572bb88f5da6ddd6e745796c48ed2a;c
+8e86748ba0e9a03f49749e82da5fe32bb44b5265a381e143466f3eeb33039592;c
+34a65d99817136bbc8c7774166860c950517b4980706cf5ed50df45bd035159c;a
+cf3af132b06c4dd936ee8f3baa0412cea03877c0c6d0e7ce15ef405d0051e06c;b
+36a370e7740b2fb85245ee37b1f5b3923d52b426e68b1a3bd8b6cb79e3e033ac;d
+32a70460581ad422fc208e11071726df3a8de1149242c63025f5fe0cbcf13afd;c
+87bc7576ac739abe46e9e6c330022594762fd87bcb14219487fe666460da21ec;b
+d7fc1d1b5eae76dd216883101221c3cb6e3faca85b76109d82f4e38cb3430cc1;e
+0ea0fd1b44491f179424849fce5f9abd6c21321e177504e8d795f563cc4a23cb;a
+99402fd5c76e95c785f59121e3f6bf20bf1c49e6a904c1186be5fe39db4c30d7;c
+ad0efd74fda1fec16e88f356b86f7dd64a89b8b030380ee68ef4d84fd7687e6c;a
+47ea8161f83251d9a8c2bc955d9eb414267951f7ec2bce9f0ed5ffc51568eafb;b
+24d202de34bc4b9ec7d8c14ed08fc5a2c3d2211d1d4ea4c60bb687f228f0effe;b
+604faf0d3d67d401384a19d85f39d7b43f8e50d77a4330fccc055b1c80cda53b;b
+50710dbc4659a2663263d5bb3221d3c02b534ed7c5c8c9b24f3094210e1f469a;e
+fb9bbde1c0b37dec46f0ffa88e4205b95ef5a082242c34342e27b8781689429f;a
+3b1f5193a0a9351cdfb367fe65952749a4f7ebcb2800414db5f66445c0c6a727;d
+bad53f81109c6bb757a8e97363f58bb6fdb2dffa30e7971a0db5d2d84d84ed14;e
+ef328070274c67c5aef4c86c97f1e419770dea6ba4d7d0db6e4786afd4a250ba;d
+a029e188ceea30290932bb322b7224efead3892415f36365c55d600733c2c2e2;a
+1947570803b43eeb00b5ba28983c27c6a60afcc8897b16c9415d383a885563d9;c
+1faedc19c13e0d4add485f4bf220fba98245710b9f27ed29ff0e7cf908921543;a
+fafcb93b10714a7628e71e4ef9e7ac7540edc6d6813d301f352f2d2669a7f572;a
+326c46ef50ec5616c69558f863a926e50056c5c68c566ad363d604a3b028ecdb;d
+14cc4daf81db8c067174f9f4a67529b0cc2c252c0c5ebc32412ef41933e1e8b1;c
+417ba23cea3798d6b1e15051715ae9eca3109ff6007d9b986dd6cf4c2a5f5229;b
+1b875ecdea273d0997c7bfa4c21c4a9fd94ce33ac062a70509b1c2185e378b4f;a
+c1aa20f8ab260c82084247cd9c464b09eca0673d76b683328bdef3c5f9cfcaf1;a
+69530a7ef20cb903da390d971e98a3c5a34099fdc830f821e532043c6f28c864;a|b|d|e
+fab894387c47445b0bba9451a5c7dacc59065e22d76049c675d43a3156ae4100;b|d
+1332c59d6543010f3322283e662f39c0cbfae2f01bfde07a8a182fb192416bf2;d
+d049c3a073dc366b2c85ad350f5d8edcff5fbfb4005ac10cea680292c94856c1;c
+a89acfd25caab9081daa7a962011dcf5507533d6432b4270e46ea826e97fc469;a
+beaabca7cf0b2923297dfc4aacacf5c9bdfdae7f9f2824e05a3e29f54cf3c877;b
+afaa4dcef442598fb74cfb5677779564667d36dd7b24701e76f5744846da3e72;b
+9116f6cdbc43537089e4a45431d7eaeabbea2bc2bcc113ac4232babe3cff0ff5;c
+76a26d83a9839196a146e8d00d88c4e03d3b3fee97340ad982d61f3814293c10;b
+1c73e02ac68c6d75a81a35cbabae08795db32c9a9e519670e7e98f48638ed3a0;e
+be14b1eeb7a06aa36d8eb999dfe4ecef2eaaf6339867e8778ee2af9faaddd3f1;a
+237fc09efcb294c0c5a543f656a5e8b19741742344d2fd7942a162ec436d56c9;e
+eea5d687b945d157bcb0dfc28ce7aaf5c27330293afdf5c32e40090446ea67d1;a
+0345bf0bd8a1aaca1e9704d0370737887480febbf9e6a549d996125cb83604dc;a
+fbd3b77b5deabf7753788d9f47e4dfddac751d2694a4cb0668f4bf0d018e1c18;e
+de22baa605917435ea0e13a5ba2d8de5ee06c67749216f031df561774293e6f9;a
+a1a2b1810949b40d9c2b2781baa31b82aa1a98d4b455466f1c7aa52b714e99a0;b
+746dab3f13dddcfb07e32befca1fc7592032ed94c9c1cca1fd879c759461b030;d
+9f07cc7bede0a6a435d87e9be275f180fe0cd4dfae811ad1643a1e0efd24d340;d
+610836362ff0a6a59667cbf408baf0ec7ae4be665fb152ba31f2315293f5afe2;e
+33bc47979ae371dab5bc5d851087e7a3c90a0aa2538c512c8ebf9dcc02cc721c;a
+c181935cd53290566a2f3ae86f29ea0c1413442c9fbacb77c3ec05d6b2eba015;b
+3e88e7ff094aa1195179e0585b1346aeecb7735892a44baac29204ba73bc2025;c
+8400c54eaf45d7798b939e11b9e9efd3c8b845b9a0f289e61baeae47d8d39d75;e
+fdf7ce542d86b47b3cf9933da481fa794c5d1bf0a8e2fe272361433de4fb9e62;a
+7f0a0aec81ee807dfbca70f201c69748a209f49a0269e3eb58c99b1588b59afd;a
+e6c792e328e9f39491a8654369b1e43bcea872ddedb3dc6142369b4ab7b2ceee;e
+f74d2d93763d2198682851b4a4ec9fa33e660b581f7c5a4f17bbe57bb04f8c1d;a
+b4c3c16bdcc103096baa3b292d82e444be403a35cea585680dc00e9f1d159c28;c
+5129ff3725a4cdc212514eea24c8aaa0ae14ce65351736a62d535a29cfc7dc81;e
+966e3d00dc64c35ece060d579eb70ce88ffe3f1870553751028ef869a8f7f7ea;c
+135280e3283f5428198674f89a25019b750d4557350dc8ee45e28cdc447bfb5b;b
+61765ee57afaa3d979073744bc3a27fd525fe2520dfe07b4aabe02351aad14ff;b
+a6a515af27c05a6896a594e73e5107aa82f4ff7692e600f545941e1c56c287e1;b
+c7915caa16a5740f0025f94b04b87d41eb28e245b5a76f3a8fa697a22a49ff38;b
+fb6778b232abf9bc5cdc9ecc394a2959db8f51949a7d4958a85411bebc335ff5;a
+31deceb22c9bd82e5c0df8946a591d42a0bdee2d9a6193937fbc45dbd1b2bfaf;d
+0de0ca01ffe9604c96b78a767e22beee58dd557de833eaee1cabb6759f2caf54;b
+c856218856160616dd96f0467abf3933a5bb1d639c570e06c21fb88918d27cd2;d
+b3737e5a9e358eb3bd9e64c35ce2a8526596faff7defad812ac5fc6a137a77e9;b
+38a2bfe4280c133640fb197b878a2a704b7cab8bd34ab3c750e8d3d14afd20c3;e
+66cbb8dd051d95d8298461b12fb64ba5fca86edb0b42fda6c0882e0469d7dca7;a
+87e74e5280ed8b5731a4c56ca31fe36c836024bb5b678c7bd23496df82c6a11b;a
+9a0836e1615f01c4e88bf0615b444b07ee1f7141d84cd97548dd6b28e0da4153;a
+439defc603f70a4aa6af853bb631316b941d7d9a1c18345f0f574176529dee36;c
+456973b49536d8a14fbaaa55e569f5375d0222639287c9dd31301e74dcba11f1;d
+9211e3d6e88c89650efaae4cdda979655b431dab230f473a23512e93ef7f1ef5;b
+dd43fd629ac8590152b3abcba6c73e6adbd682d07ab972ed828b17c8d8467463;b
+6f81adf58dedd349d52a3e1f5e5ea3f43b163c0fae1d8a40911e88d6974cab73;a
+0b06b20bfa9f128bcdd84962961f1a4d7e057ca42894d1c82be6a42eee87727e;a
+58ddff24f38dbab143c2a75f6d735ba4b6ca2d387cd82dee883a594b3e091975;b
+14145a17780f8d594f99899d428f4c6e0cc5be21148e33755abf984ed45b6709;c
+195bdf428f7887e3e07fb985c7ba0dc2f668f56d329616582b0a45f57a852206;b
+d905d12a6ae2e7c98c8835bdfc4d4cfb42b2bec4a93dedbcd4b8a42f46daf14c;c
+f406422497d29c9442500e6337ab47a9941b2db9d005a01172c01ddfadf1525d;e
+6570061c2ad92f3d689c4ab28e8093eb8399b6b76b44435f4adb361404ce1f7b;a
+76d3acf326e0c2c4ca6c8c2c2f6f354b75cdb0206cc6ff4b32b206fb95cab092;c
+bdf55a539367baad889261f532b4095629bb8a55a56057208345086f815d2365;b
+420764f0aad6f1805275f7f6292f31ef7c7619e08a009f8f63424b31290840eb;b
+5ef2e6b12c7f3f7f12294a19723fc84250b60c1d5917fa2b2b851e62aed20f2e;b
+e0b9625efbd4c8870865a1177116200861be5f936d2c856a74e29e6da4842350;a|b|c|d|e
+87b84fcbdaba6de905ee84f63a496918db85929f419053f66442588d02ebd023;a|b|c|d|e
+338dc01e04cc90fa72bd1a50da182c18826bd143b19512dc8f96f2d198940d84;a|d
+6cc0e1d8c29ed23f72b420de096531da1d15fc40388f7ce91c2797a138811fed;a
+195e5754ede2181ccef511fa988b2638656ef0a1eb0628646317544ea42bc5e3;a|b|d
+5fe1d4496ca612e0dc349a7acc8bf96899db8633a2d3647b4174a48d4143fd18;a|b|d|e
+e928c0f55034f1541bb66320357c80aff6f88722329da018e28d9ef4f5d9c141;b|e
+800ba5869a420a0106caef36683950c48760777d48780004d68faee41f4ad4ee;e
+7c3dbb14cb3ae55c79b00278feda05faa7aafc07f9e4e226cacfdf84177ea0ac;b
+be70d95722cc2b388ab07038b68c09ea6d4443b821685400556969f64e12e52e;d|e
+2b160a974c8a6c6c19d95471b57ac755b5ac1d2da297f989e606ceb9e4738b0a;a|c|e
+52bfd4782ec560e906b9b70b92012f1a49a6e5043ebeae2594e3a5d8a1ced7c8;b
+0f2dd8c44337e917c0c44820af38d59cec8375859e848c477d27231e78156ef1;e
+7e763b6d772186309a9058a601c56f41130bf4f4205b857883b5d0e4af69ebad;a
+4354821d639a3d0e400475877f098c2987e645d62d2c9ebcc3585fc442cba8ce;a|b|c|d|e
+52fc10ab59fb50a7857fcb6cbb52e02731f84aa5db3d6b41162828574f5a18f6;b|d
+043952256271ed4fbb18e10429815cba53b793e6110d41d3a752e4e564f2a885;c|d
+961b1deb75dadac38f2cfad6eb159b4349604d3eb984f7549790293f085670ba;e
+d185be848832e5567b61a6b42a4214cc4e0c984bd5d212df2bb9c53675973aa3;a|b|d
+02e087764de5d81b20655ac9823f5a5eb3b797aabdd109813eef4262a59cc31a;e
+d5c0a8fb8450b30c0f6e20ad686e6711843aeb6dae575b7b6b3af37ed393d3f7;a|b|d
+3bef891507315739f09a37394890b023f5c24606a8b16bb3d1f724538e5af860;a|b|c|d|e
+5d78d2d1df1afb24d4db21881e6ddea37a94ffb2d8a666ef93091d1bb120472c;b|e
+8e92d2774fb4a6b81b0ff85acc7dfdfe474fee38382337944ac25452537b1f60;b
+07d4ac3c5b796f20e10f01368f38ca467b006ab28049102ae2ce736c01283dc3;a
+49bfca2fbe61c0333bf79232551d1e3a3efe428041cb48ee2b65d472d74f1263;e
+668f19600647b0ab4f22c4a7050e64da757c5ff9e1284f407e39aa1043f1d041;e
+aef3df138b6e05dc613f27487ad2a8507b1e5e6a4435858a7e5bd0bd296d78a3;a
+1ba3cbd8ea59c36448da96a1456eaf9129c847966fa62daff6cfac6af1472bd0;a|b|c|d|e
+a27eabc6b991f953901d3d11ab017dd77952e51a7caa5523db83875bc64a5a42;c
+84baa00ad2e909339265a320fb5997ca9c03bf468dee99d34af58c721d099296;c|e
+5d8f9a4d6737f9feed9e6a80669bb3f78c80c8b2d2818daad389672d2ac2da4b;a|b|d
+da878415013cb9a49ca44d18764a0dcf05ed017adc125c3382e9711958b00e0f;a|b|d|e
+e18acb66481acb3791b882fd3de246d8ddf256f62dcf791039e41050267aeac4;d
+df70c1e171b5cbba43d0c9298e9ed516f91440bec5c49e7f3bd7fcabea28face;c
+f07c9ef6cfe48da8958cc63ba389ebec718928223a1f43308a781cc5ca9f4aa7;b
+6915637ab17e7ebb74e33a48ca2a143137271bec33fe3c778bf9177a7467d89d;b
+79c4aa6d714fb4a3827eb7296bbfafa3c26131fabeca82c12028de9032d504ce;a|b|c|d|e
+0d939a4cc603d981d33efc677cedb7ee74b697ea1d2c4798d7acfa712f5f6396;b
+7ea047c3a4cf2c53ad510891c5efd1481ae1d8fc47acfc1cad2244a2d3301143;b|d
+7f82201f8e951176fe772fa146044fc580bb3300be8f0ef00381a4851937e93a;a|d
+dbb245f85aa1b274e4f9024a2bde3f34ce5d03f4a7e7413a6fd6e3c5517cbd92;a|b|c|d
+e46dc0347181d93f248363bc3e7fa70c3d17aa953139b5a8e5fd9d0e2e4764ec;b
+8559a56b2e8c1cfdd663bc51bfec6f4eae1efad2e3cecbc19d5ee5f6b2a4857f;c
+3a8b302d83751be0b2d526bda25a759bc758049da567ea31ff7fef75e8ccf4d7;b
+e2c38292dfa0f9d1f62f26fc4debf2871875799054fdccbf59e9d8762ab8d368;b|c
+7bf4d48d5901be3554657a94547e13f04dd3adafc1f725fd5b3b1c51855fcde6;b
+24b174904db6a075d78d4a2b8a588f23bed8a101eea5c230c97102ff8d48c818;a|b|d
+e544ed315fe466a7e0916cf8f80d713165cb017c37b866ca3012998480d46211;b
+06e4db4d10817c0b7e7fdcc2170d549f6a72c98c27ed788c0e4926757673e98c;b
+a12e8697bfcada1526bf27f34ed2c5d20ad76756c46e1e2e49a3cb540b8e53c3;a
+f961df911e75c60d11559cc39347555c576f85c397fde969ad65a84040823faa;b
+c564cf55f36459cfe7ad2745c040f54b7807b079b19136daa0e7abbf6215e20d;b|d
+f37e8bf4ddf7ce7792c358fb8f93aee3f675a9dbdd3ef722e605b4d7742b3d89;a|b|c|d
+9a0b9b24457b7fa4e33ceb8f7282a35de8ca727e0bf18a76637554866ee7e975;a
+244504f6e8d5293c9cb9da6d5fc89d9810952e02ecd9bc00cbebb49836569a26;a|b|d
+4aa50e3562d694d110d6710b97a1af3025d572cd0c11e8294eaa5869dae58769;c
+2024771905c23da63fc6b11380c3812fbec8b11e55086ab62372ee1135611734;d
+a7378260414081672121a364c535f35478803aaa0dad5aca5ec512d9bcc0e0d4;a
+26ba5861d6fd2bc75b09723189a7d5d5b64f975223f8b189dbb189b421452e1c;a|b|d|e
+988d086a1674fca659e5d00c9b51170095cb4cebcb5452559f50bbafca8bae43;b
+37ae0c3e8aa916314beb740694581acdc2061d70aed61d874a28f3b680623a40;c
+45c1c85d7fd96ce20028ed98fe79b7008af8e86ec87123cd7cb8f1c97c4e30ad;b|d
+65238b968e10f323f14f8936377baed357a8ba36639e7dd21eed82a2b28150b4;c
+ba5b9031614af75c32703054d1303042dc240e7f5b42791c475f56e9577beebb;a|b|d|e
+e31616f519942fb91adea6a1b2b396bc2e8bfc444d82a4d414679d7791dc2d3b;c
+7fb8420411994bd7a2707097a2b4054c67f1ffdba986fa1583c2e4f07b2f686d;a|b|d|e
+47911646b7d53bb2440917501716bae5113e0d38688d409d9eacd017218a4726;d
+785bb92e737b5e40c5249d682853fef682a4e427e25266373a4416681d6f8ed4;b|d
+18ae17a337da010d23dd8787764c442b27f669f703b3f8c4711d2e4b10bb8485;b|d
+b764bd4de66377aeda35a1e4efab2ea16c5c5ac1705b9c4930be43ad1549317b;a
+fa6e7b38de02210f20947b4fc5d594f27ecf0443e440bbeaad79a1426afb294b;a|b|c|d|e
+113f89272407baf4500e16e37eec1e24c874193095176d55a89d12de9c630509;e
+dacd885396c098ecf5703507c9dfb1384d78d996801dc68c5eb3dddb17b6e441;a
+3a8d98a854af52b692274d9b5635e2806313dd6f308e26e16b949034af6e6aae;a
+651db8b949dfc98830ce4052087faa9886eba1e064391c1d6e4fa706b97b6cf0;e
+06a1c62442c4d5c836ed0e31cd7399509c9fdc9d63501354759bb788352b7647;a
+19d270a9503ac4f41c1bb34ebd1c7b85402dbcd18f595e896526dbe2b25990a1;a
+af6daf22e611a504665e8f331bf23713a740d21f342d625baf4ba77fa2039881;a|b|d|e
+595b0d513d3aafca130df6da921f2dbdc690e62ba4a68ef2cdbfd9971d65f96a;c
+c015a9c119a1384a3aed6e37e67002a73aa08d0672c8a96efe4d30b3d1414741;a|b
+89b491c518842ec2625ee51bd3380e28fb85dd2c0dd6995d61b91f766692c4c7;a|b|c|d|e
+cbb8b2aadbaa97e72a0528b34a248d85ef1c3866323fb94e427ca85158300a4f;e
+08af77bf5b02c4878e2f42abfa3a35c8a54521c97ebe39c0c0dd225f3edbdb6a;b|d
+0fb98a3bd298c481559ccb5b5327050fbcc487d9f78b8ee09873d4e681a3b146;e
+2451ab155f5a63efe19b3480a0f9002c01cefccd14e083f1ccfe657715b0e392;a|b|c|d|e
+7f53dd13c966af4b6329a1607d7eb02a2a8fd8edc022a43a1ad637ea9c15ad66;b
+a5d80312297d8c94ea568212d026c4a547d4c4b09d7889c28da8f7223420036b;b
+a6dead265a5cadaf5b495bc77a980d6da774c160a3c6a98033739b151d459072;c|e
+a8039aea329080b9f4a96e97ed2d95dac36acbf07e67ca48758ec5b40d91966a;a|b|d
+cc6efcca2fc3b84ec3c68a2ca14d1c786db62cb5fb582a8b4b23709dae4a09ac;b|d
+60aae73dbe85e0e6a804ed7bda8850bfeda2f864ed99bf534f3fbd8d5e52c335;a|c|e
+811e0b2610837a61be63620b6a03db7c72b9a00dfffadcd1f0e2734458bd1f48;c
+ed4e02e98a3f22e27df664fae5dba3d0957ed474dbd6b56bba639e0f9f1a3660;e
+b316eda5016efc3b6643c845d1466a48bb5eba31ef8fa223a1846c82c3047bdc;c
+25d89c3ba2467c0048a96652227d6df13d9287a0b3bf7a2ff5aa52d62885e90c;e
+f508faf23563a40d1fc0f8d3e61ad6f369a743291e8e91f1736ee60dc273828e;b|c|d
+252e1d1fb4f26aa2b99d67dab601d08cacdd228bdc5364c1a7bf3ffe7ce3ef14;a|b|c|d
+08bac060f2bb389a56f9beeecadbe686098f352e17ce58eececd99f36d31137a;b|c
+0589c1714b455b2b3534535f1fadbb7a6f2e852e7ae6781b0664024f1af49a19;a
+d768718bfa54a8621d582eece3c6be255fe459f30f7693ee436b26c670a0cc7e;a
+71d0afb87971beebdf06e39b843330233dcacd8eaea6029512bbeee4d14a1d5b;e
+6b27ea0623d38c20c6a9c14039bc51a0ad24ee00e154e640573625126606a493;a|b|e
+ecb3cfd556f7a53a783eec0ed689e5dc0dc72231a0422ca27ca1666b0575c3fe;b|d
+2cf4c0539dac5cb9c34b44eebb7fa905420505fd96fc203c28215dd6f4bc96a7;d
+a4c75c5fcde839e79c02af67895ab1f89faa569dad88d4c8421ebdbafa83652f;e
+330abd90c66c17c328b85a7916e1cfa6e3aa6f8467bc71b25db083051a43ba7e;a
+a6a68b1745d6291d48bc08a68003cc70e53e4aa79ba288c12a26dcbc0d0c764a;b
+3cadf62b5eea5f454b0939f236b3284aa9a710e0aa8744007f33854e7069df9b;a
+82dc47e77e7db30e387746234a87750460ab69ba7ee39ff6b07c95ee001586f4;a
+cb59493d2e51039f25ba0c964735fd57cb33aa61b47c576f74c1ff8fb195fd36;a
+4f650808336a9dbe0b7685e6a51dbf37723d8060262e4270189e71672b373322;a|b|d
+03ec466d6ed2bbd0ab68dcc7c342c50defa91cc964e3ad146668c64eb2130a35;d
+25519a98bbc8dda2c8cc5c3bd5b75056013884c0cf64b7b43964a9415d9624d6;d
+4396f8a46a0992bf0a5fa8c602903537d911dd8a7016731ad08cab8f6eb7fc17;a|b|c|d|e
+ec2a97bac0115abc05ae3ab1e9fe641eae0d2a5d2766a7dfe9b6f33b1fd14225;a|b|d
+5af8c8b50ceec88fcf4466f210b3a6db051f4020ac3f9b0d39dc4e3a1861c81a;b
+b9d7c46fde21bc9032e327bfe2579971035f8060807ea4cfd4daa16b45f3c052;c
+ef552a5d9aab5af706afaf60f2e88fd10112c4b565f1c08f4cfb7a155aa21aea;a|b|d
+27dcf62037e2551e4412f598d02fb50c116c8ffd8a044373e3903c92f5920685;a|b|d
+c8631a03d9f5d85f1414c0c25778c195f2e3cbaf5179c2ff994972c0c75b6b74;a|b|d
+b29a4d3dc5a81cfafe650cb72260b14648247e850b54986752f4be5b75137f4e;a
+e7de0950990565f1279f10f0f3b2ae8382ec0b300a10abfb813c2ce1acf89eb5;d
+6f598d882baeeb8a24ea0a56827009eea3247e0f6defc934f1aafbec1baa7a3f;e
+2c9e7b88b27fd45d2a3fcd00524d978bc559c2eed0946f0f0d43f820b57e77c6;b|c|d
+685ed7055068fc7f2309622719e48ac9e8866c95f4fd74e3a1117ddd8e10008b;a|b|c|e
+41422f287d44bfa74c9529d2d96c378781e592c3a9458f5c53c4976dbaef977b;e
+b2da7cb787bd6a4fbc8072750bea5552112301c7860ae3111bfe69c016fb5134;d|e
+47317942e017254732adcb94af1468b77aa3a903d0294ded99a848cb534293cc;a
+6b562e2fcf87bf4ac49b275685eb540863073d062545ad30a9c20dd71bb6168b;b
+dc62bf6f887b5317c23e28545e2b5f05101c8910ea18bc6f285ac344c0935af8;a
+4d6f8a5b9cea8e4c3ab336e47756fe732100a37d3aece1cdcfcd2db7149045fd;a|b|d|e
+9f7a9487e77f91fd70cd7e43420778a6a57a3611b8a01f2ff958c8fa314a8785;b
+165c43f00aa499c817075878fc3c9915062cc6ac1a09f8e5c9f5a9f2a9b60577;b
+d7ae41161cc0c351bff40b8999eb900d36fd856786c7bea010f95c14215b7242;a
+5e0b25ca7dd41571c5e6edab642607597704f9ae84a5cb027b32daef02c7858d;e
+0422a534c831aa652c7e5f1dda4de7f90375da12a57ee7b4b901a627c37898a3;b|d
+f01a9fe35b914c01703411dd16e6375768c45358d7583b36a4648c45d43b2e6e;a
+6196273cc22106fe17fa9663b9062da65045872cc0b4503818bae4ebb857251c;a|b|d
+204c0a57eac1d926036a24189551bb8c7cbb47f7fcfdc7c66bcd98e76c8bfd9b;a
+c0fde34ea7353c30d8562af8da4bb5d1b43de46ead5662bd9589cef917af0857;a|b|d
+d9ce4b47f06c4c1e58f070a1984987ea9b46db3e0eba2754207795dd817a2e75;a|b|c
+072590481ff31559f6877212ff870ffd2eac74084e43fbce132962e5bac7154e;e
+34986780304a69ab4a9a1a0a79ab7c821c4acf7a9c8e62345867f2042baef52b;a
+58d539b03a5e99db10474facf1bca56801fa01cfbfaa699a204f25515815251c;a|b|d
+c221f47062d023a4594d27b3b41378d7c2d7e3658a3a7a5961cbb4d283a2b7fa;a|b|c|d|e
+4ea8d6c979f6d9c05f039e667e8c73841dac51c286bb959973b7efa2f15743a5;a|b|c|d|e
+fe789f9cde64509f81fe199ae5076902087b80bc9d3c1534f7bed8544c504133;a
+e4919631ac162d6ba89b1aa2956ca3cc6135b239a662e3f5b7b9f012a71db99d;a|c|e
+75ba6c5467b1583fb4c2fba6cf21163588a74fd69fe3bd84bcb96f08b3a4a55a;b|d
+c049c294998dd5c32f11e21016ceca34d2d70fb71b5bc0159752e769e3d2258a;a
+ca58b47c71d26e84ee12edc3f0aa7530384b660a86c7c64f4e0aaeb741aafa93;b|d
+b4ee814e2a7d58cdda55a429fa800e9185a2a0641f1f7fc5cd15a900486867c6;a
+8321f7d1f6ebefd577ed8719a449a0940aba570ef0bc74694680cd3966a627f2;a|b|c|d|e
+541510ce8e8ecfa98a94918416e28f49d18aea0f296728b6a9ff6522a6f2cc77;a|b|d
+cf604684dacba8e6fd3fb0583d01f35015f9c3360b207fba077070dcfed38c63;a

run_galactica.py

+#https://huggingface.co/facebook/opt-iml-30b
+from transformers import AutoTokenizer, OPTForCausalLM
+import torch
+
+def main(result_path: str, corpus_path: str, model: str = 'galactica-30b', load_in_8bit=False, template_num: int = 0):
+    checkpoint = 'facebook/' + model
+    llm = OPTForCausalLM.from_pretrained(checkpoint, device_map="auto", load_in_8bit=load_in_8bit)
+    tokenizer = AutoTokenizer.from_pretrained(checkpoint)
+
+    def generate(input_string):
+        with torch.no_grad():
+            inputs = tokenizer(input_string, return_tensors="pt").input_ids.to("cuda")
+            outputs = llm.generate(inputs, max_new_tokens=32)
+
+            generated = tokenizer.decode(outputs[0], skip_special_tokens=True)
+        return generated[len(input_string):]
+
+    import deft
+    results = deft.run_inference(generate, corpus_path, deft.lm_templates[template_num], add_left_parenthesis=False)
+    deft.write_results(results, result_path)
+
+if __name__ == '__main__':
+    import fire
+    fire.Fire(main)

runs

@@ -5,6 +5,7 @@ python run_bloomz.py output/bloomz-560m_prompt0.txt ../json/dev.json bloomz-560m
 python run_tkinstruct.py output/tk-instruct-3b-def_prompt0.txt ../json/dev.json tk-instruct-3b-def | tee logs/tk-instruct-3b-def_prompt0.txt
 python run_tkinstruct.py output/tk-instruct-11b-def_prompt0.txt ../json/dev.json tk-instruct-11b-def | tee logs/tk-instruct-11b-def_prompt0.txt
 python run_opt-iml.py output/opt-iml-30b_prompt0.txt ../json/dev.json opt-iml-30b | tee logs/opt-iml-30b_prompt0.txt
+python run_galactica.py output/galactica-30b_prompt0.txt ../json/dev.json galactica-30b | tee logs/galactica-30b_prompt0.txt
 
 python run_api.py output/code-cushman-001_prompt0.txt ../json/dev.json openai/code-cushman-001 | tee logs/code-cushman-001_prompt0.txt
 python run_api.py output/code-davinci-002_prompt0.txt ../json/dev.json openai/code-davinci-002 | tee logs/code-davinci-002_prompt0.txt