<unk> in pretrained embeddgins is used for Dict::unknownValueStr

torch_modules/src/Submodule.cpp

@@ -45,8 +45,10 @@ void Submodule::loadPretrainedW2vEmbeddings(torch::nn::Embedding & embeddings, s
         util::myThrow(fmt::format("invalid w2v line '{}' less than 2 columns", buffer));
 
       auto dictIndex = getDict().getIndexOrInsert(splited[0]);
+      if (splited[0] == "<unk>")
+        dictIndex = getDict().getIndexOrInsert(Dict::unknownValueStr);
 
-      if (dictIndex == getDict().getIndexOrInsert(Dict::unknownValueStr) or dictIndex == getDict().getIndexOrInsert(Dict::nullValueStr) or dictIndex == getDict().getIndexOrInsert(Dict::emptyValueStr))
+      if (splited[0] != "<unk>" and splited[0] != Dict::unknownValueStr and (dictIndex == getDict().getIndexOrInsert(Dict::unknownValueStr) or dictIndex == getDict().getIndexOrInsert(Dict::nullValueStr) or dictIndex == getDict().getIndexOrInsert(Dict::emptyValueStr)))
         continue;
 
       if (embeddingsSize != splited.size()-1)