Do not split for unknown tokens but replace instead

torch_modules/src/ContextLSTM.cpp

@@ -42,17 +42,12 @@ void ContextLSTMImpl::addToContext(std::vector<std::vector<long>> & context, Dic
         for (auto & contextElement : context)
           contextElement.push_back(dictIndex);
 
-
-        if (splitUnknown)
         for (auto & targetCol : unknownValueColumns)
           if (col == targetCol)
             if (dict.getNbOccs(dictIndex) <= unknownValueThreshold)
-              {
-                context.emplace_back(context.back());
               context.back().back() = dict.getIndexOrInsert(Dict::unknownValueStr);
       }
 }
-}
 
 torch::Tensor ContextLSTMImpl::forward(torch::Tensor input)
 {