From ce0503c84319a5918dee1dba2e6949d68d109397 Mon Sep 17 00:00:00 2001
From: Franck Dary <franck.dary@lis-lab.fr>
Date: Mon, 20 Apr 2020 10:28:42 +0200
Subject: [PATCH] Do not split for unknown tokens but replace instead
---
torch_modules/src/ContextLSTM.cpp | 13 ++++---------
1 file changed, 4 insertions(+), 9 deletions(-)
diff --git a/torch_modules/src/ContextLSTM.cpp b/torch_modules/src/ContextLSTM.cpp
index 5da02e7..d247788 100644
--- a/torch_modules/src/ContextLSTM.cpp
+++ b/torch_modules/src/ContextLSTM.cpp
@@ -42,15 +42,10 @@ void ContextLSTMImpl::addToContext(std::vector<std::vector<long>> & context, Dic
for (auto & contextElement : context)
contextElement.push_back(dictIndex);
-
- if (splitUnknown)
- for (auto & targetCol : unknownValueColumns)
- if (col == targetCol)
- if (dict.getNbOccs(dictIndex) <= unknownValueThreshold)
- {
- context.emplace_back(context.back());
- context.back().back() = dict.getIndexOrInsert(Dict::unknownValueStr);
- }
+ for (auto & targetCol : unknownValueColumns)
+ if (col == targetCol)
+ if (dict.getNbOccs(dictIndex) <= unknownValueThreshold)
+ context.back().back() = dict.getIndexOrInsert(Dict::unknownValueStr);
}
}
--
GitLab