Commit 2a0a58f1 authored by Franck Dary's avatar Franck Dary
Browse files

Do not lock dicts when the pretraiend file is empty

parent b13669bd
......@@ -54,7 +54,7 @@ class Dict
std::size_t size() const;
int getNbOccs(int index) const;
void removeRareElements();
void loadWord2Vec(std::filesystem::path path, std::string prefix);
bool loadWord2Vec(std::filesystem::path path, std::string prefix);
bool isSpecialValue(const std::string & value);
};
......
......@@ -223,10 +223,10 @@ void Dict::removeRareElements()
nbOccs = newNbOccs;
}
void Dict::loadWord2Vec(std::filesystem::path path, std::string prefix)
bool Dict::loadWord2Vec(std::filesystem::path path, std::string prefix)
{
if (path.empty())
return;
return false;
if (!std::filesystem::exists(path))
util::myThrow(fmt::format("pretrained word2vec file '{}' do not exist", path.string()));
......@@ -238,6 +238,7 @@ void Dict::loadWord2Vec(std::filesystem::path path, std::string prefix)
char buffer[100000];
bool firstLine = true;
bool pretrained = false;
try
{
......@@ -262,6 +263,7 @@ void Dict::loadWord2Vec(std::filesystem::path path, std::string prefix)
continue;
}
pretrained = true;
auto splited = util::split(util::strip(buffer), ' ');
if (splited.size() < 2)
......@@ -287,6 +289,8 @@ void Dict::loadWord2Vec(std::filesystem::path path, std::string prefix)
util::myThrow(fmt::format("file '{}' is empty", path.string()));
setState(originalState);
return pretrained;
}
bool Dict::isSpecialValue(const std::string & value)
......
......@@ -58,9 +58,13 @@ ContextModuleImpl::ContextModuleImpl(std::string name, const std::string & defin
auto splited = util::split(p, ',');
if (splited.size() != 2)
util::myThrow("expected 'prefix,pretrained.w2v'");
getDict().loadWord2Vec(this->path / splited[1], splited[0]);
getDict().setState(Dict::State::Closed);
dictSetPretrained(true);
auto pretrained = getDict().loadWord2Vec(this->path / splited[1], splited[0]);
if (pretrained)
{
getDict().setState(Dict::State::Closed);
dictSetPretrained(true);
}
}
}
......
......@@ -63,9 +63,12 @@ ContextualModuleImpl::ContextualModuleImpl(std::string name, const std::string &
auto splited = util::split(p, ',');
if (splited.size() != 2)
util::myThrow("expected 'prefix,file.w2v'");
getDict().loadWord2Vec(this->path / splited[1], splited[0]);
getDict().setState(Dict::State::Closed);
dictSetPretrained(true);
auto pretrained = getDict().loadWord2Vec(this->path / splited[1], splited[0]);
if (pretrained)
{
getDict().setState(Dict::State::Closed);
dictSetPretrained(true);
}
}
}
......
......@@ -49,9 +49,12 @@ FocusedColumnModuleImpl::FocusedColumnModuleImpl(std::string name, const std::st
auto splited = util::split(p, ',');
if (splited.size() != 2)
util::myThrow("expected 'prefix,pretrained.w2v'");
getDict().loadWord2Vec(this->path / splited[1], splited[0]);
getDict().setState(Dict::State::Closed);
dictSetPretrained(true);
auto pretrained = getDict().loadWord2Vec(this->path / splited[1], splited[0]);
if (pretrained)
{
getDict().setState(Dict::State::Closed);
dictSetPretrained(true);
}
}
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment