Skip to content
Snippets Groups Projects
Commit 2a0a58f1 authored by Franck Dary's avatar Franck Dary
Browse files

Do not lock dicts when the pretraiend file is empty

parent b13669bd
No related branches found
No related tags found
No related merge requests found
......@@ -54,7 +54,7 @@ class Dict
std::size_t size() const;
int getNbOccs(int index) const;
void removeRareElements();
void loadWord2Vec(std::filesystem::path path, std::string prefix);
bool loadWord2Vec(std::filesystem::path path, std::string prefix);
bool isSpecialValue(const std::string & value);
};
......
......@@ -223,10 +223,10 @@ void Dict::removeRareElements()
nbOccs = newNbOccs;
}
void Dict::loadWord2Vec(std::filesystem::path path, std::string prefix)
bool Dict::loadWord2Vec(std::filesystem::path path, std::string prefix)
{
if (path.empty())
return;
return false;
if (!std::filesystem::exists(path))
util::myThrow(fmt::format("pretrained word2vec file '{}' do not exist", path.string()));
......@@ -238,6 +238,7 @@ void Dict::loadWord2Vec(std::filesystem::path path, std::string prefix)
char buffer[100000];
bool firstLine = true;
bool pretrained = false;
try
{
......@@ -262,6 +263,7 @@ void Dict::loadWord2Vec(std::filesystem::path path, std::string prefix)
continue;
}
pretrained = true;
auto splited = util::split(util::strip(buffer), ' ');
if (splited.size() < 2)
......@@ -287,6 +289,8 @@ void Dict::loadWord2Vec(std::filesystem::path path, std::string prefix)
util::myThrow(fmt::format("file '{}' is empty", path.string()));
setState(originalState);
return pretrained;
}
bool Dict::isSpecialValue(const std::string & value)
......
......@@ -58,11 +58,15 @@ ContextModuleImpl::ContextModuleImpl(std::string name, const std::string & defin
auto splited = util::split(p, ',');
if (splited.size() != 2)
util::myThrow("expected 'prefix,pretrained.w2v'");
getDict().loadWord2Vec(this->path / splited[1], splited[0]);
auto pretrained = getDict().loadWord2Vec(this->path / splited[1], splited[0]);
if (pretrained)
{
getDict().setState(Dict::State::Closed);
dictSetPretrained(true);
}
}
}
} catch (std::exception & e) {util::myThrow(fmt::format("{} in '{}'",e.what(),definition));}
}))
......
......@@ -63,11 +63,14 @@ ContextualModuleImpl::ContextualModuleImpl(std::string name, const std::string &
auto splited = util::split(p, ',');
if (splited.size() != 2)
util::myThrow("expected 'prefix,file.w2v'");
getDict().loadWord2Vec(this->path / splited[1], splited[0]);
auto pretrained = getDict().loadWord2Vec(this->path / splited[1], splited[0]);
if (pretrained)
{
getDict().setState(Dict::State::Closed);
dictSetPretrained(true);
}
}
}
} catch (std::exception & e) {util::myThrow(fmt::format("{} in '{}'",e.what(),definition));}
}))
......
......@@ -49,11 +49,14 @@ FocusedColumnModuleImpl::FocusedColumnModuleImpl(std::string name, const std::st
auto splited = util::split(p, ',');
if (splited.size() != 2)
util::myThrow("expected 'prefix,pretrained.w2v'");
getDict().loadWord2Vec(this->path / splited[1], splited[0]);
auto pretrained = getDict().loadWord2Vec(this->path / splited[1], splited[0]);
if (pretrained)
{
getDict().setState(Dict::State::Closed);
dictSetPretrained(true);
}
}
}
} catch (std::exception & e) {util::myThrow(fmt::format("{} in '{}'",e.what(),definition));}
}))
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment