Class DefaultVocab¶

Defined in File default_vocab.cpp

Nested Relationships¶

Nested Types¶

Class DefaultVocab::VocabFreqOrderer

Inheritance Relationships¶

Base Type¶

public marian::IVocab (Class IVocab)

Derived Type¶

public marian::ClassVocab (Class ClassVocab)

Class Documentation¶

class DefaultVocab : public marian::IVocab ¶

Subclassed by marian::ClassVocab

Public Functions

virtual ~DefaultVocab()¶

virtual const std::string &canonicalExtension() const¶

virtual const std::vector<std::string> &suffixes() const¶

virtual Word operator[](const std::string &word) const¶

Words encode(const std::string &line, bool addEOS, bool) const¶

std::string decode(const Words &sentence, bool ignoreEOS) const¶

std::string surfaceForm(const Words &sentence) const¶

void addSpecialWords(std::vector<Word> &special) const¶

virtual std::string type() const¶

virtual Word getEosId() const¶

virtual Word getUnkId() const¶

const std::string &operator[](Word word) const¶

size_t size() const¶

size_t load(const std::string &vocabPath, size_t maxSize)¶

virtual void createFake()¶

virtual void create(const std::string &vocabPath, const std::vector<std::string> &trainPaths, size_t maxSize = 0)¶

Protected Types

typedef std::map<std::string, Word> Str2Id¶

typedef std::vector<std::string> Id2Str¶

Protected Attributes

Str2Id str2id_¶

Id2Str id2str_¶

Word eosId_ = Word::NONE ¶

Word unkId_ = Word::NONE ¶

std::vector<std::string> suffixes_ = {".yml", ".yaml", ".json"}¶

std::vector<Word> controlChars_¶

class VocabFreqOrderer¶

Public Functions

VocabFreqOrderer(const std::unordered_map<std::string, size_t> &counter)¶

bool operator()(const std::string &a, const std::string &b) const¶