Template Class Transformer

Inheritance Relationships

Base Type

  • public EncoderOrDecoderBase

Class Documentation

template<class EncoderOrDecoderBase>
class Transformer : public EncoderOrDecoderBase

Public Functions

Transformer(Ptr<ExpressionGraph> graph, Ptr<Options> options)
Expr addPositionalEmbeddings(Expr input, int start = 0, bool trainPosEmbeddings = false) const
virtual Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr<data::CorpusBatch> = nullptr) const
Expr triangleMask(int length) const
Expr preProcess(std::string prefix, std::string ops, Expr input, float dropProb = 0.0f) const
Expr postProcess(std::string prefix, std::string ops, Expr input, Expr prevInput, float dropProb = 0.0f) const
void collectOneHead(Expr weights, int dimBeam)
Expr Attention(std::string, Expr q, Expr k, Expr v, Expr mask = nullptr, bool saveAttentionWeights = false, int dimBeam = 1)
Expr MultiHead(std::string prefix, int dimOut, int dimHeads, Expr q, const Expr &keys, const Expr &values, const Expr &mask, bool cache = false, bool saveAttentionWeights = false)
Expr LayerPooling(std::string prefix, Expr input, const Expr &values)
Expr LayerAttention(std::string prefix, Expr input, Expr keys, Expr values, Expr mask, int dimHeads, bool cache = false, bool saveAttentionWeights = false)
Expr DecoderLayerSelfAttention(rnn::State &decoderLayerState, const rnn::State &prevdecoderLayerState, std::string prefix, Expr input, Expr selfMask, int startPos)
Expr LayerFFN(std::string prefix, Expr input, bool isDecoder = false) const
Expr LayerAAN(std::string prefix, Expr x, Expr y) const
Expr DecoderLayerAAN(rnn::State &decoderState, const rnn::State &prevDecoderState, std::string prefix, Expr input, Expr selfMask, int startPos) const
Expr DecoderLayerRNN(std::unordered_map<std::string, Ptr<rnn::RNN>> &perLayerRnn, rnn::State &decoderState, const rnn::State &prevDecoderState, std::string prefix, Expr input, Expr, int) const

Public Static Functions

static Expr transposeTimeBatch(Expr input)
static Expr transposedLogMask(Expr mask)
static Expr SplitHeads(Expr input, int dimHeads)
static Expr JoinHeads(Expr input, int dimBeam = 1)

Protected Functions

template<typename T>
T opt(const char *const key) const
template<typename T>
T opt(const std::string &key) const
template<typename T>
T opt(const char *const key, const T &def) const
template<typename T>
T opt(const std::string &key, const T &def) const

Protected Attributes

std::unordered_map<std::pair<std::string, Shape>, Expr> cache_
mutable std::vector<float> sinusoidalEmbeddingsFreq_
mutable std::vector<float> sinusoidalEmbeddingsOffs_
bool depthScaling_ = {false}
size_t depth_ = {0}
std::vector<Expr> alignments_