#pragma once #include #include #include #include #include #include "vocabulary.h" namespace ctranslate2 { // This class loads a vocabulary mapping model, a text file associating n-grams // with list of possible target candidates: // // \t candidate1 candidate2 ... candidateN // // and provides methods to map input tokens to possible target tokens. class VocabularyMap { public: VocabularyMap(std::istream& map_file, const Vocabulary& vocabulary); // The returned vector is ordered. std::vector get_candidates(const std::vector>& source_tokens, const std::vector>& target_prefix_ids) const; private: const size_t _vocabulary_size; std::set _fixed_candidates; std::vector>> _map_rules; }; }