// © 2022 and later: Unicode, Inc. and others. // License & terms of use: http://www.unicode.org/copyright.html #ifndef MLBREAKENGINE_H #define MLBREAKENGINE_H #include "hash.h" #include "unicode/resbund.h" #include "unicode/uniset.h" #include "unicode/utext.h" #include "uvectr32.h" U_NAMESPACE_BEGIN #if !UCONFIG_NO_BREAK_ITERATION /** * A machine learning break engine for the phrase breaking in Japanese. */ class MlBreakEngine : public UMemory { public: /** * Constructor. * * @param digitOrOpenPunctuationOrAlphabetSet An UnicodeSet with the digit, open punctuation and * alphabet. * @param closePunctuationSet An UnicodeSet with close punctuation. * @param status Information on any errors encountered. */ MlBreakEngine(const UnicodeSet &digitOrOpenPunctuationOrAlphabetSet, const UnicodeSet &closePunctuationSet, UErrorCode &status); /** * Virtual destructor. */ virtual ~MlBreakEngine(); public: /** * Divide up a range of characters handled by this break engine. * * @param inText A UText representing the text * @param rangeStart The start of the range of the characters * @param rangeEnd The end of the range of the characters * @param foundBreaks Output of C array of int32_t break positions, or 0 * @param inString The normalized string of text ranging from rangeStart to rangeEnd * @param inputMap The vector storing the native index of inText * @param status Information on any errors encountered. * @return The number of breaks found */ int32_t divideUpRange(UText *inText, int32_t rangeStart, int32_t rangeEnd, UVector32 &foundBreaks, const UnicodeString &inString, const LocalPointer &inputMap, UErrorCode &status) const; private: /** * Load the machine learning's model file. * * @param error Information on any errors encountered. */ void loadMLModel(UErrorCode &error); /** * In the machine learning's model file, specify the name of the key and value to load the * corresponding feature and its score. * * @param rb A ResouceBundle corresponding to the model file. * @param keyName The kay name in the model file. * @param valueName The value name in the model file. * @param model A hashtable to store the pairs of the feature and its score. * @param error Information on any errors encountered. */ void initKeyValue(UResourceBundle *rb, const char *keyName, const char *valueName, Hashtable &model, UErrorCode &error); /** * Initialize the index list from the input string. * * @param inString A input string to be segmented. * @param indexList A code unit index list of inString. * @param status Information on any errors encountered. * @return The number of code units of the first four characters in inString. */ int32_t initIndexList(const UnicodeString &inString, int32_t *indexList, UErrorCode &status) const; /** * Evaluate whether the index is a potential breakpoint. * * @param inString A input string to be segmented. * @param indexList A code unit index list of the inString. * @param startIdx The start index of the indexList. * @param numCodeUnits The current code unit boundary of the indexList. * @param numBreaks The accumulated number of breakpoints. * @param boundary A vector including the index of the breakpoint. * @param status Information on any errors encountered. * @return The number of breakpoints */ int32_t evaluateBreakpoint(const UnicodeString &inString, int32_t *indexList, int32_t startIdx, int32_t numCodeUnits, int32_t numBreaks, UVector32 &boundary, UErrorCode &status) const; void printUnicodeString(const UnicodeString &s) const; UnicodeSet fDigitOrOpenPunctuationOrAlphabetSet; UnicodeSet fClosePunctuationSet; Hashtable fModel[13]; // {UW1, UW2, ... UW6, BW1, ... BW3, TW1, TW2, ... TW4} 6+3+4= 13 int32_t fNegativeSum; }; #endif U_NAMESPACE_END /* MLBREAKENGINE_H */ #endif