/**
 * Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License.
 */


#include "FAConfig.h"
#include "FAAllocator.h"
#include "FAFsmConst.h"
#include "FAUtils.h"
#include "FAUtf8Utils.h"
#include "FAUtf32ToEnc.h"
#include "FAStr2Int_hash.h"
#include "FAException.h"

#include <iostream>
#include <string>
#include <fstream>
#include <iomanip>
#include <sstream>


const char * __PROG__ = "";

FAAllocator g_alloc;

const char * g_pInFile = NULL;
const char * g_pOutFile = NULL;

std::istream * g_pIs = &std::cin;
std::ifstream g_ifs;

std::ostream * g_pOs = &std::cout;
std::ofstream g_ofs;

int g_min_order = 3;
int g_max_order = 3;
const char * g_pOutEnc = "UTF-8";
int g_line_step = -1;

bool g_no_output = false;
bool g_no_process = false;

const int MaxBuffSize = 4096 * 1024;
int g_Buff1 [MaxBuffSize];
char g_Buff2 [MaxBuffSize];


void usage ()
{
  std::cout << "\n\
Usage: fa_utf8tongrams_cp [OPTIONS]\n\
\n\
This tool collects byte ngrams in the specified encoding.\n\
\n\
  --in=<input> - reads input text from the <input> file,\n\
    if omited stdin is used\n\
\n\
  --out=<output> - writes output to the <output> file,\n\
    if omited stdout is used\n\
\n\
  --output-enc=<encoding> - input encoding name or codepage number\n\
    UTF-8 is used by default\n\
\n\
  --max-order=N - takes all the n-grams of the order N and below,\n\
    3 is used by default\n\
\n\
  --min-order=N - takes all the n-grams of the order N and above,\n\
    3 is used by default\n\
\n\
  --line-step=N - the amount of lines processed at once,\n\
    by default full input is processed at once\n\
\n\
";
}


void process_args (int& argc, char**& argv)
{
    for (; argc--; ++argv) {

        if (!strcmp ("--help", *argv)) {
            usage ();
            exit (0);
        }
        if (0 == strncmp ("--in=", *argv, 5)) {
            g_pInFile = &((*argv) [5]);
            continue;
        }
        if (0 == strncmp ("--out=", *argv, 6)) {
            g_pOutFile = &((*argv) [6]);
            continue;
        }
        if (0 == strncmp ("--output-enc=", *argv, 13)) {
            g_pOutEnc = &((*argv) [13]);
            continue;
        }
        if (0 == strncmp ("--max-order=", *argv, 12)) {
            g_max_order = atoi (&((*argv) [12]));
            continue;
        }
        if (0 == strncmp ("--min-order=", *argv, 12)) {
            g_min_order = atoi (&((*argv) [12]));
            continue;
        }
        if (0 == strncmp ("--line-step=", *argv, 12)) {
            g_line_step = atoi (&((*argv) [12]));
            continue;
        }
    }
}


static void PrintNgram (
        std::ostream& os, 
        const unsigned char * pBuff, 
        const int Size,
        const int Freq
    )
{
    for (int i = 0; i < Size; ++i) {

        if (0 != i) {
            os << ' ';
        }

        const int B = pBuff [i];
        os << std::setw (2) << std::setfill ('0') << std::hex << B;
    }

    os << '\t' << std::dec << Freq ;
}


static void PrintNgrams (std::ostream& os, const FAStr2IntA * pMap)
{
    DebugLogAssert (pMap);

    const int Count = pMap->GetStrCount ();

    for (int i = 0; i < Count; ++i) {

        const char * pStr = NULL;
        const int StrLen = pMap->GetStr (i, &pStr);
        const int Freq = pMap->GetValue (i);

        PrintNgram (os, (const unsigned char*) pStr, StrLen, Freq);
        os << '\n';
    }
}


int __cdecl main (int argc, char ** argv)
{
    __PROG__ = argv [0];

    --argc, ++argv;

    ::FAIOSetup ();

    // process command line
    process_args (argc, argv);

    int LineNum = -1;
    std::string line;

    try {

        if (g_pInFile) {
            g_ifs.open (g_pInFile, std::ios::in);
            FAAssertStream (&g_ifs, g_pInFile);
            g_pIs = &g_ifs;
        }
        if (g_pOutFile) {
            g_ofs.open (g_pOutFile, std::ios::out);
            g_pOs = &g_ofs;
        }

        FAUtf32ToEnc conv (&g_alloc);
        conv.SetEncodingName (g_pOutEnc);

        FAStr2Int_hash stats (&g_alloc);

        while (!(g_pIs->eof ())) {

            if (!std::getline (*g_pIs, line))
                break;

            LineNum++;

            const char * pLine = line.c_str ();
            int LineLen = (const int) line.length ();

            if (0 < LineLen) {
                DebugLogAssert (pLine);
                if (0x0D == (unsigned char) pLine [LineLen - 1])
                    LineLen--;
            }

            if (0 >= LineLen)
                continue;

            // UTF-8 --> UTF-32LE
            const int Count1 = \
                ::FAStrUtf8ToArray (pLine, LineLen, g_Buff1, MaxBuffSize);
            if (0 >= Count1 || MaxBuffSize < Count1) {
                continue;
            }

            // UTF-32LE --> ENC
            const int Count2 = \
                conv.Process (g_Buff1, Count1, g_Buff2, MaxBuffSize);
            if (0 >= Count2 || MaxBuffSize < Count2) {
                continue;
            }

            // update statistics
            for (int i = 0; i < Count2; ++i) {

                const char * pNgram = g_Buff2 + i;

                for (int N = g_min_order; N <= g_max_order; ++N) {

                    if (Count2 < N + i) {
                        break;
                    }

                    int Value = 0;

                    if (stats.Get (pNgram, N, &Value)) {
                        stats.Add (pNgram, N, Value + 1);
                    } else {
                        stats.Add (pNgram, N, 1);
                    }
                }
            }

            // print statistics
            if (-1 != g_line_step && 0 == (LineNum % g_line_step)) {
                PrintNgrams (*g_pOs, &stats);
                stats.Clear ();
            }

        } // of  while (!(g_pIs->eof ())) ...

        // print statistics
        PrintNgrams (*g_pOs, &stats);

    } catch (const FAException & e) {

        const char * const pErrMsg = e.GetErrMsg ();
        const char * const pFile = e.GetSourceName ();
        const int Line = e.GetSourceLine ();

        std::cerr << "ERROR: " << pErrMsg << " in " << pFile \
            << " at line " << Line << " in program " << __PROG__ << '\n';

        std::cerr << "ERROR: in data at line: " << LineNum << " in \"" \
            << line << "\"\n";

        return 2;

    } catch (...) {

        std::cerr << "ERROR: Unknown error in program " << __PROG__ << '\n';

        std::cerr << "ERROR: in data at line: " << LineNum << " in \"" \
            << line << "\"\n";

        return 1;
    }

    // print out memory leaks, if any
    FAPrintLeaks(&g_alloc, std::cerr);

    return 0;
}