'''
scripts.py

This code uses the latest copy of Scripts.txt from unicode.org
to generate a C file (and header) defining which script every character
belongs to.
'''

import csv
import os
import requests
import re
import sys
import tempfile
import requests
import subprocess

from cStringIO import StringIO

from collections import OrderedDict, defaultdict
from itertools import islice

from lxml import etree

from operator import itemgetter

from zipfile import ZipFile

this_dir = os.path.realpath(os.path.dirname(__file__))
sys.path.append(os.path.realpath(os.path.join(os.pardir, os.pardir)))

from geodata.encoding import safe_encode, safe_decode
from geodata.file_utils import ensure_dir, download_file
from geodata.string_utils import NUM_CODEPOINTS, wide_unichr

from cldr_languages import *
from download_cldr import download_cldr
from languages import get_country_languages
from unicode_paths import UNICODE_DATA_DIR
from word_breaks import script_regex, regex_char_range

SRC_DIR = os.path.join(this_dir, os.pardir, os.pardir, os.pardir, 'src')

SCRIPTS_DATA_DIR = os.path.join(UNICODE_DATA_DIR, 'scripts')
LOCAL_SCRIPTS_FILE = os.path.join(SCRIPTS_DATA_DIR, 'Scripts.txt')
LOCAL_ISO_15924_FILE = os.path.join(SCRIPTS_DATA_DIR, 'iso15924.txt')

BLOCKS_DATA_DIR = os.path.join(UNICODE_DATA_DIR, 'blocks')
LOCAL_BLOCKS_FILE = os.path.join(BLOCKS_DATA_DIR, 'Blocks.txt')

PROPS_DATA_DIR = os.path.join(UNICODE_DATA_DIR, 'props')
LOCAL_PROPS_FILE = os.path.join(PROPS_DATA_DIR, 'PropList.txt')
LOCAL_PROP_ALIASES_FILE = os.path.join(PROPS_DATA_DIR, 'PropertyAliases.txt')
LOCAL_PROP_VALUE_ALIASES_FILE = os.path.join(PROPS_DATA_DIR, 'PropertyValueAliases.txt')
LOCAL_DERIVED_CORE_PROPS_FILE = os.path.join(PROPS_DATA_DIR, 'DerivedCoreProperties.txt')

WORD_BREAKS_DIR = os.path.join(UNICODE_DATA_DIR, 'word_breaks')
LOCAL_WORD_BREAKS_FILE = os.path.join(WORD_BREAKS_DIR, 'WordBreakProperty.txt')

SCRIPTS_HEADER = 'unicode_script_types.h'
SCRIPTS_DATA_FILENAME = 'unicode_scripts_data.c'

SCRIPTS_URL = 'http://unicode.org/Public/UNIDATA/Scripts.txt'
BLOCKS_URL = 'http://unicode.org/Public/UNIDATA/Blocks.txt'
PROPS_URL = 'http://unicode.org/Public/UNIDATA/PropList.txt'
PROP_ALIASES_URL = 'http://unicode.org/Public/UNIDATA/PropertyAliases.txt'
PROP_VALUE_ALIASES_URL = 'http://unicode.org/Public/UNIDATA/PropertyValueAliases.txt'
DERIVED_CORE_PROPS_URL = 'http://unicode.org/Public/UNIDATA/DerivedCoreProperties.txt'
WORD_BREAKS_URL = 'http://unicode.org/Public/UNIDATA/auxiliary/WordBreakProperty.txt'

ISO_15924_URL = 'http://unicode.org/iso15924/iso15924.txt.zip'

scripts_header_template = u'''#ifndef UNICODE_SCRIPT_TYPES_H
#define UNICODE_SCRIPT_TYPES_H

#include <stdlib.h>

#define NUM_CODEPOINTS {num_codepoints}
#define MAX_LANGS {max_langs}

typedef enum {{
    {script_enum}
    NUM_SCRIPTS
}} script_t;

#endif
'''

scripts_c_data_template = u'''
script_t char_scripts[] = {{
    {char_scripts}
}};

script_code_t script_codes[] = {{
    {script_codes}
}};

script_languages_t script_languages[] = {{
    {script_languages}
}};
'''

script_code_template = '{{SCRIPT_{name}, "{code}"}}'

script_language_template = '{{{num_langs}, {languages}}}'


def unicode_to_integer(u):
    return int('0x{}'.format(u), 16)


def script_name_constant(i, u):
    return u'SCRIPT_{} = {}'.format(u.upper(), i)


UNKNOWN_SCRIPT = 'Unknown'
COMMON_SCRIPT = 'Common'


def parse_char_range(r):
    return [unicode_to_integer(u) for u in r.split('..')]


def get_chars_by_script():
    scripts_file = open(LOCAL_SCRIPTS_FILE)
    scripts = [None] * NUM_CODEPOINTS

    # Lines look like:
    # 0041..005A    ; Latin # L&  [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z
    for char_range, script, char_class in script_regex.findall(scripts_file.read()):
        script_range = parse_char_range(char_range)
        if len(script_range) == 2:
            for i in xrange(script_range[0], script_range[1] + 1):
                scripts[i] = script
        elif script_range:
            scripts[script_range[0]] = script

    return scripts


COMMENT_CHAR = '#'
DELIMITER_CHAR = ';'


def parse_file(f):
    for line in f:
        line = line.split(COMMENT_CHAR)[0].strip()
        if not line:
            continue
        tokens = line.split(DELIMITER_CHAR)
        if tokens:
            yield [t.strip() for t in tokens]


def get_property_aliases():
    prop_aliases_file = open(LOCAL_PROP_ALIASES_FILE)

    aliases = {}

    for line in parse_file(prop_aliases_file):
        prop = line[1]
        prop_aliases = [line[0]] + line[2:]

        for alias in prop_aliases:
            aliases[alias.lower()] = prop.lower()

    return aliases


def get_property_value_aliases():
    prop_value_aliases_file = open(LOCAL_PROP_VALUE_ALIASES_FILE)

    value_aliases = defaultdict(dict)

    for line in parse_file(prop_value_aliases_file):
        prop = line[0]
        if prop not in ('ccc', 'gc'):
            value = line[2]
            aliases = [line[1]] + line[3:]
        else:
            value = line[1]
            aliases = line[2:]

        for alias in aliases:
            value_aliases[prop.lower()][alias] = value

    return dict(value_aliases)


def get_unicode_blocks():
    blocks_file = open(LOCAL_BLOCKS_FILE)

    blocks = defaultdict(list)

    for line in parse_file(blocks_file):
        char_range, block = line
        char_range = parse_char_range(char_range)

        if len(char_range) == 2:
            for i in xrange(char_range[0], char_range[1] + 1):
                blocks[block.lower()].append(wide_unichr(i))
        elif char_range:
            blocks[block.lower()].append(wide_unichr(char_range[0]))

    return dict(blocks)


def get_unicode_properties():
    props_file = open(LOCAL_PROPS_FILE)

    props = defaultdict(list)

    for line in parse_file(props_file):
        char_range, prop = line

        char_range = parse_char_range(char_range)

        if len(char_range) == 2:
            for i in xrange(char_range[0], char_range[1] + 1):
                props[prop.lower()].append(wide_unichr(i))
        elif char_range:
            props[prop.lower()].append(wide_unichr(char_range[0]))

    derived_props_file = open(LOCAL_DERIVED_CORE_PROPS_FILE)
    for line in parse_file(derived_props_file):
        char_range, prop = line
        char_range = parse_char_range(char_range)

        if len(char_range) == 2:
            for i in xrange(char_range[0], char_range[1] + 1):
                props[prop.lower()].append(wide_unichr(i))
        elif char_range:
            props[prop.lower()].append(wide_unichr(char_range[0]))

    return dict(props)


def get_word_break_properties():
    props_file = open(LOCAL_WORD_BREAKS_FILE)

    props = defaultdict(list)

    for line in parse_file(props_file):
        char_range, prop = line

        char_range = parse_char_range(char_range)

        if len(char_range) == 2:
            for i in xrange(char_range[0], char_range[1] + 1):
                props[prop].append(wide_unichr(i))
        elif char_range:
            props[prop].append(wide_unichr(char_range[0]))

    return dict(props)


def build_master_scripts_list(chars):
    all_scripts = OrderedDict.fromkeys(filter(bool, chars))

    for i, script in enumerate(all_scripts.keys()):
        all_scripts[script] = i + 1

    # Unknown script for all characters not covered
    all_scripts[UNKNOWN_SCRIPT] = 0

    return all_scripts


SCRIPT_ALIASES_SUPPLEMENTAL = {
    'Hant': 'Han',
    'Hans': 'Han'
}


def get_script_codes(all_scripts):

    if not os.path.exists(LOCAL_ISO_15924_FILE):
        temp_dir = tempfile.gettempdir()

        script_codes_filename = os.path.join(temp_dir, ISO_15924_URL.rsplit('/')[-1])

        # This comes as a .zip
        script_codes_response = requests.get(ISO_15924_URL)
        zf = ZipFile(StringIO(script_codes_response.content))
        iso15924_filename = [name for name in zf.namelist() if name.startswith('iso15924')][0]

        # Strip out the comments, etc.
        temp_iso15924_file = u'\n'.join([line.rstrip() for line in safe_decode(zf.read(iso15924_filename)).split('\n')
                                        if line.strip() and not line.strip().startswith('#')])

        f = open(LOCAL_ISO_15924_FILE, 'w')
        f.write(safe_encode(temp_iso15924_file))
        f.close()

    script_codes_file = open(LOCAL_ISO_15924_FILE)

    script_codes = {}
    seen_scripts = set()

    # Scripts in the CLDR repos use 4-letter ISO-15924 codes, so map those
    for code, _, name, _, _, _ in csv.reader(script_codes_file, delimiter=';'):
        if name in all_scripts:
            script_codes[code] = name
            seen_scripts.add(name)
        else:
            normalized_name = name.split('(')[0].strip()
            if normalized_name in all_scripts and normalized_name not in seen_scripts:
                script_codes[code] = normalized_name
                seen_scripts.add(normalized_name)

    value_aliases = get_property_value_aliases()
    script_aliases = value_aliases['sc']

    for code, script in script_aliases.iteritems():
        if code not in script_codes and script in all_scripts:
            script_codes[code] = script

    script_codes.update(SCRIPT_ALIASES_SUPPLEMENTAL)

    return script_codes


SCRIPT_CODE_ALIASES = {
    'Jpan': ['Hani', 'Hira', 'Kana'],
    'Kore': ['Hang', 'Han']
}


def extract_language_scripts(xml):
    language_scripts = defaultdict(list)

    for lang in xml.xpath('//languageData/language'):
        language_code = lang.attrib['type'].lower()
        scripts = lang.get('scripts')
        if not scripts:
            continue
        for script in scripts.split():
            script_aliases = SCRIPT_CODE_ALIASES.get(script)
            if not script_aliases:
                language_scripts[language_code].append(script)
            else:
                language_scripts[language_code].extend(script_aliases)

    return language_scripts


def batch_iter(iterable, batch_size):
    source_iter = iter(iterable)
    while True:
        batch = list(islice(source_iter, batch_size))
        if len(batch) > 0:
            yield batch
        else:
            return


def get_script_languages():
    # For some languages (Greek, Thai, etc.), use of an unambiguous script is sufficient
    # to identify the language. We keep track of those single language scripts to inform
    # the language classifier

    chars = get_chars_by_script()
    all_scripts = build_master_scripts_list(chars)
    script_codes = get_script_codes(all_scripts)

    cldr_supplemental_data = open(CLDR_SUPPLEMENTAL_DATA)
    cldr_xml = etree.parse(cldr_supplemental_data)
    language_scripts = extract_language_scripts(cldr_xml)

    country_languages_path = os.path.join(DEFAULT_LANGUAGES_DIR, COUNTRY_LANGUAGES_FILENAME)
    if not os.path.exists(country_languages_path):
        fetch_cldr_languages(DEFAULT_LANGUAGES_DIR)

    country_language_file = open(country_languages_path)
    country_language_reader = csv.reader(country_language_file, delimiter='\t')

    countries = set([country for country, lang, script, pct, is_official
                     in country_language_reader])

    spoken_languages = set.union(*(set(get_country_languages(country)) for country in countries))

    script_code_languages = defaultdict(list)
    for language, scripts in language_scripts.iteritems():
        if language not in spoken_languages:
            continue
        for script in scripts:
            script_code_languages[script].append(language)

    script_languages = defaultdict(list)

    for script_code, script_name in script_codes.iteritems():
        langs = script_code_languages.get(script_code, [])
        script_languages[script_name].extend(langs)

    for name in all_scripts.iterkeys():
        script_languages.setdefault(name, [])

    return script_languages


def main(out_dir=SRC_DIR):
    # Output is a C header and data file, see templates
    out_file = open(os.path.join(out_dir, SCRIPTS_DATA_FILENAME), 'w')
    out_header = open(os.path.join(out_dir, SCRIPTS_HEADER), 'w')

    download_file(SCRIPTS_URL, LOCAL_SCRIPTS_FILE)
    download_file(BLOCKS_URL, LOCAL_BLOCKS_FILE)
    download_file(PROPS_URL, LOCAL_PROPS_FILE)
    download_file(PROP_ALIASES_URL, LOCAL_PROP_ALIASES_FILE)
    download_file(PROP_VALUE_ALIASES_URL, LOCAL_PROP_VALUE_ALIASES_FILE)
    download_file(DERIVED_CORE_PROPS_URL, LOCAL_DERIVED_CORE_PROPS_FILE)
    download_file(WORD_BREAKS_URL, LOCAL_WORD_BREAKS_FILE)

    if not os.path.exists(CLDR_SUPPLEMENTAL_DATA):
        download_cldr()

    chars = get_chars_by_script()
    all_scripts = build_master_scripts_list(chars)
    script_codes = get_script_codes(all_scripts)

    script_languages = get_script_languages()

    max_langs = 0

    for script, langs in script_languages.iteritems():
        num_langs = len(langs)
        if num_langs > max_langs:
            max_langs = num_langs

    # Generate C header and constants

    script_enum = u'''
    '''.join(['SCRIPT_{} = {},'.format(s.upper(), i) for s, i in sorted(all_scripts.iteritems(), key=itemgetter(1))])

    out_header.write(scripts_header_template.format(num_codepoints=NUM_CODEPOINTS,
                     max_langs=max_langs,
                     script_enum=script_enum))
    out_header.close()

    # Generate C data file

    char_scripts_data = u''',
    '''.join([', '.join([str(all_scripts[sc or UNKNOWN_SCRIPT]) for sc in batch]) for batch in batch_iter(chars, 25)])

    script_codes_data = u''',
    '''.join([script_code_template.format(name=name.upper(), code=code) for code, name in script_codes.iteritems()])

    sorted_lang_scripts = [script_languages[s] for s, i in sorted(all_scripts.iteritems(), key=itemgetter(1))]

    script_language_data = u''',
    '''.join([script_language_template.format(num_langs=len(langs),
              languages='{{{}}}'.format(', '.join(['"{}"'.format(l) for l in langs]) if langs else 'NULL'))
              for langs in sorted_lang_scripts])

    out_file.write(scripts_c_data_template.format(header_name=SCRIPTS_HEADER,
                   char_scripts=char_scripts_data,
                   script_codes=script_codes_data,
                   script_languages=script_language_data))
    out_file.close()


if __name__ == '__main__':
    main(*sys.argv[1:])