#!/usr/bin/env python3.6

import codecs
import csv
import logging
import logging.config
import os
import shutil

import common

logger = None

COL_HEADER_FEDAS = 'FEDAS'

def read_lookup(ref_file):
    ref_dict = dict()

    header_row = None
    header_read = False
    fedas_idx = -1

    with codecs.open(ref_file, "rb", encoding='utf-8') as f:
        reader = csv.reader(f, delimiter=';')

        for row in reader:
            if header_read:
                fedas = row[fedas_idx]
                row.pop(fedas_idx)
                ref_dict[fedas] = row
            else:
                header_row = row
                row_upper = [x.upper() for x in row]
                if COL_HEADER_FEDAS in row_upper:
                    fedas_idx = row_upper.index(COL_HEADER_FEDAS)

                if fedas_idx < 0:
                    raise Exception("Missing FEDAS column in ref.")

                header_row.pop(fedas_idx)
                header_read = True

    return header_row, ref_dict


def read_in_file(in_file):
    header_read = False
    header_row = None
    fedas_col_index = -1

    fedas_list = list()
    with codecs.open(in_file, "r", encoding='utf-8') as f:
        reader = csv.reader(f, delimiter=';')

        for row in reader:
            if header_read:
                fedas_num = row[fedas_col_index]
                if not fedas_num:
                    fedas_num = "ERROR_NO_FEDAS"
                    row[fedas_col_index] = "ERROR_NO_FEDAS"

                fedas_list.append(row)
            else:
                header_read = True
                header_row = row
                row_upper = [x.upper() for x in row]
                fedas_col_index = row_upper.index(COL_HEADER_FEDAS)
                if fedas_col_index < 0:
                    raise Exception("Missing Fedas column")

    return header_row, fedas_list


def match_fedas(in_list, ref_folder, in_header):
    ref_files = os.listdir(ref_folder)
    num_files = len(ref_files)
    matched_header = in_header
    print("Found {0} files...".format(num_files))

    if num_files == 0:
        return

    for f in ref_files:
        full_path = os.path.join(ref_folder, f)

        if not os.path.isfile(full_path):
            continue

        print("Reading ref file {}".format(f))
        ref_header, ref_dict = read_lookup(full_path)
        matched_header += ref_header
        num_cols = len(ref_header)
        for row in in_list:
            k = row[0]
            if k in ref_dict:
                row += ref_dict[k]
            else:
                row += ["ERROR"] * num_cols

    return matched_header, in_list


def write_matches(out_file, in_list, header_row):
    c1_idx = -1
    c2_idx = -1
    c3_idx = -1
    write_all = False

    if "CategoryLevel1Name" in header_row:
        c1_idx = header_row.index("CategoryLevel1Name")
    if "CategoryLevel2Name" in header_row:    
        c2_idx = header_row.index("CategoryLevel2Name")
    if "CategoryLevel3Name" in header_row:    
        c3_idx = header_row.index("CategoryLevel3Name")

    with codecs.open(out_file, "w", encoding='utf-8') as of:
        writer = csv.writer(of, delimiter=';', quoting=csv.QUOTE_ALL)

        if c1_idx > -1 and c2_idx > -1 and c3_idx > -1:
            header_row = header_row + ["CategoryLevelAll"]
            write_all = True

        writer.writerow(header_row)
        for v in in_list:
            if write_all:
                v = v + [ ";".join([v[c1_idx],v[c2_idx], v[c3_idx]]) ]
            writer.writerow(v)


def process(data_root):
    global logger

    path_ref = os.path.join(data_root, "ref")
    path_in = os.path.join(data_root, "in")
    path_out = os.path.join(data_root, "out")
    path_proc = os.path.join(data_root, "processed")
    path_err = os.path.join(data_root, "error")
    path_log = os.path.join(data_root, "log")

    log_file = os.path.join(path_log, os.path.basename(__file__) + ".log")
    logging.config.dictConfig(common.get_logging_config(log_file))

    logger = logging.getLogger(__file__)

    files = os.listdir(path_in)
    num_files = len(files)

    logger.info("Found {0} files...".format(num_files))

    if num_files == 0:
        return

    for f in files:
        log_file_name = os.path.splitext(f)[0] + '.log'

        in_file = os.path.join(path_in, f)
        out_file = os.path.join(path_out, f)
        proc_file = os.path.join(path_proc, f)
        err_file = os.path.join(path_err, f)
        log_file = os.path.join(path_log, log_file_name)

        if os.path.isfile(in_file):
            logger.debug("Performing FEDAS Lookup...%s", in_file)

            try:
                print("Reading in file...")
                in_header, in_list = read_in_file(in_file)
                matched_header, matched_list = match_fedas(in_list, path_ref, in_header)
                write_matches(out_file, matched_list, matched_header)
                shutil.move(in_file, proc_file)
            except Exception as e:
                print(e)
                logger.error(e)
                shutil.move(in_file, err_file)


def main():
    if os.path.exists(r"/ftproot/"):
        data_root = r"/ftproot/"
    elif os.path.exists(r"/home/holger/git/bitbucket/thelen/data/"):
        data_root = r"/home/holger/git/bitbucket/thelen/data/"
    else:
        print("Unknown environment. Please set data_root to data folder location. Aborting.")
        return

    fedas_data_root = os.path.join(data_root, "Matching_Fedas")

    process(fedas_data_root)

    print("Done.")

if __name__ == "__main__":
    main()