#!/usr/bin/env python3.6

import codecs
import csv
import logging
import logging.config
import os
import shutil

import common

logger = None

HEADER_KEY = "EXISTING_HEADERS"
MAX_ROWS = 1000000

COL_VAR_ID = 'VariationID'

def write_csv(path_out, data):
    file_num = 1
    row_count = 0
    header_row = data.pop(HEADER_KEY, [])

    out_file = os.path.join(path_out, "hauptvariante-{}.csv".format(file_num))
    of = codecs.open(out_file, "w", encoding='utf-8')
    writer = csv.writer(of, delimiter=';', quoting=csv.QUOTE_ALL)

    for _,v in data.items():
        if row_count == 0:
            writer.writerow(header_row)
        
        writer.writerow(v)
        row_count += 1
        if row_count == MAX_ROWS:
            row_count = 0
            file_num += 1
            out_file = os.path.join(path_out, "hauptvariante-{}.csv".format(file_num))
            of = codecs.open(out_file, "w", encoding='utf-8')
            writer = csv.writer(of, delimiter=';', quoting=csv.QUOTE_ALL)


def read_file(in_file, data_dict):
    header_read = False
    var_id_idx = -1
    drop_column_list = list()
    existing_headers = data_dict.get(HEADER_KEY, [])
    first_file = len(existing_headers) == 0

    with codecs.open(in_file, "r", encoding='utf-8') as f:
        reader = csv.reader(f, delimiter=';')

        for row in reader:
            if header_read:
                var_id  = row[var_id_idx]
                if var_id:
                    if first_file:
                        data_dict[var_id] = row
                    elif var_id in data_dict:
                        num_dropped = 0
                        for didx in drop_column_list:
                            row.pop(didx - num_dropped)
                            num_dropped += 1

                        data_dict[var_id] = data_dict[var_id] + row
            else:
                if COL_VAR_ID in row:
                    var_id_idx = row.index(COL_VAR_ID)

                if first_file:
                    existing_headers = row
                else:
                    for x in row:
                        if x in existing_headers:
                            drop_column_list.append(row.index(x))
                        else:
                            existing_headers.append(x)
                data_dict[HEADER_KEY] = existing_headers

                header_read = True


def match_variations(data_root):    
    matched_data = dict()

    path_in = os.path.join(data_root, "in")
    path_out = os.path.join(data_root, "out")
    path_proc = os.path.join(data_root, "processed")
    path_err = os.path.join(data_root, "error")

    in_files = os.listdir(path_in)
    num_files = len(in_files)

    logger.info("Found {0} files...".format(num_files))

    if num_files == 0:
        return

    for f in in_files:
        in_file = os.path.join(path_in, f)
        proc_file = os.path.join(path_proc, f)

        if not os.path.isfile(in_file):
            continue

        logger.info("Reading in file %s", f)

        try:
            read_file(in_file, matched_data)
            shutil.move(in_file, proc_file)
        except Exception as ex:
            logger.error(ex)
            err_file = os.path.join(path_err, f)
            shutil.move(in_file, err_file)

    write_csv(path_out, matched_data)


def process(data_root):
    global logger

    path_log = os.path.join(data_root, "log")

    log_file = os.path.join(path_log, os.path.basename(__file__) + ".log")
    logging.config.dictConfig(common.get_logging_config(log_file))

    logger = logging.getLogger(__file__)

    try:
        match_variations(data_root)
    except Exception as e:
        logger.error(e)


def main():
    if os.path.exists(r"/ftproot/"):
        data_root = r"/ftproot/"
    elif os.path.exists(r"/home/holger/git/bitbucket/thelen/data/"):
        data_root = r"/home/holger/git/bitbucket/thelen/data/"
    else:
        print("Unknown environment. Please set data_root to data folder location. Aborting.")
        return

    var_data_root = os.path.join(data_root, "Matching_Hauptvariante")

    process(var_data_root)

    print("Done.")

if __name__ == "__main__":
    main()