#!/usr/bin/env python
# -*- coding: utf-8 -*-

import csv
import codecs
import logging
import os
import shutil
import sys
import time
import glob

logger = None

def setup_logging(log_path):
    # Set up logging
    l = logging.getLogger()

    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

    stdout_hdlr = logging.StreamHandler(sys.stdout)
    stdout_hdlr.setLevel(logging.DEBUG)
    stdout_hdlr.setFormatter(formatter)

    datetime_file_name = time.strftime("%Y%m%d-%H%M%S")
    log_file_name = datetime_file_name + '.log'
    log_file = os.path.join(log_path, log_file_name)

    file_hdlr = logging.FileHandler(log_file)
    file_hdlr.setLevel(logging.DEBUG)
    file_hdlr.setFormatter(formatter)

    l.addHandler(stdout_hdlr)
    l.addHandler(file_hdlr)

    return l


def read_ref(match_file):
    global logger

    ref_dict = dict()

    header_read = False
    header_row = None

    with codecs.open(match_file, "rb", encoding='utf-8') as f:
        reader = csv.reader(f, delimiter=';')

        for row in reader:
            if header_read:
                key = row[0]

                if len(key.strip()) == 0:
                    logger.warning("Ignoring empty key field")
                    continue

                if key in ref_dict:
                    logger.warning("Duplicate matching key {}".format(key))
                else:
                    ref_dict[key] = row[1:]
            else:
                header_row = row[1:]
                header_read = True

    return ref_dict, header_row


def match_files(source_file, matched_file, unmatched_file, ref_dict, headers):
    global logger

    header_read = False

    with codecs.open(source_file, "r", encoding='utf-8') as f, \
            codecs.open(matched_file, "w", encoding='utf-8') as mf, \
            codecs.open(unmatched_file, "w", encoding='utf-8') as uf:

        reader = csv.reader(f, delimiter=';')
        writer_matched = csv.writer(mf, delimiter=';', quoting=csv.QUOTE_NONNUMERIC)
        writer_unmatched = csv.writer(uf, delimiter=';', quoting=csv.QUOTE_NONNUMERIC)

        for row in reader:
            if header_read:
                key = row[0]

                if key in ref_dict:
                    writer_matched.writerow(row + ref_dict[key])
                else:
                    writer_unmatched.writerow(row)
            else:
                writer_matched.writerow(row + headers)
                writer_unmatched.writerow(row)

                header_read = True


def process_files(data_root):
    global logger

    path_in = os.path.join(data_root, "in")
    path_out = os.path.join(data_root, "out")
    path_prc = os.path.join(data_root, "processed")
    path_err = os.path.join(data_root, "error")
    path_log = os.path.join(data_root, "log")

    logger = setup_logging(path_log)

    source_file_list = glob.glob(os.path.join(path_in, "1_*"))
    match_file_list = glob.glob(os.path.join(path_in, "2_*"))

    if len(source_file_list) != 1:
        logger.error("Could not find a source file starting with 1_")
        exit(1)

    if len(match_file_list) != 1:
        logger.error("Could not find a matched file starting with 2_")
        exit(1)

    source_file = source_file_list[0]
    match_file = match_file_list[0]
    source_filename = os.path.basename(source_file)
    match_filename = os.path.basename(match_file)

    proc_source_file = os.path.join(path_prc, source_filename)
    err_source_file = os.path.join(path_err, source_filename)

    proc_match_file = os.path.join(path_prc, match_filename)
    err_match_file = os.path.join(path_err, match_filename)

    out_matched_file = os.path.join(path_out, "matched.csv")
    out_unmatched_file = os.path.join(path_out, "errors.csv")

    try:
        logger.debug("Processing {0} & {1}...".format(source_filename, match_filename))

        ref_dict, headers = read_ref(match_file)

        match_files(source_file, out_matched_file, out_unmatched_file, ref_dict, headers)

        if os.path.isfile(source_file):
            shutil.move(source_file, proc_source_file)

        if os.path.isfile(match_file):
            shutil.move(match_file, proc_match_file)

    except Exception as e:
        logger.error(e)

        # Move in file to error
        shutil.move(source_file, err_source_file)
        shutil.move(match_file, err_match_file)

        # Clean up. Remove broken out files if they exist
        if os.path.isfile(out_matched_file):
            os.remove(out_matched_file)

        if os.path.isfile(out_unmatched_file):
            os.remove(out_unmatched_file)


def main():
    data_root = r"/ftproot/"
    # data_root = r"/ftproot/"
    # data_root = r"/home/holger/git/thelen/data/"
    # data_root = r"c:/temp/thelen/"

    data_root = os.path.join(data_root, "Matching_Sverweis")
    process_files(data_root)

if __name__ == "__main__":
    main()