#!/usr/bin/env python
import os
import shutil
import codecs
import csv
import logging


def read_ref(ref_path):
    ref_dict = {}

    header_read = False
    header_names = None
    full_ref_path = os.path.join(ref_path, "1000_Ebay_Ref.csv")
    with codecs.open(full_ref_path, "rb", encoding='utf-8') as f:

        reader = csv.reader(f, delimiter=';')
        ean_index = -1
        for row in reader:
            if header_read:
                ean = (row.pop(ean_index)).strip()
                ref_dict[ean] = row
            else:
                lc_row = [x.lower() for x in row]
                if not "ean" in lc_row:
                    raise Exception("EAN column not found")

                ean_index = lc_row.index("ean")
                del row[ean_index]
                header_names = row

                header_read = True

    return ref_dict, header_names


def merge(in_file, out_file, missing_file, ean_dict, ean_header):
    header_read = False
    ean_index = -1

    with codecs.open(in_file, "r", encoding='utf-8') as f, \
            codecs.open(out_file, "w", encoding='utf-8') as of, \
            codecs.open(missing_file, "w", encoding='utf-8') as df:

        reader = csv.reader(f, delimiter="\t")
        writer = csv.writer(of, delimiter=";")
        missing = csv.writer(df, delimiter=";")

        for row in reader:
            if header_read:
                ean = row[ean_index]

                if not ean.strip():
                    continue

                if ean in ean_dict:
                    writer.writerow(row + ean_dict[ean])
                else:
                    missing.writerow([ean] + row)
            else:

                lc_row = [x.lower() for x in row]

                if 'ean' not in lc_row:
                    raise Exception("EAN column column not found")

                ean_index = lc_row.index('ean')
                writer.writerow(row + ean_header)
                missing.writerow(["EAN"] + ean_header)
                header_read = True



def process_files(data_root):
    path_in = os.path.join(data_root, "in")
    path_out = os.path.join(data_root, "out")
    path_prc = os.path.join(data_root, "processed")
    path_err = os.path.join(data_root, "error")
    path_log = os.path.join(data_root, "log")
    path_ref = os.path.join(data_root, "ref")

    files = os.listdir(path_in)
    num_files = len(files)

    print("Found {0} files...".format(num_files))

    if num_files == 0:
        return

    ean_dict, ean_hdr = read_ref(path_ref)

    for f in files:
        log_filename = os.path.splitext(f)[0] + '.log'
        missing_filename = os.path.splitext(f)[0] + '_missing.csv'

        in_file = os.path.join(path_in, f)
        out_file = os.path.join(path_out, f)
        dupe_file = os.path.join(path_out, missing_filename)
        prc_file = os.path.join(path_prc, f)
        err_file = os.path.join(path_err, f)
        log_file = os.path.join(path_log, log_filename)

        if os.path.isfile(in_file):
            logging.basicConfig(filename=log_file, level=logging.DEBUG)

            logging.debug("Checking %s for duplicates.", in_file)
            try:
                merge(in_file, out_file, dupe_file, ean_dict, ean_hdr)
                shutil.move(in_file, prc_file)
            except Exception as e:
                logging.error(e)
                shutil.move(in_file, err_file)


def main():
    data_root = r"/ftproot/"
    # data_root = r"/home/holger/git/bitbucket/thelen/data/"

    dupe_data_root = os.path.join(data_root, "Matching_Ebay")
    process_files(dupe_data_root)


if __name__ == "__main__":
    main()