#!/usr/bin/env python
import os
import shutil
import codecs
import csv
import logging

__author__ = 'Holger Knust'


def check_for_dupes(in_file, out_file, dupe_file):
    header_read = False
    ean_index = -1
    purchase_price_index = -1
    ean_dict = dict()
    dupe_dict = dict()
    header_row = None

    with codecs.open(in_file, "r", encoding='utf-8') as f, codecs.open(out_file, "w", encoding='utf-8') as of, \
            codecs.open(dupe_file, "w", encoding='utf-8') as df:
        reader = csv.reader(f, delimiter=";")
        writer = csv.writer(of, delimiter=";")
        dupes = csv.writer(df, delimiter=";")

        for row in reader:
            if header_read:
                ean = row[ean_index]

                if len(ean.strip()) == 0:
                    continue

                pp_text = row[purchase_price_index]

                try:
                    purchase_price = float(pp_text.replace(",", "."))
                except ValueError:
                    purchase_price = 0.0

                if ean in ean_dict:
                    if purchase_price < ean_dict[ean][0]:
                        if ean in dupe_dict:
                            dupe_dict[ean].append(row)
                        else:
                            dupe_dict[ean] = [row]

                        logging.info("Duplicate of {0} found!".format(row[ean_index]))
                    else:
                        # Append EAN row to dupes and replace with current row
                        if ean in dupe_dict:
                            dupe_dict[ean].append(ean_dict[ean][1])
                        else:
                            dupe_dict[ean] = [ean_dict[ean][1]]
                        ean_dict[ean] = (purchase_price, row)
                else:
                    ean_dict[ean] = (purchase_price, row)
            else:
                header_read = True
                header_row = row
                lower_row = list(map(str.lower, row))

                if 'ean' in lower_row:
                    ean_index = lower_row.index('ean')
                if 'ns1:ean' in lower_row:
                    ean_index = lower_row.index('ns1:ean')
                if 'sku' in lower_row:
                    ean_index = lower_row.index('sku')
                if 'ek' in lower_row:
                    purchase_price_index = lower_row.index('ek')

                if ean_index < 0 or purchase_price_index < 0:
                    raise Exception("Not all necessary columns found")

        if len(ean_dict):
            writer.writerow(header_row)
            for _, row in ean_dict.items():
                writer.writerow(row[1])

        if len(dupe_dict):
            dupes.writerow(header_row)
            for ean in dupe_dict.keys():
                for row in dupe_dict[ean]:
                    dupes.writerow(row)
        else:
            # delete empty dupes file
            os.remove(out_file)
            logging.debug("No duplicates found.")


def process_files(data_root):
    path_in = os.path.join(data_root, "in")
    path_out = os.path.join(data_root, "out")
    path_prc = os.path.join(data_root, "processed")
    path_err = os.path.join(data_root, "error")
    path_log = os.path.join(data_root, "log")

    files = os.listdir(path_in)
    num_files = len(files)

    print("Found {0} files...".format(num_files))

    if num_files == 0:
        return

    for f in files:
        log_file_name = os.path.splitext(f)[0] + '.log'
        dupes_file_name = os.path.splitext(f)[0] + '_dupes.csv'
        print(dupes_file_name)
        in_file = os.path.join(path_in, f)
        out_file = os.path.join(path_out, f)
        dupe_file = os.path.join(path_out, dupes_file_name)
        prc_file = os.path.join(path_prc, f)
        err_file = os.path.join(path_err, f)
        log_file = os.path.join(path_log, log_file_name)

        if os.path.isfile(in_file):
            logging.basicConfig(filename=log_file, level=logging.DEBUG)

            logging.debug("Checking {0} for duplicates.".format(in_file))
            try:

                check_for_dupes(in_file, out_file, dupe_file)
                shutil.move(in_file, prc_file)
            except Exception as e:
                logging.error(e)
                shutil.move(in_file, err_file)


def main():
    # data_root = r"/ftproot/"
    data_root = r"/ftproot/"
    # data_root = r"/home/holger/git/thelen/data/"

    dupe_data_root = os.path.join(data_root, "Matching_DOPPELTE")
    process_files(dupe_data_root)


if __name__ == "__main__":
    main()