#!/usr/bin/env python

__author__ = 'holger.knust'

import codecs
import csv
from datetime import datetime
import logging
import os
from pathlib import Path
import shutil


processed_models = set()

def merge_dedupe(in_file, out_file, write_header):
    header_read = False
    just_concatenate = False
    match_index = -1

    with codecs.open(in_file, "r", encoding='utf-8') as f, \
        codecs.open(out_file, "a", encoding='utf-8') as of:

        reader = csv.reader(f, delimiter=';')
        writer = csv.writer(of, delimiter=';')

        for row in reader:
            if any(row):
                if header_read:
                    if not just_concatenate:
                        match_field = row[match_index]

                        if not match_field and match_field.isspace() or \
                            match_field in processed_models:
                            continue

                        processed_models.add(match_field)

                    writer.writerow(row)
                else:
                    header_read = True
                    if "EAN" in row:
                        match_index = row.index('EAN')

                    if "SKU" in row:
                        match_index = row.index('SKU')

                    if match_index < 0:
                        just_concatenate = True

                    if write_header:
                        writer.writerow(row)

def process(data_root):
    path_in = os.path.join(data_root, "in")
    path_out = os.path.join(data_root, "out")
    path_proc = os.path.join(data_root, "processed")
    path_err = os.path.join(data_root, "error")
    path_log = os.path.join(data_root, "log")


    files = sorted(Path(path_in).iterdir(), key=lambda f: f.stat().st_mtime)
    num_files = len(files)

    print("Found {0} files...".format(num_files))

    if num_files == 0:
        return

    first_file = True
    log_file_name = datetime.now().strftime("merge_dedupe_%Y%m%d%H%M%S%f.log")
    out_file_name = datetime.now().strftime("merged_%Y%m%d%H%M%S%f.csv")
    out_file = os.path.join(path_out, out_file_name)
    log_file = os.path.join(path_log, log_file_name)

    for f in files:
        filename = os.path.basename(str(f))
        in_file = os.path.join(path_in, filename)
        proc_file = os.path.join(path_proc, filename)
        err_file = os.path.join(path_err, filename)

        logging.basicConfig(filename=log_file, level=logging.DEBUG)

        logging.debug("Merging and deduplicating %s...", filename)

        if os.path.isfile(in_file):
            logging.debug("Converting %s to %s...", filename, out_file_name)
            try:
                merge_dedupe(in_file, out_file, first_file)

                # Write header first file only
                if first_file:
                    first_file = False

                shutil.move(in_file, proc_file)
            except Exception as e:
                logging.error(e)
                print(e)
                shutil.move(in_file, err_file)

def main():
    if os.path.exists(r"/ftproot/"):
        data_root = r"/ftproot/"
    elif os.path.exists(r"/home/holger/git/bitbucket/thelen/data/"):
        data_root = r"/home/holger/git/bitbucket/thelen/data/"
    else:
        print(
            "Unknown environment. Please set data_root to data folder location. Aborting."
        )
        return

    file_root = os.path.join(data_root, "Matching_CSV_zusammenfuehren")

    process(file_root)

    print("Done.")


if __name__ == "__main__":
    main()