#!/usr/bin/env python
# -*- coding: utf-8 -*-

import datetime
import os
import codecs
import csv
import logging
import shutil
import time

__author__ = 'holger'


def split_file(in_file, out_file_template, num_lines_per_file):
    of = None
    line_number = 1
    header_read = False

    file_part_number = 1
    out_file = out_file_template.format(file_part_number)
    print("Processing {}".format(out_file))

    with codecs.open(in_file, "r", encoding='utf-8') as f:
        reader = csv.reader(f, delimiter=';')

        try:
            of  = codecs.open(out_file, "w", encoding='utf-8')
            writer = csv.writer(of, delimiter=';')

            for row in reader:
                if header_read:
                    writer.writerow(row)
                    line_number += 1

                    if line_number > num_lines_per_file:
                        # Close the old file, open a new one and write header
                        of.close()

                        file_part_number += 1
                        out_file = out_file_template.format(file_part_number)
                        print("Processing {}".format(out_file))
                        of = codecs.open(out_file, "w", encoding='utf-8')
                        writer = csv.writer(of, delimiter=';')
                        writer.writerow(header)

                        line_number = 1
                else:
                    header = row
                    header_read = True
                    writer.writerow(row)
            of.close()
            of = None
        except Exception as ex:
            logging.error(ex)
        finally:
            if of:
                of.close()

def process_files(data_root):
    path_in = os.path.join(data_root, "in")
    path_out = os.path.join(data_root, "out")
    path_prc = os.path.join(data_root, "processed")
    path_err = os.path.join(data_root, "error")
    path_log = os.path.join(data_root, "log")
    path_prcin = os.path.join(data_root, "in_process")
    path_done = os.path.join(data_root, "done")

    # Set up logging
    log_file_name = time.strftime("%Y%m%d-%H%M%S") + '.log'
    log_file = os.path.join(path_log, log_file_name)
    logging.basicConfig(filename=log_file, level=logging.DEBUG)

    prev_files = sorted(os.listdir(path_out))
    proc_in_files = os.listdir(path_prcin)

    if len(prev_files) > 0 or len(proc_in_files) > 0:
        logging.info("Found remaining {0} files...".format(len(prev_files)))

        for pif in proc_in_files:
            shutil.move(os.path.join(path_prcin, pif), os.path.join(path_done, pif))

        if len(prev_files) > 0:
            prev_filename = prev_files[0]

            filename, extension = os.path.splitext(prev_filename)
            new_filename = filename[:filename.rfind("_")] + extension
            print("Moving {} to {}".format(prev_filename, new_filename))
            shutil.move(os.path.join(path_out, prev_filename), os.path.join(path_prcin, new_filename))
    else:
        new_files = os.listdir(path_in)
        num_in_files = len(new_files)

        logging.info("Found new {0} files...".format(num_in_files))

        if num_in_files == 0:
            return

        for f in new_files:
            in_file = os.path.join(path_in, f)
            filename, extension = os.path.splitext(f)
            output_filename_template = filename + '_{}' + extension
            out_file = os.path.join(path_out, output_filename_template)
            prc_file = os.path.join(path_prc, f)
            err_file = os.path.join(path_err, f)
            log_file = os.path.join(path_log, 'matching_splitter_' + datetime.datetime.now().strftime(
                "%Y%m%d%H%M%S")) + '.log'

            if os.path.isfile(in_file):
                logging.basicConfig(filename=log_file, level=logging.DEBUG)

                logging.debug("Splitting and processing file...".format(in_file))
                try:
                    split_file(in_file, out_file, 50000)
                    shutil.move(in_file, prc_file)
                except Exception as e:
                    logging.error(e)
                    shutil.move(in_file, err_file)


def main():
    data_root = r"/ftproot/"
    # data_root = r"/home/holger/git/thelen/data"
    #data_root = r"c:/temp/thelen/"

    data_root = os.path.join(data_root, "Matching_Splitter")
    process_files(data_root)

if __name__ == "__main__":
    main()