#!/usr/bin/env python

__author__ = 'holger'

import csv
import codecs
import locale
import logging
import os
import shutil
import time

def filter_deposits(in_file, out_file, is_first_file):
    amount_col_index = 2
    header_read = False

    if is_first_file:
        file_mode = "w"
    else:
        file_mode = "a"

    loc = locale.getlocale()
    # getlocale() may return (None,None) if not set before.
    # Workaround by setting locale to default before changing it.
    if not loc[0]:
        # Set default locale
        locale.setlocale(locale.LC_ALL, '')
        loc = locale.getlocale()

    # Set to German locale
    locale.setlocale(locale.LC_ALL, 'de_DE.utf8')

    remove_start_col_index = 22 # column indexes in sample data
    remove_end_col_index = 27
    usage_start_col_index = 25  # these have been adjusted for when the unused column have been removed
    usage_end_col_index = 37
    final_col_index = 27

    with codecs.open(in_file, "r", encoding='cp1252') as f, codecs.open(out_file, file_mode, encoding='cp1252') as of:
        reader = csv.reader(f, delimiter=';')
        writer = csv.writer(of, delimiter=';')
        for row in reader:
            if any(row):
                skip_row = False
                if header_read:
                    if row and len(row) >= amount_col_index:
                        try:
                            # Parse the amount
                            amount = locale.atof(row[amount_col_index])
                        except Exception as ex:
                            logging.error(ex)
                            amount = 0.0

                        if amount <= 0:
                            skip_row = True
                        else:
                            # Remove unnecessary header columns
                            col_idx = remove_start_col_index
                            for idx in range(num_removal_columns):
                                row.pop(col_idx)

                            # Consolidate usage colunns into one
                            usage_text = row[usage_start_col_index]

                            col_idx = usage_start_col_index  + 1
                            for idx in range(num_usage_columns):

                                if len(row[col_idx]):
                                    usage_text = usage_text + row[col_idx]

                                # Remove column
                                row.pop(col_idx)

                            row[usage_start_col_index] = usage_text

                            for idx in range(num_end_of_row_cols):
                                row.pop(final_col_index)

                else:
                    header_read = True
                    if 'Betrag' in row:
                        amount_col_index = row.index('Betrag')

                    if 'Splittbuchung - Auftraggeber / Name' in row:
                        remove_start_col_index = row.index('Splittbuchung - Auftraggeber / Name')
                        #print('remove start {0}'.format(remove_start_col_index))

                    if 'Splittbuchung - Verwendungszweckzeile 1' in row:
                        remove_end_col_index = row.index('Splittbuchung - Verwendungszweckzeile 1')
                        #print('remove start {0}'.format(remove_end_col_index))

                    num_removal_columns = remove_end_col_index - remove_start_col_index + 1

                    # Remove unnecessary header columns
                    col_idx = remove_start_col_index
                    for idx in range(num_removal_columns):
                        row.pop(col_idx)


                    if 'Verwendungszweckzeile 2' in row:
                        usage_start_col_index = row.index('Verwendungszweckzeile 2')
                        #print('usage start {0}'.format(usage_start_col_index))

                    if 'Verwendungszweckzeile 14' in row:
                        usage_end_col_index = row.index('Verwendungszweckzeile 14')
                        #print('usage end {0}'.format(usage_end_col_index))

                    num_usage_columns = usage_end_col_index - usage_start_col_index

                    # Remove additional header columns
                    col_idx = usage_start_col_index + 1
                    for idx in range(num_usage_columns):
                        row.pop(col_idx)

                    if 'Steuersatz' in row:
                        final_col_index = row.index('Steuersatz')
                        num_end_of_row_cols = len(row) - final_col_index
                        #print('final {0}'.format(final_col_index))
                        for idx in range(num_end_of_row_cols):
                            row.pop(final_col_index)

                    # Don't write header again if this is not the first file
                    skip_row = not is_first_file

                if not skip_row:
                    writer.writerow(row)

    # Restore original locale
    if loc[0]:
        locale.setlocale(locale.LC_ALL, loc)

def process_files(data_root):
    path_in = os.path.join(data_root, "in")
    path_out = os.path.join(data_root, "out")
    path_prc = os.path.join(data_root, "processed")
    path_err = os.path.join(data_root, "error")
    path_log = os.path.join(data_root, "log")

    # Set up logging
    datetime_file_name = time.strftime("%Y%m%d-%H%M%S")
    log_file_name = datetime_file_name + '.log'
    log_file = os.path.join(path_log, log_file_name)
    logging.basicConfig(filename=log_file, level=logging.DEBUG)

    files = os.listdir(path_in)
    num_files = len(files)

    logging.info("Found {0} files...".format(num_files))

    if num_files == 0:
        return

    # This is reset after the first file is processed
    is_first_file = True

    for f in files:
        in_file = os.path.join(path_in, f)
        proc_file = os.path.join(path_prc, f)
        err_file = os.path.join(path_err, f)

        out_file_name = datetime_file_name + ".csv"
        out_file = os.path.join(path_out, out_file_name)

        if os.path.isfile(in_file):
            try:
                logging.debug("Processing {0}...".format(in_file))

                filter_deposits(in_file, out_file, is_first_file)

                if os.path.isfile(in_file):
                    shutil.move(in_file, proc_file)

                if is_first_file:
                    is_first_file = False
            except Exception as e:
                logging.error(e)

                # Move in file to error
                shutil.move(in_file, err_file)

                # Clean up. Remove broken out files if they exist
                if os.path.isfile(out_file):
                    os.remove(out_file)

def main():
    data_root = r"/ftproot/"
    #data_root = r"/data/projects/pycharm/thelen/conversion/data/"
    #data_root = r"c:/temp/thelen/"

    data_root = os.path.join(data_root, "Matching_Bankdaten")
    process_files(data_root)

if __name__ == "__main__":
    main()