#!/usr/bin/env python3.7

import codecs
import csv
import logging.config
import os
import re
import shutil
import sys
import time

import requests
from typing import Dict, List, Tuple

from common import get_column_index_csv, get_data_root, setup_logging

DATA_FOLDER_NAME = "Matching_Basic_Purchase_Price"


def read_ref(ref_path: str) -> dict:
    """ Read the endpoint ref. """
    ref_dict = dict()
    url_idx = -1
    prefix_idx = -1
    size_idx = -1

    header_read = False
    full_ref_path = os.path.join(ref_path, "endpoints.csv")

    with codecs.open(full_ref_path, "r", encoding='utf-8-sig') as f:
        reader = csv.reader(f, delimiter=';')

        for row in reader:
            if header_read:
                url = row[url_idx]
                ref_dict[url] = {"prefix": row[prefix_idx],
                  "size": row[size_idx]}
            else:
                url_idx = get_column_index_csv(row, "url")
                if url_idx < 0:
                    raise Exception(
                        f"Url column not found in '{full_ref_path}'.")

                prefix_idx = get_column_index_csv(row, "prefix")
                if prefix_idx < 0:
                    raise Exception(
                        f"Prefix column not found in '{full_ref_path}'.")

                size_idx = get_column_index_csv(row, "size")
                if size_idx < 0:
                    raise Exception(
                        f"Size column not found in '{full_ref_path}'.")

                header_read = True

    return ref_dict


def get_filename_from_header(cd):
    """
    Get filename from content-disposition
    """
    if not cd:
        return ""
    fname = re.findall("filename=\"(.+)\"", cd)

    if len(fname) == 0:
        return ""

    return fname[0]


def download_file(url, path_in, prefix, num_lines):
    try:
        logging.debug("Starting download of %s", url)
        r = requests.get(url, allow_redirects=True, timeout=600)
        logging.debug("Finished download of %s", url)

        if r.status_code != 200:
            logging.warning("Server returned status {}".format(r.status_code))
            return

        filename = f"{prefix}-{num_lines}.csv"

        full_in = os.path.join(path_in, filename)

        logging.debug("Saving data to %s", full_in)
        content_txt = r.content.decode('utf-8')
        content = content_txt.replace('\0', ' ')
        open(full_in, 'wb').write(content.encode('utf-8'))
    except Exception as ex:
        logging.exception(ex)


def write_csv(output_path: str, data: list, header=None, delimiter=';'):
    logging.info("Writing %s with %d rows.", output_path, len(data))
    with codecs.open(output_path, "w", encoding='utf-8') as of:
      writer = csv.writer(of, delimiter=delimiter, quoting=csv.QUOTE_ALL)

      if header:
        writer.writerow(header)

      for r in data:
          writer.writerow(r)


def split(input_path: str,
          delimiter:str=',',
          row_limit:int=1000,
          output_name_template:str='chunk_%04d.csv',
          output_path='.',
          keep_headers=True
    ):

    chunk_rows = []
    header = None
    header_read = False
    with open(input_path, "r") as input_fp:
      reader = csv.reader(input_fp, delimiter=delimiter)
      current_chunk = 1
      rows_read = 0

      for row in reader:
        if header_read:
          if rows_read < row_limit:
            chunk_rows.append(row)
            rows_read += 1
          else:
            current_out_path = os.path.join(output_path, output_name_template \
              % current_chunk)
            current_chunk += 1
            write_csv(current_out_path, chunk_rows, header)
            chunk_rows = []
            rows_read = 0
        else:
          header = row
          header_read = True

      if rows_read > 0:
        current_out_path = os.path.join(output_path, output_name_template \
          % current_chunk)
        current_chunk += 1
        write_csv(current_out_path, chunk_rows, header)
        chunk_rows = []
        rows_read = 0


def main() -> None:
  data_folder = os.path.join(get_data_root(), DATA_FOLDER_NAME)

  path_in = os.path.join(data_folder, "in")
  path_out = os.path.join(data_folder, "out")
  path_prc = os.path.join(data_folder, "processed")
  path_err = os.path.join(data_folder, "error")
  path_log = os.path.join(data_folder, "log")
  path_ref = os.path.join(data_folder, "ref")

  setup_logging(path_log)

  logging.debug("Importing endpoints...")
  endpoints = read_ref(path_ref)

  if len(endpoints) == 0:
      logging.error("No endpoints found. Aborting...")
      sys.exit(1)

  logging.debug(f"Found {len(endpoints)} endpoints...")

  for url in endpoints.keys():
    prefix = endpoints[url].get('prefix', 'chunk')
    num_lines = endpoints[url].get('size', 10000)

    logging.info(f"Downloading file from {url}...")
    download_file(url, path_in, prefix, num_lines)

  files = os.listdir(path_in)
  num_files = len(files)

  logging.debug(f"Found {num_files} files...")

  if num_files == 0:
    return

  for f in files:
    in_file = os.path.join(path_in, f)
    prc_file = os.path.join(path_prc, f)
    err_file = os.path.join(path_err, f)

    if not os.path.isfile(in_file):
        continue

    if not "-" in f:
      logging.warning("'%s' is not a correct file name. Expected - in name.")
      continue

    chunk_prefix = f.split("-")[0]
    chunk_num_lines = int((f.split("-")[1]).split(".")[0])
    logging.debug(f"Splitting file '{f}'...")

    try:
      out_file_templ=f"{chunk_prefix}-%04d.csv"
      split(in_file, delimiter=';', row_limit=chunk_num_lines,
        output_name_template=out_file_templ, output_path=path_out,
        keep_headers=True)

      if os.path.isfile(prc_file):
          os.remove(prc_file)

      shutil.move(in_file, prc_file)
    except Exception as e:
        logging.exception(e)
        try:
            shutil.move(in_file, err_file)
        except Exception as e:
            logging.exception(e)

  print("Done.")


if __name__ == "__main__":
    main()