# -*- coding: utf-8 -*-
"""
BuildWebApp - batch conversion and webpage building for EyeRIS rois and data
produced from RaytrixParticleProc.py

"""


import os
import cv2
import sys
import glob
import time
import json
import math
import piexif
import pandas
import shutil
import tarfile
import cvtools
import pystache
import datetime
import numpy as np
import xmlsettings
import piexif.helper
import multiprocessing
from scipy import stats
from operator import itemgetter
from collections import OrderedDict
from scipy.interpolate import interp1d
from multiprocessing import Pool, Process, Queue


def lmap(f, l):
    return list(map(f, l))

def lzip(a,b):
    return list(zip(a, b))

flow_frames = True

class Counter(object):
    def __init__(self):
        self.val = multiprocessing.Value('i', 0)

    def increment(self, n=1):
        with self.val.get_lock():
            self.val.value += n

    @property
    def value(self):
        return self.val.value

def parse_eyeris_stamp(filename, start_time, start_unixtime):
    # Example: Exported-78420070.212200-000000102874-2154-1596-1252-1672-1398.png
    timestamp = None
    for substr in filename.split('-'):
        try:
            timestamp = float(substr)
            break
        except ValueError:
            pass

    timestamp = (timestamp - start_time)/1000 + start_unixtime

    # print(timestamp)

    timestring = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')

    return timestamp, timestring

def parse_mini_rov_line(line):
    # Example: 1565962637.856,4.156,15.285,5.302,33.521,-68.889,0.000
    data = np.zeros((1, 7))
    for i, tok in enumerate(line.split(",")):
        data[0, i] = float(tok)
    return data


def build_ctd_data(log_file, data_source="MiniROV"):

    with open(log_file, "r") as f:
        lines = f.readlines()

    # strip out header
    while lines[0][0] == "#":
        lines = lines[1:]

    if data_source == "MiniROV":

        COLS = 7

        data = np.zeros((len(lines), COLS))
        for i, line in enumerate(lines):
            data[i, :] = parse_mini_rov_line(line)

        # Interp data to every second
        x = data[:, 0]
        new_x = np.arange(int(data[0, 0])-1, int(data[-1, 0])+1, 1)

        data_new = np.zeros((len(new_x), COLS))

        # interp data 4x
        for col in range(0, COLS):
            y = data[:, col]
            f = interp1d(x, y, fill_value="extrapolate")
            data_new[:, col] = f(new_x)

        return data_new


def parse_new_spc_stamp(filename):


    # Patch bug in PCAM where timestamp string is somtimes incorrectly set to
    # 0 or a small value. Use the file creation time instead.
    #
    # This is okay so long as the queue in PCAM mostly empty. We can adapt
    # the frame counter to fix this in the future.

    timestamp = 0
    for substr in filename.split('-'):
        try:
            timestamp = int(substr)
            break
        except ValueError:
            pass

    # handle new file formwat with unixtime in microseconds
    if timestamp > 1498093400000000:
        timestamp = timestamp / 1000000

    # Range check the timestamp
    if timestamp < 100000 or timestamp > time.time():
        print("" + filename + " strange timestamp.")
        # timestamp = os.path.getctime(image_path)
        return None, ""
    # print "Timestamp: " + str(timestamp)

    timestring = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')

    return timestamp, timestring

def parse_timestamp(filename,camera_type, cfg):

    if camera_type == "SPC":
        return parse_new_spc_stamp(filename)
    if camera_type == "EYERIS-R26":
        return parse_eyeris_stamp(filename, cfg.get("StartTimestamp", 0.0), cfg.get("StartUnixtime", 0))

def process_image(bundle):

    image_path = bundle['image_path']
    camera_type = bundle['camera_type']
    image = bundle['image']
    image_dir = bundle['image_dir']
    cfg = bundle['cfg']

    filename = os.path.basename(image_path)

    timestamp, timestring = parse_timestamp(filename, camera_type, cfg)
    if timestamp is None:
        output = {}
        return output

    prefix = os.path.splitext(filename)[0]

    # get the relative target range from the last field of the filename
    # 0 is at the focal plane and positive moves closer to the camera.
    targ_range_cm = int(prefix.split('-')[-1])/10.0

    # pixels are largest at focus plane and then smaller as we move closer to the camera
    eff_res = cfg.get("PixelSize", 101.0) * (cfg.get("FocusRange", 75.0) - targ_range_cm) / cfg.get("FocusRange", 75.0)

    # image is preloaded so no need to load here
    #image = cvtools.import_image(data_path,filename,bayer_pattern=cv2.COLOR_BAYER_BG2RGB)
    img_c_8bit = cvtools.convert_to_8bit(image)

    # images will be saved out later so set save_to_disk to False
    features = cvtools.quick_features(
        img_c_8bit,
        save_to_disk=False,
        abs_path=image_dir,
        file_prefix=prefix,
        cfg=cfg
    )

    use_jpeg = use_jpeg = cfg.get("UseJpeg").lower() == 'true'
    if use_jpeg:
        filename = prefix + '.jpeg'
    else:
        filename = prefix + '.png'



    entry = {}
    entry['length_unit'] = 'um'
    entry['range'] = cfg.get("FocusRange", 75.0) - targ_range_cm
    entry['pixel_size'] = eff_res
    entry['maj_axis_len'] = features['major_axis_length'] * eff_res
    entry['min_axis_len'] = features['minor_axis_length'] * eff_res
    entry['aspect_ratio'] = features['aspect_ratio']
    entry['area'] = features['area'] * eff_res * eff_res
    entry['clipped_fraction'] = features['clipped_fraction']
    entry['orientation'] = features['orientation']*180/math.pi
    entry['eccentricity'] = features['eccentricity']
    entry['solidity'] = features['solidity']
    entry['estimated_volume'] = features['estimated_volume']

    entry['intensity_gray'] = features['intensity_gray']
    entry['intensity_red'] = features['intensity_red']
    entry['intensity_green'] = features['intensity_green']
    entry['intensity_blue'] = features['intensity_blue']

    entry['timestring'] = timestring
    entry['timestamp'] = timestamp
    entry['width'] = img_c_8bit.shape[1]
    entry['height'] = img_c_8bit.shape[0]
    entry['url'] = bundle['reldir'] + '/' + filename
    entry['file_size'] = os.path.getsize(image_path)

    output = {}
    output['entry'] = entry
    output['image_path'] = image_dir
    output['prefix'] = prefix
    output['features'] = features

    return output

# threaded function for each process to call
# queues are used to sync processes
def process_bundle_list(bundle_queue,output_queue):

    while True:
        try:
            output_queue.put(process_image(bundle_queue.get()))
        except:
            time.sleep(0.02*np.random.rand())
# Split a list into sublists
def chunks(l, n):
    n = max(1, n)
    return [l[i:i+n] for i in range(0, len(l), n)]

def format_user_comment(lat, lng, mini_rov_data, entry):

    entry['lat'] = lat
    entry['lon'] = lng
    entry['minirov'] = mini_rov_data.tolist()
    return json.dumps(entry, sort_keys=True, indent=4, separators=(',', ': '))

# Process a directory of images
def run(data_path, cfg):

    print("Running EyeRIS image conversion...")

    print('getting CTD Data...')

    ctd_data = build_ctd_data(os.path.join('data', cfg.get("MiniROVCTD", ".")))

    # get the base name of the directory
    base_dir_name = os.path.basename(os.path.abspath(data_path))

    # list the directory for tif images
    print("Listing directory " + base_dir_name + "...")

    image_list = []
    image_pattern = "*." + cfg.get("ImageExt", "png")
    if cfg.get('MergeSubDirs', "false").lower() == "true":
        sub_directory_list = sorted(glob.glob(os.path.join(data_path, "[0-9]"*cfg.get("SubdirZeros", 10))))
        for sub_directory in sub_directory_list:
            print("Listing sub directory " + sub_directory + "...")
            image_list += glob.glob(os.path.join(sub_directory, image_pattern))
    else:
        image_list += glob.glob(os.path.join(data_path, image_pattern))

    image_list = sorted(image_list)

    # skip if no images were found
    if len(image_list) == 0:
        print ("No images were found. skipping this directory.")
        return

    # Get the total number of images in the directory
    total_images = len(image_list)

    # Create the output directories for the images and web app files
    subdir = os.path.join(data_path, '..', base_dir_name + '_static_html')
    if not os.path.exists(subdir):
        os.makedirs(subdir)
    image_dir = os.path.join(subdir, 'images')
    if not os.path.exists(image_dir):
        os.makedirs(image_dir)

    print("Starting image conversion and page generation...")

    # loop over the images and do the processing
    images_per_dir = cfg.get('ImagesPerDir', 2000)

    is_raw = True
    if cfg.get("BayerPattern").lower() == "rg":
        bayer_conv = cv2.COLOR_BAYER_RG2RGB
    if cfg.get("BayerPattern").lower() == "bg":
        bayer_conv = cv2.COLOR_BAYER_BG2RGB
    if cfg.get("BayerPattern").lower() == "none":
        bayer_conv = None
        is_raw = False

    print("Loading images...\n")
    bundle_queue = Queue()
    for index, image in enumerate(image_list):

        reldir = 'images/' + str(images_per_dir*int(index/images_per_dir)).zfill(5)
        absdir = os.path.join(image_dir, str(images_per_dir*int(index/images_per_dir)).zfill(5))

        filename = os.path.basename(image)

        if not os.path.exists(absdir):
            os.makedirs(absdir)

        bundle = {}
        bundle['image_path'] = image
        bundle['camera_type'] = cfg.get('CameraType', 'EYERIS-R26')
        bundle['image'] = cvtools.import_image(os.path.dirname(image), filename, raw=is_raw, bayer_pattern=bayer_conv)
        # bundle['image'] = np.flipud(bundle['image']) # Assume RaytrixParticleProc fixes flipped image
        bundle['data_path'] = data_path
        bundle['image_dir'] = absdir
        bundle['reldir'] = reldir
        bundle['cfg'] = cfg
        bundle['total_images'] = total_images

        # Us this to test processing code for errors. It's much easier than after all the processes have been
        # spawned
        # output = process_image(bundle)

        bundle_queue.put(bundle)
        print("Loading images... (" + str(index) + " of " + str(total_images) + ")\r", end=" "),

        #if index > 2000:
        #    total_images = index
        #    break

    # Get the number to process to use based on CPUs
    n_threads = multiprocessing.cpu_count() - 1
    if n_threads < 1:
        n_threads = 1

    # Create the set of processes and start them
    start_time = time.time()
    output_queue = Queue()
    processes = []
    for i in range(0, n_threads):
        p = Process(target=process_bundle_list, args=(bundle_queue, output_queue))
        p.start()
        processes.append(p)

    # Monitor processing of the images and save processed images to disk as they become available
    print("\nProcessing Images...\r"),
    counter = 0
    entry_list = []
    use_jpeg = use_jpeg = cfg.get("UseJpeg").lower() == 'true'
    raw_color = cfg.get("SaveRawColor").lower() == 'true'
    ctd_start_time = cfg.get("StartUnixtime", 0)
    ctd_offset = np.squeeze(np.argwhere(ctd_data[:, 0] == ctd_start_time))
    lat = cfg.get("Latitude", 0.0)
    lng = cfg.get("Longitude", 0.0)
    while True:
        print("Processing and saving images... (" + str(counter).zfill(5) + " of " + str(total_images).zfill(5) + ")\r", end=" ")

        if counter >= total_images:
            break

        try:
            output = output_queue.get()
            if output:
                # add in CTD data using timestamp
                output['entry']['pressure'] = ctd_data[ctd_offset + int(output['entry']['timestamp']) - ctd_start_time, 3]
                output['entry']['temperature'] = ctd_data[ctd_offset + int(output['entry']['timestamp']) - ctd_start_time, 2]
                output['entry']['salinity'] = ctd_data[ctd_offset + int(output['entry']['timestamp']) - ctd_start_time, 4]
                output['entry']['o2'] = ctd_data[ctd_offset + int(output['entry']['timestamp']) - ctd_start_time, 5]
                output['entry']['lat'] = lat
                output['entry']['lng'] = lng
                entry_list.append(output['entry'])
                output_path = os.path.join(output['image_path'], output['prefix'])
                if use_jpeg:
                    if raw_color:
                        cv2.imwrite(os.path.join(output_path+"_rawcolor.jpeg"), output['features']['rawcolor'])
                    cv2.imwrite(os.path.join(output_path+".jpeg"), output['features']['image'])
                    # Set exif data from miniROV and lat/lon
                    exif_dict = piexif.load(os.path.join(output_path+".jpeg"))
                    user_string = format_user_comment(
                        lat,
                        lng,
                        ctd_data[ctd_offset + int(output['entry']['timestamp']) - ctd_start_time, :],
                        output['entry']
                    )
                    exif_dict["Exif"][piexif.ExifIFD.UserComment] = piexif.helper.UserComment.dump(user_string)
                    piexif.insert(piexif.dump(exif_dict), os.path.join(output_path+".jpeg"))
                else:
                    if raw_color:
                        cv2.imwrite(os.path.join(output_path+"_rawcolor.png"), output['features']['rawcolor'])
                    cv2.imwrite(os.path.join(output_path+".png"), output['features']['image'])
                
                cv2.imwrite(os.path.join(output_path+"_binary.png"), output['features']['binary'])

            counter = counter + 1
        except:
            time.sleep(0.5)

    # Record the total time for processing
    proc_time = int(math.floor(time.time()-start_time))

    # Terminate the processes in case they are stuck
    for p in processes:
        p.terminate()

    print("\nPostprocessing...")

    # sort the entries by height and build the output
    entry_list.sort(key=itemgetter('maj_axis_len'), reverse=True)

    # Create histograms of several key features

    # image resolution in mm/pixel
    image_res = cfg.get('PixelSize', 22.1)/1000

    #print "Image resolution is set to: " + str(image_res) + " mm/pixel."

    # Get arrays from the dict of features
    total_images = len(entry_list)
    nbins = int(np.ceil(np.sqrt(total_images)))
    maj_len = np.array(lmap(itemgetter('maj_axis_len'), entry_list))*image_res
    min_len = np.array(lmap(itemgetter('min_axis_len'), entry_list))*image_res
    volume = np.array(lmap(itemgetter('estimated_volume'), entry_list))*image_res*image_res*image_res
    aspect_ratio = np.array(lmap(itemgetter('aspect_ratio'), entry_list))
    orientation = np.array(lmap(itemgetter('orientation'), entry_list))
    area = np.array(lmap(itemgetter('area'), entry_list))*image_res*image_res
    unixtime = np.array(lmap(itemgetter('timestamp'), entry_list))
    elapsed_seconds = unixtime - np.min(unixtime)
    file_size = np.array(lmap(itemgetter('file_size'), entry_list))/1000.0

    total_seconds = max(elapsed_seconds)
    print ("Total seconds recorded: " + str(total_seconds))
    if total_seconds < 1:
        total_seconds = 1

    print ("\nComputing histograms...")

    # Compute histograms
    all_hists = {}
    hist = np.histogram(area,nbins)
    all_hists['area'] = json.dumps(lzip(hist[1].tolist(),hist[0].tolist()))
    hist = np.histogram(maj_len,nbins)
    all_hists['major_axis_length'] = json.dumps(lzip(hist[1].tolist(),hist[0].tolist()))
    hist = np.histogram(min_len,nbins)
    all_hists['minor_axis_length'] = json.dumps(lzip(hist[1].tolist(),hist[0].tolist()))
    hist = np.histogram(aspect_ratio,nbins)
    all_hists['aspect_ratio'] = json.dumps(lzip(hist[1].tolist(),hist[0].tolist()))
    hist = np.histogram(elapsed_seconds,np.uint32(total_seconds))
    all_hists['elapsed_seconds'] = json.dumps(lzip(hist[1].tolist(),hist[0].tolist()))
    hist = np.histogram(orientation,nbins)
    all_hists['orientation'] = json.dumps(lzip(hist[1].tolist(),hist[0].tolist()))
    hist = np.histogram(file_size,nbins)

    print ("\nComputing stats...")

    all_hists['file_size'] = json.dumps(lzip(hist[1].tolist(),hist[0].tolist()))
    # Compute general stats from features
    all_stats = {}
    all_stats['area'] = stats.describe(area)
    all_stats['major_axis_length'] = stats.describe(maj_len)
    all_stats['minor_axis_length'] = stats.describe(min_len)
    all_stats['aspect_ratio'] = stats.describe(aspect_ratio)
    all_stats['elapsed_seconds'] = stats.describe(elapsed_seconds)
    all_stats['orientation'] = stats.describe(orientation)
    all_stats['file_size'] = stats.describe(file_size)


    # print ("Exporting spreadsheet results...")

    # df = pandas.DataFrame(entry_list_scaled)
    # df.to_csv(os.path.join(subdir,'features.tsv'), index=False, sep='\t')

    print("Building web app...")

    # Load html template for rendering
    template = ""
    with open(os.path.join('app', 'index.html'), "r") as fconv:
        template = fconv.read()

    # Define the render context from the processed histograms, images, and stats
    context = {}
    context['version'] = '0.2'
    context['total_images'] = total_images
    context['proc_time'] = proc_time
    context['duration'] = total_seconds
    context['compression_ratio'] = int((1000.0*24*total_images)/np.sum(file_size))
    context['rois_per_second'] = total_images/context['duration']
    context['kb_per_second'] = int(np.sum(file_size)/context['duration'])
    context['recording_started'] = datetime.datetime.fromtimestamp(np.min(unixtime)).strftime('%Y-%m-%d %H:%M:%S')
    context['app_title'] = "EyeRIS Convert: " + base_dir_name
    context['dir_name'] = base_dir_name
    context['raw_color'] = raw_color
    context['image_res'] = image_res
    context['really_big'] = "{:0.3f}".format(10) + " mm"
    context['big'] = "{:0.3f}".format(5) + " mm"
    context['small'] = "{:0.3f}".format(5) + " mm"
    context['really_small'] = "{:0.3f}".format(2) + " mm"
    if use_jpeg:
        context['image_ext'] = '.jpeg'
    else:
        context['image_ext'] = '.png'
    context['stats_names'] = [{"name":"Min"}, {"name":"Max"}, {"name":"Mean"}, {"name":"Standard Deviation"}, {"name":"Skewness"}, {"name":"Kurtosis"}]

        # definie the charts to display from the histogram data
    charts = []
    for chart_name, data_values in all_hists.items():
        chart = {}
        chart['source'] = 'js/' + chart_name + '.js'
        chart['name'] = chart_name
        units = ""
        if chart_name == 'area':
            units = " (mm*mm)"
        if chart_name == 'major_axis_length' or chart_name == 'minor_axis_length':
            units = " (mm)"
        if chart_name == 'file_size':
            units = " (kB)"
        if chart_name == 'elapsed_seconds':
            units = " (s)"
        if chart_name == 'orientation':
            units = " (deg)"
        chart['title'] = 'Histogram of ' + chart_name + units
        chart['x_title'] = chart_name + units
        chart['y_title'] = 'counts'
        chart['stats_title'] = chart_name
        chart['data'] = data_values
        chart['stats'] = []
        chart['stats'].append({"name":"Min", "value":"{:10.3f}".format(all_stats[chart_name][1][0])})
        chart['stats'].append({"name":"Max", "value":"{:10.3f}".format(all_stats[chart_name][1][1])})
        chart['stats'].append({"name":"Mean", "value":"{:10.3f}".format(all_stats[chart_name][2])})
        chart['stats'].append({"name":"Standard Deviation", "value":"{:10.3f}".format(math.sqrt(all_stats[chart_name][3]))})
        chart['stats'].append({"name":"Skewness", "value":"{:10.3f}".format(all_stats[chart_name][4])})
        chart['stats'].append({"name":"Kurtosis", "value":"{:10.3f}".format(all_stats[chart_name][5])})
        charts.append(chart)

    context['charts'] = charts
    
    # render the html page and save to disk
    page = pystache.render(template, context)

    with open(os.path.join(subdir, 'eyerisdata.html'), "w") as fconv:
        fconv.write(page)

    # remove any old app files and try to copy over new ones
    try:
        shutil.rmtree(os.path.join(subdir, "css"), ignore_errors=True)
        shutil.copytree("app/css", os.path.join(subdir, "css"))
        shutil.rmtree(os.path.join(subdir, "js"), ignore_errors=True)
        shutil.copytree("app/js", os.path.join(subdir, "js"))
    except:
        print ("Error copying supporting files for html.")

    # Load roistore.js database for rendering
    template = ""
    with open(os.path.join('app', 'js', 'database-template.js'), "r") as fconv:
        template = fconv.read()

    context = {}
    context['image_items'] = entry_list
    context['table'] = base_dir_name

    # render the javascript page and save to disk
    page = pystache.render(template, context)

    with open(os.path.join(subdir, 'js', 'database.js'), "w") as fconv:
        fconv.write(page)

    print ("Done.")
    
def valid_image_dir(test_path, image_pattern):

    list = glob.glob(os.path.join(test_path, image_pattern))
    
    if len(list) > 0:
        return True
    else:  
        return False
    
    
# Module multiprocessing is organized differently in Python 3.4+
try:
    # Python 3.4+
    if sys.platform.startswith('win'):
        import multiprocessing.popen_spawn_win32 as forking
    else:
        import multiprocessing.popen_fork as forking
except ImportError:
    import multiprocessing.forking as forking

if sys.platform.startswith('win'):
    # First define a modified version of Popen.
    class _Popen(forking.Popen):
        def __init__(self, *args, **kw):
            if hasattr(sys, 'frozen'):
                # We have to set original _MEIPASS2 value from sys._MEIPASS
                # to get --onefile mode working.
                os.putenv('_MEIPASS2', sys._MEIPASS)
            try:
                super(_Popen, self).__init__(*args, **kw)
            finally:
                if hasattr(sys, 'frozen'):
                    # On some platforms (e.g. AIX) 'os.unsetenv()' is not
                    # available. In those cases we cannot delete the variable
                    # but only set it to the empty string. The bootloader
                    # can handle this case.
                    if hasattr(os, 'unsetenv'):
                        os.unsetenv('_MEIPASS2')
                    else:
                        os.putenv('_MEIPASS2', '')

    # Second override 'Popen' class with our modified version.
    forking.Popen = _Popen

if __name__ == '__main__':

    multiprocessing.freeze_support()

    if len(sys.argv) < 3:
        print("Usage: python BuildWebApp.py [settings_path] [data_path]")
    else:
        data_path = sys.argv[2]
        settings_path = sys.argv[1]

        # load the config file
        cfg = xmlsettings.XMLSettings(settings_path)

        combine_subdirs = cfg.get('MergeSubDirs', "False").lower() == "true"

        image_pattern = "*." + cfg.get("ImageExt", "tif")

        print("Settings file: " + settings_path)

        # If file given try to unpack
        if os.path.isfile(data_path):
            extracted_path = data_path + "_unpacked"
            with tarfile.open(data_path) as archive:
                archive.extractall(path=extracted_path)
            data_path = extracted_path
            to_clean = extracted_path

        # If given directory is a single data directory, just process it
        if valid_image_dir(data_path, image_pattern):
            run(data_path, cfg)
            sys.exit(0)

        # Otherwise look for data directories in the given directory
        if combine_subdirs:
            run(data_path, cfg)
        else:
            # List data directories and process each one
            # expect the directories to be in the unixtime format
            directory_list = sorted(glob.glob(os.path.join(data_path, "[0-9]" * cfg.get("SubdirZeros", 10))))
            # Process the data directories in order
            print('Processing each data directory...')
            for directory in directory_list:
                if valid_image_dir(directory, image_pattern):
                    run(directory, cfg)