#!/usr/bin/env python3 import os from shutil import copyfile import pandas as pd from scripts.syslog_tools_vis import VisTool class SysIO(object): def __init__(self): super().__init__() self.vis = VisTool() # define path to data folders self.csv_path = './data/csv' self.json_path = './data/json' self.fig_path = './data/fig' self.cp_csv_path = None self.cp_json_path = None # enable sync to /var/www/html/ data folders if executed from tethysdata self.sync = False # change to True to enable this functionality if 'tethysdata' in os.uname()[1]: self.csv_path = '/home/tethysadmin/lrauv-tools/handle-lrauv-logs/lrauv-log-critical/data/csv' self.json_path = '/home/tethysadmin/lrauv-tools/handle-lrauv-logs/lrauv-log-critical/data/json' self.fig_path = '/home/tethysadmin/lrauv-tools/handle-lrauv-logs/lrauv-log-critical/data/fig' self.cp_json_path = '/var/www/html/reliability/data/json' self.cp_csv_path = '/var/www/html/reliability/data/csv' self.cp_fig_path = '/var/www/html/reliability/data/fig' # define data-types for loading DataFrame columns (automates conversions) self.dtype = { # note: pandas DataFrames treat strings as objects 'comp': 'object', 'deployment': 'object', 'deployment_name': 'object', 'log': 'object', 'msg': 'object', 'vehicle': 'object', 'year': 'object', 'fail_count': 'int', 'tbcf': 'timedelta64[ns]', 'mtbcf': 'timedelta64[ns]', 'runtime': 'timedelta64[ns]', 'timestamp': 'datetime64[ns]', } @staticmethod def hr2td(t): return pd.to_timedelta(t, unit='ns') @staticmethod def td2hr(t): return t.total_seconds() / 3600 def set_sync(self, sync_): self.sync = sync_ # I/O # ------------------------------------------------------------------------------------------------- def init_dataframes(self): """Initialize the pandas DataFrames. :return: empty DataFrames: logs, cr, cr_all """ # define column names logs_col = ['vehicle', 'year', 'deployment_name', 'deployment', 'log', 'runtime', 'fail_count', 'mtbcf'] cr_col = logs_col[:-1] + ['comp', 'timestamp', 'msg', 'tbcf'] # initialize DataFrames logs = pd.DataFrame(columns=sorted(logs_col)) cr = pd.DataFrame(columns=sorted(cr_col)) cr_all = pd.DataFrame(columns=sorted(cr_col[:-1])) return logs, cr, cr_all def load_json(self, vehicle): """Load data from json to pandas DataFrames. :param vehicle: working vehicle name (string) :return: loaded DataFrames: logs, cr, cr_all """ # assemble path to json folder load_path = os.path.join(self.json_path, vehicle) # read-in json files to pandas DataFrame logs = pd.read_json(load_path + '_logs.json', dtype=self.dtype, orient='records') cr = pd.read_json(load_path + '_critical.json', dtype=self.dtype, orient='records') cr_all = pd.read_json(load_path + '_critical_all.json', dtype=self.dtype, orient='records') # convert columns containing float variables with NaNs to Timedelta logs['mtbcf'] = logs['mtbcf'].apply(self.hr2td) return logs, cr, cr_all def export_json(self, logs, cr, cr_all, vehicle): """Write data from pandas DataFrames to json. :param logs: DataFrame to export :param cr: DataFrame to export :param cr_all: DataFrame to export :param vehicle: vehicle name (string) :return: N/A """ # assemble path to json folder (create if needed) export_path = os.path.join(self.json_path, vehicle) if not os.path.exists(self.json_path): os.makedirs(self.json_path) # export to json logs.to_json(export_path + '_logs.json', date_unit='ns', orient='records') cr.to_json(export_path + '_critical.json', date_unit='ns', orient='records') cr_all.to_json(export_path + '_critical_all.json', date_unit='ns', orient='records') # visualize data self.vis.generate_plot(logs.copy(), cr.copy(), vehicle) self.vis.export_fig(vehicle, self.fig_path) # sync json files to /var/www/html/ data folders (tethysdata only) if self.cp_json_path and self.sync: cp_path_end = ['_logs.json', '_critical.json', '_critical_all.json'] for p in cp_path_end: copyfile(export_path + p, self.cp_json_path + '/' + vehicle + p) # export figure to /var/www/html/ data folder self.vis.export_fig(vehicle, self.cp_fig_path) def export_csv(self, logs, cr, cr_all, vehicle, yr=''): """Write data from pandas DataFrames to .csv files. :param logs: DataFrame to export :param cr: DataFrame to export :param cr_all: DataFrame to export :param vehicle: vehicle name (string) :param yr: year (string) :return: N/A """ # assemble path to csv folder (create if needed) export_path = os.path.join(self.csv_path, vehicle, yr) if not os.path.exists(export_path): os.makedirs(export_path) # assemble file paths ex_path = [] path_end = ['_logs.csv', '_critical.csv', '_critical_all.csv'] for p in path_end: ex_path.append(os.path.join(export_path, vehicle + '_' + yr + p)) # define columns to be written (and their order) logs_col = ['vehicle', 'year', 'deployment_name', 'deployment', 'log', 'runtime', 'fail_count', 'mtbcf'] cr_col = logs_col[:-3] + ['comp', 'timestamp', 'tbcf', 'msg'] cr_all_col = logs_col[:-3] + ['comp', 'timestamp', 'msg'] # re-order DataFrame columns (make sync's to avoid altering original df) cr = cr[cr_col].copy() logs = logs[logs_col].copy() cr_all = cr_all[cr_all_col].copy() # convert columns containing Timedelta variables to hours cr['tbcf'] = cr['tbcf'].fillna(pd.NaT).apply(self.td2hr) logs['mtbcf'] = logs['mtbcf'].fillna(pd.NaT).apply(self.td2hr) logs['runtime'] = logs['runtime'].fillna(pd.NaT).apply(self.td2hr) # write csv logs[logs['year'].str.contains(yr)].to_csv(ex_path[0], index=False) cr[cr['year'].str.contains(yr)].to_csv(ex_path[1], index=False) cr_all[cr_all['year'].str.contains(yr)].to_csv(ex_path[2], index=False) # sync csv files to /var/www/html/ data folders (tethysdata only) if self.cp_csv_path and self.sync: # assemble file paths (create if needed) cp_path = os.path.join(self.cp_csv_path, vehicle, yr) if not os.path.exists(cp_path): os.makedirs(cp_path) # sync files for p in path_end: ex_path = os.path.join(export_path, vehicle + '_' + yr + p) cp_path_out = os.path.join(cp_path, vehicle + '_' + yr + p) copyfile(ex_path, cp_path_out)