#!/usr/bin/env python3 """lrauv-data-file-audit.py Retrieve and process reliability information from LRAUV syslogs. - walk server and locate all .dlist files - parse .dlist - get deployment/log names - identify missing .nc4 files - identify follow-on 'sci' and 'eng' .nc files that need to be regenerated Mike McCann, MBARI, 18 December 2018 """ import argparse import logging import os import re import sys from collections import namedtuple from glob import glob from scripts.syslog_tools import SysTools logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', handlers=[logging.StreamHandler(sys.stdout)]) logger = logging.getLogger(__name__) report = namedtuple('Report', 'dlist errors') audit_report = {} nc4_codes = { 'zero_sized_syslog': 'z', 'missing': 'x', 'present': '.', } scieng_codes = { 'missing_no_known_reason': 'x', 'missing_no_variables': 'v', 'missing_no_time_time': 't', 'log_file_missing': 'l', 'present': '.', } def parse_command_line(): parser = argparse.ArgumentParser(description='Audit LRAUV processed data files.') parser.add_argument("-p", "--path", default='/mbari/LRAUV', type=str, help="set path to LRAUV data root") parser.add_argument("-v", "--verbose", action="store_true", default=False, help="enable output verbosity") parser.add_argument("-f", "--force", action="store_true", default=False, help="force fresh scan") parser.add_argument("-vh", "--vehicle", default='tethys', help="vehicle to process") parser.add_argument("-i", "--ignore", default=['latest', 'lab', 'tank', 'tow', 'battery', 'cal', 'none', 'logging'], nargs='+', help="list of deployment name key-words to ignore") args = parser.parse_args() return args def print_legend(): logger.info('Starting syslog scan for LRAUV %s:' % args.vehicle) print(' .nc4 file codes:') for k,v in nc4_codes.items(): print(' {} {}'.format(v, k)) print(' scieng file codes:') for k,v in scieng_codes.items(): print(' {} {}'.format(v, k)) def nc4_file_state(log_path): '''Check other contents in the log_path to test whether there really should be a .nc4 file Return character code indicating presence or reason why not. ''' try: if os.stat(os.path.join(log_path, 'syslog')).st_size: return nc4_codes['present'] else: return nc4_codes['zero_sized_syslog'] except FileNotFoundError: return nc4_codes['missing'] def scieng_file_state(log_path, log_file): '''Check other contents in the log_path to test whether there really should be a scieng.nc file. Return character code indicating presence or reason why not. ''' not_creating_line = "ERROR .* Not creating" no_start_and_end = "WARNING .* Can't get start and end date from .nc4" try: with open(log_file) as fp: for line in fp.readlines(): if re.match(not_creating_line, line): # Likely no variables available in .nc4 to produce the scieng.no file return scieng_codes['missing_no_variables'] if re.match(no_start_and_end, line): # Likely no time_time variable in the scieng.nc file return scieng_codes['missing_no_time_time'] return scieng_codes['present'] except FileNotFoundError: return scieng_codes['log_file_missing'] def check_logs(depl_name, dlist_path, log_dirs, file_type): '''Loop through the log directories and print out success or fail indicators ''' print(' {}: '.format(file_type), end='') print(' ' * (10 - len(file_type)), end='') error_messages = '' for log in log_dirs: log_path = os.path.join(dlist_path.split('.dlist')[0], log) if not os.path.isdir(log_path): msg = " Directory does not exist: {}\n".format(log_path) error_messages += msg else: files = glob(os.path.join(log_path, '*{}'.format(file_type))) if files: file = files[0] print('.', end='') else: if 'nc4' in file_type: nc4_code = nc4_file_state(log_path) print(nc4_code, end='') if nc4_code == nc4_codes['missing']: msg = " No {} file in {}\n".format(file_type, log_path) error_messages += msg elif 'scieng' in file_type: scieng_log_files = glob(os.path.join(log_path, '*scieng.log')) if scieng_log_files: scieng_code = scieng_file_state(log_path, scieng_log_files[0]) print(scieng_code, end='') if (scieng_code == scieng_codes['missing_no_known_reason'] and nc4_code == nc4_code['present']): msg = " No {} file in {}\n".format(file_type, log_path) error_messages += msg else: if os.stat(os.path.join(log_path, 'syslog')).st_size: msg = " 2S_scieng.log file missing: {}\n".format(log_path) error_messages += msg print('') if error_messages: audit_report[depl_name] = report(dlist_path, error_messages) def crawl_dirs(args): stl = SysTools() stl.walkserver(args.path, args.vehicle) folder_name = os.path.join(args.path, args.vehicle, "missionlogs") for yr in sorted(stl.deployments[args.vehicle].keys()): if stl.deployments[args.vehicle][yr]: for deployment in sorted(stl.deployments[args.vehicle][yr]): dlist_path = os.path.join(folder_name, yr, deployment) stl.clear_dlist() stl.parse_dlist(dlist_path) if not any(i in stl.dlist['name'].lower() for i in args.ignore): depl_name = stl.dlist['name'] log_dirs = sorted(stl.dlist['log']) logger.info("Deployment {}:".format(depl_name)) print(' {} - {} log dirs'.format(dlist_path, len(log_dirs))) check_logs(depl_name, dlist_path, log_dirs, '.nc4') check_logs(depl_name, dlist_path, log_dirs, 'scieng.nc') def print_report(args): '''Print details of things that need fixing ''' logger.info('-----------------------------------------------------') logger.info('Detailed data file audit for LRAUV {}:'.format(args.vehicle)) sorted_report = sorted(audit_report.items(), key=lambda kv: kv[1].dlist) for depl_name, report in sorted_report: print('{}:'.format(depl_name)) print(' {}'.format(report.dlist)) print('{}'.format(report.errors)) logger.info('Completed data file audit for LRAUV %s.' % args.vehicle) if __name__ == '__main__': args = parse_command_line() print_legend() crawl_dirs(args) print_report(args)