"""These functions create .json and .kml files from CPF SBD files. AUTHOR : Brian Ha (summer intern) MENTOR : Gene Massion MBARI TEAM : Chemical Sensor Group PROJECT : Coastal Profiling Floats (CPF) DATE CREATED : 2018-06-06 LAST REVISION : 2018-07-18 """ # Python Standard Library Imports # ============================================================================= import os import csv import json import shutil import logging # 3rd Party Library Imports # ============================================================================= import simplekml # Constants # ============================================================================= from cpf_constants import CPF_IDs, CPF_SBD_FORMAT_VERSION_NUMBERS from cpf_constants import SBD_SOURCE_DIRECTORY, SBD_DESTINATION_DIRECTORY from cpf_constants import SBD_DUPLICATES_DIRECTORY from cpf_constants import JSON_DIRECTORY, KML_DIRECTORY from cpf_constants import LOG_DIRECTORY, PROCESSOR_LOG_FILE_NAME from cpf_constants import CPF_PNG_PATH from cpf_constants import COORDINATE_PRECISION, SURFACE_DEPTH # Settings # ============================================================================= # If a "LOG_DIRECTORY" folder doesn't already exist, create a new one if not os.path.exists(LOG_DIRECTORY): os.makedirs(LOG_DIRECTORY) logging.basicConfig(filename=LOG_DIRECTORY+PROCESSOR_LOG_FILE_NAME, level=logging.WARNING, format="%(asctime)s:%(levelname)s:%(message)s") # Functions # ============================================================================= def set_sbd_format_v1_constants(): '''Use these constants if the SBD format version number is 1.''' # Declare engineering constants as global global TIMESTAMP_INDEX global LAT_INDEX global LAT_CARDINAL_DIR_INDEX global LONG_INDEX global LONG_CARDINAL_DIR_INDEX global NUM_SAT_VEHIC_INDEX global ACK_INDEX global SIGNAL_QUAL_INDEX global MO_STATUS_INDEX global MISSION_TIMEOUT_INDEX global PARK_TIME_INDEX global PARK_PRES_INDEX global PROFILE_NUM_INDEX global MAX_PRES_INDEX global RECOV_MODE_INDEX global BATT_BUS_VOLT_INDEX global CAN_PRES_INDEX global CAN_HUMID_INDEX # Declare scientific constants as global global SCI_PAYLOAD_STARTING_INDEX global SCI_PRES_INDEX global SCI_TEMP_INDEX global SCI_SALIN_INDEX global SCI_NUM_SCANS_INDEX global SCI_END_INDEX global HEX_DATA_LENGTH # Declare misc. SBD constants as global global ACK_MESSAGE global SBD_DELIMITER global COORDINATE_PRECISION # Assign engineering constants TIMESTAMP_INDEX = 0 LAT_INDEX = 2 LAT_CARDINAL_DIR_INDEX = 3 LONG_INDEX = 5 LONG_CARDINAL_DIR_INDEX = 6 NUM_SAT_VEHIC_INDEX = 8 ACK_INDEX = 9 SIGNAL_QUAL_INDEX = 11 MO_STATUS_INDEX = 13 MISSION_TIMEOUT_INDEX = 16 PARK_TIME_INDEX = 18 PARK_PRES_INDEX = 20 PROFILE_NUM_INDEX = 22 MAX_PRES_INDEX = 24 RECOV_MODE_INDEX = 26 BATT_BUS_VOLT_INDEX = 28 CAN_PRES_INDEX = 30 CAN_HUMID_INDEX = 32 # Assign scientific hex data constants SCI_PAYLOAD_STARTING_INDEX = 35 SCI_PRES_INDEX = 0 SCI_TEMP_INDEX = 4 SCI_SALIN_INDEX = 8 SCI_NUM_SCANS_INDEX = 12 SCI_END_INDEX = 14 HEX_DATA_LENGTH = 14 # Assign misc. SBD constants ACK_MESSAGE = 'ACK SBD' SBD_DELIMITER = ',' def get_json_file_path(imei_num): '''Returns the JSON file path for a CPF based on its IMEI number.''' cpf_id = CPF_IDs[int(imei_num)] json_file_path = JSON_DIRECTORY + 'CPF_' + str(cpf_id).zfill(3) + '.json' return json_file_path def create_template_json_file(json_file_path): ''' Creates a new JSON file (without data) for a CPF.''' sbd_data = { # Engineering Scalar Quantities (one measurement per profile) "Timestamps" : [], "Latitudes_[dec_deg]" : [], "Longitudes_[dec_deg]" : [], "Num_Satellite_Vehicles" : [], "Signal_Quality" : [], "Mobile_Originated_Status" : [], "Mission_Timeout" : [], "Park_Time" : [], "Park_Pressure_[decibar]" : [], "Profile_Numbers" : [], "Max_Pressure_[decibar]" : [], "Recovery_Mode" : [], "Battery_Bus_Voltage_[volts]" : [], "Can_Pressure_[millibar]" : [], "Can_Humidity_[rel_%]" : [], # Scientific Vector Quantities (multiple measurements per profile) "Pressure_[decibar]" : [], "Temperature_[deg_C]" : [], "Salinity_[ppt]" : [], "Num_Scans" : [], # Misc. Info "ACKs" : [] } with open(json_file_path, 'w') as outfile: json.dump(sbd_data, outfile) def deg_dec_min_to_dec_deg(deg_dec_min, cardinal_direction): '''Converts coordinates from degree decimal minutes to decimal degrees.''' # Sanitize input deg_dec_min = str(deg_dec_min) # Find the decimal point try: decimal_point_index = deg_dec_min.index('.') except ValueError: raise ValueError("ERROR: The coordinate '%s' must contain a " \ "decimal point. Check the SBD." % deg_dec_min) # Determine the minute index minute_index = decimal_point_index - 2 # Convert to decimal degrees deg = int(deg_dec_min[0:minute_index]) dec_min = float(deg_dec_min[minute_index:]) dec_deg = round((deg + dec_min/60.0), COORDINATE_PRECISION) # Multiply by -1 if the coordinate is to the south or west if cardinal_direction.upper() in ['S', 'W']: dec_deg *= -1 return dec_deg def append_new_sbd_data(json_data, sbd_data): '''Appends raw SBD data to the JSON data dictionary for a single CPF.''' # Check if an "ACK SBD" was received. # If so, append the whole SBD string within the "ACK" field and return. if sbd_data[ACK_INDEX].upper() == ACK_MESSAGE: json_data['ACKs'].append(str(sbd_data)) return json_data # Convert latitude and longitude measurements lat_dec_deg = deg_dec_min_to_dec_deg(sbd_data[LAT_INDEX], sbd_data[LAT_CARDINAL_DIR_INDEX]) long_dec_deg = deg_dec_min_to_dec_deg(sbd_data[LONG_INDEX], sbd_data[LONG_CARDINAL_DIR_INDEX]) # Append Scalar Engineering Quantities json_data['Timestamps'].append(sbd_data[TIMESTAMP_INDEX]) json_data['Latitudes_[dec_deg]'].append(lat_dec_deg) json_data['Longitudes_[dec_deg]'].append(long_dec_deg) json_data['Num_Satellite_Vehicles'].append(int(sbd_data[NUM_SAT_VEHIC_INDEX])) json_data['Signal_Quality'].append(int(sbd_data[SIGNAL_QUAL_INDEX])) json_data['Mobile_Originated_Status'].append(sbd_data[MO_STATUS_INDEX]) json_data['Mission_Timeout'].append(sbd_data[MISSION_TIMEOUT_INDEX]) json_data['Park_Time'].append(sbd_data[PARK_TIME_INDEX]) json_data['Park_Pressure_[decibar]'].append(float(sbd_data[PARK_PRES_INDEX])) json_data['Profile_Numbers'].append(sbd_data[PROFILE_NUM_INDEX]) json_data['Max_Pressure_[decibar]'].append(float(sbd_data[MAX_PRES_INDEX])) if sbd_data[RECOV_MODE_INDEX] == 'True': json_data['Recovery_Mode'].append(True) else: json_data['Recovery_Mode'].append(False) json_data['Battery_Bus_Voltage_[volts]'].append(float(sbd_data[BATT_BUS_VOLT_INDEX])) json_data['Can_Pressure_[millibar]'].append(float(sbd_data[CAN_PRES_INDEX])) json_data['Can_Humidity_[rel_%]'].append(float(sbd_data[CAN_HUMID_INDEX])) # Vector Quantities json_data['Pressure_[decibar]'].append([]) json_data['Temperature_[deg_C]'].append([]) json_data['Salinity_[ppt]'].append([]) json_data['Num_Scans'].append([]) science_payload = sbd_data[SCI_PAYLOAD_STARTING_INDEX:] for hex_data in science_payload: if len(hex_data) != HEX_DATA_LENGTH: continue pressure_hex = hex_data[SCI_PRES_INDEX:SCI_TEMP_INDEX] temperature_hex = hex_data[SCI_TEMP_INDEX:SCI_SALIN_INDEX] salinity_hex = hex_data[SCI_SALIN_INDEX:SCI_NUM_SCANS_INDEX] num_scans_hex = hex_data[SCI_NUM_SCANS_INDEX:SCI_END_INDEX] pressure_decimal = int(pressure_hex, 16)/10.0 temperature_decimal = int(temperature_hex, 16)/1000.0 salinity_decimal = int(salinity_hex, 16)/1000.0 num_scans_decimal = int(num_scans_hex, 16) # The CPF sometimes reports erroneous scientific data. # Recall, CPF scientific data is reported backwards (i.e. the first # value represents the end of the profile and the last value # represents the start of the profile. If the CPF reports a pressure # measurement that is less than the previous measurement, the hex measurement # will be ignored. Decision made 2018-07-16 by Brian Ha and Gene Massion. if len(json_data['Pressure_[decibar]'][-1]) == 0: pass elif pressure_decimal < json_data['Pressure_[decibar]'][-1][-1]: continue json_data['Pressure_[decibar]'][-1].append(pressure_decimal) json_data['Temperature_[deg_C]'][-1].append(temperature_decimal) json_data['Salinity_[ppt]'][-1].append(salinity_decimal) json_data['Num_Scans'][-1].append(num_scans_decimal) return json_data def update_jsons(): '''Processes new SBDs for all CPFs.''' # Create a list of SBD files in the SBD_SOURCE_DIRECTORY sbd_files = [fname for fname in os.listdir(SBD_SOURCE_DIRECTORY) if fname.endswith(".sbd")] # If there are no SBD files to process, quit. if len(sbd_files) < 1: kml_update_needed = False return kml_update_needed # If there are multiple SBD files to process, sort them numerically. sbd_files.sort() # For each SBD file... for sbd_file_name in sbd_files: imei_num, sbd_num = sbd_file_name.split('_') # Determine SBD format version number cpf_id = CPF_IDs[int(imei_num)] sbd_format_version_num = CPF_SBD_FORMAT_VERSION_NUMBERS[cpf_id] # Use constants associated with SBD format version if sbd_format_version_num == 1: set_sbd_format_v1_constants() # Open the SBD as a CSV. Parse the SBD string. sbd_file_path = "{}{}".format(SBD_SOURCE_DIRECTORY, sbd_file_name) with open(sbd_file_path, newline='') as csvfile: sbd_data = csv.reader(csvfile, delimiter=SBD_DELIMITER).__next__() # If a JSON_DIRECTORY folder doesn't exist, create a new one. if not os.path.exists(JSON_DIRECTORY): os.makedirs(JSON_DIRECTORY) # Determine which CPF .json to update based on the CPF's IMEI number json_file_path = get_json_file_path(imei_num) # If a .json file doesn't already exist for this CPF, create a new one. if not os.path.exists(json_file_path): create_template_json_file(json_file_path) # Open the CPF .json file with open(json_file_path, 'r') as infile: raw_string = infile.read() json_data = json.loads(raw_string) # Check if the SBD's timestamp matches a previously processed SBD. # If so, the SBD is assumed to be a duplicate. # It is transferred to a separate directory and logged. sbd_timestamp = sbd_data[TIMESTAMP_INDEX] if sbd_timestamp not in json_data['Timestamps']: # Append SBD data from the latest profile to .json file new_json_data = append_new_sbd_data(json_data, sbd_data) # Save updated .json file with open(json_file_path, 'w') as outfile: json.dump(new_json_data, outfile) # If a SBD_DESTINATION_DIRECTORY doesn't exist, create one. if not os.path.exists(SBD_DESTINATION_DIRECTORY): os.makedirs(SBD_DESTINATION_DIRECTORY) # Move the SBD file to the SBD_DESTINATION_DIRECTORY. sbd_dest_path = "{}{}".format(SBD_DESTINATION_DIRECTORY, sbd_file_name) shutil.move(sbd_file_path, sbd_dest_path) else: # If a SBD_DUPLICATES_DIRECTORY doesn't exist, create one. if not os.path.exists(SBD_DUPLICATES_DIRECTORY): os.makedirs(SBD_DUPLICATES_DIRECTORY) # Move the SBD file to the SBD_DUPLICATES_DIRECTORY. sbd_duplicates_path = "{}{}".format(SBD_DUPLICATES_DIRECTORY, sbd_file_name) shutil.move(sbd_file_path, sbd_duplicates_path) # Log the occurrence logging.warning('"{}" is a duplicate. It was moved to {}'.format(sbd_file_name, SBD_DUPLICATES_DIRECTORY)) # Return true to indicate that the KML files need to be updated kml_update_needed = True return kml_update_needed def update_kmls(): '''Creates a new KML file for each CPF based on each CPF's JSON file. ''' style = simplekml.Style() style.iconstyle.icon.href = CPF_PNG_PATH style.iconstyle.scale = 3 cpf_imeis = list(CPF_IDs.keys()) for imei_num in cpf_imeis: cpf_id = CPF_IDs[imei_num] # Try to read CPF JSON file try: json_file_path = get_json_file_path(imei_num) with open(json_file_path, 'r') as infile: raw_string = infile.read() json_data = json.loads(raw_string) except FileNotFoundError: #print('WARNING: .json not found for CPF-%i. ' \ # 'Skipping to next CPF.' % cpf_id) continue # Store timestamp data when = json_data["Timestamps"] # Format coordinate data coordinates = [] for index in range(len(when)): long = json_data['Longitudes_[dec_deg]'][index] lat = json_data['Latitudes_[dec_deg]'][index] coordinates.append((long, lat , SURFACE_DEPTH)) # Create new KML document kml = simplekml.Kml() # Create new KML folder element fol = kml.newfolder() # Create new KML gx:Track trk = fol.newgxtrack(name='CPF-%i' % cpf_id) # Set gx:altitudeMode to "absolute" trk.altitudemode = 'absolute' # Add timestamps to the gx:Track trk.newwhen(when) # Add coordinates to the gx:Track trk.newgxcoord(coordinates) # Set gx:angles to 0.0 angles = [0.0] * len(when) trk.newgxangle(str(angles)) # Make the track icon a CPF float PNG. trk.style = style # Add KML pins representing each CPF surfacing profile_numbers = json_data['Profile_Numbers'] for index in range(len(coordinates)): if index == 0: pnt = fol.newpoint(name='Deploy') else: profile_number, _, _ = profile_numbers[index].split(' ') pnt = fol.newpoint(name='P-%s' % profile_number) pnt.coords = [coordinates[index]] # If a "CPF_KMLs" folder doesn't already exist, create a new one if not os.path.exists(KML_DIRECTORY): os.makedirs(KML_DIRECTORY) # Save the new KML file kml_file_path = KML_DIRECTORY + 'CPF_' + str(cpf_id).zfill(3) + '.kml' kml.save(kml_file_path)