//========================================================================= // Summary : Supervisor Component // Filename : Supervisor.cc // Author : Tom O'Reilly // Project : */ // Revision : 1 // Created : 2000/08/19 // Modified : 2000/08/19 //========================================================================= // Description : //========================================================================= #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "Supervisor.h" #include "SupervisorIF.h" #include "TaskNames.h" #include "VehicleConfigurator.h" #include "VehicleConfigurationIF.h" #include "LayeredControlIF.h" #include "ThalesPowerSystemIF.h" #include "ExternalCommsIF.h" #include "NavigationIF.h" #include "Syslog.h" #include "System.h" #include "StringConverter.h" #include "Profiler.h" #include #include "DeviceIF.h" #include "RemoteLogIF.h" #include "RemoteAgentIF.h" #define MaxExecArgs 10 #define KILLMISSION_TIMEOUT 5 Supervisor *Supervisor::_theSupervisor = 0; Supervisor::Supervisor(const char *umbrellaLogDirName, Boolean simulated, Boolean realTailCone, const char *planFileName, const char *abortFileName, char **env /* = NULL */) { Boolean debug = True; char Nodes[10] = { 0,0,0,0,0,0,0,0,0,0 }; int numNodes; #ifdef USE_WD _watchdog = NULL; _watchdog = new AmproCommand( MessageQueue:Write ); AmproCommand::Command cmd; cmd.cmd = AmproCommand::SetWatchdog; cmd.secs = 30; // Nice generous margin _watchdog->write( &cmd ); #endif strcpy(_planFileName,""); strcpy(_abortFileName,""); strcpy(_umbrellaLogDirName, umbrellaLogDirName); _output = NULL; _command = NULL; _power = NULL; _useNodeTwo = True; // Keep track of the most recently created Supervisor for cleanup // on signal/exit _theSupervisor = this; _env=env; _simulated = simulated; _realTailCone = realTailCone; _attemptProcRestart = True; _layeredControl = NULL; _rem = NULL; _stopSupervisor = false; for (int i=0; i<100; i++) _faultCount[i] = 0; dprintf("Setting signal handlers."); signal(SIGINT, Supervisor::signalHandler); signal(SIGTERM, Supervisor::signalHandler); signal(SIGQUIT, Supervisor::signalHandler); // We don't wait for children to die, and we don't // want any zombies either... // signal delete serverIF;(SIGCHLD, Supervisor::signalChildHandler); signal(SIGCHLD, SIG_IGN); dprintf("Clearing task lists."); _serverTasks.clear(); _driverTasks.clear(); dprintf("Setting up output and command structures."); // Clear out the output shared mem (assume it's been cleared on constr.) _output = new SupervisorOutput( SharedData::ReadWrite, False ); _output->write(); _command = new SupervisorCommand( MessageQueue::ReadWrite ); _command->flush(); //_input.flush(); _missionState = NotRunning; _killMissionSent = 0; strcpy( _output->data.directoryName, _umbrellaLogDirName); strcat( _output->data.directoryName, "/inter-mission"); _output->data.isMissionRunning = False; _output->write(); dprintf("Checking node status."); if ( getnid() != 1 ) { Syslog::write("Supervisor should only be run on node 1."); stopSupervisor(1); } _useNodeTwo = System::checkNode( 2 ); // create interface to vehicleConfigurationServer try { _vconfig = new VehicleConfigurationIF("supervisor"); } catch(...) { Syslog::write("Had trouble talking to VehicleConfigurationServer"); return; } Syslog::write("Spawning servers..."); createServerTasks(); //chill out for a bit and wait for things to get settled sleep(2); createDriverTasks(Persistent); Syslog::write("Creating interfaces to power and remoteLog"); // create interface to powerServer try { _power = new PowerSystemIF("supervisor"); } catch(...) { Syslog::write("Had trouble talking to powerServer"); } try { _rem = new RemoteLogIF("supervisor"); } catch(...) { Syslog::write("Had trouble talking to remoteLog"); } _eventTriggers = new EventTriggers( VehicleConfigurator::getServerPidFromIFName(SupervisorIFServerName)); dprintf("Event triggers object is 0x%x, observer pid = %d.", _eventTriggers, VehicleConfigurator::getServerPidFromIFName(SupervisorIFServerName)); } Supervisor::~Supervisor() { debug_t debug = 3; dwrite(2)("Destructing"); // This stuff had better have already happened... //killMission(); //stopSupervisor(0); // We used to do this, but now only servers should delete shared memory //if( _output ) delete _output; // Clean up other memory bits, though. if( _command ) delete _command; if ( _power ) delete _power; if ( _eventTriggers ) delete _eventTriggers; if( _planFileName ) free((void *)_planFileName); if( _abortFileName ) free((void *)_abortFileName); #ifdef USE_WD // Hope you get here AmproCommand::Command cmd; cmd.cmd = AmproCommand::DisableWatchdog; _watchdog->write( &cmd ); if( _watchdog ) { delete( _watchdog ); _watchdog = NULL; } #endif _theSupervisor = 0; } void Supervisor::stopSupervisor(int exitCode) { // make sure that we finish up with whatever we were doing if (!_stopSupervisor) { _stopSupervisor = 1; return; } Boolean debug = true; dprintf("Supervisor::stopSupervisor()"); // This prints out some info about the processes (before we kill them) reportprocesstimes(); // Kill the driver tasks, then the server tasks. slayTasks(&_driverTasks,1, True); slayTasks(&_serverTasks,1, True); // Finish up... // system("slaySupervisor"); exit(exitCode); } // Loop through the tasks and find out if they're still alive. int Supervisor::checkTasks(TaskList * taskList) { int nDied = 0; debug_t debug = 1; TaskEntry *task; list::iterator it = taskList->begin(); if(taskList->empty()) return -1; for ( ; it != taskList->end(); ++it) { task = *it; // Check to see if process exists if (kill(task->_pid, 0) == -1) { if (errno == ESRCH) { // Task isn't there anymore! dwrite(1)("checkTasks --Task \"%s\" (%d on node %d) died.", task->_interface->name(), task->_pid, task->_node); // wait for the #@$@#$ zombie int status; int dead_pid = waitpid(task->_pid,&status,WNOHANG); //int dead_pid = wait(&status); nDied++; // copy out important info int kill = task->_kill; VehicleConfigurationIF::Name taskName; strcpy(taskName,task->_interface->name()); // remove from list it = taskList->erase(it); dwrite(2)("checkTasks -- Removed task %s",taskName); // use copied info if (!kill) { dealWithDeadTask(taskName); } } } else { // it exists, check to see if we should kill it if (task->_kill) { // kill the task if (task->_kill == 1) { if (kill(task->_pid, SIGQUIT) == -1) { dwrite(2)("checkTasks -- SIGQUIT didn't cut it"); } } // kill it harder later on if (task->_kill == 10) { dwrite(2)("checkTasks -- Process still alive after SIGQUIT: sending SIGKILL"); if (kill(task->_pid, SIGKILL) == -1) { dwrite(2)("checkTasks -- SIGKILL didn't cut it"); } } task->_kill++; } } } return nDied; } // Really this is just for drivers - servers shouldn't die (and if they do, we // just shutdown the whole system). void Supervisor::dealWithDeadTask( VehicleConfigurationIF::Name driverName ) { debug_t debug = 1; VehicleConfigurationIF::Name processName; VehicleConfigurationIF::Name ifName; VehicleConfigurationIF::Name onFault; VehicleConfigurationIF::Name onFailure; //this returns 0 on success if (_vconfig->getProcessNameFromDriver(driverName, processName) != 0) { Syslog::write("Error: dealWithDeadTask - getProcessNameFromDriver : %s", driverName); return; }; Syslog::write("Dealing with dead task %s, process %s",driverName,processName); if(_rem != NULL) { char temp[128]; sprintf(temp, "Supervisor::Driver %s died", driverName); _rem->remoteWrite(2, temp); } //this returns 0 on success if (_vconfig->getProcessFaultAction(processName, onFault) != 0){ Syslog::write("Error: dealWithDeadTask - getProcessFaultAction"); return; } //this returns 0 on success if (_vconfig->getProcessFailureAction(processName, onFailure) != 0) { Syslog::write("Error: dealWithDeadTask - getProcessFailureAction"); return; } Syslog::write("Process: %s OnFault: %s, OnFailure: %s", processName, onFault, onFailure); //this returns 0 on success if (_vconfig->getIFName(processName, ifName) != 0) { Syslog::write("Error: dealWithDeadTask - getIFName"); return; } DeviceIF * serverIF = new DeviceIF("supervisor",0,(const char *)ifName); // check its status switch (serverIF->status()) { case DeviceIF::Ok: case DeviceIF::Initializing: case DeviceIF::Offline: case DeviceIF::Error: //serverIF->clearOutput(); break; case DeviceIF::FailedInit: case DeviceIF::FailedRead: case DeviceIF::Failed: break; } dwrite(1)("Server Status: %d",serverIF->status()); delete serverIF; _faultCount[_vconfig->findProcess(processName)] += 1; if ( _faultCount[_vconfig->findProcess(processName)] > 3 ) { // do failure dwrite(1)("driver failure"); if (stricmp(onFailure, "ignore") == 0) { // Kill it commandDriver(processName,Both,0); if(_rem != NULL) _rem->remoteWrite(2, "Supervisor::onFailure -- Ignorning"); } if (stricmp(onFailure, "abortMission") == 0) { abortMission(); if(_rem != NULL) _rem->remoteWrite(2, "Supervisor::onFailure -- Aborting Mission"); } if (stricmp(onFailure, "killMission") == 0) { killMission(); if(_rem != NULL) _rem->remoteWrite(2, "Supervisor::onFailure -- Killing Mission"); } if (stricmp(onFailure, "stopSupervisor") == 0) { stopSupervisor(0); if(_rem != NULL) _rem->remoteWrite(2, "Supervisor::onFailure -- Stopping Supervisor"); } } else { // do fault dwrite(1)("driver fault: %d", _faultCount[_vconfig->findProcess(processName)]); if (stricmp(onFault, "ignore") == 0) { // Kill it persistent or not commandDriver(processName,Both,0); if(_rem != NULL) _rem->remoteWrite(2, "Supervisor::onFault -- Ignorning"); } if (stricmp(onFault, "respawn") == 0) { // Kill it commandDriver(processName,Both,0); // Respawn commandDriver(processName,Both,1); if(_rem != NULL) _rem->remoteWrite(2, "Supervisor::onFault -- Respawning"); } if (stricmp(onFault, "powercycleRespawn") == 0) { dwrite(3)("Powercycling and respawn"); // Kill it commandDriver(processName,Both,0); // Powercycle VehicleConfigurationIF::DeviceInfo devInfo; _vconfig->deviceInfo(processName, &devInfo); _power->netOff(devInfo.powerCircuit); sleep(1); _power->netOn(devInfo.powerCircuit); // Respawn commandDriver(processName,Both,1); if(_rem != NULL) _rem->remoteWrite(2, "Supervisor::onFault -- Power Cycling and Respawning"); } if (stricmp(onFault, "abortMission") == 0) { abortMission(); if(_rem != NULL) _rem->remoteWrite(2, "Supervisor::onFault -- Aborting Mission"); } if (stricmp(onFault, "killMission") == 0) { killMission(); if(_rem != NULL) _rem->remoteWrite(2, "Supervisor::onFault -- Killing Mission"); } if (stricmp(onFault, "stopSupervisor") == 0) { // For now just shutdown the supervisor stopSupervisor(0); if(_rem != NULL) _rem->remoteWrite(2, "Supervisor::onFault -- Stopping supervisor"); } } } void Supervisor::keepAlive() { Boolean debug = False; dprintf("Supervisor::keepAlive()\n"); #ifdef USE_WD // Hope you get here AmproCommand::Command cmd; cmd.cmd = AmproCommand::SetWatchdog; cmd.secs = 5; // Enough? _watchdog->write( &cmd ); #endif } void Supervisor::createServerTasks() { Boolean debug = False; char errorBuf[256]; short deviceCount, i; VehicleConfigurationIF::Name progName; VehicleConfigurationIF::Driver server; VehicleConfigurationIF::Name ifName; int node=0; if( (deviceCount = _vconfig->processCount()) < 0 ) return; dprintf("Supervisor::createServerTasks -- Found %d devices", deviceCount ); for( i=0; igetProcessName( i, progName ) ) continue; node = _vconfig->getNode( i ); dprintf("Supervisor::createServerTasks -- Now loading process #%d: %s", i, progName ); if( _vconfig->getProcessServer( i, server) ) continue; if (strlen(server) == 0) continue; dprintf("Supervisor::createServerTasks -- Starting server %s", server ); if (createTask(&_serverTasks, server, "", node, False) == -1) { dprintf("Supervisor::createServerTasks() - \n" "Exec of \"%s\" failed", server); } } return; } void Supervisor::createDriverTasks(Supervisor::DriverType dtype) { Boolean debug = False; char errorBuf[256]; short deviceCount, i; VehicleConfigurationIF::Name progName; VehicleConfigurationIF::Driver driver; VehicleConfigurationIF::Name ifName; int node=0; if( (deviceCount = _vconfig->processCount()) < 0 ) return; dprintf("Supervisor::createDriverTasks -- Found %d devices", deviceCount ); for( i=0; igetProcessName( i, progName ) ) continue; commandDriver(progName,dtype,1); } return; } void Supervisor::slayTasks(TaskList * taskList, bool immediate, bool persistent) { TaskEntry *taskEntry; list::iterator it = taskList->begin(); for ( ; it != taskList->end(); ++it) { taskEntry = *it; //if the entry is not persistent or if we want to kill everything //which is denoted by persistent is true if(!taskEntry->_persistent || persistent) { taskEntry->_kill = 1; Syslog::write("_kill set for %s (%d) %d %d",taskEntry->_interface->name(),taskEntry->_pid, taskEntry->_persistent, persistent); if (immediate) { kill(taskEntry->_pid,SIGQUIT); // int status; // int dead_pid = wait(&status); } } } } // This function loops through the devices in the configuration and turns them all on. int Supervisor::powerOnDevices() { short deviceCount, i; short poweredCount = 0; VehicleConfigurationIF::DeviceInfo devInfo; Boolean debug = True; // Check to see if we have an interface to the power system. if ( _power == NULL ) { Syslog::write("Supervisor::powerOnDevices() - no power system interface."); return -1; } // Check to see if the power driver is ready. for (int j=0;j<50;j++) { if ( _power->getStatus() == DeviceIF::Ok ) break; System::milliSleep(100); } if ( _power->getStatus() != DeviceIF::Ok ) { Syslog::write("powerOnDevices() - power driver not ready yet"); return -1; } // Get a count of the devices... deviceCount = _vconfig->processCount(); if ( deviceCount < 0 ) { Syslog::write("Supervisor::powerOnDevices() - Could not get device count."); return -1; } Syslog::write("Supervisor::powerOnDevices() - Powering on %d devices.", deviceCount); // Loop through the devices in the configuration for( i=0; i < deviceCount; i++ ) { try { // Get the circuit number for this device and turn it on. _vconfig->deviceInfo( i, &devInfo ); dprintf("Supervisor::powerOnDevices() - device %d has power circuit %d.", i , devInfo.powerCircuit); if ( devInfo.powerCircuit >= 0 ) { dprintf("Supervisor::powerOnDevices() - powering on device %d, circuit %d.", i, devInfo.powerCircuit); if ( _power->netOn( devInfo.powerCircuit) != DeviceIF::Ok ) Syslog::write("Supervisor::powerOnDevices() - Error powering on device %d. Continuing.", i); else poweredCount++; } } catch (...) { Syslog::write("Supervisor::powerOnDevices() - Error powering on device %d.", i); return -1; } } Syslog::write("Supervisor::powerOnDevices() - Successfully powered on %d devices.", poweredCount); return 1; } void Supervisor::createPowerDriverTask() { VehicleConfigurationIF::Name processName; VehicleConfigurationIF::Driver driverName; Boolean debug = True; //first we want to see if it exists //need to change if ever declared with more than one driver _vconfig->getProcessName( ThalesPowerSystemIFServerName, processName ); _vconfig->getProcessDriver(processName,driverName); TaskEntry *taskEntry; list::iterator it = _driverTasks.begin(); for ( ; it != _driverTasks.end(); ++it) { taskEntry = *it; //fprintf(stdout, "Comparing %s to %s\n", driverName, taskEntry->_interface->name()); if (!strcmp(driverName, taskEntry->_interface->name())) { dprintf("Supervisor -- createPowerDriverTask(): Power driver already exists: %d", taskEntry->_pid); return; } } //otherwise we need to create it try { dprintf("Supervisor::createPowerDriverTask() - process name %s.", processName); commandDriver(processName,Both,1); } catch (...) { Syslog::write("Supervisor::createPowerDriverTask() - error starting up power driver."); return; } //make sure that powerDriver is ready to go no matter what //otherwise you can't turn circuits on sleep(8); Syslog::write("Supervisor::createPowerDriverTask() - started power driver."); } // This function loops through the devices in the configuration and powers them all off. // By the time this function is called, the power driver has been started. int Supervisor::powerOffDevices() { short deviceCount, i; short poweredCount = 0; VehicleConfigurationIF::DeviceInfo devInfo; Boolean debug = True; // Check to see if we have an interface to the power system. if ( _power == NULL ) { Syslog::write("Supervisor::powerOffDevices() - no power system interface."); return -1; } // Get a count of the devices... deviceCount = _vconfig->processCount(); if ( deviceCount < 0 ) { Syslog::write("Supervisor::powerOffDevices() - Could not get device count."); return -1; } Syslog::write("Supervisor::powerOffDevices() - Powering off %d devices.", deviceCount); // Loop through the devices in the configuration for ( i=0; i < deviceCount; i++ ) { try { // Get the circuit number for this device and turn it off _vconfig->deviceInfo( i, &devInfo ); dprintf("Supervisor::powerOffDevices() - device %d has power circuit %d.", i, devInfo.powerCircuit); if ( devInfo.powerCircuit >= 0 ) { dprintf("Supervisor::powerOffDevices() - powering on device %d, circuit %d.", i, devInfo.powerCircuit); if ( _power->netOff( devInfo.powerCircuit ) != DeviceIF::Ok ) Syslog::write("Supervisor::powerOffDevices() - Error powering off device %d. Continuing.", i); else poweredCount++; } } catch (...) { Syslog::write("Supervisor::powerOffDevices() - Error powering off device %d.", i); return -1; } } Syslog::write("Supervisor::powerOffDevices() - Successfully powered off %d devices.", poweredCount); return 1; } int Supervisor::createTask(TaskList * taskList, const char *taskName, const char *args, int node, Boolean restarted, bool persistent) { Boolean debug = False; if ( checkTaskEntry( taskList, taskName, args, node) == True ) { dprintf("Found task %s on taskList - no need to rerun.", taskName); return 0; } dprintf("About to spawn %s", taskName); pid_t taskPid = System::spawn(taskName, args, node); dprintf("Spawned %s, PID = %d", taskName, taskPid); if ( taskPid == -1 ) { Syslog::write("Could not spawn %s",taskName); perror("Supervisor::createTask() - qnx_spawn() failed"); return -1; } dprintf("Supervisor::createTask() - parent task; add TaskEntry"); addTaskEntry(taskList, taskName, taskPid, node, args, persistent); dprintf("Supervisor::createTask() - added TaskEntry."); return 0; } Boolean Supervisor::checkTaskEntry(TaskList *taskList, const char *taskName, const char *args, int node) { Boolean debug = False; TaskEntry *taskEntry; dprintf("Supervisor::checkTaskEntry -- looking through %d tasks for '%s'.", taskList->size(), taskName); list::iterator it = taskList->begin(); for ( ; it != taskList->end(); ++it) { taskEntry = *it; if ( ( ! strcmp(taskName, taskEntry->_interface->name() ) ) && ( node == taskEntry->_node ) && ( ! strcmp( args, taskEntry->_args ) ) ) { // Task is already in list. Use same interface, just replace pid dprintf("Supervisor::checkTaskEntry -- task %s found in list (process id %d).\n", taskName, taskEntry->_pid); return True; } } return False; } void Supervisor::addTaskEntry(TaskList * taskList, const char *taskName, pid_t pid, int node, const char *args, bool persistent) { Boolean debug = False; TaskEntry *taskEntry; dprintf("Supervisor::addTaskEntry -- There are %d tasks in the table", taskList->size() ); // Check to see whether task is already in table list::iterator it = taskList->begin(); for ( ; it != taskList->end(); ++it) { taskEntry = *it; if (!strcmp(taskName, taskEntry->_interface->name())) { // Task is already in list. Use same interface, just replace pid dprintf("Supervisor -- task %s already in list. Just updating pid from %d to %d.\n", taskName, taskEntry->_pid, pid); taskEntry->_pid = pid; return; } } dprintf("Supervisor::addTaskEntry -- Adding TaskInterface of name %s", taskName ); // Task is not yet in list; add it TaskInterface *interface = new TaskInterface(taskName, 0, taskName, DoNotConnect); dprintf("Supervisor::addTaskEntry -- Creating taskEntry pid %d, interface %s, args: %s", pid, interface->name(), args ); taskEntry = new TaskEntry(pid, interface, node, args, persistent); dprintf("Supervisor::addTaskEntry -- Now adding to table"); taskList->push_back(taskEntry); dprintf("Supervisor::addTaskEntry -- Finished"); // Don't know what this was used for, but it fills up under the // current paradigm... // _output->addEntry( taskName, interface->name(), pid ); // _output->setStatus( pid, Sysinfo::Initializing ); } void Supervisor::commandDriver(char* name, Supervisor::DriverType dtype, Boolean turnOn) { Boolean debug = False; VehicleConfigurationIF::Name v_name; VehicleConfigurationIF::Driver v_driver; int node=0; int kill_pid=0; TaskEntry *task; strcpy(v_name, name); node = _vconfig->getNode(v_name); dprintf("Supervisor::commandDriver -- Node %d found for name %s", node, v_name); for (int i = 0; i<3; i++) { if (i == 0) { if( _vconfig->getProcessDriver( v_name, v_driver) ) return; } if (i == 1) { if( _vconfig->getProcessDriver2( v_name, v_driver) ) return; } if (i == 2) { if( _vconfig->getProcessDriver3( v_name, v_driver) ) return; } dprintf("Supervisor::commandDriver -- Driver %s found for name %s", v_driver, v_name); if (strlen(v_driver) == 0) return; Boolean persists = _vconfig->getDriverPersists(v_name, i+1); if(dtype == Both || (persists && dtype == Persistent) || (!persists && dtype == Normal)) { if ( turnOn ) { dprintf("Supervisor::commandDriver -- Starting driver %s", v_driver ); if (createTask(&_driverTasks, v_driver, "", node, False, persists) == -1) { dprintf("Supervisor::commandTasks() - Exec of \"%s\" failed", v_driver); } } else { list::iterator it = _driverTasks.begin(); for ( ; it != _driverTasks.end(); ++it) { task = *it; if (strcmp(task->_interface->name(), v_driver) == 0) { task->_kill = 1; dprintf("Supervisor::commandTasks -- _kill set for %s (%d)", task->_interface->name(), task->_pid); break; } } } } } } void Supervisor::startMission() { debug_t debug = True ; VehicleConfigurationIF::Name ifName; char newLogName[64]; if((_missionState == Running) || (_missionState == AbortSent)) { Syslog::write("Start mission ignored as mission is already running"); return; } Syslog::write("simulated=%s, realTailCone=%s", (_simulated) ? "Yes" : "No", (_realTailCone) ? "Yes" : "No"); // Actually launch the new mission // Create data log subdirectory for this run System::createLogDir(newLogName, _umbrellaLogDirName, True); dprintf("Created log directory %s in %s.\n", newLogName, _umbrellaLogDirName); // Reset driver fault counts for (int i=0; i<100; i++) _faultCount[i] = 0; if( strlen(_planFileName) > 1 ) { dprintf("_planFileName valid, proceeding.\n"); // Power on the devices createPowerDriverTask(); powerOnDevices(); sleep(3); // Spin up the drivers createDriverTasks(Both); sleep(3); if(_layeredControl == NULL) _layeredControl = new LayeredControlIF("supervisor"); if( strlen(_planFileName) > 1 ) { Syslog::write("*** Starting mission %s", _planFileName ); if( strlen(_abortFileName) > 1 ) { Syslog::write("*** With abort plan: %s", _abortFileName ); _layeredControl->startNewMission( _planFileName, _abortFileName ); } else { Syslog::write("*** With default abort plan"); _layeredControl->startNewMission( _planFileName, "abortSimple.cfg"); } } } _output->data.systemStatus = Sysinfo::Unknown; _missionState = Running; _output->data.isMissionRunning = True; strcpy(_output->data.directoryName, _umbrellaLogDirName); strcat(_output->data.directoryName, "/"); strcat(_output->data.directoryName, newLogName); // Toss the new table into the shared memory block _output->write(); triggerEvent(SupervisorIF::MissionStarted); return; } void Supervisor::abortMission() { Boolean debug = True; Syslog::write("*** ABORT MISSION RECEIVED ***"); if(_missionState != Running) { Syslog::write("Abort mission ignored as mission is not in Running state\n"); return; } if(_rem != NULL) { _rem->remoteWrite(2, "Supervisor::Sending abortMission to LayeredControl"); } _missionState = AbortSent; if( _layeredControl != NULL) _layeredControl->abortMission(); } void Supervisor::killMission() { Boolean debug = True; Syslog::write("*** KILL MISSION RECEIVED ***"); if((_missionState != Running) && (_missionState != AbortSent)) { Syslog::write("Abort mission ignored as mission is not in either Running or AbortSent states\n"); return; } if(_rem != NULL) { _rem->remoteWrite(2, "Supervisor::Sending killMission to LayeredControl"); } //we're giving layeredControl a certain amount of time to call //missionCompleted or we are going to do it anyway _killMissionSent = time(NULL); _missionState = KillSent; if( _layeredControl != NULL) _layeredControl->killMission(); //now we should get a missionCompleted in just a second } void Supervisor::missionCompleted() { Boolean debug = True; char newLogName[64]; memset(newLogName,0,64); Syslog::write("*** MISSION COMPLETED RECEIVED ***"); _killMissionSent = 0; if(_missionState == NotRunning) { Syslog::write("Mission completed ignored as no mission is running\n"); return; } if(_rem != NULL) { _rem->remoteWrite(2, "Supervisor::Mission Completed"); } _missionState = NotRunning; System::createLogDir(newLogName, _umbrellaLogDirName, False); // Shut down the drivers. slayTasks(&_driverTasks); checkTasks(&_driverTasks); sleep(5); // Power off the devices //powerOffDevices(); _output->data.isMissionRunning = False; sprintf(_output->data.directoryName, "%s/%s", _umbrellaLogDirName,newLogName); _output->write(); triggerEvent(SupervisorIF::MissionFinished); } void Supervisor::run() { while (!_stopSupervisor) { // Check status of serverTasks if (checkTasks(&_serverTasks) > 0) { // Something died Syslog::write("Server(s) died\n"); stopSupervisor(1); } // Check status of driverTasks if (checkTasks(&_driverTasks) > 0) { // Something died Syslog::write("Driver(s) died\n"); } if(_killMissionSent != 0) { if(time(NULL) - _killMissionSent > KILLMISSION_TIMEOUT) { Syslog::write("Supervisor::LayeredControl too slow. Sending missionCompleted."); _killMissionSent = 0; missionCompleted(); } } if(_missionState == AbortSent) { if(_layeredControl != NULL) { if(_layeredControl->getMissionState() == LayeredControlIF::NormalState) _layeredControl->abortMission(); } } // Check the incoming command queue SupervisorCommand::Command cmdIn; while( (_command->read( &cmdIn )) > 0 ) { switch( cmdIn.cmd ) { case SupervisorCommand::StartMission: Syslog::write("Starting mission..."); startMission(); break; case SupervisorCommand::AbortMission: Syslog::write("Aborting mission..."); abortMission(); break; case SupervisorCommand::KillMission: Syslog::write("Killing mission..."); killMission(); break; case SupervisorCommand::MissionCompleted: Syslog::write("Mission Completed"); missionCompleted(); break; case SupervisorCommand::StopSupervisor: Syslog::write("Stopping supervisor"); stopSupervisor(0); break; case SupervisorCommand::SetMission: Syslog::write("Setting mission files..."); if (strlen(cmdIn.missionPlan) > 1) { strcpy(_planFileName, cmdIn.missionPlan); Syslog::write("Set MissionFileName: %s", _planFileName); } if (strlen(cmdIn.abortPlan) > 1) { strcpy(_abortFileName, cmdIn.abortPlan); Syslog::write("Set AbortFileName: %s", _abortFileName); } break; case SupervisorCommand::SetFlags: Syslog::write("Setting flags"); _simulated = cmdIn.simulated; _realTailCone = cmdIn.realTailCone; break; case SupervisorCommand::DeviceFailed: // commandDriver(cmdIn.deviceName, 0); Syslog::write("Nav reports device %s is acting up.",cmdIn.deviceName); break; case SupervisorCommand::SetDriverCommand: commandDriver(cmdIn.driverName, Both, cmdIn.turnOn); break; case SupervisorCommand::ReloadConfig: Syslog::write("Reloading configuration from vcs..."); reloadConfig(); break; } } // Reset watchdog timer keepAlive(); // Increment cycleCount (watched by dynamicControl, Powerboard, etc) _output->data.cycleCount++; _output->write(); System::milliSleep(500); } Syslog::write("Finished run"); stopSupervisor(0); } void Supervisor::reloadConfig() { Boolean debug = True; TaskEntry *rasEntry; TaskEntry *taskEntry; list::iterator it; // This function is a bit of a hack. The right way to go about this // kind of thing would be to reload the vehicleConfiguration intelligently // (that is - without losing information about PIDs & interfaces), and then // only reloading the servers that need to be reloaded. // // Instead, this function implements a rather brute-force approach to the // problem - it shuts nearly everything down, and restarts servers. // // This function is called from RemoteAgentServer::reloadVehicle() which has already // called VehicleConfigurationServer::reloadTables(). Syslog::write("Supervisor::reloadConfig() - Reloading configuration & restarting servers."); // The order of operations: // 1. Kill drivers // 2. Kill servers. // 3. Start servers. // This probably won't be valid anymore. if ( _power ) delete _power; // Here we go killing all the drivers. Syslog::write("Supervisor::reloadConfig() - Killing off drivers."); slayTasks(&_driverTasks,0, True); int j = 0; while (j != -1) { System::milliSleep(500); j = checkTasks(&_driverTasks); } // Okay, now kill off the servers (except the select core) Syslog::write("Supervisor::reloadConfig() - Killing off servers."); slayTasks(&_serverTasks,0, True); sleep(1); int i = 0; while (i != -1) { System::milliSleep(1000); i = checkTasks(&_serverTasks); } // Now start the servers up. Syslog::write("Supervisor::reloadConfig() - Starting servers."); createServerTasks(); sleep(2); createDriverTasks(Persistent); // Create interface to powerServer try { _power = new PowerSystemIF("supervisor"); } catch(...) { Syslog::write("Had trouble talking to powerServer"); } // Done! dprintf("Supervisor::reloadConfig() - Done reloading the configuration."); return; } void Supervisor::triggerEvent(EventCode event) { Boolean debug = False; dprintf("Supervisor::triggerEvent() - _eventTriggers ptr = 0x%x\n", _eventTriggers); _eventTriggers->notify(event); } void Supervisor::signalHandler(int sigNo) { Boolean debug = True; if (!_theSupervisor) // Doesn't exist return; dprintf("Supervisor::signalHandler(); caught signal %d\n", sigNo); Supervisor::cleanup(); _theSupervisor->stopSupervisor(0); } void Supervisor::signalChildHandler( int sigNo ) { /* Boolean debug = True; dprintf("Supervisor::signalChildHandler() -- A child died..."); // First, check to make sure i exist if( !_theSupervisor ) return; //int status; //int dead_pid = wait(&status); if (_theSupervisor->checkTasks(&(_theSupervisor->_serverTasks)) > 0) { // Something died Syslog::write("Server(s) died\n"); _theSupervisor->stopSupervisor(1); } // Check status of driverTasks if (_theSupervisor->checkTasks(&(_theSupervisor->_driverTasks)) > 0) { // Something died Syslog::write("Driver(s) died\n"); } */ return; } void Supervisor::cleanup() { Boolean debug = False; dprintf("Supervisor::cleanup() - delete _theSupervisor"); delete _theSupervisor; dprintf("Supervisor::cleanup() - done"); return; } void Supervisor::reportprocesstimes() { Syslog::write("******** REPORTPROCESSTIMES ********"); int sid = getsid(getpid()); char sidptr[10]; sidptr[0] = 0x00; itoa(sid,sidptr,10); int sinReturn = spawnlp(P_WAIT,"reportprocesstimes.sh", "reportprocesstimes.sh",NULL ); Syslog::write("sin returned %d", sinReturn ); if( sinReturn < 0 ) { Syslog::write("sin error (%d): %s", errno, strerror( errno ) ); } }