########################################################################### # EyeRISUDPCLIent.py # # Monterey Bay Aquarium Research Institute 2019 # All rights reserved. # # Command-line Interface (CLI) client app for the EyeRIS controller. # Intended to be a temporary client app for the controller while a # more user-friendly GUI app is developed. # A number of parameters are hard-coded and intended to be changed # by the user: # # NLights - currently set to two. Will eventually grow to six. # UDP_IP - currently set to 192.168.1.199. Must match controller IP. # UDP_PORT - currently set to 1111. Must match controller port. # # Created: May 2019 Paul Roberts # Modified: Jul 26 2019 Rich Henthorn - Initial functionality/formats # ########################################################################### import time import pyqtgraph as pg from pyqtgraph.Qt import QtCore, QtGui import numpy as np import os import lcm import glob import cv2 import imageio ## Global variables for the App pg.mkQApp() ## Stylesheets styleNormalProg = 'QProgressBar {border: 2px solid grey;border-radius: 5px;text-align: center;font: 75 12pt "MS Shell Dlg 2"; } QProgressBar::chunk { background-color: #adcdff;}"' styleWarnProg = 'QProgressBar {border: 2px solid grey;border-radius: 5px;text-align: center;font: 75 12pt "MS Shell Dlg 2"; } QProgressBar::chunk { background-color: #f5c842;}"' styleErrProg = 'QProgressBar {border: 2px solid grey;border-radius: 5px;text-align: center;font: 75 12pt "MS Shell Dlg 2"; } QProgressBar::chunk { background-color: #f54242;}"' ## Define main window class from template path = os.path.dirname(os.path.abspath(__file__)) uiFile = os.path.join(path, 'RxFlowProc.ui') WindowTemplate, TemplateBaseClass = pg.Qt.loadUiType(uiFile) class MovingAverage: def __init__(self, data_shape, samples=10, method='average'): self.samples = samples self.method = method self.data_shape = data_shape self.data_dims = len(data_shape) + 1 self.buffer_dims = data_shape + [samples] self.buffer = np.zeros(self.buffer_dims) self.buffer_index = 0 def add_sample(self, d): if self.buffer_index < self.samples: self.buffer[..., self.buffer_index] = d self.buffer_index += 1 else: self.buffer[..., 0:self.samples-1] = self.buffer[..., 1:self.samples] self.buffer[..., self.samples-1] = d def get_average(self): if self.buffer_index <= 0: return self.buffer[..., 0] if self.buffer_index < self.samples: return self.buffer[..., self.buffer_index-1] if self.method == 'average': return np.mean(self.buffer, self.data_dims-1) if self.method == 'median': return np.median(self.buffer, self.data_dims-1) class ImageHandler(QtCore.QThread): totalFocus = QtCore.Signal(object) depthMap = QtCore.Signal(object) procImage = QtCore.Signal(object) estimatedFPS = QtCore.Signal(float) estimatedDataRate = QtCore.Signal(float) fpsPlot = QtCore.Signal(object) imagesLoaded = QtCore.Signal(float) currentImage = QtCore.Signal(int) def __init__(self, data_path): QtCore.QThread.__init__(self) self.data_path = data_path self.tf_images = glob.glob(os.path.join(self.data_path,'TotalFocus','*TotalFocus*.png')) self.depth_images = glob.glob(os.path.join(self.data_path,'DepthMap','*DepthMap*.tiff')) self.nimages = len(self.tf_images) self.loaded_tf_images = {} self.loaded_depth_images = {} self.processed_images = {} self.loadedCounter = 0 self.image_index = 0 self.last_displayed_image = 0 self.frameRate = 30.0 self.playing = False self.avg = MovingAverage([2], samples=11, method='median') self.feat = cv2.ORB_create() # Change thresholds params = cv2.SimpleBlobDetector_Params() params.minDistBetweenBlobs = 256 #params.minThreshold = 10; #params.maxThreshold = 200; # Filter by Area. """params.filterByArea = True params.minArea = 1500 # Filter by Circularity params.filterByCircularity = False params.minCircularity = 0.1 # Filter by Convexity params.filterByConvexity = False params.minConvexity = 0.87 # Filter by Inertia params.filterByInertia = False params.minInertiaRatio = 0.01 """ self.detector = cv2.SimpleBlobDetector_create(params) self.tracker = cv2.TrackerKCF_create() # set LCM handling and messaging self.lc = lcm.LCM() self.tfSub = self.lc.subscribe("TFIMAGE", self.tfImageHandler) self.fpsLog = [] self.fpsLogHistory = 512 self.offset = 0 self.lastTimestamp = 0 def __del__(self): self.wait() def setImageIndex(self, index): self.image_index = index % self.nimages if self.image_index < 0: self.image_index = 0 def previousFrame(self): self.setImageIndex(self.image_index-1) def nextFrame(self): self.setImageIndex(self.image_index+1) def setFrame(self, frameNumber): self.setImageIndex(frameNumber) def constrainROI(self,img,roi): if (roi[0] < 0): roi[0] = 0 if (roi[1] < 0): roi[1] = 0 if (roi[0] > img.shape[1]): roi[0] = img.shape[1] if (roi[1] > img.shape[0]): roi[1] = img.shape[0] if (roi[0]+roi[2] > img.shape[1]): roi[2] = img.shape[1]-roi[0] if (roi[2] < 0): roi[2] = 0 if (roi[1]+roi[3] > img.shape[0]): roi[3] = img.shape[0]-roi[1] if (roi[3] < 0): roi[3] = 0 return roi def orb_features(self, img, init=False): kp, des = self.feat.detectAndCompute(img, None) print(len(kp)) img = cv2.drawKeypoints(img, kp, None, (255, 0, 0), 4) return img def centroid_and_shift(self, img, init=False): # resize gray = img.copy() #gray_filtered = cv2.bilateralFilter(gray, 7, 50, 50) edges_filtered = cv2.Canny(gray, 20, 120) kernel = np.ones((5, 5), np.uint8) edges_filtered = cv2.morphologyEx(edges_filtered, cv2.MORPH_CLOSE, kernel) contours, hierarchy = cv2.findContours(edges_filtered, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) cont_image = cv2.cvtColor(gray, cv2.COLOR_GRAY2RGB) # find the contour with the largest area max_area_index = 0 max_area = -1 for ind, cont in enumerate(contours): if cv2.contourArea(cont) > max_area: max_area = cv2.contourArea(cont) max_area_index = ind #ok = True """if init: x, y, w, h = cv2.boundingRect(contours[max_area_index]) bbox = [x, y, w, h] self.tracker.init(gray, tuple(bbox)) else: ok, bbox = self.tracker.update(gray) """ #if not ok: # print('lost track, using bbox') #x, y, w, h = cv2.boundingRect(contours[max_area_index]) #bbox = [x, y, w, h] #bbox = self.constrainROI(img,list(bbox)) #self.avg.add_sample(bbox) #bbox = self.avg.get_average() #print(bbox) #p1 = (int(bbox[0]), int(bbox[1])) #p2 = (int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3])) #cv2.rectangle(cont_image, p1, p2, (255, 0, 0), 2, 1) # find the convex hull of the contour #hull = cv2.convexHull(contours[max_area_index]) (x,y),(MA,ma),angle = cv2.fitEllipse(contours[max_area_index]) # get centroid of contour #mom = cv2.moments(contours[max_area_index]) #cx = int(mom['m10'] / mom['m00']) #cy = int(mom['m01'] / mom['m00']) self.avg.add_sample([x, y]) cent = self.avg.get_average() bbox = (cent[0]-256,cent[1]-256,512,512) bbox = self.constrainROI(img,bbox) #cv2.ellipse(cont_image, el, (0, 255, 0), 2) #cv2.drawContours(cont_image, hull, -1, (0, 255, 0), 3) cv2.circle(cont_image, (int(cent[0]), int(cent[1])), 5, (255, 0, 0), 2) #kps = self.detector.detect(edges_filtered) #tmp = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB) #im_with_kps = cv2.drawKeypoints(tmp, kps, np.array([]), (0, 0, 255), cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS) # crop to size cont_image = cont_image[int(bbox[1]): int(bbox[1] + bbox[3]), int(bbox[0]): int(bbox[0] + bbox[2])] return cont_image def tfImageHandler(self, channel, data): msg = total_focus_image.decode(data) img = np.frombuffer(msg.data, dtype='B').reshape(msg.width, msg.height) # img = cv2.GaussianBlur(img,(11,11),0) # img = img + self.offset # self.offset = (self.offset + 1) % 255 self.signal.emit(img) if self.lastTimestamp != 0: delta_t = msg.timestamp - self.lastTimestamp if delta_t > 0: fpsEst = 1000.0 / delta_t if (len(self.fpsLog)) > 10: newVal = np.mean(self.fpsLog[-10:] + [fpsEst]) else: newVal = fpsEst if len(self.fpsLog) >= self.fpsLogHistory: self.fpsLog = self.fpsLog[1:] + [newVal] else: self.fpsLog = self.fpsLog + [newVal] self.estimatedFPS.emit(newVal) self.estimatedDataRate.emit(msg.width * msg.height * newVal / 1000000) self.fpsPlot.emit(self.fpsLog) self.lastTimestamp = msg.timestamp def run(self): # your logic here first_run = True loaded_image_idx = 0 while self.isRunning: #self.lc.handle() # only emit an image if it is new or has changed or we are playing if self.last_displayed_image != self.image_index or self.playing: # save the index in case it gets modified outside the loop during # this iteration idx = self.image_index # load image if needed if idx not in self.loaded_tf_images: try: print("loading image... " + str(idx)) self.loaded_tf_images[idx] = imageio.imread(self.tf_images[idx]) self.loaded_depth_images[idx] = imageio.imread(self.depth_images[idx]) self.loadedCounter += 1 self.imagesLoaded.emit(self.loadedCounter) except Exception as e: print(e) # convert images tf_image = cv2.resize(np.uint8(self.loaded_tf_images[idx]/256), (1024, 1024)) depth_image = cv2.resize(self.loaded_depth_images[idx], (1024, 1024)) # process image if not already done if idx not in self.processed_images: self.processed_images[idx] = self.orb_features(tf_image, first_run) first_run = False # emit image as signal from handler self.totalFocus.emit(tf_image) self.depthMap.emit(depth_image) self.procImage.emit(self.processed_images[idx]) # save the image index to see if we should update next iteration self.last_displayed_image = idx # automatically increment the index if playing and sleep for given rate if self.playing: self.image_index = (idx + 1) % self.nimages time.sleep(1 / self.frameRate) else: # load more images until done if loaded_image_idx not in self.loaded_tf_images: try: print("loading image... " + str(loaded_image_idx)) self.loaded_tf_images[loaded_image_idx] = imageio.imread(self.tf_images[loaded_image_idx]) self.loaded_depth_images[loaded_image_idx] = imageio.imread(self.depth_images[loaded_image_idx]) self.loadedCounter += 1 self.imagesLoaded.emit(self.loadedCounter) loaded_image_idx += 1 except Exception as e: print(e) class MainWindow(TemplateBaseClass): previousFrameSignal = QtCore.Signal(int) nextFrameSignal = QtCore.Signal(int) setFrameSignal = QtCore.Signal(int) def __init__(self, argv): self.data_path = argv[1] self.playing = False TemplateBaseClass.__init__(self) self.setWindowTitle('RxFlowProx') self.rawImageDisplayScale = 255 self.lastImage = [] # Create the main window self.ui = WindowTemplate() self.ui.setupUi(self) # image input thread self.image_handler = ImageHandler(self.data_path) self.image_handler.totalFocus.connect(self.total_focus_image_update) self.image_handler.depthMap.connect(self.depth_image_update) self.image_handler.procImage.connect(self.proc_image_update) self.image_handler.imagesLoaded.connect(self.ui.imagesLoadedProgressBar.setValue) self.image_handler.start() self.previousFrameSignal.connect(self.image_handler.previousFrame) self.nextFrameSignal.connect(self.image_handler.nextFrame) self.setFrameSignal.connect(self.image_handler.setFrame) self.ui.imagesLoadedProgressBar.setMaximum(self.image_handler.nimages) self.ui.frameSelector.setMaximum(self.image_handler.nimages) self.ui.frameSelector.valueChanged.connect(self.ui.frameNumberSpinBox.setValue) self.ui.frameNumberSpinBox.setMaximum(self.image_handler.nimages) self.ui.imagesLoadedProgressBar.setValue(0) # UI handlers self.ui.playButton.clicked.connect(self.togglePlay) self.ui.prevButton.clicked.connect(self.prevFrame) self.ui.nextButton.clicked.connect(self.nextFrame) self.ui.rawDisplayScale.valueChanged.connect(self.setScale) self.ui.frameSelector.valueChanged.connect(self.setFrame) self.ui.playbackRate.editingFinished.connect(self.updatePlaybackRate) self.ui.playbackRate.setText('30.0') vb = pg.ViewBox() self.ui.totalFocusGraphicsView.setCentralItem(vb) vb.setAspectLocked() self.totalFocusImage = pg.ImageItem() vb.addItem( self.totalFocusImage) vb.setRange(QtCore.QRectF(0, 0, 1024, 1024)) self.tfMouseMoved = pg.SignalProxy(vb.scene().sigMouseMoved, rateLimit=60, slot=self.tfMouseMoved) self.tfMouseMoved = pg.SignalProxy(vb.scene().sigMouseClicked, rateLimit=60, slot=self.tfMouseClicked) self.tfvb = vb vb = pg.ViewBox() self.ui.depthGraphicsView.setCentralItem(vb) vb.setAspectLocked() self.depthImage = pg.ImageItem() vb.addItem(self.depthImage) vb.setRange(QtCore.QRectF(0, 0, 1024, 1024)) vb = pg.ViewBox() self.ui.procGraphicsView.setCentralItem(vb) vb.setAspectLocked() self.procImage = pg.ImageItem() vb.addItem(self.procImage) vb.setRange(QtCore.QRectF(0, 0, 1024, 1024)) self.show() def tfMouseClicked(self,evt): pos = evt[0].pos() if self.tfvb.sceneBoundingRect().contains(pos): mousePoint = self.tfvb.mapSceneToView(pos) print([mousePoint.x(),mousePoint.y()]) def tfMouseMoved(self,evt): pos = evt[0] if self.tfvb.sceneBoundingRect().contains(pos): mousePoint = self.tfvb.mapSceneToView(pos) print([mousePoint.x(),mousePoint.y()]) def updatePlaybackRate(self): try: self.image_handler.frameRate = float(self.ui.playbackRate.text()) print(self.image_handler.frameRate) except TypeError: pass def togglePlay(self): if self.playing: self.playing = False self.image_handler.playing = False self.ui.playButton.setStyleSheet("") else: self.playing = True self.image_handler.playing = True self.ui.playButton.setStyleSheet("background-color: #999;") def setScale(self): self.rawImageDisplayScale = self.ui.rawDisplayScale.value() scale = [0, self.rawImageDisplayScale] if len(self.lastImage): self.totalFocusImage.setImage(self.lastImage, autoLevels=False, levels=scale, autoDownsample=False) # print(self.rawImageDisplayScale) def setFrame(self): frameNumber = self.ui.frameSelector.value() % self.image_handler.nimages self.setFrameSignal.emit(frameNumber) def prevFrame(self): self.previousFrameSignal.emit(-1) def nextFrame(self): self.nextFrameSignal.emit(-1) def total_focus_image_update(self, img): scale = [0, self.rawImageDisplayScale] self.totalFocusImage.setImage(img, autoLevels=False, levels=scale, autoDownsample=False) self.lastImage = img self.ui.frameSelector.setValue(self.image_handler.image_index) def depth_image_update(self, img): lower = float(self.ui.minDepthSpinBox.value()) upper = float(self.ui.maxDepthSpinBox.value()) img = 255*(np.abs(img)-lower)/upper img[img<0] = 0 img[img > 255] = 255 #img = cv2.convertScaleAbs(np.abs(img), img, 255.0/upper, -lower*255/upper) img = np.uint8(img) img = 255 - img img = cv2.applyColorMap(img, cv2.COLORMAP_JET) scale = [0, self.rawImageDisplayScale] self.depthImage.setImage(img, autoLevels=False, levels=scale, autoDownsample=False) def proc_image_update(self, img): scale = [0, self.rawImageDisplayScale] self.procImage.setImage(img, autoLevels=False, levels=scale, autoDownsample=False) ########################################################################### # Main program ########################################################################### ## Start Qt event loop unless running in interactive mode or using pyside. if __name__ == '__main__': import sys win = MainWindow(sys.argv) if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'): QtGui.QApplication.instance().exec_()