///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Copyright (c) 2016 Raytrix GmbH. All rights reserved. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #pragma once #include "DLLInterface.h" #include "ParametersPimpl.h" #include "Calibration.h" #include "CudaDevice.h" #include "ICudaData.h" #include "RayImage.h" #include "Rx.Core/RxString.h" #include "Rx.Core.Ex/RxImage.h" ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // namespace: Rx.LFR ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// namespace Rx { namespace LFR { class CCudaCompute_Impl; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Provides methods for using a CUDA device as the computation unit of the LFR. /// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class RX_API CCudaCompute : public CParametersPimpl { public: ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Default constructor. /// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// CCudaCompute(); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Move constructor. /// /// /// [in,out] The CUDA compute instance. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// CCudaCompute(CCudaCompute&& xCudaCompute); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Destructor. /// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ~CCudaCompute(); public: ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Sets the CUDA device that allocates the memory required for the operations and that runs the algorithms. /// /// /// The CUDA device. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void SetCudaDevice(const CCudaDevice& xCudaDevice); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Gets the CUDA device that has been set by calling SetCudaDevice. Throws an exception if not set yet. /// /// /// The CUDA device. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// const CCudaDevice& GetCudaDevice() const; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Applies the given ray image. /// /// This uploads the raw and the gray image to the CUDA device and applies the calibration and the parameters from the meta /// data to the CUDA algorithms. /// /// /// The ray image. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void ApplyRayImage(const CRayImage& xRayImage); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Applies the given calibration to the algorithms. This must be done every time the calibration changes. /// /// /// The calibration. /// (Optional) True to upload the gray image of the calibration to CUDA. False to use the current /// gray image even if it is invalid. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void ApplyCalibration(const CCalibration& xCalib, bool bUploadGrayImg = true); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Uploads the given image as the new raw image of all computations. /// /// /// The source image. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void UploadRawImage(const Interop::Runtime28::IImage& xSrcImg); public: ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Sets the computation filter mask image. /// /// This image is used to reduce the number of lenses used for depth estimation. /// /// It must have the same dimension as the raw image and a pixel type of Rx::Interop::Runtime28::EPixelType::Lum and a data /// type of Rx::Interop::Runtime28::EDataType::UByte. Otherwise the image is converted. /// /// Each lens of the raw image has a lens center that can be rounded to a pixel. If this pixel has the value 255 in the /// filter mask image, the corresponding lens is used. If this pixel has another value, the lens is not used. /// /// /// The computation filter mask image. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void SetComputationFilterMask(const CRxImage& xImage); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Sets the computation filter mask image. /// /// This image is used to reduce the number of lenses used for depth estimation. /// /// It must have the same dimension as the raw image and a pixel type of Rx::Interop::Runtime28::EPixelType::Lum and a data /// type of Rx::Interop::Runtime28::EDataType::UByte. Otherwise the image is converted. /// /// Each lens of the raw image has a lens center that can be rounded to a pixel. If this pixel has the value 255 in the /// filter mask image, the corresponding lens is used. If this pixel has another value, the lens is not used. /// /// /// The image file to load. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void SetComputationFilterMask(const CRxString& sxFilename); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Removes the computation filter mask set by SetComputationFilterMask. /// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void RemoveComputationFilterMask(); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Reinitializes the depth database. /// /// This can be useful when calling Compute_DepthMap after changing the ray depth image manually. /// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void ReinitializeDepthDB(); public: ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Pre-Process a ray image. This can apply denoise, brightness and sharpness adjustments. /// /// Result image: /// - EImage::Processed_Normalized /// /// Required images: /// - EImage::Raw /// - EImage::Gray (optional) /// /// Influencing parameters: /// - ECCPar::PreProc_Demosaic_Enable /// - ECCPar::PreProc_Demosaic_R /// - ECCPar::PreProc_Demosaic_G /// - ECCPar::PreProc_Demosaic_B /// - ECCPar::PreProc_DenoiseNLM_Enable /// - ECCPar::PreProc_DenoiseNLM_FilterDia /// - ECCPar::PreProc_DenoiseNLM_NoiseLevel /// - ECCPar::PreProc_DenoiseNLM_BlendFactor /// - ECCPar::PreProc_Std_Enable /// - ECCPar::PreProc_Std_Gradation /// - ECCPar::PreProc_Sharp1_Enable /// - ECCPar::PreProc_Sharp1_OnlyCalc /// - ECCPar::PreProc_Sharp1_BlurStdDev /// - ECCPar::PreProc_Sharp1_Factor. /// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void Compute_PreProcess(); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Calculates the grid image. This is the raw or the gray image with an overlay that shows the current grid calibration. /// /// Result image: /// - EImage::Grid /// /// Required images: /// - EImage::Raw or EImage::Gray /// /// Influencing parameters: /// - ECCPar::Grid_DrawOnGrayImage /// - ECCPar::Grid_DrawMlaGrid. /// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void Compute_Grid(); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Pre-Process a ray image with a minimal set of preprocessing parameters. /// /// Result image: /// - EImage::GridData /// /// Required images: /// - EImage::Raw /// /// Influencing parameters: /// - ECCPar::PreProc_Demosaic_R /// - ECCPar::PreProc_Demosaic_G /// - ECCPar::PreProc_Demosaic_B. /// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void Compute_GridDataImage(); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Focus image to a plane perpendicular to the optical axis, i.e. parallel to the image plane. /// /// This focuses to a plane perpendicular to the optical axis. You can change the relative focal plane. A relative focus /// value of zero refocuses to the plane furthest away from the viewer and a value of 1 would focus to a plane that passes /// through the object side focal point of the recording lens. /// /// Result image: /// - EImage::RefocusBasic /// /// Required images: /// - EImage::Processed_Normalized /// /// Influencing parameters: /// - ECCPar::Focus_RelativeFocusPlane /// - ECCPar::Depth_MinVirtualDepth /// - ECCPar::Depth_MaxVirtualDepth. /// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void Compute_RefocusBasic(); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Estimates the depth of the current raw image and creates the ray depth image. /// /// Result image: /// - EImage::DepthRay /// /// Required images: /// - EImage::Processed_Normalized /// /// Influencing parameters: /// - ECCPar::PreProc_Sharp1_Enable /// - ECCPar::PreProc_Sharp1_OnlyCalc /// - ECCPar::PreProc_Sharp1_BlurStdDev /// - ECCPar::PreProc_Sharp1_Factor /// - ECCPar::Depth_SparseLensSet /// - ECCPar::Depth_NearResolutionLevel /// - ECCPar::Depth_PatchDiameter /// - ECCPar::Depth_PatchStride /// - ECCPar::Depth_MinVirtualDepth /// - ECCPar::Depth_MaxVirtualDepth /// - ECCPar::Depth_MinCorrelation /// - ECCPar::Depth_MinStdDeviation /// - ECCPar::Depth_LensFilterMask. /// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void Compute_DepthRay(); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Calculates the depth map image. This image can be filled to produce a complete filled depth map. /// /// Result image: /// - EImage::DepthMap_View_Virtual (if eSpace == LF::ESpace::View_Virtual) or /// - EImage::DepthMap_View_Object_Pinhole (if eSpace == LF::ESpace::View_Object_Pinhole) /// /// Required images: /// - EImage::DepthRay /// /// Influencing parameters: /// - ECCPar::Depth_ConsistencyCheck /// - ECCPar::Depth_Fill_Enabled /// - ECCPar::Depth_Fill_IterCnt /// - ECCPar::Depth_Fill_IterSize /// - ECCPar::Depth_Fill_Complete /// - ECCPar::Depth_Fill_ShrinkRangePX /// - ECCPar::Depth_Fill_Filter_Enabled /// - ECCPar::Depth_Fill_Bilateral_Enabled /// - ECCPar::Depth_Fill_Bilateral_Edge /// - ECCPar::Depth_Fill_Bilateral_Range /// - ECCPar::Depth_Fill_Bilateral_FilterRadius. /// /// /// (Optional) The space. This is LF::ESpace::View_Virtual or LF::ESpace::View_Object_Pinhole. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void Compute_DepthMap(LF::ESpace::ID eSpace = LF::ESpace::View_Object_Pinhole); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Calculates the depth 3D image. /// /// This creates a RGBA float image which maps each pixel in the view camera to a 3D position in object space. The R /// channel is the X coordinate, the G channel is the Y coordinate and the B channel is the Z coordinate. The Z coordinate /// is relative to the global coordinate system which typically coincides with the light field camera sensor. All /// coordinate values are given in millimeters. The A channel is the Z coordinate in virtual depths. /// /// Result image: /// - EImage::Depth3D /// /// Required images: /// - EImage::DepthMap_View_Object_Pinhole /// /// Influencing parameters: /// - none. /// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void Compute_Depth3D(); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Calculates the colored depth map image. /// /// Result image: /// - EImage::DepthMapColored_View_Virtual (if eSpace == LF::ESpace::View_Virtual) or /// - EImage::DepthMapColored_View_Object_Pinhole (if eSpace == LF::ESpace::View_Object_Pinhole) /// /// Required images: /// - EImage::DepthMap_View_Virtual (if eSpace == LF::ESpace::View_Virtual) or /// - EImage::Depth3D (if eSpace == LF::ESpace::View_Object_Pinhole) /// /// Influencing parameters: /// - ECCPar::DepthMap_Color_Mode /// - ECCPar::DepthMap_Color_Max /// - ECCPar::DepthMap_Color_Min /// - ECCPar::DepthMap_Color_Map. /// /// /// (Optional) The space. This is LF::ESpace::View_Virtual or LF::ESpace::View_Object_Pinhole. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void Compute_DepthColorCode(LF::ESpace::ID eSpace = LF::ESpace::View_Object_Pinhole); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Focus on a depth surface. /// /// In principle, each pixel of the result image can be focused individually. This function makes this property available /// by focusing the ray image to the depth surface given in the internal depth image related to the given space. /// /// Result image: /// - EImage::TotalFocus_View_Virtual (if eSpace == LF::ESpace::View_Virtual) or /// - EImage::TotalFocus_View_Object_Pinhole (if eSpace == LF::ESpace::View_Object_Pinhole) /// /// Required images: /// - EImage::Processed_Normalized /// - EImage::DepthMap_View_Virtual (if eSpace == LF::ESpace::View_Virtual) or /// - EImage::DepthMap_View_Object_Pinhole (if eSpace == LF::ESpace::View_Object_Pinhole) /// /// Influencing parameters: /// - ECCPar::Focus_RelativeFocusPlane /// - ECCPar::Focus_DepthBlendingScale /// - ECCPar::Depth_MinVirtualDepth /// - ECCPar::Depth_MaxVirtualDepth. /// /// /// (Optional) The space. This is LF::ESpace::View_Virtual or LF::ESpace::View_Object_Pinhole. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void Compute_TotalFocus(LF::ESpace::ID eSpace = LF::ESpace::View_Object_Pinhole); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Calculates the multi view image. /// /// Result image: /// - EImage::Multiview /// /// Required images: /// - EImage::DepthRay /// - EImage::Processed_Normalized /// /// Influencing parameters: /// - ECCPar::Multiview_Type /// - ECCPar::Multiview_EyeSeparation /// - ECCPar::Depth_ConsistencyCheck /// - ECCPar::Depth_Fill_Enabled /// - ECCPar::Depth_Fill_IterCnt /// - ECCPar::Depth_Fill_IterSize /// - ECCPar::Depth_Fill_Complete /// - ECCPar::Depth_Fill_ShrinkRangePX /// - ECCPar::Depth_Fill_Filter_Enabled /// - ECCPar::Depth_Fill_Bilateral_Enabled /// - ECCPar::Depth_Fill_Bilateral_Edge /// - ECCPar::Depth_Fill_Bilateral_Range /// - ECCPar::Depth_Fill_Bilateral_FilterRadius. /// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void Compute_Multiview(); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Calculates the histogram of the image with the given image ID and stores the histogram data in the given array. This /// array must be large enough to hold 4 * 256 numbers. The number of calculated channels is returned in uChannels. /// /// /// The image ID. /// [out] The histogram. /// [out] The number of channels. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void Compute_Histogram(EImage::ID eImgID, CRxArrayUInt& xHistogram, unsigned& uChannels); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Calculates the minimum and the maximum value in the given image. /// /// /// The image ID. The image must be of type float. /// The channel of each pixel that is used to get the values. /// [out] The minimum. /// [out] The maximum. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void Compute_ImageMinMax(EImage::ID eImgID, unsigned uChannel, float& fMin, float& fMax); public: ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Resize the given source image and store the result in the given target image using the given divisor. /// /// /// [in] The target image. This must not be a moniker. It's created inside. /// The source image. /// The divisor. A value of 2 means that the target image is half the source size. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void ResizeImage(Interop::Runtime28::IImage* pxTrgImg, const Interop::Runtime28::IImage* pxSrcImg, int iDivisor); public: ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Creates the image format of the refocus and the total focus image. This considers the Focus_ImageDivisor parameter. /// /// /// The raw image format. /// True to force landscape. Otherwise the calibration is used. /// /// The image format. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// CRxImageFormat CreateFocusImageFormat(const CRxImageFormat& xRawFormat, bool bForceLandscape) const; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Gets the cuda compute interface. /// /// /// The interface type. /// /// Null if it fails, else the interface. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void* GetInterface(Interfaces::ECudaCompute::ID eData); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Gets the cuda compute interface. /// /// /// The interface type. /// /// Null if it fails, else the interface. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// const void* GetInterface(Interfaces::ECudaCompute::ID eData) const; public: ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// /// Move assignment operator. /// /// /// [in,out] The cuda compute instance. /// /// The instance. ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// CCudaCompute& operator=(CCudaCompute&& xCudaCompute); }; } }