KinectV2PointCloudProvider.h

Go to the documentation of this file.

/*
 * This file is part of ArmarX.
 *
 * Copyright (C) 2011-2016, High Performance Humanoid Technologies (H2T), Karlsruhe Institute of Technology (KIT), all rights reserved.
 *
 * ArmarX is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * ArmarX is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * @package    VisionX::ArmarXObjects::KinectV2PointCloudProvider
 * @author     Christoph Pohl (christoph dot pohl at kit dot edu)
 * @date       2019
 * @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
 *             GNU General Public License
 */
 
#pragma once
 
 
#include <ArmarXCore/core/Component.h>
#include <ArmarXCore/core/system/ArmarXDataPath.h>
#include <VisionX/core/ImageProcessor.h>
#include <VisionX/components/pointcloud_core/CapturingPointCloudProvider.h>
 
#include <VisionX/interface/components/RGBDImageProvider.h>
//#include <VisionX/interface/components/Calibration.h>
 
#include <pcl/common/transforms.h>
#include <pcl/point_cloud.h>
#include <pcl/point_types.h>
 
#include <libfreenect2/libfreenect2.hpp>
#include <libfreenect2/frame_listener_impl.h>
#include <libfreenect2/registration.h>
#include <libfreenect2/packet_pipeline.h>
 
#include <Eigen/Geometry>
 
#include <VisionX/components/pointcloud_provider/KinectV2PointCloudProvider/KinectHelpers.h>
 
 
//#include <Image/IplImageAdaptor.h>
 
namespace visionx
{
    enum KinectProcessorType
    {
        CPU, OPENCL, OPENCLKDE, OPENGL, CUDA, CUDAKDE
    };
 
    /**
         * @class KinectV2PointCloudProviderPropertyDefinitions
         * @brief
         */
    class KinectV2PointCloudProviderPropertyDefinitions :
        public CapturingPointCloudProviderPropertyDefinitions
    {
    public:
        KinectV2PointCloudProviderPropertyDefinitions(std::string prefix) :
            CapturingPointCloudProviderPropertyDefinitions(prefix)
        {
            defineOptionalProperty<bool>("EnableRGB", true, "Enable color image.");
            defineOptionalProperty<bool>("EnableDepth", true, "Enable depth image.");
            defineOptionalProperty<KinectProcessorType>("Pipeline", CPU,
                    "Kinect Pipeline for image processing. Can be one of CPU, GL, CL, CLKDE, CUDA, CUDAKDE")
            .map("CPU", CPU)
            .map("GL", OPENGL)
            .map("CL", OPENCL)
            .map("CLKDE", OPENCLKDE)
            .map("CUDA", CUDA)
            .map("CUDAKDE", CUDAKDE);
            defineOptionalProperty<std::string>("Serial", "",
                                                "Serial Number of the Camera to use. If empty the serial will be chosen automatically");
            defineOptionalProperty<bool>("Mirror", false,
                                         "Mirrors the resulting images");
            defineOptionalProperty<std::string>("ReferenceFrameName", "DepthCamera", "Optional reference frame name.");
        }
    };
 
    /**
         * @class KinectV2PointCloudProvider
         * @ingroup VisionX-Components
         * @brief A brief description
         *
         * Detailed Description
         */
    class KinectV2PointCloudProvider :
        virtual public RGBDPointCloudProviderInterface,
        virtual public CapturingPointCloudProvider,
        virtual public ImageProvider
    {
    public:
        /**
             * @see armarx::ManagedIceObject::getDefaultName()
             */
        KinectV2PointCloudProvider() : listener_(
                libfreenect2::Frame::Color | libfreenect2::Frame::Ir | libfreenect2::Frame::Depth),
            undistorted_(512, 424, 4), registered_(512, 424, 4) {};
 
        std::string getDefaultName() const override
        {
            return "KinectV2PointCloudProvider";
        }
 
    protected:
        /**
             * @see visionx::PointCloudProviderBase::onInitPointCloudProvider()
             */
        void onInitCapturingPointCloudProvider() override;
 
        /**
             * @see visionx::PointCloudProviderBase::onExitPointCloudProvider()
             */
        void onExitCapturingPointCloudProvider() override;
 
        /**
             * @see visionx::PointCloudProviderBase::onStartCapture()
             */
        void onStartCapture(float frameRate) override;
 
        /**
             * @see visionx::PointCloudProviderBase::onStopCapture()
             */
        void onStopCapture() override;
 
        /**
             * @see visionx::PointCloudProviderBase::doCapture()
            */
        bool doCapture() override;
 
        /**
             * @see visionx::CapturingImageProvider::onInitImageProvider()
             */
        void onInitImageProvider() override;
 
        /**
             * @see visionx::CapturingImageProvider::onExitImageProvider()
             */
        void onExitImageProvider() override {}
 
        bool hasSharedMemorySupport(const Ice::Current& c = Ice::emptyCurrent) override
        {
            return true;
        }
 
        /**
             * @see PropertyUser::createPropertyDefinitions()
             */
        armarx::PropertyDefinitionsPtr createPropertyDefinitions() override;
 
        // mixed inherited stuff
        void onInitComponent() override;
 
        void onConnectComponent() override;
 
        void onDisconnectComponent() override;
 
        void onExitComponent() override;
 
        StereoCalibration getStereoCalibration(const Ice::Current& c = Ice::emptyCurrent) override;
 
        bool getImagesAreUndistorted(const ::Ice::Current& c = Ice::emptyCurrent) override
        {
            return false;
        }
 
 
        std::string getReferenceFrame(const Ice::Current& c = Ice::emptyCurrent) override
        {
            return getProperty<std::string>("ReferenceFrameName");
        };
 
    private:
        typedef pcl::PointXYZRGBA PointT;
 
        libfreenect2::Freenect2Device::IrCameraParams getCameraIrParameters();
 
        libfreenect2::Freenect2Device::ColorCameraParams getCameraRGBParameters();
 
        void initializeCameraCalibration();
 
        libfreenect2::Freenect2 freenect2_;
        libfreenect2::Freenect2Device* dev_;
        libfreenect2::PacketPipeline* pipeline_;
        libfreenect2::SyncMultiFrameListener listener_;
        libfreenect2::FrameMap frames_;
        libfreenect2::Registration* registration_;
        libfreenect2::Frame undistorted_, registered_;
        bool enable_rgb_, enable_depth_, mirror_;
        int deviceId_;
        std::string serial_;
        KinectProcessorType pipeline_type_;
 
        KinectToPCLHelper<PointT>* pclHelper;
        KinectToIVTHelper* ivtHelper;
 
        ImageProviderInterfacePrx imageProviderPrx;
        std::string providerName;
        CByteImage** rgbImages_;
        int rgb_width_, rgb_height_, d_width_, d_height_;
        StereoCalibration calibration;
 
    };
}