Component.h

Go to the documentation of this file.

/*
 * This file is part of ArmarX.
 *
 * ArmarX is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * ArmarX is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * @package    VisionX::ArmarXObjects::Yolo
 * @author     Christian R. G. Dreher <c.dreher@kit.edu>
 * @date       2019
 * @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
 *             GNU General Public License
 */
 
 
#pragma once
 
 
// STD/STL
#include <memory>
#include <mutex>
#include <string>
#include <vector>
 
// IVT
#include <Image/ByteImage.h>
 
// Darknet
#include <darknet.h>
 
// Ice
#include <Ice/Current.h>
#include <IceUtil/Time.h>
 
// ArmarX
#include <ArmarXCore/core/logging/Logging.h>
#include <ArmarXCore/core/services/tasks/RunningTask.h>
 
#include <VisionX/core/ImageProcessor.h>
#include <VisionX/interface/components/YoloComponentInterface.h>
#include <VisionX/interface/components/YoloObjectListener.h>
#include <VisionX/interface/core/ImageProviderInterface.h>
#include <VisionX/tools/ImageUtil.h>
 
namespace visionx::yolo
{
 
    class Component :
        virtual public visionx::ImageProcessor,
        virtual public ComponentInterface,
        virtual public armarx::Logging
    {
 
    public:
        static const std::string default_name;
 
    private:
        bool m_image_received;
 
        std::mutex m_input_image_mutex;
 
        std::string m_image_provider_id;
        visionx::ImageProviderInfo m_image_provider_info;
 
        IceUtil::Time m_timestamp_last_image;
 
        ::CByteImage** m_input_image_buf;
        CByteImageUPtr m_input_image;
        CByteImageUPtr m_output_image;
 
        armarx::RunningTask<Component>::pointer_type m_task_object_detection;
 
        visionx::ImageProviderInterfacePrx m_image_provider;
        unsigned int m_image_provider_channel;
 
        ObjectListener::ProxyType m_object_detection_listener;
 
        IceUtil::Time m_minimum_loop_time;
 
        /**
         * @brief Darknet's network.
         */
        darknet::network m_network;
 
        std::vector<std::string> m_darknet_classes;
 
    public:
        virtual ~Component() override;
 
        /**
         * @see visionx::ImageProcessor::onInitImageProcessor()
         */
        virtual void onInitImageProcessor() override;
 
        /**
         * @see visionx::ImageProcessor::onConnectImageProcessor()
         */
        virtual void onConnectImageProcessor() override;
 
        /**
         * @see visionx::ImageProcessor::onDisconnectImageProcessor()
         */
        virtual void onDisconnectImageProcessor() override;
 
        /**
         * @see visionx::ImageProcessor::onExitImageProcessor()
         */
        virtual void onExitImageProcessor() override;
 
        /**
         * @see armarx::ManagedIceObject::getDefaultName()
         */
        virtual std::string getDefaultName() const override;
 
        /**
         * @see visionx::ImageProcessor::process()
         */
        virtual void process() override;
 
        /**
         * @see PropertyUser::createPropertyDefinitions()
         */
        virtual armarx::PropertyDefinitionsPtr createPropertyDefinitions() override;
 
        /**
         * @brief Converts Darknet types to VisionX types.
         * @param detections Array of detections.
         * @param classes Available classes as strings in correct order.
         * @return Converted Ice types as vector of DetectedObjects.
         */
        static std::vector<DetectedObject>
        convertOutput(const std::vector<darknet::detection>& detections,
                      const std::vector<std::string>& classes);
 
        /**
         * @brief Renders the objects detectedObjects, belonging to one of a class defined in
         *        classes onto the outputImage (initialsed from inputImage).
         * @param inputImage The base image output_image is initialised from.
         * @param detectedObjects All detected objects to be rendered.
         * @param classes List of all classes an object can belong to.
         * @param outputImage Ouput variable of the resulting rendered image.
         */
        static void renderOutput(const ::CByteImage* inputImage,
                                 const std::vector<DetectedObject>& detectedObjects,
                                 ::CByteImage* outputImage);
 
        /**
         * @brief Restores the default parameter values.
         */
        virtual void restoreDefaults(const Ice::Current& = Ice::Current{}) override;
 
        /**
         * @brief Returns the currently set thresh parameter.
         * @return Currently set thresh parameter.
         */
        virtual double getThresh(const Ice::Current&) override;
 
        /**
         * @brief Sets the new thresh parameter.
         * @param thresh New thresh.
         */
        virtual void setThresh(double thresh, const Ice::Current&) override;
 
        /**
         * @brief Returns the currently set hierThresh parameter.
         * @return Currently set hierThresh parameter.
         */
        virtual double getHierThresh(const Ice::Current&) override;
 
        virtual float getFpsCap(const Ice::Current&) override;
 
        /**
         * @brief Sets the new hierThresh parameter.
         * @param hier_thresh New hierThresh.
         */
        virtual void setHierThresh(double hier_thresh, const Ice::Current&) override;
 
        /**
         * @brief Returns the currently set nms parameter.
         * @return Currently set nms parameter.
         */
        virtual double getNms(const Ice::Current&) override;
 
        /**
         * @brief Sets the new nms parameter.
         * @param nms New nms.
         */
        virtual void setNms(double nms, const Ice::Current&) override;
 
        virtual void setFpsCap(float fps_cap, const Ice::Current& = Ice::Current{}) override;
 
        /**
         * @brief Returns the list of all known classes.
         * @return List of all known classes.
         */
        virtual std::vector<std::string> getClasses(const Ice::Current&) override;
 
    private:
        /**
         * @brief Bootstraps Darknet.
         *
         * Initialises Darknet so it can quickly be accessed afterwards.  Bootstrapping usually
         * takes a while.
         */
        virtual void bootstrap_darknet();
 
        /**
         * @brief Run Darknet to detect objects in the current image.
         */
        virtual void run_object_detection_task();
    };
 
} // namespace visionx::yolo
 
namespace visionx
{
    using Yolo = visionx::yolo::Component;
}