ImageProcessor.h

Go to the documentation of this file.

/*
 * This file is part of ArmarX.
 *
 * Copyright (C) 2011-2016, High Performance Humanoid Technologies (H2T), Karlsruhe Institute of Technology (KIT), all rights reserved.
 *
 * ArmarX is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * ArmarX is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * @package    VisionX::Core
 * @author     Kai Welke (kai dot welke at kit dot edu)
 * @author     Jan Issac (jan dot issac at gmx dot net)
 * @date       2011
 * @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
 *             GNU General Public License
 */
 
#pragma once
 
// ArmarXCore
#include <ArmarXCore/core/IceManager.h>
#include <ArmarXCore/core/services/sharedmemory/IceSharedMemoryConsumer.h>
#include <ArmarXCore/core/services/tasks/PeriodicTask.h>
#include <ArmarXCore/interface/core/SharedMemory.h>
 
// IVT
#include <Image/ByteImage.h>
#include <Image/FloatImage.h>
 
// VisionXInterfaces
#include <VisionX/interface/core/DataTypes.h>
#include <VisionX/interface/core/ImageProcessorInterface.h>
 
// VisionXTools
#include <condition_variable>
#include <mutex>
#include <shared_mutex>
 
#include <VisionX/core/ImageProvider.h>
#include <VisionX/tools/FPSCounter.h>
 
namespace visionx
{
    // forward declaration
    class ImageTransferStats;
    class ImageProviderInfo;
    class ResultImageProvider;
 
    /**
     * @class ImageProcessorPropertyDefinitions
     * @brief
     */
    class ImageProcessorPropertyDefinitions : public armarx::ComponentPropertyDefinitions
    {
    public:
        ImageProcessorPropertyDefinitions(std::string prefix) :
            armarx::ComponentPropertyDefinitions(prefix)
        {
            //defineRequiredProperty<std::string>("PropertyName", "Description");
            defineOptionalProperty<CompressionType>(
                "CompressionType",
                eNoCompression,
                "Compression algorithms to be used. Values: None, PNG, JPEG",
                armarx::PropertyDefinitionBase::eModifiable)
                .setCaseInsensitive(false)
                .map("None", eNoCompression)
                .map("PNG", ePNG)
                .map("JPEG", eJPEG)
                .map("JPG", eJPEG);
            defineOptionalProperty<int>(
                "CompressionQuality",
                95,
                "Quality of the compression: PNG: 0-9 (9: best compression, but slowest), JPEG: "
                "0-100 (100: best quality) ",
                armarx::PropertyDefinitionBase::eModifiable);
            defineOptionalProperty<bool>("ForceIceTransfer",
                                         false,
                                         "If set to true, this image processor will always use the "
                                         "Ice transfer for images instead of shared memory.");
        }
    };
 
    /**
     * The ImageProcessor class provides an interface for access to
     * ImageProviders via Ice and shared memory. The interface defines a set of
     * convenience methods which simplify the image access. ImageProcessor
     * implements the ImageProcessorBase Ice interface for new image announcement
     * and the armarx::ImageProvider providing visualizations.
     */
    class ImageProcessor : virtual public armarx::Component, virtual public ImageProcessorInterface
    {
    public:
        static std::string ListenerSuffix;
        std::string getImageListenerTopicName(std::string providerName) const;
 
    protected:
        /**
         * Registers a delayed topic subscription and a delayed provider proxy
         * retrieval which all will be available on the start of the component.
         *
         * @param name                  Provider name
         */
        void usingImageProvider(std::string name);
        void releaseImageProvider(std::string providerName);
 
        /**
         * Select an ImageProvider.
         *
         * This method subscribes to an an ImageProvider and makes the provider
         * available in the waitForImages() and getImages() methods.
         *
         * @param name                  Ice adapter name of the ImageProvider
         * @param destinationImageType  Image's type after image transmission
         *
         * @return Information of the image provider
         */
        ImageProviderInfo getImageProvider(std::string name,
                                           ImageType destinationImageType = eRgb,
                                           bool waitForProxy = false);
        ImageProviderInfo getImageProvider(std::string name,
                                           bool waitForProxy,
                                           ImageType destinationImageType = eRgb);
 
        /**
         * Enables visualization
         *
         * @param numberImages number of images provided by the visualization
         * @param imageDimension size of images
         * @param imageType type of images
         * @param name The topic name to use (if empty getName() + "Result" is used)
         *
         * @return Information of the image provider
         */
        void enableResultImages(int numberImages,
                                ImageDimension imageDimension,
                                ImageType imageType,
                                const std::string& name = "");
 
        /**
         * sends result images for visualization
         * @see enableVisualization
         *
         * @param images array of images to send
         */
        void provideResultImages(CByteImage** images, armarx::MetaInfoSizeBasePtr info = nullptr);
        void provideResultImages(const std::vector<CByteImageUPtr>& images,
                                 armarx::MetaInfoSizeBasePtr info = nullptr);
 
        /**
         * Wait for new images.
         *
         * Wait for new image of an image provider. Use if only one
         * ImageProvider is used (see useImageProvider).
         *
         * @param milliseconds          Timeout for waiting
         *
         * @return True if new images are available. False in case of error or
         *         timeout
         */
        bool waitForImages(int milliseconds = 1000);
 
        /**
         * Wait for new images.
         *
         * Wait for new image of an image provider. Use if multiple
         * ImageProviders are used (see useImageProvider).
         *
         * @param providerName          Name of provider to wait for images
         * @param milliseconds          Timeout for waiting
         *
         * @return True if new images are available. False in case of error or
         *         timeout
         */
        bool waitForImages(std::string providerName, int milliseconds = 1000);
 
        bool waitForImages(std::string providerName, IceUtil::Time waitTime);
 
        bool isNewImageAvailable();
        bool isNewImageAvailable(const std::string& providerName);
 
        /**
         * Poll images from provider.
         *
         * Polls images from a used ImageProvider either via shared memory or
         * via Ice. If both components run on the same machine, shared memory
         * transfer is used. Otherwise Ice is used for image transmission. The
         * transfer type is decided in the useImageProvider method and is set
         * in the corresponding ImageFormatInfo.
         *
         * Use this method if only one ImageProvider is used.
         *
         * @param ppImages              Image buffers where the images are
         *                              copied to. The buffers have to be
         *                              initialized by the component. All
         *                              required information for the allocation
         *                              of the buffers can be found in the
         *                              corresponding ImageFormatInfo.
         *
         * @return Number of images copied. Zero if no new images have been
         *         available.
         */
        int getImages(CByteImage** ppImages);
 
        /**
         * Poll images from provider.
         *
         * Polls images from a used ImageProvider either via shared memory or
         * via Ice. If both components run on the same machine, shared memory
         * transfer is used. Otherwise Ice is used for image transmission. The
         * transfer type is decided in the useImageProvider method and is set
         * in the corresponding ImageFormatInfo.
         *
         * Use this method if multiple ImageProviders are used.
         *
         * @param providerName          Name of provider to poll from
         *
         * @param ppImages              Image buffers where the images are
         *                              copied to. The buffers have to be
         *                              initialized by the component. All
         *                              required information for the allocation
         *                              of the buffers can be found in the
         *                              corresponding ImageFormatInfo.
         *
         * @return Number of images copied. Zero if no new images have been
         *         available.
         */
        int getImages(std::string providerName,
                      CByteImage** ppImages,
                      armarx::MetaInfoSizeBasePtr& info);
        int getImages(std::string providerName,
                      const std::vector<CByteImageUPtr>& ppImages,
                      armarx::MetaInfoSizeBasePtr& info);
 
        /**
         * @see ImageProcessorBase::getImages(CByteImage*)
         */
        int getImages(CFloatImage** ppImages);
 
        /**
         * @see ImageProcessorBase::getImages(std::string, CByteImage**)
         */
        int getImages(std::string providerName,
                      CFloatImage** ppImages,
                      armarx::MetaInfoSizeBasePtr& info);
 
        /**
         * Retrieve statistics for a connection to an ImageProvider.
         *
         * @param provideNname          Name of the provider
         * @param resetStats            Reset statistics
         *
         * @return Reference to statistics for the connection to the provder
         */
        ImageTransferStats getImageTransferStats(std::string provideNname, bool resetStats = false);
 
 
        /**
         * @brief Get meta information from the image provider.
         * @param imageProviderName Name of image provider. If empty, returns first entry in ImageProvider map.
         * @return Null if no ImageProvider used or name could not be found.
         */
        armarx::MetaInfoSizeBasePtr
        getImageMetaInfo(const std::string& imageProviderName = "") const;
 
        void setFramerate(float fps);
        float getFramerate() const;
 
        /**
         * Sets the compression type and compression quality. Quality needs to fit to compressionType: PNG: 0-9 (9: best compression, but slowest), JPEG: 0-100 (100: best quality)         *
         */
        void setCompressionType(CompressionType compressionType = ePNG, int compressionQuality = 9);
        CompressionType getCompressionType() const;
        int getCompressionQuality() const;
 
 
        // ================================================================== //
        // == Interface of an ImageProcessor ================================ //
        // ================================================================== //
        /**
         * Setup the vision component.
         *
         * Implement this method in the ImageProcessor in order to setup its
         * parameters. Use this for the registration of adaptars and
         * subscription to topics
         *
         * @param argc number of filtered command line arguments
         * @param argv filtered command line arguments
         */
        virtual void onInitImageProcessor() = 0;
 
        /**
         * Implement this method in the ImageProcessor in order execute parts
         * when the component is fully initialized and about to run.
         */
        virtual void onConnectImageProcessor() = 0;
 
        /**
         * Implement this method in the ImageProcessor in order execute parts
         * when the component looses network connectivity.
         */
        virtual void
        onDisconnectImageProcessor()
        {
        }
 
        /**
         * Exit the ImapeProcessor component.
         *
         * Implement this method in order to clean up the ImageProcessor
         */
        virtual void onExitImageProcessor() = 0;
 
        /**
         * Process the vision component.
         *
         * The main loop of the imageprocessor to be implemented in the
         * subclass. Do not block this method. One process should execute
         * exactly one image processing step.
         * The rate at which this function is called can be set with setFramerate().
         * If no framerate is set, the process will run as fast as possible.
         * @see setFramerate
         */
        virtual void process() = 0;
 
        // ================================================================== //
        // == RunningComponent implementation =============================== //
        // ================================================================== //
        /**
         * @see Component::onInitComponent()
         */
        void onInitComponent() override;
 
        /**
         * @see Component::onConnectComponent()
         */
        void onConnectComponent() override;
 
        /**
         * @see Component::onDisconnectComponent()
         */
        void onDisconnectComponent() override;
 
        /**
         * @see Component::onExitComponent()
         */
        void onExitComponent() override;
 
        /**
         * @see PropertyUser::createPropertyDefinitions()
         */
        armarx::PropertyDefinitionsPtr createPropertyDefinitions() override;
        void componentPropertiesUpdated(const std::set<std::string>& changedProperties) override;
 
    protected:
        /**
         * @see PeriodicTask
         */
        virtual void runProcessor();
 
        // ================================================================== //
        // == SharedMemoryComponent Ice interface =========================== //
        // ================================================================== //
        /**
         * Returns machines hardware Id string
         */
        std::string getHardwareId(const Ice::Current& c = Ice::emptyCurrent);
 
        // ================================================================== //
        // == ImageListener Ice interface =================================== //
        // ================================================================== //
        /**
         * Listener callback function. This is called by the used image
         * providers to report the availability of a newly captured
         * image.
         *
         * @param providerName          The reporting image provider name
         */
        void reportImageAvailable(const std::string& providerName,
                                  const Ice::Current& c = Ice::emptyCurrent) override;
 
        // ================================================================== //
        // == ImageProcessor internals ====================================== //
        // ================================================================== //
        /**
         * @see ImageProcessorBase::getImages(CByteImage**)
         */
        int getImages(void** ppBuffer);
 
        /**
         * @see ImageProcessorBase::getImages(std::string, CByteImage**)
         */
        int getImages(std::string providerName, void** ppBuffer, armarx::MetaInfoSizeBasePtr& info);
 
        /**
         * Returns the number of images provided by the specified image provider
         * If no provider is specified, the first provider is taken.
         *
         * @param providerName  Requested image provider
         */
        int getNumberOfImages(const std::string& providerName = "");
 
        /**
         * clean up memory
         */
        void cleanup();
 
        // image provider
        using ImageProviderMap =
            std::map<std::string, armarx::IceSharedMemoryConsumer<unsigned char>::pointer_type>;
        ImageProviderMap usedImageProviders;
 
        std::map<std::string, ImageProviderInfo> imageProviderInfoMap;
        std::shared_mutex imageProviderInfoMutex;
 
        // result image visualization
        IceInternal::Handle<ResultImageProvider> resultImageProvider;
 
        // statistics
        std::mutex statisticsMutex;
        std::map<std::string, ImageTransferStats> statistics;
 
        armarx::PeriodicTask<ImageProcessor>::pointer_type processorTask;
 
        float desiredFps = 0;
        bool startProcessorTask = true;
 
 
        CompressionType compressionType = eNoCompression;
        int compressionQuality = 9;
    };
 
    /**
     * Shared pointer for convenience
     */
    using ImageProcessorPtr = IceInternal::Handle<ImageProcessor>;
 
    // ====================================================================== //
    // == class ImageTransferStats declaration ============================== //
    // ====================================================================== //
 
    /**
     * The ImageTransferStats class provides information on the connection
     * between ImageProvider and ImageProcessor. Use
     * ImageProcessorBase::getImageTransferStats() in order to retrieve the
     * statistics
     */
    class ImageTransferStats
    {
    public:
        ImageTransferStats()
        {
            imageProviderFPS.reset();
            pollingFPS.reset();
        }
 
        /**
         * Statistics for the images announced by the ImageProvider.
         */
        FPSCounter imageProviderFPS;
 
        /**
         * Statistics for the images polled by the ImageProcessor.
         */
        FPSCounter pollingFPS;
    };
 
    // ====================================================================== //
    // == class ImageProviderInfo declaration =============================== //
    // ====================================================================== //
 
    class ImageProviderInfo
    {
    public:
        /**
         * proxy to image provider
         */
        ImageProviderInterfacePrx proxy;
 
        /**
         * memory block
         */
        std::vector<unsigned char> buffer;
 
        /**
         * meta info
         */
        armarx::MetaInfoSizeBasePtr info;
 
        /**
         * Required image destination type.
         *
         * This is used to convert the transmitted image into an image with the
         * desired image type if necessary, since the transmitted image type is
         * given by the provider.
         */
        ImageType destinationImageType;
 
        /**
         * Image format struct that contains all necessary image information
         */
        ImageFormatInfo imageFormat;
 
        /**
         * Transfer mode of images
         */
        ImageTransferMode imageTransferMode;
 
        /**
        * Number of images.
         */
        int numberImages;
 
        /**
         * Indicates whether an image is available.
         */
        bool imageAvailable;
 
        /**
         * Conditional variable used internally for synchronization purposes
         */
        std::shared_ptr<std::condition_variable> imageAvailableEvent;
    };
 
    // ====================================================================== //
    // == class ResultImageProvider declaration ============================= //
    // ====================================================================== //
    /**
     * The ResultImageProvider is used by the ImageProcessor to stream
     * result images to any other processor (e.g. ImageMonitor)
     * Use ImageProcessor::enableVisualization() and ImageProcessor::provideResultImages()
     * in order to offer result images in an image processor.
     */
    class ResultImageProvider : public ImageProvider
    {
        friend class ImageProcessor;
 
    public:
        ResultImageProvider() : resultImageProviderName("ResultImageProvider")
        {
        }
 
        void
        setNumberResultImages(int numberResultImages)
        {
            this->numberResultImages = numberResultImages;
        }
 
        void
        setResultImageFormat(ImageDimension resultImageDimension, ImageType resultImageType)
        {
            this->resultImageDimension = resultImageDimension;
            this->resultImageType = resultImageType;
        }
 
        void
        provideResultImages(CByteImage** images, Ice::Long timestamp)
        {
            updateTimestamp(timestamp);
            provideImages(images, IceUtil::Time::microSeconds(timestamp));
        }
 
        void
        provideResultImages(const std::vector<CByteImageUPtr>& images, Ice::Long timestamp)
        {
            updateTimestamp(timestamp);
            provideImages(images, IceUtil::Time::microSeconds(timestamp));
        }
 
 
    protected:
        void
        setResultImageProviderName(const std::string& name)
        {
            this->resultImageProviderName = name;
        }
 
        std::string
        getDefaultName() const override
        {
            return resultImageProviderName;
        }
 
        void
        onInitImageProvider() override
        {
            setNumberImages(numberResultImages);
            setImageFormat(resultImageDimension, resultImageType);
        }
 
        void
        onExitImageProvider() override
        {
        }
 
        int numberResultImages;
 
    private:
        std::string resultImageProviderName;
 
        visionx::ImageDimension resultImageDimension;
        visionx::ImageType resultImageType;
    };
 
} // namespace visionx