SegmentableTemplateRecognition.h

Go to the documentation of this file.

/*
* This file is part of ArmarX.
*
* ArmarX is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* ArmarX is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package    VisionX::Component
* @author    Kai Welke <welke at kit dot edu>
* @copyright  2013 Humanoids Group, HIS, KIT
* @license    http://www.gnu.org/licenses/gpl-2.0.txt
*             GNU General Public License
*/
 
#pragma once
 
 
// VisionX
#include <ArmarXCore/observers/DebugObserver.h>
 
#include <VisionX/components/object_perception/ObjectLocalizerProcessor.h>
 
// IVT
#include <Eigen/Core>
 
#include <Color/ColorParameterSet.h>
#include <ObjectFinder/ObjectFinderStereo.h>
#include <Visualizer/OpenGLVisualizer.h>
 
// global forward declarations
class CByteImage;
class CGLContext;
 
namespace visionx
{
    static const std::size_t SegmentableBitmapWidth = 64;
    static const std::size_t SegmentableBitmapSize =
        SegmentableBitmapWidth * SegmentableBitmapWidth;
    // How much uint32_t do we need to store all the bits
    static const std::size_t SegmentableBitmapUInt32Size = SegmentableBitmapSize / 32;
 
    struct SegmentableTemplate
    {
        float rotationX;
        float rotationY;
        float rotationZ;
        std::uint32_t bitmap[SegmentableBitmapUInt32Size];
    };
 
    struct SegmentableTemplateHeader
    {
        char magic[4];
        char name[256];
        std::uint32_t templateCount;
        SegmentableTemplate templates[1];
    };
 
    struct SegmentableTemplateEntry
    {
        std::unique_ptr<SegmentableTemplateHeader> data;
        std::unique_ptr<CFloatMatrix> model;
        ObjectColor color;
    };
 
    // properties of SegmentableTemplateRecognition
    class SegmentableTemplateRecognitionPropertyDefinitions :
        public ObjectLocalizerProcessorPropertyDefinitions
    {
        static inline Eigen::Vector3f
        stringToVector3f(std::string propertyValue)
        {
            Eigen::Vector3f vec;
            sscanf(propertyValue.c_str(),
                   "%f, %f, %f",
                   &vec.data()[0],
                   &vec.data()[1],
                   &vec.data()[2]);
            return vec;
        }
 
    public:
        SegmentableTemplateRecognitionPropertyDefinitions(std::string prefix) :
            ObjectLocalizerProcessorPropertyDefinitions(prefix)
        {
            defineOptionalProperty<std::string>(
                "ColorParameterFile",
                "VisionX/examples/cebit-colors.txt",
                "The color parameter file configures the colors used for segmentable recognition "
                "(usually colors.txt)");
            defineOptionalProperty<int>(
                "MinPixelsPerRegion",
                6000,
                "Minimum number of pixels per region for detecting a uniformly colored object");
            defineOptionalProperty<float>(
                "MaxEpipolarDistance",
                12,
                "Maximum epipolar line distance allowed for a valid 3D recognition result");
 
            defineOptionalProperty<std::string>("TemplatePath", "VisionX/templates");
 
 
            armarx::PropertyDefinition<Eigen::Vector3f>::PropertyFactoryFunction f =
                &stringToVector3f;
 
            defineOptionalProperty<Eigen::Vector3f>("MinPoint",
                                                    Eigen::Vector3f(-3000.0f, -3000.0f, 100.0f),
                                                    "min point for valid result bounding box")
                .setFactory(f);
            defineOptionalProperty<Eigen::Vector3f>("MaxPoint",
                                                    Eigen::Vector3f(3000.0f, 3000.0f, 3500.0f),
                                                    "max point for valid result bounding box")
                .setFactory(f);
 
            defineOptionalProperty<float>("TemplateMatchThreshold", 0.7f, "");
            defineOptionalProperty<float>("SizeRatioThreshold", 0.7f, "");
            defineOptionalProperty<float>("CorrelationThreshold", 0.7f, "");
 
            defineOptionalProperty<bool>("SingleInstance", true, "");
        }
    };
 
    /**
     * SegmentableTemplateRecognition uses CSegmentableRecognition from IVT to recognize and localize single-colored objects based on their color and shape.
     * The object data is read from PriorKnowledge and CommonStorage via MemoryX.
     * The object localization is invoked automatically by the working memory if the object has been requested there.
     *
     */
    class SegmentableTemplateRecognition : virtual public ObjectLocalizerProcessor
    {
    public:
        /**
         * @see PropertyUser::createPropertyDefinitions()
         */
        armarx::PropertyDefinitionsPtr
        createPropertyDefinitions() override
        {
            return armarx::PropertyDefinitionsPtr(
                new SegmentableTemplateRecognitionPropertyDefinitions(getConfigIdentifier()));
        }
 
        void
        reportStereoCalibrationChanged(const StereoCalibration& stereoCalibration,
                                       bool x,
                                       const std::string& referenceName,
                                       const Ice::Current& c = Ice::emptyCurrent) override
        {
            ObjectLocalizerProcessor::reportStereoCalibrationChanged(
                stereoCalibration, x, referenceName, c);
 
            initRecognizer();
        }
 
        /**
        * @see Component::getDefaultName()
        */
        std::string
        getDefaultName() const override
        {
            return "SegmentableTemplateRecognition";
        }
 
        SegmentableTemplateRecognition();
        ~SegmentableTemplateRecognition() override;
 
    protected:
        /**
         * @see ObjectLocalizerProcessor::onInitObjectLocalizerProcessor()
         */
        void onInitObjectLocalizerProcessor() override;
 
        /**
         * Initializes the CSegmentableRecognition
         *
         * @see ObjectLocalizerProcessor::onConnectObjectLocalizerProcessor()
         */
        void
        onConnectObjectLocalizerProcessor() override
        {
 
            debugObserver = getTopic<armarx::DebugObserverInterfacePrx>(
                getProperty<std::string>("DebugObserverName").getValue());
        }
 
        /**
         * @see ObjectLocalizerProcessor::onExitObjectLocalizerProcessor()
         */
        void onExitObjectLocalizerProcessor() override;
 
        /**
         * Initializes segmentable recognition
         *
         * @return success
         */
        bool initRecognizer() override;
 
        /**
         * Add object class to segmentable object recognition.
         *
         * @param objectClassEntity entity containing all information available for the object class
         * @param fileManager GridFileManager required to read files associated to prior knowledge from the database.
         *
         * @return success of adding this entity to the TexturedObjectDatabase
         */
        bool addObjectClass(const memoryx::EntityPtr& objectClassEntity,
                            const memoryx::GridFileManagerPtr& fileManager) override;
 
        /**
         * localizes segmentable object instances
         *
         * @param objectClassNames names of the class to localize
         * @param cameraImages the two input images
         * @param resultImages the two result images. are provided if result images are enabled.
         *
         * @return list of object instances
         */
        memoryx::ObjectLocalizationResultList
        localizeObjectClasses(const std::vector<std::string>& objectClassNames,
                              CByteImage** cameraImages,
                              armarx::MetaInfoSizeBasePtr imageMetaInfo,
                              CByteImage** resultImages) override;
 
    private:
        void visualizeResults(const Object3DList& objectList, CByteImage** resultImages);
 
        float calculateRecognitionCertainty(const std::string& objectClassName,
                                            const Object3DEntry& entry);
 
    private:
        std::shared_ptr<CGLContext> contextGL;
        std::shared_ptr<COpenGLVisualizer> m_pOpenGLVisualizer;
        std::shared_ptr<CObjectFinderStereo> m_pObjectFinderStereo;
 
        // params
        int paramMinPixelsPerRegion = 200;
        float paramTemplateMatchThreshold = 0.7f;
        float paramSizeRatioThreshold = 0.7f;
        float paramCorrelationThreshold = 0.7f;
        bool paramSingleInstance = true;
 
        // settings
        std::string templatePath;
        float minPixelsPerRegion;
        float maxEpipolarDistance;
        CColorParameterSet colorParameters;
        // Object name => template data
        std::map<std::string, SegmentableTemplateEntry> database;
 
        Vec3d validResultBoundingBoxMin, validResultBoundingBoxMax;
 
 
        std::map<std::string, int> seq;
 
 
        armarx::DebugObserverInterfacePrx debugObserver;
    };
} // namespace visionx