ObjectLocalizationSaliency.h

Go to the documentation of this file.

/*
 * This file is part of ArmarX.
 *
 * ArmarX is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * ArmarX is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * @package    RobotComponents::ArmarXObjects::ObjectLocalizationSaliency
 * @author     David Schiebener (schiebener at kit dot edu)
 * @author     Markus Grotz ( markus dot grotz at kit dot edu )
 * @date       2016
 * @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
 *             GNU General Public License
 */
 
#pragma once
 
 
#include <condition_variable>
#include <mutex>
 
#include <ArmarXCore/core/Component.h>
#include <ArmarXCore/core/services/tasks/PeriodicTask.h>
 
#include <RobotAPI/components/EarlyVisionGraph/GraphPyramidLookupTable.h>
#include <RobotAPI/components/EarlyVisionGraph/IntensityGraph.h>
#include <RobotAPI/components/EarlyVisionGraph/MathTools.h>
#include <RobotAPI/interface/components/ViewSelectionInterface.h>
 
#include <MemoryX/core/MemoryXCoreObjectFactories.h>
#include <MemoryX/interface/components/WorkingMemoryInterface.h>
#include <MemoryX/interface/memorytypes/MemorySegments.h>
#include <MemoryX/interface/workingmemory/AbstractWorkingMemoryInterface.h>
#include <MemoryX/libraries/helpers/ObjectRecognitionHelpers/ObjectRecognitionWrapper.h>
#include <MemoryX/libraries/memorytypes/MemoryXTypesObjectFactories.h>
#include <MemoryX/libraries/memorytypes/entity/ObjectClass.h>
#include <MemoryX/libraries/memorytypes/entity/ObjectInstance.h>
#include <MemoryX/libraries/observers/ObjectMemoryObserver.h>
#include <MemoryX/libraries/updater/ObjectLocalization/MemoryXUpdaterObjectFactories.h>
 
namespace armarx
{
    /**
     * @class ObjectLocalizationSaliencyPropertyDefinitions
     * @brief
     */
    class ObjectLocalizationSaliencyPropertyDefinitions :
        public armarx::ComponentPropertyDefinitions
    {
    public:
        ObjectLocalizationSaliencyPropertyDefinitions(std::string prefix) :
            armarx::ComponentPropertyDefinitions(prefix)
        {
            defineOptionalProperty<std::string>(
                "RobotStateComponentName",
                "RobotStateComponent",
                "Name of the robot state component that should be used");
            defineOptionalProperty<std::string>(
                "HeadIKUnitName",
                "HeadIKUnit",
                "Name of the head IK unit component that should be used");
            defineOptionalProperty<std::string>(
                "WorkingMemoryName",
                "WorkingMemory",
                "Name of the WorkingMemory component that should be used");
            defineOptionalProperty<std::string>(
                "PriorKnowledgeName",
                "PriorKnowledge",
                "Name of the PriorKnowledge component that should be used");
            defineOptionalProperty<std::string>(
                "ViewSelectionName",
                "ViewSelection",
                "Name of the ViewSelection component that should be used");
            defineOptionalProperty<std::string>(
                "CameraFrameName",
                "VirtualCentralGaze",
                "Name of the frame of the head base in the robot model");
 
            defineOptionalProperty<std::string>(
                "HeadFrameName",
                "Head Base",
                "Name of the frame of the head base in the robot model");
            defineOptionalProperty<float>(
                "RandomNoiseLevel",
                1.0f,
                "Maximum for the random noise that will be added to the localization necessities");
            defineOptionalProperty<float>("HalfCameraOpeningAngle",
                                          12.0 * M_PI / 180.0,
                                          "0.5 * minimal opening angles of the used cameras");
            defineOptionalProperty<float>(
                "MaxObjectDistance",
                1500.f,
                "Maximum distance of objects the head to be considered for observation");
            defineOptionalProperty<float>(
                "CentralHeadTiltAngle",
                110.0f,
                "Defines the height direction that will be considered 'central' in the reachable "
                "area of the head (in degrees). Default is looking 20 degrees downwards");
            defineOptionalProperty<float>(
                "ProbabilityToLookForALostObject",
                0.03f,
                "Probability that one of the objects that have been seen but could later not been "
                "localized again will be included in the view selection");
            defineOptionalProperty<float>(
                "UpdateFrequency", 10.0f, "Frequency of the saliency update");
        }
    };
 
    /**
     * @defgroup Component-ObjectLocalizationSaliency ObjectLocalizationSaliency
     * @ingroup RobotComponents-Components
     * A description of the component ObjectLocalizationSaliency.
     *
     * @class ObjectLocalizationSaliency
     * @ingroup Component-ObjectLocalizationSaliency
     * @brief Brief description of class ObjectLocalizationSaliency.
     *
     * Detailed description of class ObjectLocalizationSaliency.
     */
    class ObjectLocalizationSaliency :
        virtual public armarx::Component,
        virtual public armarx::ViewSelectionObserver
    {
    public:
        /**
         * @see armarx::ManagedIceObject::getDefaultName()
         */
        std::string
        getDefaultName() const override
        {
            return "ObjectLocalizationSaliency";
        }
 
        void
        onActivateAutomaticViewSelection(const Ice::Current& c = Ice::emptyCurrent) override
        {
            ARMARX_LOG << "enabling object saliency computation";
 
            next = IceUtil::Time::seconds(0);
 
            if (!(processorTask && processorTask->isRunning()))
            {
                processorTask->start();
            }
            else
            {
                ARMARX_INFO << "view selection already active";
            }
        }
 
        void
        onDeactivateAutomaticViewSelection(const Ice::Current& c = Ice::emptyCurrent) override
        {
 
            ARMARX_LOG << "disabling object saliency computation";
            if (processorTask && processorTask->isRunning())
            {
                processorTask->stop();
            }
        }
 
        void nextViewTarget(Ice::Long timestamp,
                            const Ice::Current& c = Ice::emptyCurrent) override;
 
 
    protected:
        /**
         * @see armarx::ManagedIceObject::onInitComponent()
         */
        void onInitComponent() override;
 
        /**
         * @see armarx::ManagedIceObject::onConnectComponent()
         */
        void onConnectComponent() override;
 
        /**
         * @see armarx::ManagedIceObject::onDisconnectComponent()
         */
        void onDisconnectComponent() override;
 
        /**
         * @see armarx::ManagedIceObject::onExitComponent()
         */
        void onExitComponent() override;
 
        /**
         * @see PropertyUser::createPropertyDefinitions()
         */
        armarx::PropertyDefinitionsPtr createPropertyDefinitions() override;
 
 
    private:
        void process();
 
        void generateObjectLocalizationSaliency();
 
        void addSaliencyRecursive(const int currentNodeIndex,
                                  const float saliency,
                                  const TSphereCoord objectSphereCoord,
                                  const int objectIndex,
                                  const float maxDistanceOnArc);
 
        void setRandomNoise(const float centralAngleForVerticalDirection,
                            const float directionVariabilityFactor = 1.0f);
        void generateRandomNoise(std::vector<memoryx::ObjectInstancePtr>& localizableObjects,
                                 memoryx::EntityBaseList& objectInstances);
 
 
        std::mutex mutex;
        IceUtil::Time next;
        IceUtil::Time lastDiff;
 
        RobotStateComponentInterfacePrx robotStateComponent;
 
        memoryx::WorkingMemoryInterfacePrx memoryProxy;
        memoryx::PriorKnowledgeInterfacePrx priorKnowledge;
        memoryx::ObjectInstanceMemorySegmentBasePrx objectInstancesProxy;
        memoryx::ObjectClassMemorySegmentBasePrx objectClassesProxy;
        VirtualRobot::RobotPtr robot;
 
 
        ViewSelectionInterfacePrx viewSelection;
 
        armarx::PeriodicTask<ObjectLocalizationSaliency>::pointer_type processorTask;
 
 
        std::map<std::string, memoryx::ObjectClassPtr> recognizableObjClasses;
 
        CIntensityGraph* saliencyEgosphereGraph;
        CIntensityGraph* randomNoiseGraph;
        CGraphPyramidLookupTable* graphLookupTable;
 
        std::vector<int> nodeVisitedForObject;
 
 
        Eigen::Vector3f offsetToHeadCenter;
 
        float centralHeadTiltAngle;
 
        float halfCameraOpeningAngle;
        float deviationFromCameraCenterFactor;
        float randomNoiseLevel;
 
        std::string headFrameName, cameraFrameName;
    };
} // namespace armarx