HumanMemory.cpp

Go to the documentation of this file.

/*
 * This file is part of ArmarX.
 *
 * ArmarX is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * ArmarX is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * @package    RobotAPI::ArmarXObjects::ExampleMemory
 * @author     Rainer Kartmann ( rainer dot kartmann at kit dot edu )
 * @date       2020
 * @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
 *             GNU General Public License
 */
 
#include "HumanMemory.h"
 
#include <algorithm>
#include <iterator>
#include <string>
#include <vector>
 
#include <Ice/Current.h>
 
#include <SimoxUtility/algorithm/apply.hpp>
#include <VirtualRobot/VirtualRobot.h>
 
#include "ArmarXCore/core/time/DateTime.h"
#include <ArmarXCore/core/Component.h>
#include <ArmarXCore/core/application/properties/PropertyDefinitionContainer.h>
#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
#include <ArmarXCore/core/logging/Logging.h>
#include <ArmarXCore/core/time/ice_conversions.h>
#include <ArmarXCore/interface/core/time.h>
 
#include "RobotAPI/libraries/armem/core/Commit.h"
#include "RobotAPI/libraries/armem/core/MemoryID.h"
#include "RobotAPI/libraries/aron/core/data/variant/container/Dict.h"
#include "RobotAPI/libraries/core/FramedPose.h"
#include <RobotAPI/interface/aron/Aron.h>
#include <RobotAPI/libraries/armem/server/MemoryToIceAdapter.h>
#include <RobotAPI/libraries/armem/server/wm/memory_definitions.h>
#include <RobotAPI/libraries/aron/common/aron_conversions/framed.h>
 
#include "VisionX/libraries/armem_human/server/face_recognition/Segment.h"
#include "VisionX/libraries/armem_human/server/profile/Segment.h"
#include <VisionX/libraries/armem_human/aron/FaceRecognition.aron.generated.h>
#include <VisionX/libraries/armem_human/aron/HumanPose.aron.generated.h>
#include <VisionX/libraries/armem_human/aron/Person.aron.generated.h>
#include <VisionX/libraries/armem_human/server/PoseSegment.h>
 
namespace armarx::armem::server::human_memory
{
 
    armarx::PropertyDefinitionsPtr
    HumanMemory::createPropertyDefinitions()
    {
        armarx::PropertyDefinitionsPtr defs =
            new ComponentPropertyDefinitions(getConfigIdentifier());
 
        setMemoryName("Human");
 
        profileSegment.defineProperties(defs, "profile.");
        personInstanceSegment.defineProperties(defs, "instance");
        poseSegment.defineProperties(defs, "pose.");
        faceRecognitionSegment.defineProperties(defs, "face.");
        identificationSegment.defineProperties(defs, "ident.");
        humanActivitySegment.defineProperties(defs, "activity.");
        humanRobotInteractionSegment.defineProperties(defs, "interaction.");
 
        return defs;
    }
 
    HumanMemory::HumanMemory() :
        profileSegment(iceAdapter()),
        personInstanceSegment(iceAdapter()),
        poseSegment(iceAdapter()),
        faceRecognitionSegment(iceAdapter()),
        identificationSegment(iceAdapter()),
        humanActivitySegment(iceAdapter()),
        humanRobotInteractionSegment(iceAdapter()),
        visu(poseSegment, faceRecognitionSegment, personInstanceSegment)
    {
        addPlugin(virtualRobotReaderPlugin);
    }
 
    std::string
    HumanMemory::getDefaultName() const
    {
        return "HumanMemory";
    }
 
    void
    HumanMemory::onInitComponent()
    {
        profileSegment.init();
        personInstanceSegment.init();
        poseSegment.init();
        faceRecognitionSegment.init();
        identificationSegment.init();
        humanActivitySegment.init();
        humanRobotInteractionSegment.init();
        visu.init();
 
        workingMemory().addCoreSegment("RecognizedAction");
    }
 
    void
    HumanMemory::onConnectComponent()
    {
        visu.connect(arviz, getDebugObserver());
    }
 
    void
    HumanMemory::onDisconnectComponent()
    {
    }
 
    void
    HumanMemory::onExitComponent()
    {
    }
 
    VirtualRobot::RobotPtr
    HumanMemory::getSynchronizedRobot(const std::string& robotName,
                                      const armarx::DateTime& timestamp)
    {
        // seach and retrieve robot
        auto& robot = [&]() -> VirtualRobot::RobotPtr&
        {
            // retrieve robot from memory if it is not in cache
            if (robots.count(robotName) == 0)
            {
                auto newRobot = virtualRobotReaderPlugin->get().getRobot(robotName);
                ARMARX_CHECK_NOT_NULL(newRobot)
                    << "Failed to obtain robot with name `" << robotName << "`.";
 
                robots[robotName] = newRobot;
            }
 
            return robots.at(robotName);
        }();
 
        // synchronize robot
        bool const success = virtualRobotReaderPlugin->get().synchronizeRobot(*robot, timestamp);
 
        if (not success)
        {
            //            throw armarx::LocalException(
            //                "Failed synchronizing the robot to calculate the global human pose from a reported "
            //                "human pose in camera coordinate system. Query timestamp is '")
            //                << timestamp << "'.";
            return nullptr;
        }
 
        return robot;
    }
 
    void
    HumanMemory::commitHumanPosesInCameraFrame(
        const ::armarx::aron::data::dto::AronDictSeq& dictSeq,
        const ::std::string& providerName,
        const ::armarx::core::time::dto::DateTime& iceTimestamp,
        const ::Ice::Current& /*c*/)
    {
        if (dictSeq.empty())
        {
            return;
        }
 
        // convert from ice
        armarx::DateTime timestamp;
        fromIce(iceTimestamp, timestamp);
 
        // convert to our aron type
        const auto toHumanPose = [](const ::armarx::aron::data::dto::DictPtr& dict)
        { return armarx::human::arondto::HumanPose::FromAron(dict); };
 
        std::vector<armarx::human::arondto::HumanPose> humanPoses =
            simox::alg::apply(dictSeq, toHumanPose);
 
        // synchronize robot
        const std::string robotName =
            humanPoses.front().keypoints.begin()->second.positionCamera.header.agent;
 
        const auto robot = getSynchronizedRobot(robotName, timestamp);
 
        if (not robot)
        {
            ARMARX_INFO << deactivateSpam(1) << "Failed to obtain robot `" << robotName
                        << "`. Won't commit poses";
            return;
        }
 
        ARMARX_CHECK_NOT_NULL(robot) << "Failed to obtain robot with name `" << robotName << "`.";
 
        // fill global pose info
        {
            for (armarx::human::arondto::HumanPose& humanPose : humanPoses)
            {
                for (auto& [_, keyPoint] : humanPose.keypoints)
                {
                    const auto& cameraFrame = keyPoint.positionCamera.header.frame;
                    const auto& agent = keyPoint.positionCamera.header.agent;
 
                    const FramedPosition keypoint(
                        keyPoint.positionCamera.position, cameraFrame, agent);
                    // ARMARX_IMPORTANT << VAROUT(keyPoint.positionCamera.position.transpose());
 
                    // global pose
                    {
                        keyPoint.positionGlobal = armarx::arondto::FramedPosition();
                        armarx::toAron(keyPoint.positionGlobal.value(),
                                       armarx::FramedPosition(keypoint.toGlobalEigen(robot),
                                                              armarx::GlobalFrame,
                                                              agent));
                    }
 
                    // pose in root frame
                    {
                        keyPoint.positionRobot = armarx::arondto::FramedPosition();
                        armarx::toAron(keyPoint.positionRobot.value(),
                                       armarx::FramedPosition(keypoint.toRootEigen(robot),
                                                              robot->getRootNode()->getName(),
                                                              agent));
                    }
 
                    // Orientation information available.
                    if (keyPoint.orientationCamera.has_value())
                    {
                        ARMARX_CHECK_EQUAL(keyPoint.orientationCamera->header.frame,
                                           keyPoint.positionCamera.header.frame);
 
                        const FramedPose keypointPose(
                            keyPoint.orientationCamera->orientation.toRotationMatrix(),
                            keyPoint.positionCamera.position,
                            cameraFrame,
                            agent);
 
                        // global pose
                        {
                            const Eigen::Isometry3f global_T_human(
                                keypointPose.toGlobalEigen(robot));
 
                            keyPoint.orientationGlobal = armarx::arondto::FramedOrientation();
 
                            armarx::toAron(
                                keyPoint.orientationGlobal.value(),
                                armarx::FramedOrientation(Eigen::Matrix3f{global_T_human.linear()},
                                                          armarx::GlobalFrame,
                                                          agent));
                        }
 
                        // pose in root frame
                        {
                            const Eigen::Isometry3f robot_root_T_human(
                                keypointPose.toRootEigen(robot));
 
 
                            keyPoint.orientationRobot = armarx::arondto::FramedOrientation();
 
                            armarx::toAron(keyPoint.orientationRobot.value(),
                                           armarx::FramedOrientation(
                                               Eigen::Matrix3f{robot_root_T_human.linear()},
                                               robot->getRootNode()->getName(),
                                               agent));
                        }
                    }
                }
            }
        }
 
        // commit human poses
        commitHumanPoses(humanPoses, providerName, timestamp);
    }
 
    void
    HumanMemory::commitPersonFacesInCameraFrame(
        const ::armarx::aron::data::dto::AronDictSeq& dictSeq,
        const ::std::string& providerName,
        const ::armarx::core::time::dto::DateTime& iceTimestamp,
        const ::Ice::Current& /*c*/)
    {
        if (dictSeq.empty())
        {
            return;
        }
 
        // convert from ice
        armarx::DateTime timestamp;
        fromIce(iceTimestamp, timestamp);
 
        // convert to our aron type
        const auto toPersonFace = [](const ::armarx::aron::data::dto::DictPtr& dict)
        { return armarx::human::arondto::FaceRecognition::FromAron(dict); };
 
        std::vector<armarx::human::arondto::FaceRecognition> personFaces =
            simox::alg::apply(dictSeq, toPersonFace);
 
        // synchronize robot
        const std::string robotName = "Armar7"; // TODO: Get robot name
 
        const auto robot = getSynchronizedRobot(robotName, timestamp);
 
        if (not robot)
        {
            ARMARX_INFO << deactivateSpam(1) << "Failed to obtain robot `" << robotName
                        << "`. Won't commit poses";
            return;
        }
 
        ARMARX_CHECK_NOT_NULL(robot) << "Failed to obtain robot with name `" << robotName << "`.";
 
        // fill global pose info
        {
            for (armarx::human::arondto::FaceRecognition& personFace : personFaces)
            {
                armarx::FramedPosition fp;
                fromAron(personFace.framedPosition3D, fp);
                personFace.position3DGlobal = fp.toGlobalEigen(robot);
            }
        }
 
        // commit human poses
        commitPersonFaces(personFaces, providerName, timestamp);
    }
 
    void
    HumanMemory::commitHumanPoses(
        const std::vector<::armarx::human::arondto::HumanPose>& humanPoses,
        const std::string& providerName,
        const armarx::DateTime& timestamp)
    {
        armem::Commit commit;
 
        const auto providerId = armem::MemoryID(
            workingMemory().name(), human::PoseSegment::CORE_SEGMENT_NAME, providerName);
        const auto entityID = providerId.withEntityName("human_poses").withTimestamp(timestamp);
 
        armem::EntityUpdate update;
        update.entityID = entityID;
        update.sentTime = timestamp; // TODO this is not fully correct
        update.referencedTime = timestamp;
 
        std::transform(
            humanPoses.begin(),
            humanPoses.end(),
            std::back_inserter(update.instancesData),
            [](const ::armarx::human::arondto::HumanPose& humanPose) -> armarx::aron::data::DictPtr
            { return humanPose.toAron(); });
 
        commit.add(update);
 
        ARMARX_DEBUG << "Committing " << update << " at time " << timestamp;
 
        iceAdapter().commitLocking(commit);
    }
 
    void
    HumanMemory::commitPersonFaces(
        const std::vector<::armarx::human::arondto::FaceRecognition>& personFaces,
        const std::string& providerName,
        const armarx::DateTime& timestamp)
    {
        armem::Commit commit;
 
        const auto providerId = armem::MemoryID(
            workingMemory().name(), human::FaceRecognitionSegment::CORE_SEGMENT_NAME, providerName);
        const auto entityID = providerId.withEntityName("human_poses").withTimestamp(timestamp);
 
        armem::EntityUpdate update;
        update.entityID = entityID;
        update.sentTime = timestamp; // TODO this is not fully correct
        update.referencedTime = timestamp;
 
        std::transform(personFaces.begin(),
                       personFaces.end(),
                       std::back_inserter(update.instancesData),
                       [](const ::armarx::human::arondto::FaceRecognition& personFace)
                           -> armarx::aron::data::DictPtr { return personFace.toAron(); });
 
        commit.add(update);
 
        ARMARX_DEBUG << "Committing " << update << " at time " << timestamp;
 
        iceAdapter().commitLocking(commit);
    }
 
    void
    HumanMemory::commitHumanProfiles(const ::armarx::aron::data::dto::AronDictSeq& dictSeq,
                                     const ::std::string& providerName,
                                     const ::armarx::core::time::dto::DateTime& iceTimestamp,
                                     const ::std::string& entityName,
                                     const ::Ice::Current& /*unused*/)
    {
        if (dictSeq.empty())
        {
            return;
        }
 
        // convert from ice
        armarx::DateTime timestamp;
        fromIce(iceTimestamp, timestamp);
 
        // convert to our aron type
        const auto toHumanProfile = [](const ::armarx::aron::data::dto::DictPtr& dict)
        { return armarx::human::arondto::Person::FromAron(dict); };
 
        std::vector<armarx::human::arondto::Person> humanProfiles =
            simox::alg::apply(dictSeq, toHumanProfile);
 
        ARMARX_DEBUG << "Collected all information on human profile from ice call"
                     << "\n"
                     << "entityName: " << entityName;
 
        //call actual commit method
        this->commitHumanProfilesA(humanProfiles, providerName, timestamp, entityName);
    }
 
    void
    HumanMemory::commitHumanProfilesA(
        const std::vector<armarx::human::arondto::Person>& humanProfiles,
        const ::std::string& providerName,
        const armarx::DateTime& timestamp,
        const ::std::string& entityName)
    {
        armem::Commit commit;
        const auto providerId = armem::MemoryID(
            workingMemory().name(), human::ProfileSegment::CORE_SEGMENT_NAME, providerName);
        const auto entityID = providerId.withEntityName(entityName).withTimestamp(timestamp);
 
        armem::EntityUpdate update;
        update.entityID = entityID;
        update.sentTime = timestamp; // TODO this is not fully correct
        update.referencedTime = timestamp;
 
        for (const auto& humanProfile : humanProfiles)
        {
            update.instancesData.push_back(humanProfile.toAron());
        }
 
        commit.add(update);
 
        ARMARX_DEBUG << "Committing " << update << " at time " << timestamp;
 
        iceAdapter().commitLocking(commit);
    }
} // namespace armarx::armem::server::human_memory