OpenPoseAdapter.h

Go to the documentation of this file.

/*
 * This file is part of ArmarX.
 *
 * ArmarX is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * ArmarX is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * @package    visionx::MonocularOpenPoseEstimation
 * @author     Fabian Peller <fabian.peller@kit.edu>
 * @date       2020
 * @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
 *             GNU General Public License
 */
 
 
#pragma once
 
 
// STD/STL
#include <filesystem>
#include <memory>
#include <mutex>
 
// OpenCV 2
#include <opencv2/core/core.hpp>
 
// IVT
#include <Image/ByteImage.h>
#include <Math/Math2d.h>
 
// OpenPose
#include <openpose/core/headers.hpp>
#include <openpose/pose/headers.hpp>
#include <openpose/utilities/headers.hpp>
 
// ugly hack to check op version. Since 1.6.0 they used custom headers to reduce the amounts of includes (E.g. string or cv2)
#if __has_include(<openpose/core/string.hpp>)
#define OP_CUSTOM_STRING_HEADERS = 1
#endif
#if __has_include(<openpose/core/matrix.hpp>)
#define OP_CUSTOM_MATRIX_HEADERS = 1
#endif
 
// ArmarX
#include <ArmarXCore/core/logging/Logging.h>
 
#include <RobotAPI/interface/visualization/DebugDrawerInterface.h>
 
#include <VisionX/interface/components/OpenPoseEstimationInterface.h>
#include <VisionX/libraries/human/pose/model/openpose_body_25.h>
 
namespace armarx
{
    class OpenPoseAdapter;
    typedef std::shared_ptr<OpenPoseAdapter> MonocularOpenPoseEstimationPtr;
 
    class OpenPoseAdapter
    {
    public:
        struct OpenPoseSettings
        {
            std::string net_resolution = "-1x368";
            std::string output_resolution = "-1x-1";
            std::string model_pose = human::pose::model::openpose_body_25::ModelId;
            std::string model_folder = "models/";
            double scale_gap = 0.3;
            int scale_number = 1;
            int num_gpu_start = 0;
            unsigned int minimum_number_of_valid_keypoints_per_entitiy = 5;
        };
 
    public:
        OpenPoseAdapter(const OpenPoseSettings&);
 
#if defined(OP_CUSTOM_STRING_HEADERS)
        static op::String
        ConvertToOPCustomType(const std::string& x)
        {
            return op::String(x);
        }
#else
        static std::string
        ConvertToOPCustomType(const std::string& x)
        {
            return x;
        }
#endif
 
#if defined(OP_CUSTOM_MATRIX_HEADERS)
        static op::Matrix
        ConvertToOPCustomType(const cv::Mat& x)
        {
            return op::Matrix(&x);
        }
#else
        static cv::Mat
        ConvertToOPCustomType(const cv::Mat& x)
        {
            return x;
        }
#endif
 
        op::Array<float> getOpenposeKeypoints(const CByteImage* imageBuffer);
        HumanPose2DMap convert2DKeypointsToIce(const op::Array<float>& op_keypoints,
                                               const CByteImage* rgbImage) const;
 
        std::vector<float> getPoseColors() const;
        std::vector<unsigned int> getPoseBodyPartPairsRender() const;
 
        // generate images out of pose
        void render2DResultImage(const CByteImage& inputImage,
                                 const op::Array<float>& keypoints,
                                 CByteImage& resultImage,
                                 float renderThreshold);
 
        // get color
        armarx::DrawColor24Bit
        getDominantColorOfPatch(const CByteImage& image, const Vec2d& point, int windowSize) const;
 
    private:
        void setupOpenPoseEnvironment();
 
 
    public:
        OpenPoseSettings settings;
 
        // OpenPose
        std::mutex op_running_mutex;
        std::shared_ptr<op::ScaleAndSizeExtractor> scaleAndSizeExtractor;
        std::shared_ptr<op::CvMatToOpInput> cvMatToOpInput;
        std::shared_ptr<op::CvMatToOpOutput> cvMatToOpOutput;
        std::shared_ptr<op::PoseExtractorCaffe> poseExtractorCaffe;
        std::shared_ptr<op::OpOutputToCvMat> opOutputToCvMat;
        op::PoseModel poseModel;
    };
} // namespace armarx