CMakePackageFinder.cpp

Go to the documentation of this file.

/*
* This file is part of ArmarX.
*
* Copyright (C) 2011-2016, High Performance Humanoid Technologies (H2T), Karlsruhe Institute of Technology (KIT), all rights reserved.
*
* ArmarX is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* ArmarX is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package    ArmarX::
* @author     Mirko Waechter ( mirko.waechter at kit dot edu)
* @date       2014
* @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
*             GNU General Public License
*/
 
#include "CMakePackageFinder.h"
#include <ArmarXCore/core/logging/Logging.h>
#include <ArmarXCore/core/util/StringHelpers.h>
#include <ArmarXCore/core/system/ArmarXDataPath.h>
#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
#include <ArmarXCore/core/services/sharedmemory/exceptions/SharedMemoryExceptions.h>
#include <ArmarXCore/util/CPPUtility/filesystem.h>
 
#include "../../rapidxml/wrapper/RapidXmlReader.h"
#include "CMakePackageFinderCache.h"
 
#include <boost/interprocess/managed_shared_memory.hpp>
#include <boost/interprocess/sync/interprocess_upgradable_mutex.hpp>
#include <boost/interprocess/sync/interprocess_condition.hpp>
 
#include <SimoxUtility/algorithm/string/string_tools.h>
#include <boost/algorithm/string/regex.hpp>
#include <boost/regex.hpp>
#include <boost/interprocess/sync/file_lock.hpp>
#include <boost/interprocess/sync/scoped_lock.hpp>
#include <boost/interprocess/sync/sharable_lock.hpp>
#include <boost/thread/thread_time.hpp>
 
#include <filesystem>
#include <sstream>
#include <memory>
#include <ctime>
#include <mutex>
#include <cstdlib>
 
#define SCRIPT_PATH "ArmarXCore/core/system/cmake/FindPackageX.cmake"
#define CACHE_PATH "/ArmarXCMakeCache_" + (getenv("USER")?getenv("USER"):"DUMMY_USER")
 
extern char ** environ;
 
 
namespace armarx
{
    std::string getHomeDir()
    {
        char* homePathC = getenv("HOME");
        std::string homePath;
        if (homePathC)
        {
            homePath = homePathC;
        }
        return homePath;
    }
 
    std::shared_ptr<boost::interprocess::file_lock> getFileLock(std::string lockName, bool verbose = false)
    {
        std::string path = remove_trailing_separator(fs::temp_directory_path().string() + CACHE_PATH);
        if (verbose)
        {
            std::cout << "getFileLock( path = " << path << ")" << std::endl;
        }
        if (!std::filesystem::exists(path))
        {
            if (verbose)
            {
                std::cout << "path does not exist" << std::endl;
            }
            if (!std::filesystem::create_directories(path))
            {
                if (verbose)
                {
                    std::cout << "failed to create directories" << std::endl;
                }
                return std::shared_ptr<boost::interprocess::file_lock>();
            }
        }
        path += "/" + lockName;
        if (verbose)
        {
            std::cout << "lock file = " << path << std::endl;
        }
        if (!std::filesystem::exists(path))
        {
            if (verbose)
            {
                std::cout << "touch " << path << std::endl;
            }
            //touch file
            std::ofstream file(path);
            file.close();
        }
        if (verbose)
        {
            std::cout << "build lock with  " << path << std::endl;
        }
        return std::shared_ptr<boost::interprocess::file_lock>(new boost::interprocess::file_lock(path.c_str()));
    }
 
    std::shared_ptr<boost::interprocess::file_lock> CreateAndCheckFileLock(const std::string& name)
    {
        ARMARX_TRACE;
        std::shared_ptr<boost::interprocess::file_lock> lock(getFileLock(name, false));
        if (!lock)
        {
            ARMARX_WARNING << "Failed to create file lock! retrying...";
            lock = getFileLock(name, true);
            ARMARX_CHECK_NOT_NULL(lock) << "Failed to get file lock '" + name + "'";
        }
        return lock;
    }
 
    const std::shared_ptr<boost::interprocess::file_lock>& CacheFileLock()
    {
        ARMARX_TRACE;
        static auto lock = CreateAndCheckFileLock(".cachelock");
        ARMARX_CHECK_NOT_NULL(lock);
        return lock;
    }
 
    const std::shared_ptr<boost::interprocess::file_lock>& CMakeFileLock()
    {
        ARMARX_TRACE;
        static auto lock = CreateAndCheckFileLock(".cmakelock");
        ARMARX_CHECK_NOT_NULL(lock);
        return lock;
    }
 
    std::mutex& CMakeMutex()
    {
        static std::mutex mx;
        return mx;
    }
 
    using ScopedFileLockPtr = std::shared_ptr<boost::interprocess::scoped_lock<boost::interprocess::file_lock>>;
 
    ScopedFileLockPtr lockCMake()
    {
        std::unique_lock lock(CMakeMutex());
 
        ScopedFileLockPtr lockPtr(new boost::interprocess::scoped_lock<boost::interprocess::file_lock>(*CMakeFileLock(),
                                  boost::get_system_time() + boost::posix_time::milliseconds(50)));
        while (!lockPtr->owns())
        {
            lockPtr->timed_lock(boost::get_system_time() + boost::posix_time::milliseconds(50));
        }
        return lockPtr;
    }
 
    bool readCMakeCache(const std::string& packageName, std::string& packageContent)
    {
        auto start = IceUtil::Time::now();
        std::string path = std::filesystem::temp_directory_path().string() + CACHE_PATH;
        path += "/" + packageName;
        if (!std::filesystem::exists(path))
        {
            return false;
        }
        else
        {
 
            const auto writeTime = std::filesystem::last_write_time(path);
            const auto now = decltype(writeTime)::clock::now();
            long age = std::chrono::duration_cast<std::chrono::seconds>(now - writeTime).count();
            //            ARMARX_INFO_S << VAROUT(packageName) << VAROUT(age) << VAROUT(std::asctime(std::localtime(&writeTime)));
            boost::interprocess::sharable_lock<boost::interprocess::file_lock> e_lock(*CacheFileLock());
            packageContent = RapidXmlReader::ReadFileContents(path);
            auto dura = (IceUtil::Time::now() - start).toMilliSecondsDouble();
            if (dura > 10000)
            {
                ARMARX_INFO_S << packageName << " from cache locked for " << dura;
            }
            if (age < 10)
            {
                return true;
            }
            else
            {
                return false;
            }
        }
    }
 
    bool updateCMakeCache(const std::string& packageName, const std::string& packageContent)
    {
        auto start = IceUtil::Time::now();
        std::string path = std::filesystem::temp_directory_path().string() + CACHE_PATH;
        if (!std::filesystem::exists(path))
        {
            if (!std::filesystem::create_directories(path))
            {
                return false;
            }
        }
        path = path + "/" + packageName;
        boost::interprocess::scoped_lock<boost::interprocess::file_lock> e_lock(*CacheFileLock());
        //        std::string existingPackageContent = RapidXmlReader::ReadFileContents(path);
        //        if (existingPackageContent != packageContent)
        {
            std::ofstream file(path);
            file << packageContent;
        }
        //        ARMARX_IMPORTANT_S << "Updateing cmakecache " << VAROUT(packageName) << VAROUT(path);
        auto dura = (IceUtil::Time::now() - start).toMilliSecondsDouble();
        if (dura > 10000)
        {
            ARMARX_INFO_S << packageName << " update cache locked for " << dura;
        }
        return true;
    }
 
    boost::interprocess::interprocess_upgradable_mutex* memoryMutex = nullptr;
    std::shared_ptr<boost::interprocess::managed_shared_memory> sharedMemorySegment;
    std::mutex cmakePackageMutex;
 
 
 
    CMakePackageFinder::CMakePackageFinder(const std::string& packageName, const std::filesystem::path& packagePath, bool suppressStdErr,  bool usePackagePathOnlyAsHint) :
        found(false),
        packageName(simox::alg::trim_copy(packageName))
    {
        if (this->packageName.empty())
        {
            ARMARX_WARNING << "CMakePackageFinder: Package name must not be empty";
        }
        static std::string scriptPath;
        {
 
            std::unique_lock lock(cmakePackageMutex);
 
            if (scriptPath.empty())
            {
                // Somehow this call fails sometimes if many components are started seperatly
                // So try on fail again with another path
                auto start = IceUtil::Time::now();
                auto list  = FindPackageIncludePathList("ArmarXCore");
                auto dura = (IceUtil::Time::now() - start);
 
                if (dura.toMilliSeconds() > 10000)
                {
                    ARMARX_INFO_S << "Cmakefinder for initial core search took long  - Duration: " << dura.toMilliSeconds();
                }
 
                if (!ArmarXDataPath::getAbsolutePath(SCRIPT_PATH, scriptPath, list, false))
                {
                    ARMARX_WARNING_S << "Finding FindPackageX.cmake failed - trying again with different path";
 
                    if (!ArmarXDataPath::getAbsolutePath(std::string("../source/") + SCRIPT_PATH, scriptPath))
                    {
                        return;
                    }
                }
            }
        }
        //        ARMARX_INFO_S << scriptPath;
        int result;
 
        auto start = IceUtil::Time::now();
        std::string resultStr;
        std::string tmpDir = getArmarXCMakeTempDir();
        try
        {
            if (!packagePath.empty())
            {
                resultStr = ExecCommand("cd " + tmpDir + "; cmake -DPACKAGE=" + this->packageName + " -DPACKAGEBUILDPATH" + (usePackagePathOnlyAsHint ? "Hint" : "") + "=" + packagePath.string() + " -P " + scriptPath, result, suppressStdErr);
            }
            else if (!readCMakeCache(this->packageName, resultStr))
            {
                resultStr = ExecCommand("cd " + tmpDir + "; cmake -DPACKAGE=" + this->packageName + " -P " + scriptPath, result, suppressStdErr);
                updateCMakeCache(this->packageName, resultStr);
 
            }
        }
        catch (...)
        {
 
 
        }
 
 
        auto dura = (IceUtil::Time::now() - start);
 
        if (dura.toMilliSeconds() > 10000)
        {
            ARMARX_INFO_S << "Cmakefinder took long for package " << packagePath << " - Duration: " << dura.toMilliSeconds();
        }
 
        std::vector<std::string> resultList;
        resultList = extractVariables(resultStr);
 
        //        ARMARX_INFO_S << resultList;
    }
 
    std::vector<std::string> CMakePackageFinder::FindPackageIncludePathList(const std::string& packageName)
    {
        //        _CreateSharedMutex();
 
        std::string output = FindPackageIncludePaths(packageName);
        std::vector<std::string> result;
        boost::split_regex(result,
                           output,
                           boost::regex("-I")
                          );
 
        for (size_t i = 0; i < result.size(); i++)
        {
            simox::alg::trim(result[i]);
 
            if (result[i].empty())
            {
                result.erase(result.begin() + i);
                i--;
            }
        }
 
        return result;
    }
 
    std::string CMakePackageFinder::FindPackageLibs(const std::string& packageName)
    {
        //        boost::interprocess::scoped_lock<boost::interprocess::file_lock> e_lock(*CMakeFileLock());
        auto lock = lockCMake();
        auto start = IceUtil::Time::now();
        try
        {
 
            std::stringstream str;
            str << "cmake --find-package -DNAME=" << packageName << " -DLANGUAGE=CXX -DCOMPILER_ID=GNU -DMODE=LINK";
            int result;
            std::string output = ExecCommand(str.str(), result);
 
            if (result != 0)
            {
                return "";
            }
 
 
            auto dura = (IceUtil::Time::now() - start).toMilliSecondsDouble();
            if (dura > 10)
            {
                //                ARMARX_INFO_S << packageName << " locked for " << dura;
            }
            return output;
 
        }
        catch (...)
        {
            return "";
        }
 
    }
 
 
    std::string CMakePackageFinder::FindPackageIncludePaths(const std::string& packageName)
    {
        auto start = IceUtil::Time::now();
        //        std::cout << start.toDateTime() << " Waiting for lock;" << std::endl;
        auto lock = lockCMake();
        //        auto dura = (IceUtil::Time::now() - start).toMilliSecondsDouble();
        //        if(dura > 1000)
        //            ARMARX_IMPORTANT_S << " waited for " << dura;
        start = IceUtil::Time::now();
        try
        {
            std::string tmpDir = getArmarXCMakeTempDir();
            std::stringstream str;
            str << "cd " + tmpDir + ";cmake --find-package -DNAME=" << packageName << " -DLANGUAGE=CXX -DCOMPILER_ID=GNU -DMODE=COMPILE";
            int result;
            std::string output = ExecCommand(str.str(), result);
 
            if (result != 0)
            {
                ARMARX_IMPORTANT_S << packageName << " search failed - output: " << output;
                return "";
            }
 
 
            auto dura = (IceUtil::Time::now() - start).toMilliSecondsDouble();
            if (dura > 10)
            {
                //                ARMARX_INFO_S << packageName << " locked for " << dura;
            }
            return output;
        }
        catch (...)
        {
 
 
            return "";
        }
 
    }
 
    std::string CMakePackageFinder::ExecCommand(std::string command, int& result, bool suppressStdErr)
    {
        auto start = IceUtil::Time::now();
        if (suppressStdErr)
        {
            command += " 2>/dev/null";
        }
        ARMARX_DEBUG << "Cmd: " << command;
        FILE* fp = popen(command.c_str(), "r");
        char line [50];
        std::string output;
 
        while (fgets(line, sizeof line, fp))
        {
            output += line;
        }
 
        result = pclose(fp) / 256;
        auto dura = (IceUtil::Time::now() - start).toMilliSecondsDouble();
        if (dura > 1000)
        {
            ARMARX_INFO_S << "ExecCommand took " << dura << " \n command: " << command;
        }
        return output;
    }
 
    std::vector<std::string> CMakePackageFinder::FindAllArmarXSourcePackages()
    {
        std::vector<std::string> result;
        using namespace std::filesystem;
 
        char ** env;
 
        env = environ;
 
        for (; *env; ++env) {
            const std::string envVar(*env);
            ARMARX_DEBUG << "Retrieved environment variable " << envVar;
 
            const std::vector<std::string> elements = simox::alg::split(envVar, "=");
            if(not (elements.size() == 2))
            {
                continue;
            }
 
            const std::string& envVarName = elements.front();
            const std::string& envVarValue = elements.back();
 
            if(simox::alg::ends_with(envVarName, "_DIR"))
            {
                const std::string packageName = simox::alg::remove_suffix(envVarName, "_DIR");
                auto pckFinder = CMakePackageFinder {packageName, envVarValue, true};
 
                ARMARX_DEBUG << "Package name: " << packageName;
                ARMARX_DEBUG << "Path: " << envVarValue;
 
                // check if this is really a package
                if (pckFinder.packageFound() && !pckFinder.getBuildDir().empty())
                {
                    ARMARX_DEBUG << "Found package " << packageName;
                    result.push_back(packageName);
                }
            }
        }
 
        return result;
    }
 
    std::string CMakePackageFinder::getName() const
    {
        return packageName;
    }
 
    bool CMakePackageFinder::packageFound() const
    {
        return found;
    }
 
    const std::map<std::string, std::string>& CMakePackageFinder::getVars() const
    {
        return vars;
    }
 
    std::string CMakePackageFinder::getVar(const std::string& varName) const
    {
        std::map<std::string, std::string>::const_iterator it =  vars.find(varName);
 
        if (it != vars.end())
        {
            return it->second;
        }
 
        return "";
    }
 
    std::vector<std::string> CMakePackageFinder::getDependencies() const
    {
        auto depListString = getVar("DEPENDENCIES");
        std::vector<std::string> resultList = armarx::Split(depListString, ";", true, true);
 
        return resultList;
    }
 
    std::map<std::string, std::string> CMakePackageFinder::getDependencyPaths() const
    {
        // is of type "package1:package1Path;package2:packagePath2"
        auto depListString = getVar("PACKAGE_DEPENDENCY_PATHS");
        std::vector<std::string> resultList = simox::alg::split(depListString, ";");
        std::map<std::string, std::string> resultMap;
 
        for (auto& depPairString : resultList)
        {
            std::vector<std::string> depPair = simox::alg::split(depPairString, ":");
 
            if (depPair.size() < 2)
            {
                continue;
            }
 
            resultMap[depPair.at(0)] = depPair.at(1);
        }
 
        return resultMap;
    }
 
    bool CMakePackageFinder::_ParseString(const std::string& input, std::string& varName, std::string& content)
    {
        //        ARMARX_INFO_S << "input: " << input;
        const boost::regex e("\\-\\- ([a-zA-Z0-9_]+):(.+)");
        boost::match_results<std::string::const_iterator> what;
 
        bool found = boost::regex_search(input, what, e);
 
        for (size_t i = 1; i < what.size(); i++)
        {
 
            if (i == 1)
            {
                varName =  what[i];
            }
            else if (i == 2)
            {
                content = what[i];
            }
 
            //            ARMARX_INFO_S << VAROUT(varName);
        }
 
        simox::alg::trim(varName);
        simox::alg::trim(content);
        return found;
 
    }
 
    void CMakePackageFinder::_CreateSharedMutex()
    {
 
    }
 
 
    std::vector<std::string> CMakePackageFinder::extractVariables(const std::string& cmakeVarString)
    {
        std::vector<std::string> resultList = simox::alg::split(cmakeVarString, "\n");
 
        for (size_t i = 0; i < resultList.size(); i++)
        {
            simox::alg::trim(resultList[i]);
 
            if (resultList[i].empty())
            {
                resultList.erase(resultList.begin() + i);
                i--;
            }
            else
            {
                std::string var;
                std::string content;
 
                if (_ParseString(resultList[i], var, content))
                {
                    found = true;
                    vars[var] = content;
                }
            }
        }
 
        return resultList;
    }
 
    std::vector<std::string> armarx::CMakePackageFinder::getIncludePathList() const
    {
        auto depListString = getIncludePaths();
        std::vector<std::string> resultList = simox::alg::split(depListString, ";");
        return resultList;
    }
 
 
    std::string CMakePackageFinder::getArmarXCMakeTempDir()
    {
        const std::string tmpDir = "/tmp";
        std::string result = tmpDir;
        char* username = getenv("USER");
        if (username)
        {
            std::string usernameString = std::string(username);
            simox::alg::trim(usernameString);
            result += "/armarxcmake-" + usernameString;
            if (!std::filesystem::exists(result))
            {
                if (!std::filesystem::create_directories(result))
                {
                    result = tmpDir;
                }
            }
        }
        return result;
    }
 
    bool CMakePackageFinder::ReplaceCMakePackageFinderVars(std::string& string)
    {
        const boost::regex e("\\$C\\{([a-zA-Z0-9_\\-]+):([a-zA-Z0-9_\\-]+)\\}");
        boost::match_results<std::string::const_iterator> what;
        bool found = boost::regex_search(string, what, e);
        std::map<std::string, CMakePackageFinder> finders;
        if (found)
        {
            for (size_t i = 1; i < what.size(); i += 3)
            {
                std::string package = what[i];
                auto it = finders.find(package);
                if (it == finders.end())
                {
                    it = finders.insert(
                             std::make_pair(package, CMakePackageFinderCache::GlobalCache.findPackage(package))).first;
                    //                it = finders.find(package);
                }
                std::string var = what[i + 1];
 
 
                auto envVar = it->second.getVar(var);
                string = boost::regex_replace(string, e, std::string(envVar));
                ARMARX_DEBUG << "Replacing '" << var << "' with '" << std::string(envVar) << "'";
            }
        }
        return found;
 
    }
 
    std::string
    CMakePackageFinder::getExecutables() const
    {
        return getVar("EXECUTABLE");
    }
    
    std::vector<std::string> CMakePackageFinder::getComponentExecutableNames() const
    {
        namespace fs = std::filesystem;
 
        const fs::path componentReportFilename =
            fs::path(getBuildDir()) / "component_executables_report.txt";
        if (fs::exists(componentReportFilename))
        {
            std::ifstream componentReportFile(componentReportFilename);
            if (componentReportFile.bad())
            {
                ARMARX_WARNING << "Could not load file: " << componentReportFilename;
                return {};
            }
 
            const std::string content(std::istreambuf_iterator<char>{componentReportFile}, {});
            return simox::alg::split(content, "\n");
        }
 
        // TODO legacy mode. Remove after full migration
        if (vars.count("EXECUTABLE") > 0)
        {
            return simox::alg::split(getVar("EXECUTABLE"), " ");
        }
 
        ARMARX_WARNING << "No component executables available. Check if `" << packageName << "/build/component_executables_report.txt` is generated properly and EXECUTABLE variable (legacy).";
        return {};
    }
} // namespace armarx