ArmarXDataPath.cpp

Go to the documentation of this file.

/*
* This file is part of ArmarX.
*
* Copyright (C) 2011-2016, High Performance Humanoid Technologies (H2T), Karlsruhe Institute of Technology (KIT), all rights reserved.
*
* ArmarX is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* ArmarX is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package    ArmarXCore::core
* @author     Nikolaus Vahrenkamp
* @date       2012
* @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
*             GNU General Public License
*/
 
#include "ArmarXDataPath.h" // for ArmarXDataPath
 
#include <algorithm> // for min, fill, find
#include <cstdlib> // for getenv
#include <filesystem>
#include <fstream>
#include <regex>
#include <sstream> // for operator<<, char_traits, etc
#include <string> // for basic_string, string, etc
#include <vector> // for vector
 
#include <boost/regex.hpp>
 
#include <IceUtil/Handle.h> // for HandleBase
 
#include <SimoxUtility/algorithm/string/string_tools.h>
#include <SimoxUtility/simox/SimoxPath.h>
 
#include "ArmarXCore/core/application/properties/Property.h"
#include "ArmarXCore/core/exceptions/Exception.h" // for LocalException
#include "ArmarXCore/core/logging/LogSender.h" // for LogSender
#include <ArmarXCore/core/application/Application.h>
#include <ArmarXCore/core/logging/Logging.h> // for ARMARX_WARNING_S, etc
#include <ArmarXCore/core/system/cmake/CMakePackageFinder.h>
#include <ArmarXCore/core/util/StringHelpers.h> // for Contains, VAROUT
#include <ArmarXCore/util/CPPUtility/filesystem.h>
 
#include "cmake/CMakePackageFinder.h"
 
//#include <execinfo.h>
//#include <stdio.h>
 
namespace armarx
{
    namespace fs = std::filesystem;
 
    //    static void printBacktrace()
    //    {
    //        void* callstack[128];
    //        int i, frames = backtrace(callstack, 128);
    //        char** strs = backtrace_symbols(callstack, frames);
    //        for (i = 0; i < frames; ++i)
    //        {
    //            printf("%s\n", strs[i]);
    //        }
    //        free(strs);
    //    }
 
    struct ArmarXDataPath_Data
    {
        ArmarXDataPath_Data()
        {
            //            printf("Creating ArmarXDataPath_Data, this: %p\n", (void*)this);
            //            printBacktrace();
        }
 
        ~ArmarXDataPath_Data()
        {
            // This is super weird!
            // This global object should exist only once.
            // However, we get two instances if run ArmarXGUIRun
            // This actually causes a double free if we put these variables
            // as static variables in the class (the vector is freed twice).
 
            // We now put these variables in this .cpp file to prevent double free.
            // However, this means we have multiple instances now!
            // How can this even happen?
            // Are there two instances of ArmarXCore.so loaded in the GUI?
 
            //            printf("Destroying ArmarXDataPath_Data(), this: %p\n", (void*)this);
            //            printBacktrace();
        }
 
        std::vector<std::string> dataPaths;
        bool initialized = false;
    };
 
    static ArmarXDataPath_Data ArmarXDataPath_data;
 
 
    ArmarXDataPath::ArmarXDataPath()
    {
    }
 
 
    bool
    ArmarXDataPath::getAbsolutePath(const std::string& relativeFilename,
                                    std::string& storeAbsoluteFilename,
                                    const std::vector<std::string>& additionalSearchPaths,
                                    bool verbose)
    {
 
        init();
 
        const std::filesystem::path filename(relativeFilename);
 
        // first: check current path
        if (std::filesystem::exists(filename))
        {
            storeAbsoluteFilename = filename.string();
            return true;
        }
 
        const auto search = [&filename, &storeAbsoluteFilename](const auto& bases)
        {
            for (const auto& currentPath : bases)
            {
                std::filesystem::path path(currentPath);
 
                std::filesystem::path filenameComplete = path / filename;
 
                if (std::filesystem::exists(filenameComplete))
                {
                    storeAbsoluteFilename = filenameComplete.string();
                    return true;
                }
            }
            return false;
        };
        const auto searchProject = [&search](const auto& project)
        {
            CMakePackageFinder finder(project);
            auto pathsString = finder.getDataDir();
            Ice::StringSeq projectDataPaths = simox::alg::split(pathsString, ";,");
            return search(projectDataPaths);
        };
        {
            if (search(additionalSearchPaths) || search(ArmarXDataPath_data.dataPaths))
            {
                return true;
            }
        }
 
        //search for a package with the first part of the path (e.g.: ArmarxCore for the path ArmarXCore/foo/bar)
        {
            std::filesystem::path p = relativeFilename;
 
            if (p.is_absolute())
            {
                p = std::filesystem::relative(p, "/");
            }
            while (p.has_parent_path())
            {
                if (p.parent_path() == ".")
                {
                    p = p.filename();
                }
                else
                {
                    p = p.parent_path();
                }
            }
            const auto pstr = p.string();
            if (!pstr.empty() && searchProject(pstr))
            {
                return true;
            }
        }
 
        //search in all projects from getDefaultPackageNames()
        if (Application::getInstance())
        {
            for (const auto& project : Application::getInstance()->getArmarXPackageNames())
            {
                if (searchProject(project))
                {
                    return true;
                }
            }
        }
 
        if (verbose)
        {
            ARMARX_WARNING_S << "Could not find file '" << relativeFilename << "'"
                             << "\n  in the following additional paths: " << additionalSearchPaths
                             << "\n  or in the following data paths: "
                             << ArmarXDataPath_data.dataPaths;
        }
 
        return false;
    }
 
    std::string
    ArmarXDataPath::getAbsolutePath(const std::string& relativeFilename,
                                    const std::vector<std::string>& additionalSearchPaths,
                                    bool verbose)
    {
        std::string storeAbsoluteFilename;
        getAbsolutePath(relativeFilename, storeAbsoluteFilename, additionalSearchPaths, verbose);
        return storeAbsoluteFilename;
    }
 
    bool
    ArmarXDataPath::SearchReadableFile(const std::string& queryFileName,
                                       std::string& resultFileName,
                                       bool verbose)
    {
        const auto file_ok = [](const std::string& f)
        { return std::filesystem::exists(f) && std::ifstream(f).good(); };
 
        //cases
        // -absolute
        //   -     readable -> done
        //   - not readable -> try to find part
        // - relative
        //   - found path
        //     -     readable -> done
        //     - not readable -> failure
        //   - failed to find path
        //     - search for sub parts -> try search again
 
        if (std::filesystem::path{queryFileName}.is_absolute() && file_ok(queryFileName))
        {
            resultFileName = queryFileName;
            return true;
        }
        //here queryFileName is always relative or the absolute is not ok
        std::string found_file;
        if (getAbsolutePath(queryFileName, found_file, {}, verbose) && file_ok(found_file))
        {
            resultFileName = found_file;
            return true;
        }
        //now we need to try to search for package names
        // try all strings ... of pattern /data/.../
        std::size_t offset = 0;
        while (offset != std::string::npos && offset < queryFileName.size())
        {
            // deal with paths like /prefix/data/foo/data/bar/file
            // -> search for 'foo/data/bar/file' and 'bar/file'
            static const std::string dataMarker = "/data/";
            offset = queryFileName.find(dataMarker, offset);
            if (offset == std::string::npos)
            {
                return false;
            }
            const std::string prefix_path = queryFileName.substr(0, offset);
            offset += dataMarker.size();
            const std::string path_in_data = queryFileName.substr(offset);
 
            // Check for '.../VirtualRobot/data/...'
            if (simox::alg::ends_with(prefix_path, "VirtualRobot"))
            {
                found_file = simox::SimoxPath::getVirtualRobotDataDir() / path_in_data;
                if (file_ok(found_file))
                {
                    resultFileName = found_file;
                    return true;
                }
            }
 
            if (getAbsolutePath(path_in_data, found_file, {}, verbose) && file_ok(found_file))
            {
                resultFileName = found_file;
                return true;
            }
            const std::size_t slash_offset = path_in_data.find("/");
            if (slash_offset == std::string::npos)
            {
                return false;
            }
            const std::string package_name = path_in_data.substr(0, slash_offset);
 
            if (FindPackageAndAddDataPath(package_name) && // found package
                getAbsolutePath(path_in_data, found_file) && // found file
                file_ok(found_file) // file ok
            )
            {
                resultFileName = found_file;
                return true;
            }
        }
        return false;
    }
 
    std::string
    ArmarXDataPath::getProject(const std::vector<std::string>& projects,
                               const std::string& relativeFilename)
    {
        std::filesystem::path fn(relativeFilename);
        for (auto currentProject : projects)
        {
            armarx::CMakePackageFinder finder(currentProject);
 
            if (!finder.packageFound())
            {
                ARMARX_VERBOSE_S << "ArmarX Package " << currentProject << " has not been found!";
                continue;
            }
 
            //ARMARX_INFO_S << "Checking datapath: " << finder.getDataDir();
            std::filesystem::path p(finder.getDataDir());
            std::filesystem::path fnComplete = p / fn;
 
            if (std::filesystem::exists(fnComplete))
            {
                //ARMARX_INFO_S << "Found in project " << currentProject;
                return currentProject;
            }
        }
 
        return std::string();
    }
 
 
    std::string
    ArmarXDataPath::cleanPath(const std::string& filepathStr)
    {
        std::filesystem::path p(filepathStr);
        std::filesystem::path result;
 
        for (std::filesystem::path::iterator it = p.begin(); it != p.end(); ++it)
        {
            if (*it == ".." && it != p.begin())
            {
                // /a/b/.. is not necessarily /a if b is a symbolic link
                bool isSymLink = false;
 
                try
                {
                    isSymLink = std::filesystem::is_symlink(result);
                }
                catch (std::filesystem::filesystem_error&)
                {
                }
 
                if (isSymLink)
                {
                    result /= *it;
                }
                // /a/b/../.. is not /a/b/.. under most circumstances
                // We can end up with ..s in our result because of symbolic links
                else if (result.filename() == "..")
                {
                    result /= *it;
                }
                // Otherwise it should be safe to resolve the parent
                else
                {
                    result = result.parent_path();
                }
            }
            else if (*it == ".")
            {
                // Ignore
            }
            else
            {
                // Just concatenate other path entries
                result /= *it;
            }
        }
 
        return result.string();
    }
 
    std::string
    ArmarXDataPath::getRelativeArmarXPath(const std::string& absolutePathString)
    {
        init();
 
        // traverse directories until root is reached
        for (auto absolutePathPart = std::filesystem::path(absolutePathString);
             absolutePathPart != absolutePathPart.parent_path();
             absolutePathPart = absolutePathPart.parent_path())
        {
            for (const auto& dataPath : ArmarXDataPath_data.dataPaths)
            {
                const std::filesystem::path p(dataPath);
 
                if (std::filesystem::equivalent(p, absolutePathPart))
                {
                    return relativeTo(p.string(), absolutePathString);
                }
            }
        }
 
        throw LocalException() << "Could not make path relative to any ArmarX data path for '"
                               << absolutePathString << "'. "
                               << "Considered the following paths: "
                               << ArmarXDataPath_data.dataPaths;
        return absolutePathString;
    }
 
    std::string
    ArmarXDataPath::relativeTo(const std::string& fromStr, const std::string& toStr)
    {
        // Start at the root path. While they are the same, do nothing.
        // When they first diverge, take the remainder of the two paths
        // and replace the entire from path with ".." segments.
        fs::path from(fromStr);
        fs::path to(toStr);
        fs::path::const_iterator fromIter = (from).begin();
        fs::path::const_iterator toIter = (to).begin();
        if (from.empty())
        {
            throw LocalException("From path is empty");
        }
        if (to.empty())
        {
            throw LocalException("To path is empty");
        }
        if (*fromIter != *toIter)
        {
            throw LocalException("From and to path do not have the same toplevel dir: ")
                << VAROUT(fromStr) << VAROUT(toStr);
        }
        // Loop through both
        while (fromIter != from.end() && toIter != to.end() && (*toIter) == (*fromIter))
        {
            ++toIter;
            ++fromIter;
        }
 
        fs::path finalPath;
        while (fromIter != from.end())
        {
            finalPath /= "..";
            ++fromIter;
        }
 
        while (toIter != to.end())
        {
            finalPath /= *toIter;
            ++toIter;
        }
 
        return finalPath.string();
    }
 
    bool
    ArmarXDataPath::mergePaths(std::string pathStr, std::string subPathStr, std::string& result)
    {
        fs::path subPath(subPathStr);
        pathStr = cleanPath(pathStr);
        subPathStr = cleanPath(subPathStr);
        fs::path strippedPath;
        while (!subPath.empty())
        {
            if (Contains(pathStr, subPath.string()))
            {
                result = remove_trailing_separator(fs::path(pathStr) / strippedPath).string();
                return true;
            }
            strippedPath = subPath.filename() / strippedPath;
            subPath = subPath.parent_path();
        }
        result.clear();
        return false;
    }
 
    void
    ArmarXDataPath::initDataPaths(const std::string& dataPathList)
    {
        addDataPaths(dataPathList);
    }
 
    bool
    ArmarXDataPath::ReplaceEnvVars(std::string& string)
    {
        const boost::regex e("\\$([a-zA-Z0-9_]+)");
        const boost::regex e2("\\$\\{([a-zA-Z0-9_]+)\\}");
        boost::match_results<std::string::const_iterator> what;
        bool found = false;
        auto replaceVars = [&](const boost::regex& e)
        {
            bool found_match = boost::regex_search(string, what, e);
            if (found_match)
            {
                for (size_t i = 1; i < what.size(); i += 2)
                {
                    std::string var = what[i];
 
                    auto envVar = getenv(var.c_str());
                    if (envVar)
                    {
                        string = boost::regex_replace(string, e, std::string(envVar));
                        found = true;
                    }
                }
            }
        };
        replaceVars(e);
        replaceVars(e2);
        return found;
    }
 
    void
    ArmarXDataPath::ReplaceVar(std::string& string,
                               const std::string varName,
                               const std::string& varValue)
    {
        string = simox::alg::replace_all(string, std::string("${") + varName + "}", varValue);
    }
 
    void
    ArmarXDataPath::ResolveHomePath(std::string& path)
    {
        ReplaceEnvVars(path);
        if (!path.empty() && path[0] == '~')
        {
            path = path.erase(0, 1);
            auto envVar = getenv("HOME");
            if (envVar)
            {
                path = std::string(envVar) + "/" + path;
            }
            else
            {
                ARMARX_WARNING << "$HOME var is not set!";
            }
        }
        path = cleanPath(path);
    }
 
 
    std::string
    ArmarXDataPath::resolvePath(const std::string& path, bool verbose)
    {
        std::string resolved = path;
        ResolveHomePath(resolved);
        if (fs::path(resolved).is_relative())
        {
            std::string absolute;
            if (getAbsolutePath(resolved, absolute, {}, verbose))
            {
                resolved = absolute;
            }
        }
        return resolved;
    }
 
 
    void
    ArmarXDataPath::addDataPaths(const std::string& dataPathList)
    {
        init();
        __addPaths(dataPathList);
    }
 
    void
    ArmarXDataPath::addDataPaths(const std::vector<std::string>& dataPathList)
    {
        init();
        for (const auto& p : dataPathList)
        {
            __addPaths(p);
        }
    }
 
    void
    ArmarXDataPath::addDataPath(const std::string& dataPath)
    {
        init();
        __addPaths(dataPath);
    }
 
    std::string
    ArmarXDataPath::getHomePath()
    {
        if (const char* home_path = std::getenv("ArmarXHome_DIR"))
        {
            return cleanPath(std::string(home_path));
        }
        return std::string();
    }
 
    void
    ArmarXDataPath::init()
    {
        if (ArmarXDataPath_data.initialized)
        {
            return;
        }
 
        //        printf("ArmarXDataPath::init(), this=%p\n", (void*)&ArmarXDataPath_data);
        //        printBacktrace();
 
        if (const char* data_path = std::getenv("ArmarXData_DIRS"))
        {
            std::string pathStr(data_path);
            __addPaths(pathStr);
        }
 
        //check the documented variable
        if (const char* data_path_dir = std::getenv("ArmarXData_DIR"))
        {
            std::string pathStr(data_path_dir);
            __addPaths(pathStr);
        }
 
        ArmarXDataPath_data.initialized = true;
    }
 
    std::vector<std::string>
    ArmarXDataPath::getDataPaths()
    {
        return ArmarXDataPath_data.dataPaths;
    }
 
    bool
    ArmarXDataPath::__addPaths(const std::string& pathList)
    {
        if (pathList == "")
        {
            return false;
        }
 
        std::vector<std::string> separatedPaths = __separatePaths(pathList);
 
        if (separatedPaths.size() == 0)
        {
            return false;
        }
 
        bool ok = true;
 
        for (int i = separatedPaths.size() - 1; i >= 0; i--)
        {
            ArmarXDataPath::ReplaceEnvVars(separatedPaths[i]);
            ok = ok & __addPath(separatedPaths[i]);
        }
 
        return ok;
    }
 
    std::vector<std::string>
    ArmarXDataPath::__separatePaths(const std::string& pathList)
    {
        std::string delimiters = ";";
        std::vector<std::string> tokens = simox::alg::split(pathList, delimiters);
        return tokens;
    }
 
    bool
    ArmarXDataPath::__pathIsValid(const std::string& path)
    {
        if (path.empty())
        {
            return false;
        }
 
        std::filesystem::path p(path);
        return std::filesystem::is_directory(p) || std::filesystem::is_symlink(p);
    }
 
    bool
    ArmarXDataPath::__addPath(const std::string& path)
    {
        if (path.empty())
        {
            return false;
        }
 
        std::vector<std::string> splitted = simox::alg::split(path, "/");
 
        if (splitted.size() < 3)
        {
            ARMARX_WARNING_S << "Not a valid data path: '" << path << "'" << std::endl;
            return false;
        }
 
        std::string root = "";
        std::string project = splitted[splitted.size() - 2];
        std::string data = splitted[splitted.size() - 1];
 
        for (unsigned int i = 0; i < splitted.size() - 2; ++i)
        {
            root += "/" + splitted[i];
        }
 
        std::filesystem::path p(root + "/" + project + "/" + data);
 
        if (!__pathIsValid(p))
        {
            ARMARX_INFO_S << "Not a valid data path: '" << p
                          << "'. Try to capitalize project folder..." << std::endl;
 
            // also check for capitalized parent folder (usually the project)
            project = simox::alg::capitalize_words(project);
            p = std::filesystem::path(root + "/" + project + "/" + data);
            if (!__pathIsValid(p))
            {
                ARMARX_INFO_S << "Not a valid data path: '" << p
                              << "'. Try to caps project folder..." << std::endl;
 
                // also check for caps parent folder (usually the project)
                project = simox::alg::to_upper(project);
                p = std::filesystem::path(root + "/" + project + "/" + data);
 
                if (!__pathIsValid(p))
                {
                    ARMARX_WARNING_S << "Not a valid data path: '" << p << "'" << std::endl;
                    return false;
                }
            }
        }
 
        if (std::find(ArmarXDataPath_data.dataPaths.begin(),
                      ArmarXDataPath_data.dataPaths.end(),
                      p) == ArmarXDataPath_data.dataPaths.end())
        {
            ARMARX_DEBUG_S << "Adding data path:" << p << std::endl;
            ArmarXDataPath_data.dataPaths.push_back(p);
        }
        return true;
    }
 
    std::string
    ArmarXDataPath::GetCachePath()
    {
        try
        {
            ApplicationPtr application = armarx::Application::getInstance();
            if (application.get() != nullptr)
            {
 
                std::string cachePathStr;
 
                cachePathStr = application->getProperty<std::string>("CachePath").getValue();
                if (std::filesystem::path(cachePathStr).is_relative())
                {
                    std::string pathPrefix;
                    if (getenv(Application::ArmarXUserConfigDirEnvVar.c_str()))
                    {
                        pathPrefix =
                            std::string(getenv(Application::ArmarXUserConfigDirEnvVar.c_str()));
                    }
                    else
                    {
                        pathPrefix = Application::GetArmarXConfigDefaultPath();
                    }
                    //                ARMARX_INFO << VAROUT(pathPrefix);
                    cachePathStr =
                        (std::filesystem::path(pathPrefix) / std::filesystem::path(cachePathStr))
                            .string();
                }
                else
                {
                    //                ARMARX_INFO << "Cache path is absolute: " << cachePathStr;
                }
 
                ReplaceEnvVars(cachePathStr);
                return cachePathStr;
            }
            else
            {
                return "";
            }
        }
        catch (LocalException& error)
        {
            return "";
        }
    }
 
    std::string
    ArmarXDataPath::GetDefaultUserConfigPath()
    {
        char* env_armarx_workspace = getenv("ARMARX_WORKSPACE");
        char* env_armarx_default_config_dir_name = getenv("ARMARX_CONFIG_DIR_NAME");
 
        std::filesystem::path armarx_workspace;
        std::filesystem::path armarx_config_dir;
 
        if (env_armarx_workspace != nullptr)
        {
            armarx_workspace = std::filesystem::path(env_armarx_workspace);
        }
        else
        {
            char* home = getenv("HOME");
 
            if (home != nullptr)
            {
                armarx_workspace = std::filesystem::path(home);
            }
            else
            {
                armarx_workspace = "~/";
            }
        }
 
        if (env_armarx_default_config_dir_name != nullptr)
        {
            armarx_config_dir = std::filesystem::path(env_armarx_default_config_dir_name);
        }
        else
        {
            if (env_armarx_workspace != nullptr)
            {
                armarx_config_dir = "armarx_config";
            }
            // Legacy mode.
            else
            {
                armarx_config_dir = ".armarx";
            }
        }
 
        return (armarx_workspace / armarx_config_dir).string();
    }
 
    bool
    ArmarXDataPath::FindPackageAndAddDataPath(const std::string& packageName)
    {
        if (packageName.empty())
        {
            return false;
        }
        armarx::CMakePackageFinder finder(packageName);
        if (finder.packageFound())
        {
            armarx::ArmarXDataPath::addDataPaths(finder.getDataDir());
            return true;
        }
        return false;
    }
} // namespace armarx