PythonApplicationManager.cpp

Go to the documentation of this file.

/*
 * This file is part of ArmarX.
 *
 * ArmarX is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * ArmarX is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * @package    ArmarXCore::ArmarXObjects::PythonApplicationManager
 * @author     Rainer Kartmann ( rainer dot kartmann at kit dot edu )
 * @date       2021
 * @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
 *             GNU General Public License
 */
 
#include "PythonApplicationManager.h"
 
#include <algorithm>
 
#include <SimoxUtility/algorithm/string.h>
 
#include <ArmarXCore/core/application/properties/PluginEnumNames.h>
#include <ArmarXCore/core/exceptions/LocalException.h>
#include <ArmarXCore/core/system/cmake/CMakePackageFinder.h>
 
namespace armarx
{
 
    const simox::meta::EnumNames<PythonApplicationManager::VenvType>
        PythonApplicationManager::VenvTypeNames{
            {PythonApplicationManager::VenvType::Auto, "Auto"},
            {PythonApplicationManager::VenvType::Dedicated, "Dedicated"},
            {PythonApplicationManager::VenvType::Shared, "Shared"},
        };
 
    void
    PythonApplicationManager::Properties::defineProperties(PropertyDefinitionsPtr defs,
                                                           const std::string& prefix)
    {
        defs->required(
            armarxPackageName, prefix + "10_ArmarXPackageName", "Name of the ArmarX package.");
        defs->optional(armarxPythonPackagesDir,
                       prefix + "11_ArmarXPythonPackagesDir",
                       "Directory where python packages are located (relative to the root of the "
                       "ArmarX package).");
 
        defs->required(pythonPackageName,
                       prefix + "20_PythonPackageName",
                       "Name of the Python package in the ArmarXPackage/python/ directory.");
        defs->required(pythonScriptPath,
                       prefix + "21_PythonScriptPath",
                       "Path to the python script (inside the python package).");
        defs->optional(pythonScriptArgumentsString,
                       prefix + "22_PythonScriptArgs",
                       "Whitespace separated list of arguments.");
        defs->optional(pythonPathEntriesString,
                       prefix + "23_PythonPathEntries",
                       "Colon-separated list of paths to add to the PYTHONPATH.");
        defs->optional(pythonPoetry, prefix + "24_PythonPoetry", "Use python poetry.");
 
        defs->optional(venvName, prefix + "30_venv.Name", "Name of the virtual environment.");
        {
            std::stringstream ss;
            ss << "Type of the virtual environment."
               << "\n\t- " << VenvTypeNames.to_name(VenvType::Auto)
               << "       \tDerive automatically."
               << "\n\t- " << VenvTypeNames.to_name(VenvType::Dedicated)
               << "\tSearch inside the python package root directory."
               << "\n\t- " << VenvTypeNames.to_name(VenvType::Shared)
               << "   \tSearch in the shared_envs directory.";
            defs->optional(venvType, prefix + "31_venv.Type", ss.str()).map(VenvTypeNames);
        }
 
 
        defs->optional(
            workingDir,
            prefix + "40_WorkingDirectory",
            "If set, this path is used as working directory for the python script (overriding "
            "BinaryPathAsWorkingDirectory)."
            "\n${HOME} for env vars, $C{RobotAPI:BINARY_DIR} for CMakePackageFinder vars");
    }
 
    void
    PythonApplicationManager::Properties::read(PropertyUser& props, const std::string& prefix)
    {
        ARMARX_TRACE;
 
        if (props.getProperty<std::string>(prefix + "40_WorkingDirectory").isSet())
        {
            workingDir =
                props.getProperty<std::string>("WorkingDirectory").getValueAndReplaceAllVars();
        };
 
        pythonScriptArgumentsVector =
            simox::alg::split(pythonScriptArgumentsString, " ", true, true);
        pythonPathEntriesVector = simox::alg::split(pythonPathEntriesString, ":", true, true);
    }
 
    PropertyDefinitionsPtr
    PythonApplicationManager::createPropertyDefinitions()
    {
        armarx::PropertyDefinitionsPtr defs =
            new ComponentPropertyDefinitions(getConfigIdentifier());
 
        properties.defineProperties(defs, "py.");
 
        defs->defineOptionalProperty<bool>(
            "BinaryPathAsWorkingDirectory",
            false,
            "If true the path of the binary is set as the working directory.");
        defs->defineOptionalProperty<bool>(
            "RestartInCaseOfCrash",
            false,
            "Whether the application should be restarted in case it crashed.");
 
        defs->defineOptionalProperty<bool>(
            "DisconnectInCaseOfCrash",
            true,
            "Whether this component should disconnect as long as the application is not running.");
        defs->defineOptionalProperty<int>(
            "FakeObjectStartDelay",
            0,
            "Delay in ms after which the fake armarx object is started on which other apps can "
            "depend. Not used if property FakeObjectDelayedStartKeyword is used.");
        defs->defineOptionalProperty<std::string>(
            "FakeObjectDelayedStartKeyword",
            "",
            "If not empty, the start up of the fake armarx object will be delayed until this "
            "keyword is found in the stdout of the subprocess.");
        defs->defineOptionalProperty<int>(
            "KillDelay",
            2000,
            "Delay ins ms before the subprocess is killed after sending the stop signal");
        defs->defineOptionalProperty<bool>(
            "PythonUnbuffered",
            true,
            "If true, PYTHONUNBUFFERED=1 is added to the environment variables.");
        defs->defineOptionalProperty<bool>("RedirectToArmarXLog",
                                           true,
                                           "If true, all outputs from the subprocess are printed "
                                           "with ARMARX_LOG, otherwise with std::cout");
        defs->defineOptionalProperty<Ice::StringSeq>(
            "AdditionalEnvVars",
            {},
            "Comma-seperated list of env-var assignment, e.g. MYVAR=1,ADDPATH=/tmp");
        defs->defineOptionalProperty<Ice::StringSeq>(
            "Dependencies",
            {},
            "Comma-seperated list of Ice Object dependencies. The external app will only be "
            "started after all dependencies have been found.");
 
        return defs;
    }
 
    std::filesystem::path
    getArmarXCliPath()
    {
        CMakePackageFinder finder("ArmarXCore");
        ARMARX_CHECK(finder.packageFound());
 
        std::filesystem::path armarxCoreRootPath = finder.getPackageDir();
        std::filesystem::path armarxCliPath = armarxCoreRootPath / "etc" / "python";
        return armarxCliPath;
    }
 
    void
    PythonApplicationManager::onInitComponent()
    {
        ARMARX_TRACE;
 
        properties.read(*this, "py.");
        paths.derive(properties);
        ARMARX_INFO << paths;
 
        {
            std::stringstream originalPythonPath;
            if (char* pp = getenv("PYTHONPATH"))
            {
                originalPythonPath << pp;
            }
            std::vector<std::string> pythonPath = simox::alg::split(originalPythonPath.str(), ":");
 
            std::stringstream message;
            {
                /* We have to remove the entry of the ArmarX command line interface
                 * (".../armarx/ArmarXCore/etc/python").
                 *
                 * Reason: The ArmarX command line tool (armarx start, armarx gui, ...)
                 * is now running from a proper virtual environment. So inside the "armarx gui"
                 * process, the path of that venv is in the PYTHONPATH.
                 * This can cause "import armarx" to try to import the command line tool
                 * instead of the ArmarX python bindings (which are probably installed
                 * in the called virtual environment).
                 */
                const std::filesystem::path armarxCliPath = getArmarXCliPath();
                if (auto it =
                        std::find(pythonPath.begin(), pythonPath.end(), armarxCliPath.string());
                    it != pythonPath.end())
                {
                    message << "(Removed '" << *it << "' from PYTHONPATH.)";
                    pythonPath.erase(it);
                }
            }
            pythonPath.push_back(this->paths.pythonPackagePath.string());
            pythonPath.push_back(properties.pythonPathEntriesString);
 
            const std::string newPythonpath = simox::alg::join(pythonPath, ":");
 
            ARMARX_INFO << "Setting PYTHONPATH:\n\t" << newPythonpath
                        << "\nOriginal PYTHONPATH:\n\t" << originalPythonPath.str() << "\n"
                        << message.str();
            ;
            setenv("PYTHONPATH", newPythonpath.c_str(), 1);
        }
 
        inputOk = true;
 
        Base::onInitComponent();
    }
 
    void
    PythonApplicationManager::onConnectComponent()
    {
        ARMARX_TRACE;
 
        if (inputOk)
        {
            Base::onConnectComponent();
        }
    }
 
    void
    PythonApplicationManager::onDisconnectComponent()
    {
        ARMARX_TRACE;
 
        if (inputOk)
        {
            Base::onDisconnectComponent();
        }
    }
 
    void
    PythonApplicationManager::onExitComponent()
    {
        ARMARX_TRACE;
 
        if (inputOk)
        {
            Base::onExitComponent();
        }
    }
 
    std::string
    PythonApplicationManager::deriveWorkingDir() const
    {
        return paths.workingDir;
    }
 
    std::string
    PythonApplicationManager::deriveApplicationPath() const
    {
        return paths.pythonBinPath;
    }
 
    void
    PythonApplicationManager::addApplicationArguments(Ice::StringSeq& args)
    {
        // Script path
        args.push_back(paths.pythonScriptPath);
        // Script args
        args.insert(args.end(),
                    properties.pythonScriptArgumentsVector.begin(),
                    properties.pythonScriptArgumentsVector.end());
    }
 
    std::ostream&
    operator<<(std::ostream& os, const PythonApplicationManager::Paths& paths)
    {
        os << "ArmarX package path: \t" << paths.armarxPackagePath;
        os << "\nPython package path: \t" << paths.pythonPackagePath;
        os << "\nPython script path:  \t" << paths.pythonScriptPath;
        os << "\nVenv path:           \t" << paths.venvPath;
        os << "\nPython binary path:  \t" << paths.pythonBinPath;
        os << "\nWorking directory:   \t" << paths.workingDir;
        return os;
    }
 
    void
    PythonApplicationManager::Paths::derive(const PythonApplicationManager::Properties& properties)
    {
        ARMARX_TRACE;
 
        namespace fs = std::filesystem;
 
        armarxPackagePath = findArmarXPackagePath(properties);
        pythonPackagePath = findPythonPackagePath(properties);
        pythonScriptPath = findPythonScriptPath(properties);
 
        if (properties.pythonPoetry)
        {
            std::string cmd = std::string("cd ") +
                              reinterpret_cast<const char*>(pythonPackagePath.u8string().c_str()) +
                              ";poetry env info -p --ansi";
 
            std::array<char, 128> buffer;
 
            // https://stackoverflow.com/questions/76867698/what-does-ignoring-attributes-on-template-argument-mean-in-this-context
            // std::unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd.c_str(), "r"), pclose);
            std::unique_ptr<FILE, int(*)(FILE*)> pipe(popen(cmd.c_str(), "r"), pclose);
            if (!pipe)
            {
                throw std::runtime_error("popen() failed!");
            }
            while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr)
            {
                std::string s = buffer.data();
                s.erase(std::remove(s.begin(), s.end(), '\n'), s.end());
                venvPath += s;
            }
        }
        else
        {
            venvPath = findVenvPath(properties);
        }
        pythonBinPath = findPythonBinaryPath(properties);
 
        if (properties.workingDir.empty())
        {
            workingDir = pythonScriptPath.parent_path();
        }
        else
        {
            workingDir = properties.workingDir;
        }
    }
 
    PythonApplicationManager::Paths::path
    PythonApplicationManager::Paths::findArmarXPackagePath(
        const PythonApplicationManager::Properties& properties) const
    {
        armarx::CMakePackageFinder finder(properties.armarxPackageName);
        if (!finder.packageFound())
        {
            std::stringstream ss;
            ss << "Could not find ArmarX package '" << properties.armarxPackageName << "'.";
            throw armarx::LocalException(ss.str());
        }
        return finder.getPackageDir();
    }
 
    PythonApplicationManager::Paths::path
    PythonApplicationManager::Paths::findPythonPackagePath(
        const PythonApplicationManager::Properties& properties) const
    {
        return armarxPackagePath / properties.armarxPythonPackagesDir /
               properties.pythonPackageName;
    }
 
    PythonApplicationManager::Paths::path
    PythonApplicationManager::Paths::findPythonScriptPath(
        const PythonApplicationManager::Properties& properties) const
    {
        std::vector<path> candidates{pythonPackagePath / properties.pythonPackageName /
                                         properties.pythonScriptPath,
                                     pythonPackagePath / properties.pythonScriptPath};
 
        if (std::optional<path> p = checkCandidateFiles(candidates))
        {
            return p.value();
        }
        else
        {
            std::stringstream ss;
            ss << "Could not find python script '" << properties.pythonScriptPath << "' "
               << "in python package '" << properties.pythonPackageName << "' "
               << "in ArmarX package '" << properties.armarxPackageName << "'. \n"
               << "Checked candidates: ";
            for (const path& candidate : candidates)
            {
                ss << "\n" << candidate;
            }
            throw armarx::LocalException(ss.str());
        }
    }
 
    PythonApplicationManager::Paths::path
    PythonApplicationManager::Paths::findVenvPath(
        const PythonApplicationManager::Properties& properties) const
    {
        ARMARX_TRACE;
 
        path venvPath;
        switch (properties.venvType)
        {
            case VenvType::Dedicated:
                venvPath = getDedicatedVenvPath(properties);
                break;
 
            case VenvType::Shared:
                venvPath = getSharedVenvPath(properties);
                break;
 
            case VenvType::Auto:
            {
                std::vector<path> candidates{
                    getDedicatedVenvPath(properties),
                    getSharedVenvPath(properties),
                };
                if (std::optional<path> p = checkCandidateDirectories(candidates))
                {
                    venvPath = p.value();
                }
                else
                {
                    std::stringstream ss;
                    ss << "Could not find dedicated or shared venv '" << properties.venvName << "' "
                       << "in python package '" << properties.pythonPackageName << "' "
                       << "in ArmarX package '" << properties.armarxPackageName << "'. \n"
                       << "Checked candidates: ";
                    for (const path& candidate : candidates)
                    {
                        ss << "\n" << candidate;
                    }
                    throw armarx::LocalException(ss.str());
                }
            }
            break;
        }
 
        if (std::filesystem::is_directory(venvPath))
        {
            return venvPath;
        }
        else
        {
            std::stringstream ss;
            ss << "Could not find "
               << simox::alg::to_lower(VenvTypeNames.to_name(properties.venvType)) << " venv '"
               << properties.venvName << "' ";
            switch (properties.venvType)
            {
                case VenvType::Auto:
                    break;
                case VenvType::Dedicated:
                    ss << "in python package '" << properties.pythonPackageName << "' ";
                    break;
                case VenvType::Shared:
                    break;
            }
            ss << "in ArmarX package '" << properties.armarxPackageName << "'. \n";
            ss << "Expected path: \n" << venvPath;
            throw armarx::LocalException(ss.str());
        }
    }
 
    auto
    PythonApplicationManager::Paths::findPythonBinaryPath(
        const PythonApplicationManager::Properties& properties) const -> path
    {
        ARMARX_TRACE;
 
        const path pythonBinPath = venvPath / "bin" / "python";
 
        if (std::filesystem::is_regular_file(pythonBinPath))
        {
            return pythonBinPath;
        }
        else
        {
            std::stringstream ss;
            ss << "Could not find python binary "
               << "in " << simox::alg::to_lower(VenvTypeNames.to_name(properties.venvType))
               << " venv '" << properties.venvName << "' ";
            switch (properties.venvType)
            {
                case VenvType::Auto:
                    break;
                case VenvType::Dedicated:
                    ss << "in python package '" << properties.pythonPackageName << "' ";
                    break;
                case VenvType::Shared:
                    break;
            }
            ss << "in ArmarX package '" << properties.armarxPackageName << "'. \n";
            ss << "Expected path: \n" << pythonBinPath;
            throw armarx::LocalException(ss.str());
        }
    }
 
    auto
    PythonApplicationManager::Paths::getDedicatedVenvPath(const Properties& properties) const
        -> path
    {
        return pythonPackagePath / properties.venvName;
    }
 
    auto
    PythonApplicationManager::Paths::getSharedVenvPath(const Properties& properties) const -> path
    {
        return armarxPackagePath / properties.armarxPythonPackagesDir / sharedVenvsDir /
               properties.venvName;
    }
 
    std::optional<PythonApplicationManager::Paths::path>
    PythonApplicationManager::Paths::checkCandidateFiles(
        const std::vector<PythonApplicationManager::Paths::path>& candidates)
    {
        return checkCandidatePaths(
            candidates, [](const path& p) { return std::filesystem::is_regular_file(p); });
    }
 
    std::optional<PythonApplicationManager::Paths::path>
    PythonApplicationManager::Paths::checkCandidateDirectories(
        const std::vector<PythonApplicationManager::Paths::path>& candidates)
    {
        return checkCandidatePaths(candidates,
                                   [](const path& p) { return std::filesystem::is_directory(p); });
    }
 
    std::optional<PythonApplicationManager::Paths::path>
    PythonApplicationManager::Paths::checkCandidatePaths(
        const std::vector<PythonApplicationManager::Paths::path>& candidates,
        std::function<bool(PythonApplicationManager::Paths::path)> existsFn)
    {
        auto it = std::find_if(candidates.begin(), candidates.end(), existsFn);
        if (it != candidates.end())
        {
            return *it;
        }
        else
        {
            return std::nullopt;
        }
    }
 
} // namespace armarx