PythonApplicationManager.cpp

Go to the documentation of this file.

/*
 * This file is part of ArmarX.
 *
 * ArmarX is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * ArmarX is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * @package    ArmarXCore::ArmarXObjects::PythonApplicationManager
 * @author     Rainer Kartmann ( rainer dot kartmann at kit dot edu )
 * @date       2021
 * @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
 *             GNU General Public License
 */
 
#include "PythonApplicationManager.h"
 
#include <SimoxUtility/algorithm/string.h>
 
#include <ArmarXCore/core/application/properties/PluginEnumNames.h>
#include <ArmarXCore/core/exceptions/LocalException.h>
#include <ArmarXCore/core/system/cmake/CMakePackageFinder.h>
 
#include <algorithm>
 
namespace armarx
{
 
    const simox::meta::EnumNames<PythonApplicationManager::VenvType> PythonApplicationManager::VenvTypeNames
    {
        { PythonApplicationManager::VenvType::Auto, "Auto" },
        { PythonApplicationManager::VenvType::Dedicated, "Dedicated" },
        { PythonApplicationManager::VenvType::Shared, "Shared" },
    };
 
 
    void PythonApplicationManager::Properties::defineProperties(PropertyDefinitionsPtr defs, const std::string& prefix)
    {
        defs->required(armarxPackageName, prefix + "10_ArmarXPackageName", "Name of the ArmarX package.");
        defs->optional(armarxPythonPackagesDir, prefix + "11_ArmarXPythonPackagesDir", "Directory where python packages are located (relative to the root of the ArmarX package).");
 
        defs->required(pythonPackageName, prefix + "20_PythonPackageName", "Name of the Python package in the ArmarXPackage/python/ directory.");
        defs->required(pythonScriptPath, prefix + "21_PythonScriptPath", "Path to the python script (inside the python package).");
        defs->optional(pythonScriptArgumentsString, prefix + "22_PythonScriptArgs", "Whitespace separated list of arguments.");
        defs->optional(pythonPathEntriesString, prefix + "23_PythonPathEntries", "Colon-separated list of paths to add to the PYTHONPATH.");
        defs->optional(pythonPoetry, prefix + "24_PythonPoetry", "Use python poetry.");
 
        defs->optional(venvName, prefix + "30_venv.Name", "Name of the virtual environment.");
        {
            std::stringstream ss;
            ss << "Type of the virtual environment."
               << "\n\t- " << VenvTypeNames.to_name(VenvType::Auto) << "       \tDerive automatically."
               << "\n\t- " << VenvTypeNames.to_name(VenvType::Dedicated) << "\tSearch inside the python package root directory."
               << "\n\t- " << VenvTypeNames.to_name(VenvType::Shared) << "   \tSearch in the shared_envs directory."
               ;
            defs->optional(venvType, prefix + "31_venv.Type", ss.str()).map(VenvTypeNames);
        }
 
 
        defs->optional(workingDir, prefix + "40_WorkingDirectory",
                       "If set, this path is used as working directory for the python script (overriding BinaryPathAsWorkingDirectory)."
                       "\n${HOME} for env vars, $C{RobotAPI:BINARY_DIR} for CMakePackageFinder vars");
    }
 
    void PythonApplicationManager::Properties::read(PropertyUser& props, const std::string& prefix)
    {
        ARMARX_TRACE;
 
        if (props.getProperty<std::string>(prefix + "40_WorkingDirectory").isSet())
        {
            workingDir = props.getProperty<std::string>("WorkingDirectory").getValueAndReplaceAllVars();
        };
 
        pythonScriptArgumentsVector = simox::alg::split(pythonScriptArgumentsString, " ", true, true);
        pythonPathEntriesVector = simox::alg::split(pythonPathEntriesString, ":", true, true);
    }
 
 
    PropertyDefinitionsPtr PythonApplicationManager::createPropertyDefinitions()
    {
        armarx::PropertyDefinitionsPtr defs = new ComponentPropertyDefinitions(getConfigIdentifier());
 
        properties.defineProperties(defs, "py.");
 
        defs->defineOptionalProperty<bool>("BinaryPathAsWorkingDirectory", false, "If true the path of the binary is set as the working directory.");
        defs->defineOptionalProperty<bool>("RestartInCaseOfCrash", false, "Whether the application should be restarted in case it crashed.");
 
        defs->defineOptionalProperty<bool>("DisconnectInCaseOfCrash", true, "Whether this component should disconnect as long as the application is not running.");
        defs->defineOptionalProperty<int>("FakeObjectStartDelay", 0, "Delay in ms after which the fake armarx object is started on which other apps can depend. Not used if property FakeObjectDelayedStartKeyword is used.");
        defs->defineOptionalProperty<std::string>("FakeObjectDelayedStartKeyword", "", "If not empty, the start up of the fake armarx object will be delayed until this keyword is found in the stdout of the subprocess.");
        defs->defineOptionalProperty<int>("KillDelay", 2000, "Delay ins ms before the subprocess is killed after sending the stop signal");
        defs->defineOptionalProperty<bool>("PythonUnbuffered", true, "If true, PYTHONUNBUFFERED=1 is added to the environment variables.");
        defs->defineOptionalProperty<bool>("RedirectToArmarXLog", true, "If true, all outputs from the subprocess are printed with ARMARX_LOG, otherwise with std::cout");
        defs->defineOptionalProperty<Ice::StringSeq>("AdditionalEnvVars", {}, "Comma-seperated list of env-var assignment, e.g. MYVAR=1,ADDPATH=/tmp");
        defs->defineOptionalProperty<Ice::StringSeq>("Dependencies", {}, "Comma-seperated list of Ice Object dependencies. The external app will only be started after all dependencies have been found.");
 
        return defs;
    }
 
    std::filesystem::path getArmarXCliPath()
    {
        CMakePackageFinder finder("ArmarXCore");
        ARMARX_CHECK(finder.packageFound());
 
        std::filesystem::path armarxCoreRootPath = finder.getPackageDir();
        std::filesystem::path armarxCliPath = armarxCoreRootPath / "etc" / "python";
        return armarxCliPath;
    }
 
 
    void PythonApplicationManager::onInitComponent()
    {
        ARMARX_TRACE;
 
        properties.read(*this, "py.");
        paths.derive(properties);
        ARMARX_INFO << paths;
 
        {
            std::stringstream originalPythonPath;
            if (char* pp = getenv("PYTHONPATH"))
            {
                originalPythonPath << pp;
            }
            std::vector<std::string> pythonPath = simox::alg::split(originalPythonPath.str(), ":");
 
            std::stringstream message;
            {
                /* We have to remove the entry of the ArmarX command line interface
                 * (".../armarx/ArmarXCore/etc/python").
                 *
                 * Reason: The ArmarX command line tool (armarx start, armarx gui, ...)
                 * is now running from a proper virtual environment. So inside the "armarx gui"
                 * process, the path of that venv is in the PYTHONPATH.
                 * This can cause "import armarx" to try to import the command line tool
                 * instead of the ArmarX python bindings (which are probably installed
                 * in the called virtual environment).
                 */
                const std::filesystem::path armarxCliPath = getArmarXCliPath();
                if (auto it = std::find(pythonPath.begin(), pythonPath.end(), armarxCliPath.string()); it != pythonPath.end())
                {
                    message << "(Removed '" << *it << "' from PYTHONPATH.)";
                    pythonPath.erase(it);
                }
            }
            pythonPath.push_back(this->paths.pythonPackagePath.string());
            pythonPath.push_back(properties.pythonPathEntriesString);
 
            const std::string newPythonpath = simox::alg::join(pythonPath, ":");
 
            ARMARX_INFO << "Setting PYTHONPATH:\n\t" << newPythonpath
                        << "\nOriginal PYTHONPATH:\n\t" << originalPythonPath.str()
                        << "\n" << message.str();
                ;
            setenv("PYTHONPATH", newPythonpath.c_str(), 1);
        }
 
        inputOk = true;
 
        Base::onInitComponent();
    }
 
 
    void PythonApplicationManager::onConnectComponent()
    {
        ARMARX_TRACE;
 
        if (inputOk)
        {
            Base::onConnectComponent();
        }
    }
 
 
    void PythonApplicationManager::onDisconnectComponent()
    {
        ARMARX_TRACE;
 
        if (inputOk)
        {
            Base::onDisconnectComponent();
        }
    }
 
 
    void PythonApplicationManager::onExitComponent()
    {
        ARMARX_TRACE;
 
        if (inputOk)
        {
            Base::onExitComponent();
        }
    }
 
    std::string PythonApplicationManager::deriveWorkingDir() const
    {
        return paths.workingDir;
    }
 
    std::string PythonApplicationManager::deriveApplicationPath() const
    {
        return paths.pythonBinPath;
    }
 
    void PythonApplicationManager::addApplicationArguments(Ice::StringSeq& args)
    {
        // Script path
        args.push_back(paths.pythonScriptPath);
        // Script args
        args.insert(args.end(), properties.pythonScriptArgumentsVector.begin(), properties.pythonScriptArgumentsVector.end());
    }
 
    std::ostream& operator<<(std::ostream& os, const PythonApplicationManager::Paths& paths)
    {
        os <<   "ArmarX package path: \t" << paths.armarxPackagePath;
        os << "\nPython package path: \t" << paths.pythonPackagePath;
        os << "\nPython script path:  \t" << paths.pythonScriptPath;
        os << "\nVenv path:           \t" << paths.venvPath;
        os << "\nPython binary path:  \t" << paths.pythonBinPath;
        os << "\nWorking directory:   \t" << paths.workingDir;
        return os;
    }
 
    void PythonApplicationManager::Paths::derive(const PythonApplicationManager::Properties& properties)
    {
        ARMARX_TRACE;
 
        namespace fs = std::filesystem;
 
        armarxPackagePath = findArmarXPackagePath(properties);
        pythonPackagePath = findPythonPackagePath(properties);
        pythonScriptPath = findPythonScriptPath(properties);
 
        if (properties.pythonPoetry)
        {
            std::string cmd = std::string("cd ")
                    + reinterpret_cast<const char*>(pythonPackagePath.u8string().c_str())
                    + ";poetry env info -p --ansi" ;
 
            std::array<char, 128> buffer;
            std::unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd.c_str(), "r"), pclose);
            if (!pipe)
            {
                throw std::runtime_error("popen() failed!");
            }
            while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr)
            {
                std::string s =  buffer.data();
                s.erase(std::remove(s.begin(), s.end(), '\n'), s.end());
                venvPath += s;
            }
        }
        else
        {
            venvPath = findVenvPath(properties);
        }
        pythonBinPath = findPythonBinaryPath(properties);
 
        if (properties.workingDir.empty())
        {
            workingDir = pythonScriptPath.parent_path();
        }
        else
        {
            workingDir = properties.workingDir;
        }
    }
 
    PythonApplicationManager::Paths::path PythonApplicationManager::Paths::findArmarXPackagePath(const PythonApplicationManager::Properties& properties) const
    {
        armarx::CMakePackageFinder finder(properties.armarxPackageName);
        if (!finder.packageFound())
        {
            std::stringstream ss;
            ss << "Could not find ArmarX package '" << properties.armarxPackageName << "'.";
            throw armarx::LocalException(ss.str());
        }
        return finder.getPackageDir();
    }
 
    PythonApplicationManager::Paths::path PythonApplicationManager::Paths::findPythonPackagePath(const PythonApplicationManager::Properties& properties) const
    {
        return armarxPackagePath
               / properties.armarxPythonPackagesDir
               / properties.pythonPackageName;
    }
 
    PythonApplicationManager::Paths::path PythonApplicationManager::Paths::findPythonScriptPath(const PythonApplicationManager::Properties& properties) const
    {
        std::vector<path> candidates
        {
            pythonPackagePath
            / properties.pythonPackageName
            / properties.pythonScriptPath,
            pythonPackagePath
            / properties.pythonScriptPath
        };
 
        if (std::optional<path> p = checkCandidateFiles(candidates))
        {
            return p.value();
        }
        else
        {
            std::stringstream ss;
            ss << "Could not find python script '" << properties.pythonScriptPath << "' "
               << "in python package '" << properties.pythonPackageName << "' "
               << "in ArmarX package '" << properties.armarxPackageName << "'. \n"
               << "Checked candidates: ";
            for (const path& candidate : candidates)
            {
                ss << "\n" << candidate;
            }
            throw armarx::LocalException(ss.str());
        }
    }
 
    PythonApplicationManager::Paths::path PythonApplicationManager::Paths::findVenvPath(const PythonApplicationManager::Properties& properties) const
    {
        ARMARX_TRACE;
 
        path venvPath;
        switch (properties.venvType)
        {
            case VenvType::Dedicated:
                venvPath = getDedicatedVenvPath(properties);
                break;
 
            case VenvType::Shared:
                venvPath = getSharedVenvPath(properties);
                break;
 
            case VenvType::Auto:
            {
                std::vector<path> candidates
                {
                    getDedicatedVenvPath(properties),
                    getSharedVenvPath(properties),
                };
                if (std::optional<path> p = checkCandidateDirectories(candidates))
                {
                    venvPath = p.value();
                }
                else
                {
                    std::stringstream ss;
                    ss << "Could not find dedicated or shared venv '" << properties.venvName << "' "
                       << "in python package '" << properties.pythonPackageName << "' "
                       << "in ArmarX package '" << properties.armarxPackageName << "'. \n"
                       << "Checked candidates: ";
                    for (const path& candidate : candidates)
                    {
                        ss << "\n" << candidate;
                    }
                    throw armarx::LocalException(ss.str());
                }
            }
            break;
        }
 
        if (std::filesystem::is_directory(venvPath))
        {
            return venvPath;
        }
        else
        {
            std::stringstream ss;
            ss << "Could not find " << simox::alg::to_lower(VenvTypeNames.to_name(properties.venvType))
               << " venv '" << properties.venvName << "' ";
            switch (properties.venvType)
            {
                case VenvType::Auto:
                    break;
                case VenvType::Dedicated:
                    ss << "in python package '" << properties.pythonPackageName << "' ";
                    break;
                case VenvType::Shared:
                    break;
            }
            ss << "in ArmarX package '" << properties.armarxPackageName << "'. \n";
            ss << "Expected path: \n" << venvPath;
            throw armarx::LocalException(ss.str());
        }
    }
 
    auto PythonApplicationManager::Paths::findPythonBinaryPath(const PythonApplicationManager::Properties& properties) const -> path
    {
        ARMARX_TRACE;
 
        const path pythonBinPath = venvPath / "bin" / "python";
 
        if (std::filesystem::is_regular_file(pythonBinPath))
        {
            return pythonBinPath;
        }
        else
        {
            std::stringstream ss;
            ss << "Could not find python binary "
               << "in " << simox::alg::to_lower(VenvTypeNames.to_name(properties.venvType))
               << " venv '" << properties.venvName << "' ";
            switch (properties.venvType)
            {
                case VenvType::Auto:
                    break;
                case VenvType::Dedicated:
                    ss << "in python package '" << properties.pythonPackageName << "' ";
                    break;
                case VenvType::Shared:
                    break;
            }
            ss << "in ArmarX package '" << properties.armarxPackageName << "'. \n";
            ss << "Expected path: \n" << pythonBinPath;
            throw armarx::LocalException(ss.str());
        }
    }
 
    auto PythonApplicationManager::Paths::getDedicatedVenvPath(const Properties& properties) const -> path
    {
        return pythonPackagePath
               / properties.venvName;
    }
 
    auto PythonApplicationManager::Paths::getSharedVenvPath(const Properties& properties) const -> path
    {
        return armarxPackagePath
               / properties.armarxPythonPackagesDir
               / sharedVenvsDir
               / properties.venvName;
    }
 
    std::optional<PythonApplicationManager::Paths::path> PythonApplicationManager::Paths::checkCandidateFiles(const std::vector<PythonApplicationManager::Paths::path>& candidates)
    {
        return checkCandidatePaths(candidates, [](const path & p)
        {
            return std::filesystem::is_regular_file(p);
        });
    }
 
    std::optional<PythonApplicationManager::Paths::path> PythonApplicationManager::Paths::checkCandidateDirectories(const std::vector<PythonApplicationManager::Paths::path>& candidates)
    {
        return checkCandidatePaths(candidates, [](const path & p)
        {
            return std::filesystem::is_directory(p);
        });
    }
 
    std::optional<PythonApplicationManager::Paths::path> PythonApplicationManager::Paths::checkCandidatePaths(const std::vector<PythonApplicationManager::Paths::path>& candidates, std::function<bool (PythonApplicationManager::Paths::path)> existsFn)
    {
        auto it = std::find_if(candidates.begin(), candidates.end(), existsFn);
        if (it != candidates.end())
        {
            return *it;
        }
        else
        {
            return std::nullopt;
        }
    }
 
}