LinuxStarter.cpp

Go to the documentation of this file.

/*
 * This file is part of ArmarX.
 *
 * Copyright (C) 2011-2016, High Performance Humanoid Technologies (H2T), Karlsruhe Institute of Technology (KIT), all rights reserved.
 *
 * ArmarX is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * ArmarX is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * @package    ArmarXCore::core
 * @author     Nicola Miskowiec
 * @date       2016
 * @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
 *             GNU General Public License
 */
 
 
#include "LinuxStarter.h"
 
#include <errno.h>
#include <signal.h>
#include <stdlib.h>
 
#include <filesystem>
#include <future>
#include <iostream>
#include <chrono>
#include <mutex>
#include <optional>
#include <sstream>
#include <string>
#include <thread>
#include <vector>
 
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <syslog.h>
#include <unistd.h>
 
#include <boost/process.hpp>
 
#include <SimoxUtility/algorithm/string/string_tools.h>
 
#include <ArmarXCore/core/exceptions/Exception.h>
#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
#include <ArmarXCore/core/logging/Logging.h>
#include <ArmarXCore/core/system/ArmarXDataPath.h>
#include <ArmarXCore/core/util/OnScopeExit.h>
 
#include "../parser/DependenciesGenerator.h"
#include "../parser/iceparser.h"
#include "ProcessUtil.h"
 
using namespace ScenarioManager;
using namespace Exec;
using namespace Parser;
//using namespace Data_Structure;
using namespace armarx;
 
namespace
{
    /**
     * How long a /proc snapshot may be reused. Walking /proc costs one open() per process on the
     * machine, so it must happen once for all applications rather than once per application per
     * poll. Applications do not appear and disappear behind the ScenarioManager's back often
     * enough for this to need to be tight.
     */
    constexpr std::chrono::seconds ProcessScanInterval{2};
 
    std::mutex ProcessScanMutex;
    std::vector<ProcessUtil::ProcessInfo> ProcessScanCache;
    std::chrono::steady_clock::time_point ProcessScanTime;
    bool ProcessScanned = false;
 
    /**
     * @return the processes whose executable name is `executableName`, from a /proc snapshot that
     * is at most ProcessScanInterval old.
     */
    std::vector<ProcessUtil::ProcessInfo>
    FindProcessesNamed(const std::string& executableName)
    {
        std::vector<ProcessUtil::ProcessInfo> matches;
        if (executableName.empty())
        {
            return matches;
        }
 
        std::lock_guard<std::mutex> guard(ProcessScanMutex);
 
        const auto now = std::chrono::steady_clock::now();
        if (!ProcessScanned || now - ProcessScanTime >= ProcessScanInterval)
        {
            ProcessScanCache = ProcessUtil::ScanProcesses();
            ProcessScanTime = now;
            ProcessScanned = true;
        }
 
        // Copy out only what the caller asked for - usually nothing or a single entry - so that
        // the whole snapshot is not duplicated for every application.
        for (const ProcessUtil::ProcessInfo& process : ProcessScanCache)
        {
            if (process.name == executableName)
            {
                matches.push_back(process);
            }
        }
        return matches;
    }
} // namespace
 
LinuxStarter::LinuxStarter()
{
    EnsureChildrenAreReaped();
}
 
void
LinuxStarter::EnsureChildrenAreReaped()
{
#if defined(BOOST_POSIX_API)
    // The applications we launch are never waited for, so ask the kernel to reap them itself
    // instead of leaving zombies behind.
    //
    // This changes the signal disposition of the *whole* process, and while SIGCHLD is ignored no
    // child can be waited for any more - system(), popen() and QProcess::waitForFinished() all
    // stop working. That conflict is unavoidable in a GUI that hosts other plugins, and it cuts
    // both ways: whoever installs a SIGCHLD handler later - Qt's QProcess does, on first use -
    // silently switches the kernel's auto-reaping back off, because auto-reaping is a property of
    // the disposition and not of any handler's behaviour.
    //
    // This used to be done exactly once per process, which meant that a disposition lost that way
    // was never restored: from then on every application we stopped stayed behind as a zombie, its
    // /proc entry never went away, and the ScenarioManager waited out the full termination timeout
    // and then latched on Unknown. Re-asserting it before each launch bounds the damage to the
    // applications started while the disposition was lost.
    //
    // It does not clean up retroactively: Linux discards children that terminate *after* SIGCHLD
    // is ignored, so zombies that have already accumulated stay until their parent exits. Reaping
    // them here with waitpid(-1) is not an option either, as that would swallow the exit status of
    // children belonging to other parts of the GUI. Those leftovers are instead handled where they
    // are observed - a zombie reads as dead, not as an unclear state - so they are harmless rather
    // than absent.
    signal(SIGCHLD, SIG_IGN);
#endif
}
 
void
LinuxStarter::startApplication(ApplicationInstancePtr app,
                               StatusManager statusManager,
                               const std::string& commandLineParameters,
                               bool printOnly)
{
    if (!app->getEnabled())
    {
        return;
    }
 
    std::filesystem::path componentFolderPath =
        std::filesystem::path(ArmarXDataPath::GetCachePath()) / "ComponentFiles";
 
    if (!std::filesystem::exists(componentFolderPath))
    {
        if (!std::filesystem::create_directories(componentFolderPath))
        {
            std::cout << "Could not create Cache folder for ScenarioManagerPlugin at "
                      << componentFolderPath.string() << std::endl;
        }
    }
 
 
    std::filesystem::path dependenciesFilePath =
        componentFolderPath /
        std::filesystem::path("./" + app->getScenario()->getPackage()->getName() +
                              ".dependencies.cfg");
 
    if (!std::filesystem::exists(dependenciesFilePath))
    {
        Parser::DependenciesGenerator builder;
        builder.generateDependenciesCfg(
            Data_Structure::PackagePtr(app->getScenario()->getPackage()));
    }
 
    if (!std::filesystem::exists(
            std::filesystem::path(app->getScenario()->getGlobalConfigPath())) ||
        !std::filesystem::exists(std::filesystem::path(app->getConfigPath())) ||
        !std::filesystem::exists(dependenciesFilePath))
    {
        std::cout << "Launching " << app->getName() << " in "
                  << app->getScenario()->getPackage()->getName() << " without an needed cfg file";
    }
    if (!std::filesystem::exists(app->getExecutableAbsPath()))
    {
        std::cout << "\033[1;31m"
                  << "Warning: Could not launch " << app->getName() << " in "
                  << app->getScenario()->getPackage()->getName()
                  << " because the executable is missing at " << app->getPathToExecutable()
                  << "\033[0m" << std::endl;
    }
    std::string args;
    args += app->getExecutableAbsPath();
    args += std::string(" --Ice.Config=")
                .append(app->getScenario()->getGlobalConfigPath())
                .append(",")
                .append(app->getConfigPath());
    auto configDomain = app->getConfigDomain();
    ARMARX_CHECK_EXPRESSION(!configDomain.empty());
    args += std::string(" --" + configDomain + ".DependenciesConfig=")
                .append(dependenciesFilePath.string());
    args +=
        std::string(" --" + configDomain + ".LoggingGroup=").append(app->getScenario()->getName());
 
    if (!commandLineParameters.empty())
    {
        Ice::StringSeq additional = Split(commandLineParameters, " ");
 
        bool inString = false;
        std::string inStringChar;
        std::string toPush = "";
        for (auto command : additional)
        {
            if (!inString && command.find("\"") != std::string::npos)
            {
                inString = true;
                inStringChar = "\"";
            }
            else if (!inString && command.find("\'") != std::string::npos)
            {
                inString = true;
                inStringChar = "\'";
            }
            else if (inString && command.find(inStringChar) != std::string::npos)
            {
                inString = false;
                toPush += command;
                args += " " + toPush;
                toPush = "";
                continue;
            }
 
            if (inString)
            {
                toPush += command + " ";
            }
            else
            {
                if (command != "")
                {
                    args += " " + command;
                }
            }
        }
    }
    else
    {
        Ice::StringSeq additional = Split(this->commandLineParameters, " ");
        for (auto arg : additional)
        {
            args += arg + " ";
        }
    }
 
    using namespace boost::process;
    using namespace boost::process::initializers;
 
    if (printOnly)
    {
        for (auto arg : args)
        {
            std::cout << arg;
        }
        std::cout << " &" << std::endl << std::endl;
        return;
    }
 
    //this apperantly prodcuces escaping errors
    //child c = execute(set_args(args), inherit_env());
 
    //this is the solution
    ARMARX_INFO << "Executing command `" << app->getExecutableAbsPath() << " " << args;
 
    // Right before forking, so that this child is reaped even if something else in the process has
    // taken the SIGCHLD disposition over since the last launch. We never wait for what we start.
    EnsureChildrenAreReaped();
 
    child c = execute(run_exe(app->getExecutableAbsPath()), set_cmd_line(args), inherit_env());
 
    int pid = c.pid;
 
    app->setPid(pid);
    statusManager.savePid(app);
}
 
std::string
LinuxStarter::getStatus(ApplicationInstancePtr application, StatusManager statusManager)
{
    int pid = application->getPid();
 
    //if the app has no pid ask the pid manager if there is an cached Pid for this app
    if (pid < 0)
    {
        const int cachedPid = statusManager.loadPid(application);
 
        if (cachedPid < 0)
        {
            // Nothing cached either. The application may still be running - we started it in an
            // earlier session, lost its pid, or it was started outside the ScenarioManager
            // altogether - so look for it before declaring it stopped.
            return adoptRunningProcess(application, statusManager);
        }
 
        // A cached pid may have been recycled by a completely unrelated process in the meantime,
        // so only adopt it if it still belongs to this application.
        const std::string processName = ProcessUtil::GetProcessName(cachedPid);
        if (processName.empty())
        {
            if (!ProcessUtil::IsAlive(cachedPid))
            {
                // The process is really gone - drop the stale cache entry, then check whether the
                // application is running under some other pid.
                //
                // Reset the pid only if it is still unset: a start running concurrently may have
                // assigned a real pid while we were probing the cached one, and clobbering that
                // would leave the application it just launched marked as stopped forever.
                if (application->compareAndResetPid(-1))
                {
                    statusManager.savePid(application);
                }
                return adoptRunningProcess(application, statusManager);
            }
 
            // The process exists but cannot be identified right now (it may have been forked but
            // not exec'd yet). Leave the cached pid alone and try again on the next poll: wrongly
            // discarding it here would mark a running application as stopped for good.
            return Data_Structure::ApplicationStatus::Unknown;
        }
 
        if (processName != application->getExecutableName())
        {
            //else delete the saved pid and inform the user
            ARMARX_INFO_S << "The Process name with Pid (" << cachedPid << ":" << processName
                          << ") does not correspond with app name (" << application->getName()
                          << "). Resetting Saved Pid.";
            if (application->compareAndResetPid(-1))
            {
                statusManager.savePid(application);
            }
            return adoptRunningProcess(application, statusManager);
        }
 
        if (!application->adoptPidIfUnset(cachedPid))
        {
            // A start running concurrently has assigned a pid while we were probing - that one is
            // authoritative.
            pid = application->getPid();
        }
        else
        {
            pid = cachedPid;
        }
 
        if (pid < 0)
        {
            return adoptRunningProcess(application, statusManager);
        }
    }
 
    // Everything below reasons about /proc/<pid>, so first make sure that pid is still ours.
    //
    // Pids are recycled. Once our process has exited the kernel may hand its pid to something
    // completely unrelated, and we would then report that stranger's state as the application's:
    // alive when the application is not, refusing a start because "it is already running", and
    // signalling - possibly killing - an innocent process on stop. This check used to exist only
    // on the cached-pid path below, which is why restarting the ScenarioManager "fixed" such a
    // row: it dropped the in-memory pid and forced it through that path.
    const ProcessUtil::Identity identity = ProcessUtil::IdentifyProcess(
        pid, application->getExecutableName(), application->getConfigPath());
 
    if (identity == ProcessUtil::Identity::Differs)
    {
        ARMARX_INFO_S << "The process with pid " << pid << " is no longer application "
                      << application->getName()
                      << " - the pid has been recycled by an unrelated process. Dropping it and "
                         "looking for the application again.";
        if (application->compareAndResetPid(pid))
        {
            statusManager.savePid(application);
        }
        return adoptRunningProcess(application, statusManager);
    }
 
    switch (probeProcessState(pid))
    {
        case ProcessState::Alive:
            return Data_Structure::ApplicationStatus::Running;
 
        case ProcessState::Indeterminate:
            // The process is there but we cannot say what it is doing. Keep the pid - the next
            // poll will most likely be able to read it again.
            //
            // Unless we could not confirm it is ours either. Then holding on to it is what makes
            // this state absorbing: a pid that is neither identifiable nor probeable is never
            // reconsidered, so the application stays Unknown - unstartable and unstoppable - until
            // the ScenarioManager itself is restarted. Look for the real process instead, and only
            // keep the pid if there is nothing better.
            if (identity != ProcessUtil::Identity::Matches)
            {
                if (const std::optional<std::string> status =
                        rediscover(application, statusManager, pid))
                {
                    return *status;
                }
            }
            return Data_Structure::ApplicationStatus::Unknown;
 
        case ProcessState::Dead:
            break;
    }
 
    // Only clear the pid we actually probed. A start running concurrently may have stored a
    // new pid in the meantime, and wiping that would leave a running application marked as
    // stopped until the ScenarioManager is restarted.
    if (application->compareAndResetPid(pid))
    {
        statusManager.savePid(application);
    }
    else
    {
        // Somebody else owns the pid now; report on that one instead of on the corpse we probed.
        return Data_Structure::ApplicationStatus::Unknown;
    }
 
    // The pid died, but the application may have been restarted behind our back, so do not commit
    // to "Stopped" without looking.
    return adoptRunningProcess(application, statusManager);
}
 
std::string
LinuxStarter::adoptRunningProcess(const ApplicationInstancePtr& application,
                                  StatusManager& statusManager)
{
    bool ambiguous = false;
    const int pid = findRunningProcess(application, ambiguous);
 
    if (pid < 0)
    {
        // Matching processes exist but none of them could be attributed to this application (or
        // several could). Guessing would let a stop request kill an unrelated process, so say we
        // do not know rather than picking one.
        return ambiguous ? Data_Structure::ApplicationStatus::Unknown
                         : Data_Structure::ApplicationStatus::Stopped;
    }
 
    if (!application->adoptPidIfUnset(pid))
    {
        // A start beat us to it - its pid is the authoritative one and will be probed next poll.
        return Data_Structure::ApplicationStatus::Unknown;
    }
 
    ARMARX_INFO_S << "Found a running process (pid " << pid << ") for application "
                  << application->getName()
                  << " that this ScenarioManager did not start (or whose pid it lost). Adopting "
                     "it, so that it is shown as running and can be stopped from here.";
    statusManager.savePid(application);
 
    // Report on the pid we just adopted rather than assuming it is alive: it may have exited
    // between the scan and now.
    return probeProcessState(pid) == ProcessState::Alive
               ? Data_Structure::ApplicationStatus::Running
               : Data_Structure::ApplicationStatus::Unknown;
}
 
std::optional<std::string>
LinuxStarter::rediscover(const ApplicationInstancePtr& application,
                         StatusManager& statusManager,
                         int stalePid)
{
    bool ambiguous = false;
    const int found = findRunningProcess(application, ambiguous);
 
    if (found < 0 || found == stalePid)
    {
        // Nothing to hand over to. Note that this deliberately does not report Stopped: the pid we
        // hold may still be the application, we just could not confirm it.
        return std::nullopt;
    }
 
    // Two steps rather than one exchange, so that a start running concurrently wins either way: it
    // either still holds `stalePid` and the reset succeeds, or it has already stored its own pid
    // and one of the two calls fails, leaving its pid untouched.
    if (!application->compareAndResetPid(stalePid) || !application->adoptPidIfUnset(found))
    {
        return std::nullopt;
    }
 
    ARMARX_INFO_S << "Application " << application->getName() << " is running as pid " << found
                  << ", not as pid " << stalePid
                  << " which could no longer be identified. Following it there.";
    statusManager.savePid(application);
 
    return probeProcessState(found) == ProcessState::Alive
               ? Data_Structure::ApplicationStatus::Running
               : Data_Structure::ApplicationStatus::Unknown;
}
 
int
LinuxStarter::findRunningProcess(const ApplicationInstancePtr& application, bool& ambiguous)
{
    ambiguous = false;
 
    const std::vector<ProcessUtil::ProcessInfo> candidates =
        FindProcessesNamed(application->getExecutableName());
    if (candidates.empty())
    {
        return -1;
    }
 
    const std::string ownConfigPath = application->getConfigPath();
 
    // Processes carrying an --Ice.Config that names our config file are certainly ours; ones
    // carrying an --Ice.Config that names somebody else's are certainly not. Anything else - a
    // component launched by hand from a shell, say - is a possible but unproven match, and is only
    // used when it is the sole candidate.
    std::vector<int> certain;
    std::vector<int> possible;
 
    for (const ProcessUtil::ProcessInfo& candidate : candidates)
    {
        // The snapshot is up to a couple of seconds old, so a candidate may already be gone -
        // notably the process we just stopped ourselves. Adopting a corpse would report it as
        // running for another poll.
        if (!ProcessUtil::IsAlive(candidate.pid))
        {
            continue;
        }
 
        const std::optional<std::string> configArgument =
            ProcessUtil::FindIceConfigArgument(candidate.arguments);
 
        if (!configArgument.has_value())
        {
            possible.push_back(candidate.pid);
        }
        else if (!ownConfigPath.empty() && Contains(*configArgument, ownConfigPath))
        {
            certain.push_back(candidate.pid);
        }
        // else: it belongs to a different scenario or instance of the same binary - not ours.
    }
 
    if (certain.size() == 1)
    {
        return certain.front();
    }
    if (!certain.empty())
    {
        // Several processes claim the same config. Nothing sensible to pick.
        ambiguous = true;
        return -1;
    }
    if (possible.size() == 1)
    {
        return possible.front();
    }
 
    ambiguous = !possible.empty();
    return -1;
}
 
LinuxStarter::ProcessState
LinuxStarter::probeProcessState(int pid)
{
    if (pid <= 0)
    {
        return ProcessState::Dead;
    }
 
    // Anything that goes wrong from here on is only a death if the process is really gone. A
    // /proc entry we cannot read - because file descriptors ran out, because /proc is mounted
    // with hidepid, because the read was interrupted - used to be reported as "Stopped", which
    // then made the caller throw the pid away and lose a running application for good.
    const auto stateIfUnreadable = [pid]
    { return ProcessUtil::IsAlive(pid) ? ProcessState::Indeterminate : ProcessState::Dead; };
 
    const std::string processFilePath = "/proc/" + std::to_string(pid) + "/status";
    std::ifstream t(processFilePath);
 
    if (!t.is_open())
    {
        return stateIfUnreadable();
    }
 
    std::string stateLine;
    std::string line;
    while (std::getline(t, line))
    {
        if (Contains(line, "State:"))
        {
            stateLine = line;
            break;
        }
    }
    if (stateLine.empty())
    {
        return stateIfUnreadable();
    }
 
    std::string result = stateLine.substr(stateLine.find(':') + 1);
 
    simox::alg::trim(result);
 
    if (result.empty())
    {
        return stateIfUnreadable();
    }
 
    switch (result.front())
    {
        case 'R': // running
        case 'S': // interruptible sleep
            return ProcessState::Alive;
 
        case 'X': // dead
        case 'Z': // terminated, still occupying its pid until somebody reaps it
            // A zombie is a *confirmed* death, not an unclear reading: the process has run its
            // course and will never run again. Reporting it as indeterminate kept the pid, and
            // since a zombie's /proc entry never goes away by itself the application stayed
            // Unknown - neither startable nor stoppable - until the ScenarioManager was restarted.
            return ProcessState::Dead;
 
        case 'D': // uninterruptible sleep
        case 'T': // stopped by a signal - alive, and it must not lose its pid over this
        case 't': // stopped by a debugger
            return ProcessState::Indeterminate;
 
        default:
            ARMARX_DEBUG_S << "Unhandled process state '" << result.front() << "' for pid " << pid;
            return ProcessState::Indeterminate;
    }
}