ArmarXManager.cpp

Go to the documentation of this file.

/*
 * This file is part of ArmarX.
 *
 * Copyright (C) 2011-2016, High Performance Humanoid Technologies (H2T), Karlsruhe Institute of Technology (KIT), all rights reserved.
 *
 * ArmarX is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * ArmarX is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * @package    ArmarXCore::core
 * @author     Kai Welke (kai dot welke at kit dot edu)
 * @date       2012
 * @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
 *             GNU General Public License
 */
 
#include "ArmarXManager.h" // for ArmarXManager, etc
 
#include <limits.h> // for hostname max length
 
#include <chrono> // for seconds
#include <cstddef> // for size_t, NULL
#include <iostream> // for operator<<, basic_ostream, etc
#include <map> // for _Rb_tree_iterator, etc
#include <ratio> // for ratio
#include <sstream> // for basic_stringbuf<>::int_type, etc
#include <string> // for string, allocator, etc
#include <thread> // for thread, sleep_for
#include <utility> // for pair, make_pair, move
#include <vector> // for vector, vector<>::iterator
 
#include <unistd.h> // for hostname
 
#include <Ice/BuiltinSequences.h> // for StringSeq
#include <Ice/Communicator.h> // for CommunicatorPtr, etc
#include <Ice/Identity.h> // for Identity
#include <Ice/Initialize.h> // for initialize
#include <Ice/LocalException.h> // for AlreadyRegisteredException, etc
#include <Ice/Metrics.h> // for Metrics
#include <Ice/NativePropertiesAdmin.h>
#include <Ice/ObjectAdapter.h> // for ObjectAdapterPtr
#include <Ice/ObjectF.h> // for upCast
#include <Ice/Properties.h> // for Properties
#include <Ice/PropertiesF.h> // for upCast
#include <Ice/Proxy.h> // for Object
#include <Ice/ProxyF.h> // for ObjectPrx
#include <IceGrid/Admin.h> // for Admin, ObjectObserverPrx, upCast
#include <IceGrid/Registry.h> // for RegistryPrx, upCast
#include <IceStorm/IceStorm.h> // for TopicPrx, TopicManagerPrx, etc
#include <IceUtil/Handle.h> // for HandleBase, Handle
 
#include "ArmarXCore/core/ArmarXObjectObserver.h"
#include "ArmarXCore/core/ManagedIceObjectRegistryInterface.h"
#include "ArmarXCore/core/application/properties/Property.h"
#include "ArmarXCore/core/exceptions/Exception.h" // for LocalException, etc
#include "ArmarXCore/core/logging/LogSender.h" // for LogSender
#include "ArmarXCore/core/logging/Logging.h" // for ARMARX_VERBOSE, etc
#include "ArmarXCore/core/services/tasks/ThreadList.h" // for ThreadList
#include "ArmarXCore/interface/core/BasicTypes.h"
#include "ArmarXCore/interface/core/Log.h" // for LogPrx, MessageType, etc
#include "ArmarXCore/interface/core/ManagedIceObjectDefinitions.h"
#include "ArmarXCore/interface/core/ThreadingIceBase.h" // for upCast
#include <ArmarXCore/core/ArmarXObjectScheduler.h>
#include <ArmarXCore/core/Component.h> // for ComponentPtr, Component
#include <ArmarXCore/core/IceManager.h> // for IceManager
#include <ArmarXCore/core/ManagedIceObject.h> // for ManagedIceObject
#include <ArmarXCore/core/application/Application.h>
#include <ArmarXCore/core/application/ApplicationProcessFacet.h>
#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
#include <ArmarXCore/core/services/tasks/PeriodicTask.h>
#include <ArmarXCore/core/system/ArmarXDataPath.h> // for ArmarXDataPath
#include <ArmarXCore/core/system/DynamicLibrary.h>
#include <ArmarXCore/core/system/FactoryCollectionBase.h>
#include <ArmarXCore/core/time/LocalTimeServer.h> // for LocalTimeServer
#include <ArmarXCore/core/time/TimeUtil.h> // for TimeUtil
#include <ArmarXCore/core/util/distributed/RemoteHandle/RemoteHandleControlBlock.h>
 
#include "ArmarXMultipleObjectsScheduler.h"
#include "IceGridAdmin.h" // for IceGridAdmin
 
namespace Ice
{
    struct Current;
} // namespace Ice
 
namespace armarx
{
#define MANAGEROBJNAME applicationName + "Manager"
#define OBJOBSNAME std::string("ArmarXObjectObserver_") + applicationName
 
    // *******************************************************
    // construction
    // *******************************************************
    ArmarXManager::ArmarXManager(std::string applicationName,
                                 int port,
                                 std::string host,
                                 std::string locatorName,
                                 Ice::StringSeq args) :
        applicationName(applicationName), managerState(eCreated)
    {
        // initialize communicator
        std::stringstream defaultLocator;
        defaultLocator << "--Ice.Default.Locator=" << locatorName << ":tcp -p " << port << " -h "
                       << host;
        args.push_back(defaultLocator.str());
        Ice::CommunicatorPtr communicator = Ice::initialize(args);
 
        // init members
        init(applicationName, communicator);
    }
 
    ArmarXManager::ArmarXManager(std::string applicationName,
                                 const Ice::CommunicatorPtr& communicator) :
        applicationName(applicationName), managerState(eCreated)
    {
        // init members
        init(applicationName, communicator);
    }
 
    ArmarXManager::~ArmarXManager()
    {
        shutdown();
        waitForShutdown();
    }
 
    bool
    ArmarXManager::checkIceConnection(bool printHint) const
    {
        return CheckIceConnection(iceManager->getCommunicator(), printHint);
    }
 
    bool
    ArmarXManager::CheckIceConnection(const Ice::CommunicatorPtr& communicator, bool printHint)
    {
        const std::string armarxHint = "'\nDid you start armarx?\n\nTo start armarx: armarx start\n"
                                       "To kill a hanging armarx: armarx killIce";
        try
        {
            if (communicator->getProperties()->getProperty("Ice.Default.Locator").empty())
            {
                if (printHint)
                {
                    std::cerr << "Required Ice property 'Ice.Default.Locator' not set! "
                              << "It has to has a value similar to 'IceGrid/Locator:tcp -p 4061 -h "
                                 "localhost'";
                }
                return false;
            }
            auto locatorProp = communicator->getProperties()->getProperty("Ice.Default.Locator");
            auto pos = locatorProp.find_first_of(':');
            std::string locatorId = locatorProp.substr(0, pos);
            auto proxy = communicator->stringToProxy(locatorId);
            IceGrid::LocatorPrx::checkedCast(proxy);
        }
        catch (...)
        {
            if (printHint)
            {
                std::cerr << "Could not contact default locator at '"
                          << communicator->getProperties()->getProperty("Ice.Default.Locator")
                          << armarxHint;
            }
            return false;
        }
 
        try
        {
            std::string registryId = "IceGrid/Registry";
            auto proxy = communicator->stringToProxy(registryId);
            IceGrid::RegistryPrx::checkedCast(proxy);
        }
        catch (...)
        {
            if (printHint)
            {
                std::cerr << "Could not contact IceGrid registry at '"
                          << communicator->getProperties()->getProperty(
                                 "IceGrid.Registry.Client.Endpoints")
                          << armarxHint;
            }
            return false;
        }
 
        try
        {
            IceManager::GetTopicManager(communicator);
        }
        catch (const Ice::NoEndpointException& e)
        {
            std::cout << "Caught exception: \n" << e.what() << std::endl;
 
            if (printHint)
            {
                std::cerr
                    << "Could not contact TopicManager at '"
                    // This string is actually used in IceManager::GetTopicManager()
                    << "IceStorm/TopicManager"
                    // << communicator->getProperties()->getProperty("IceStormAdmin.TopicManager.Default")
                    << armarxHint;
            }
            return false;
        }
 
        return true;
    }
 
    // *******************************************************
    // ArmarXManager property setters
    // *******************************************************
    void
    ArmarXManager::enableLogging(bool enable)
    {
        LogSender::SetLoggingActivated(enable);
    }
 
    void
    ArmarXManager::enableProfiling(bool enable)
    {
        ThreadList::getApplicationThreadList()->enableProfiler(enable);
    }
 
    void
    ArmarXManager::setGlobalMinimumLoggingLevel(MessageTypeT minimumLoggingLevel)
    {
        LogSender::SetGlobalMinimumLoggingLevel(minimumLoggingLevel);
    }
 
    void
    ArmarXManager::setDataPaths(std::string dataPaths)
    {
        ArmarXDataPath::initDataPaths(dataPaths);
    }
 
    // *******************************************************
    // adding / removing ManagedIceObjects
    // *******************************************************
 
 
    void
    ArmarXManager::addObject(const ManagedIceObjectPtr& object,
                             bool addWithOwnAdapter,
                             const std::string& objectName,
                             bool useOwnScheduleThread)
    {
        addObject(object,
                  addWithOwnAdapter ? Ice::ObjectAdapterPtr() : getAdapter(),
                  objectName,
                  useOwnScheduleThread);
    }
 
    void
    ArmarXManager::addObject(const ManagedIceObjectPtr& object,
                             const std::string& objectName,
                             bool addWithOwnAdapter,
                             bool useOwnScheduleThread)
    {
        addObject(object, addWithOwnAdapter, objectName, useOwnScheduleThread);
    }
 
    void
    ArmarXManager::addObject(const ManagedIceObjectPtr& object,
                             Ice::ObjectAdapterPtr objectAdapterToAddTo,
                             const std::string& objectName,
                             bool useOwnScheduleThread)
    {
        if (!object)
        {
            throw LocalException("Cannot add NULL object");
        }
        {
            auto cptr = ComponentPtr::dynamicCast(object);
            if (cptr && !cptr->createdByComponentCreate)
            {
                throw LocalException("Components need to be created by Component::create");
            }
        }
 
        if (!objectName.empty())
        {
            if (!object->getName().empty())
            {
                ARMARX_INFO << "Adding object with custom name: " << objectName;
            }
            object->setName(objectName);
        }
        if (object->getName().empty())
        {
            object->setName(object->getDefaultName());
        }
        if (object->getName().empty())
        {
            throw LocalException("Object name must not be empty");
        }
        auto lock = acquireManagedObjectsMutex();
        if (!lock)
        {
            return;
        }
 
        bool reachable = false;
 
        try
        {
            reachable = getIceManager()->isObjectReachable(object->getName());
        }
        catch (...)
        {
            throw;
        }
 
        if (reachable)
        {
            throw Ice::AlreadyRegisteredException(
                __FILE__, __LINE__, object->ice_id(), object->getName());
        }
 
        try
        {
            ArmarXObjectSchedulerPtr objectScheduler = new ArmarXObjectScheduler(
                this, iceManager, object, objectAdapterToAddTo, useOwnScheduleThread);
            auto pair = managedObjects.insert(
                std::make_pair(object->getName(), std::move(objectScheduler)));
            if (!pair.second)
            {
                throw Ice::AlreadyRegisteredException(
                    __FILE__, __LINE__, object->ice_id(), object->getName());
            }
            if (!useOwnScheduleThread)
            {
                std::scoped_lock lock(schedulerListMutex);
                singleThreadedSchedulers.at(rand() % singleThreadedSchedulers.size())
                    ->addObjectScheduler(pair.first->second);
            }
        }
        catch (...)
        {
            throw;
        }
    }
 
    void
    ArmarXManager::addObjectAsync(const ManagedIceObjectPtr& object,
                                  const std::string& objectName,
                                  bool addWithOwnAdapter,
                                  bool useOwnScheduleThread)
    {
        ArmarXManagerPtr manager(this);
        std::thread{
            [=]
            {
                try
                {
                    manager->addObject(object, objectName, addWithOwnAdapter, useOwnScheduleThread);
                }
                catch (...)
                {
                    armarx::handleExceptions();
                }
            }
 
        }.detach();
    }
 
    void
    ArmarXManager::removeObjectBlocking(const std::string& objectName)
    {
        ArmarXObjectSchedulerPtr scheduler;
        {
            auto lock = acquireManagedObjectsMutex();
            if (!lock)
            {
                return;
            }
            auto it = managedObjects.find(objectName);
            if (it == managedObjects.end())
            {
                ARMARX_WARNING << "Cannot remove unknown object '" << objectName << "'";
                return;
            }
            scheduler = it->second;
        }
        try
        {
            if (scheduler)
            {
                removeObject(scheduler, true);
            }
        }
        catch (...)
        {
            throw;
        }
    }
 
    void
    ArmarXManager::removeObjectNonBlocking(const std::string& objectName)
    {
        auto removal = [objectName, this]()
        {
            ArmarXObjectSchedulerPtr scheduler;
            {
                auto lock = acquireManagedObjectsMutex();
                if (!lock)
                {
                    return;
                }
                auto it = managedObjects.find(objectName);
                if (it == managedObjects.end())
                {
                    ARMARX_WARNING << "Cannot remove unknown object '" << objectName << "'";
                    return;
                }
                scheduler = it->second;
            }
            try
            {
                this->removeObject(scheduler, false);
            }
            catch (...)
            {
                throw;
            }
        };
        std::thread{removal}.detach();
    }
 
    void
    ArmarXManager::removeObjectBlocking(const ManagedIceObjectPtr& object)
    {
        removeObjectBlocking(object->getName());
    }
 
    void
    ArmarXManager::removeObjectNonBlocking(const ManagedIceObjectPtr& object)
    {
        removeObjectNonBlocking(object->getName());
    }
 
    // *******************************************************
    // shutdown handling
    // *******************************************************
    void
    ArmarXManager::waitForShutdown()
    {
        // assure no state changed
        managerStateMutex.lock();
 
        if (managerState == eShutdown)
        {
            managerStateMutex.unlock();
            return;
        }
 
        // wait for shutdown
        std::unique_lock lock(shutdownMutex);
        managerStateMutex.unlock();
        shutdownCondition.wait(lock);
        ARMARX_VERBOSE << "Waiting for shutdown finished";
    }
 
    void
    ArmarXManager::shutdown()
    {
        try
        {
            // check state
            {
                std::scoped_lock lock(managerStateMutex);
 
                // do not shutdown if shutdown is in progress
                if (managerState >= eShutdownInProgress)
                {
                    return;
                }
 
                managerState = eShutdownInProgress;
 
                // locking managedObjects ne done before state mutex is released
                managedObjectsMutex.lock();
            }
 
            ARMARX_VERBOSE << "Shutting down ArmarXManager" << std::endl;
 
            // stop cleanup task. All objects will be removed manually in shutdown
            cleanupSchedulersTask->stop();
 
            if (checkDependencyStatusTask)
            {
                checkDependencyStatusTask->stop();
            }
 
            sharedRemoteHandleState.reset();
            remoteReferenceCountControlBlockManager.reset();
 
            disconnectAllObjects();
 
            // shutdown log sender
            LogSender::SetSendLoggingActivated(false);
 
            // remove all managed objects
            removeAllObjects(true);
            try
            {
                iceManager->getIceGridSession()->getAdmin()->removeObject(
                    Ice::Identity{MANAGEROBJNAME, ""});
            }
            catch (...)
            {
            }
 
            singleThreadedSchedulers.clear();
 
            try
            {
                // deactivate all object adapters (and do internal iceManager shutdown)
                iceManager->shutdown();
 
                // wait until all adapters have been deactivated
                iceManager->waitForShutdown();
            }
            catch (...)
            {
            }
            // destroy manager and communicator
            iceManager->destroy();
 
            // set to NULL to avoid cycle in pointers ArmarXManager <-> ArmarXObjectObserver
            objObserver = nullptr;
 
            // set state to shutdown and notify waitForShutdown
            {
                std::scoped_lock lock(managerStateMutex);
 
                // inform waiting threads of shutdown
                {
                    std::unique_lock lock(shutdownMutex);
                    ARMARX_VERBOSE << "notifying shutdown waiters" << std::endl;
                    shutdownCondition.notify_all();
                }
 
                managerState = eShutdown;
            }
 
            managedObjectsMutex.unlock();
        }
        catch (std::exception& e)
        {
            ARMARX_INFO << "shutdown failed with exception!\n" << e.what() << std::endl;
        }
 
        ARMARX_INFO << "Shutdown of ArmarXManager finished!" << std::endl;
    }
 
    void
    ArmarXManager::asyncShutdown(std::size_t timeoutMs)
    {
        std::thread{[this, timeoutMs]
                    {
                        std::this_thread::sleep_for(std::chrono::milliseconds{timeoutMs});
                        shutdown();
                    }}
            .detach();
    }
 
    bool
    ArmarXManager::isShutdown()
    {
        std::scoped_lock lock(managerStateMutex);
        return (managerState == eShutdown);
    }
 
    // *******************************************************
    // getters
    // *******************************************************
 
    const IceManagerPtr&
    ArmarXManager::getIceManager() const
    {
        return iceManager;
    }
 
    const Ice::CommunicatorPtr&
    ArmarXManager::getCommunicator() const
    {
        return iceManager->getCommunicator();
    }
 
    std::vector<ManagedIceObjectPtr>
    ArmarXManager::getManagedObjects()
    {
        std::vector<ManagedIceObjectPtr> objects;
 
        auto lock = acquireManagedObjectsMutex();
        if (!lock)
        {
            return objects;
        }
 
        ObjectSchedulerMap::iterator iter = managedObjects.begin();
 
        while (iter != managedObjects.end())
        {
            objects.push_back(iter->second->getObject());
            iter++;
        }
 
 
        return objects;
    }
 
    // *******************************************************
    // Slice MiceManagerInsightProvider implementation
    // *******************************************************
 
    std::string
    ArmarXManager::getHostname(const Ice::Current&)
    {
        char hostname[HOST_NAME_MAX];
        gethostname(hostname, HOST_NAME_MAX);
        return std::string(hostname);
    }
 
    Ice::StringSeq
    ArmarXManager::getObjectNames(const Ice::Current& c)
    {
        return getManagedObjectNames();
    }
 
    mice::MiceObjectConnectivity
    ArmarXManager::getMiceObjectConnectivity(const std::string& objectName, const Ice::Current& c)
    {
        mice::MiceObjectConnectivity miceCon = mice::MiceObjectConnectivity();
        ManagedIceObjectConnectivity con = getObjectConnectivity(objectName);
        miceCon.subscribedTopics = con.usedTopics;
        miceCon.publishedTopics = con.offeredTopics;
        miceCon.usedObjects = Ice::StringSeq();
 
        for (const auto& entry : con.dependencies)
        {
            miceCon.usedObjects.push_back(entry.first);
        }
 
        return miceCon;
    }
 
    IceMX::MetricsAdminPrx
    ArmarXManager::getMetricsAdmin(const Ice::Current&)
    {
        Ice::ObjectPrx adminObj = getIceManager()->getCommunicator()->getAdmin();
        IceMX::MetricsAdminPrx metrAdmin = IceMX::MetricsAdminPrx::checkedCast(adminObj, "Metrics");
        return metrAdmin;
    }
 
    // *******************************************************
    // Slice ArmarXManagerInterface implementation
    // *******************************************************
    ManagedIceObjectState
    ArmarXManager::getObjectState(const std::string& objectName, const Ice::Current& c)
    {
        auto lock = acquireManagedObjectsMutex();
        if (!lock)
        {
            return eManagedIceObjectExiting;
        }
 
        ObjectSchedulerMap::iterator iter = managedObjects.find(objectName);
 
        if (iter == managedObjects.end())
        {
            return eManagedIceObjectExited;
        }
 
        ManagedIceObjectState state = (ManagedIceObjectState)iter->second->getObject()->getState();
 
        return state;
    }
 
    ManagedIceObjectConnectivity
    ArmarXManager::getObjectConnectivity(const std::string& objectName, const Ice::Current& c)
    {
        auto lock = acquireManagedObjectsMutex();
        if (!lock)
        {
            return ManagedIceObjectConnectivity();
        }
 
        ObjectSchedulerMap::iterator iter = managedObjects.find(objectName);
 
        if (iter == managedObjects.end())
        {
            return ManagedIceObjectConnectivity();
        }
 
        ManagedIceObjectConnectivity con = iter->second->getObject()->getConnectivity();
 
 
        return con;
    }
 
    StringStringDictionary
    ArmarXManager::getObjectProperties(const ::std::string& objectName, const ::Ice::Current&)
    {
        StringStringDictionary propertyMap;
        ObjectSchedulerMap::iterator iter = managedObjects.find(objectName);
 
        if (iter == managedObjects.end())
        {
            return propertyMap;
        }
 
        ComponentPtr component = ComponentPtr::dynamicCast(iter->second->getObject());
 
        if (!component)
        {
            return propertyMap;
        }
        ARMARX_CHECK_EXPRESSION(component->getPropertyDefinitions());
        auto result = component->getPropertyDefinitions()->getPropertyValues(
            component->getPropertyDefinitions()->getPrefix());
        return result;
    }
 
    ObjectPropertyInfos
    ArmarXManager::getObjectPropertyInfos(const ::std::string& objectName, const ::Ice::Current&)
    {
        ObjectPropertyInfos propertyMap;
        ObjectSchedulerMap::iterator iter = managedObjects.find(objectName);
 
        if (iter == managedObjects.end())
        {
            return propertyMap;
        }
 
        ComponentPtr component = ComponentPtr::dynamicCast(iter->second->getObject());
 
        if (!component)
        {
            return propertyMap;
        }
 
        ARMARX_CHECK_EXPRESSION(component->getPropertyDefinitions());
        for (auto prop : component->getPropertyDefinitions()->getPropertyValues())
        {
            propertyMap[component->getPropertyDefinitions()->getPrefix() + prop.first] = {
                component->getPropertyDefinitions()->getDefinitionBase(prop.first)->isConstant(),
                prop.second};
        }
        return propertyMap;
    }
 
    ObjectPropertyInfos
    ArmarXManager::getApplicationPropertyInfos(const Ice::Current&)
    {
        ObjectPropertyInfos propertyMap;
        if (!Application::getInstance() || !Application::getInstance()->getPropertyDefinitions())
        {
            return propertyMap;
        }
        for (auto prop : Application::getInstance()->getPropertyDefinitions()->getPropertyValues())
        {
            propertyMap[Application::getInstance()->getPropertyDefinitions()->getPrefix() +
                        prop.first] = {Application::getInstance()
                                           ->getPropertyDefinitions()
                                           ->getDefinitionBase(prop.first)
                                           ->isConstant(),
                                       prop.second};
        }
        return propertyMap;
    }
 
    Ice::PropertiesAdminPrx
    ArmarXManager::getPropertiesAdmin(const Ice::Current&)
    {
        Ice::ObjectPrx adminObj = getIceManager()->getCommunicator()->getAdmin();
        Ice::PropertiesAdminPrx propAdmin =
            Ice::PropertiesAdminPrx::checkedCast(adminObj, "Properties");
        return propAdmin;
    }
 
    void
    ArmarXManager::setComponentIceProperties(const Ice::PropertiesPtr& properties)
    {
        for (ManagedIceObjectPtr& managedObject : getManagedObjects())
        {
            ComponentPtr component = ComponentPtr::dynamicCast(managedObject);
            if (component)
            {
                component->setIceProperties(properties);
            }
        }
    }
 
    void
    ArmarXManager::updateComponentIceProperties(const Ice::PropertyDict& properties)
    {
        for (ManagedIceObjectPtr& managedObject : getManagedObjects())
        {
            ComponentPtr component = ComponentPtr::dynamicCast(managedObject);
            if (component)
            {
                component->updateIceProperties(properties);
            }
        }
    }
 
    Ice::StringSeq
    ArmarXManager::getManagedObjectNames(const Ice::Current& c)
    {
        Ice::StringSeq objectNames;
 
        auto lock = acquireManagedObjectsMutex();
        if (!lock)
        {
            return objectNames;
        }
 
        ObjectSchedulerMap::iterator iter = managedObjects.begin();
 
        while (iter != managedObjects.end())
        {
            objectNames.push_back(iter->first);
            iter++;
        }
 
 
        return objectNames;
    }
 
    class IcePropertyChangeCallback : public Ice::PropertiesAdminUpdateCallback
    {
    public:
        IcePropertyChangeCallback(armarx::ApplicationPtr application) : application(application)
        {
        }
 
        void
        updated(const Ice::PropertyDict& changes) override
        {
            //            ARMARX_INFO << "Properties were updated: " << changes;
            if (application)
            {
                application->updateIceProperties(changes);
            }
        }
 
    private:
        armarx::ApplicationPtr application;
    };
 
    using IcePropertyChangeCallbackPtr = IceUtil::Handle<IcePropertyChangeCallback>;
 
    // *******************************************************
    // private methods
    // *******************************************************
    void
    ArmarXManager::init(std::string applicationName, const Ice::CommunicatorPtr& communicator)
    {
        increaseSchedulers(1);
 
        ApplicationPtr appInstance = Application::getInstance();
 
 
        // create ice manager
        iceManager = new IceManager(
            communicator,
            applicationName,
            appInstance ? appInstance->getProperty<std::string>("TopicSuffix").getValue() : "");
 
 
        if (!checkIceConnection())
        {
            throw Ice::ConnectFailedException(__FILE__, __LINE__);
        }
 
        this->installProcessFacet();
 
        // init logging
        LogSender::setProxy(applicationName, iceManager->getTopic<LogPrx>("Log"));
 
        setTag("ArmarXManager");
 
        RegisterKnownObjectFactoriesWithIce(communicator);
 
 
        // make sure icegrid session exists before continuing
        auto icegrid = iceManager->getIceGridSession();
        // register obj observer for onDisconnect
        objObserver = new ArmarXObjectObserver(this);
        Ice::ObjectAdapterPtr observerAdapter;
        Ice::ObjectPrx oPrx =
            icegrid->registerObjectWithNewAdapter(objObserver, OBJOBSNAME, observerAdapter);
        IceGrid::ObjectObserverPrx objObsPrx = IceGrid::ObjectObserverPrx::checkedCast(oPrx);
        icegrid->setObjectObserver(objObsPrx);
 
        // register manager to ice
        icegrid->registerObjectWithNewAdapter(this, MANAGEROBJNAME, armarxManagerAdapter);
 
 
        // create periodic task for starter cleanup
        cleanupSchedulersTask = new PeriodicTask<ArmarXManager>(this,
                                                                &ArmarXManager::cleanupSchedulers,
                                                                500,
                                                                false,
                                                                "ArmarXManager::cleanupSchedulers");
        cleanupSchedulersTask->start();
 
        checkDependencyStatusTask =
            new PeriodicTask<ArmarXManager>(this,
                                            &ArmarXManager::checkDependencies,
                                            1000,
                                            false,
                                            "ArmarXManager::DependenciesChecker");
        checkDependencyStatusTask->start();
 
 
        // set manager state to running
        {
            std::scoped_lock lock(managerStateMutex);
            managerState = eRunning;
        }
 
        // create ThreadList for this process
        ThreadList::getApplicationThreadList()->setApplicationThreadListName(applicationName +
                                                                             "ThreadList");
 
        try
        {
            addObject(ThreadList::getApplicationThreadList());
        }
        catch (...)
        {
        }
 
        // create a local TimeServer for this process
        if (appInstance && appInstance->getProperty<bool>("UseTimeServer").getValue())
        {
            ARMARX_VERBOSE << "Using time from global time server.";
            LocalTimeServer::getApplicationTimeServer()->setApplicationTimeServerName(
                applicationName + "LocalTimeServer");
            addObject(LocalTimeServer::getApplicationTimeServer());
            TimeUtil::SetTimeServer(LocalTimeServer::getApplicationTimeServer());
        }
        else
        {
            ARMARX_VERBOSE << "Using time from local system clock.";
        }
 
        sharedRemoteHandleState.reset(new SharedRemoteHandleState{
            appInstance
                ? appInstance->getProperty<unsigned int>("RemoteHandlesDeletionTimeout").getValue()
                : SharedRemoteHandleState::DEFAULT_DELETION_DELAY});
        remoteReferenceCountControlBlockManager.reset(
            new RemoteReferenceCountControlBlockManager{IceUtil::Time::milliSeconds(100)});
    }
 
    void
    ArmarXManager::cleanupSchedulers()
    {
        std::scoped_lock lock(terminatingObjectsMutex);
 
        ObjectSchedulerList::iterator iter = terminatingObjects.begin();
 
        while (iter != terminatingObjects.end())
        {
            const ArmarXObjectSchedulerPtr& sched = *iter;
            ARMARX_VERBOSE << deactivateSpam(1, sched->getObject()->getName())
                           << "Checking termination state of " << sched->getObject()->getName()
                           << ": "
                           << ManagedIceObject::GetObjectStateAsString(sched->getObjectState());
            // remove terminated starters
            if ((*iter)->isTerminated())
            {
                ARMARX_VERBOSE << "Delayed Removal of ManagedIceObject "
                               << (*iter)->getObject()->getName() << " finished";
                iter = terminatingObjects.erase(iter);
            }
            else
            {
                iter++;
            }
        }
    }
 
    void
    ArmarXManager::disconnectDependees(const std::string& object)
    {
        try
        {
            std::vector<std::string> dependees = getDependendees(object);
 
            auto lock = acquireManagedObjectsMutex();
            if (!lock)
            {
                return;
            }
 
            for (const auto& dependee : dependees)
            {
                ArmarXManager::ObjectSchedulerMap::iterator it = managedObjects.find(dependee);
                ARMARX_INFO << deactivateSpam(10, dependee + object) << "'" << dependee
                            << "' disconnected because of '" << object << "'";
 
                if (it != managedObjects.end())
                {
                    it->second->disconnected(true);
                }
            }
        }
        catch (...)
        {
            throw;
        }
    }
 
    void
    ArmarXManager::disconnectAllObjects()
    {
        auto lock = acquireManagedObjectsMutex();
        if (!lock)
        {
            return;
        }
        ObjectSchedulerMap::iterator iter = managedObjects.begin();
 
        for (; iter != managedObjects.end(); iter++)
        {
            iter->second->disconnected(false);
        }
    }
 
    std::vector<std::string>
    ArmarXManager::getDependendees(const std::string& removedObject)
    {
        std::vector<std::string> result;
 
        auto lock = acquireManagedObjectsMutex();
        if (!lock)
        {
            return result;
        }
 
        try
        {
 
            ObjectSchedulerMap::const_iterator it = managedObjects.begin();
 
            for (; it != managedObjects.end(); it++)
            {
                ArmarXObjectSchedulerPtr scheduler = it->second;
 
                if (scheduler->dependsOn(removedObject))
                {
                    result.push_back(it->first);
                }
            }
        }
        catch (...)
        {
            throw;
        }
 
        return result;
    }
 
    void
    ArmarXManager::wakeupWaitingSchedulers()
    {
        auto lock = acquireManagedObjectsMutex();
        if (!lock)
        {
            return;
        }
 
        try
        {
 
            ObjectSchedulerMap::const_iterator it = managedObjects.begin();
 
            for (; it != managedObjects.end(); it++)
            {
                ArmarXObjectSchedulerPtr scheduler = it->second;
                scheduler->wakeupDependencyCheck();
            }
        }
        catch (...)
        {
            throw;
        }
    }
 
    void
    ArmarXManager::removeAllObjects(bool blocking)
    {
        ObjectSchedulerMap tempMap;
        {
            std::scoped_lock lock(managedObjectsMutex);
            tempMap = managedObjects;
        }
 
        for (auto& it : tempMap)
        {
            removeObject(it.second, false);
        }
 
        if (blocking)
        {
            for (auto objectScheduler : terminatingObjects)
            {
                objectScheduler->waitForTermination();
            }
 
            terminatingObjects.clear();
        }
    }
 
    bool
    ArmarXManager::removeObject(const ArmarXObjectSchedulerPtr& objectScheduler, bool blocking)
    {
        if (!objectScheduler)
        {
            return true;
        }
 
        const std::string objName = objectScheduler->getObject()->getName();
 
        try
        {
            iceManager->removeObject(objName);
            {
                std::scoped_lock lock2(managedObjectsMutex);
                managedObjects.erase(objectScheduler->getObject()->getName());
            }
            // terminate
            objectScheduler->terminate();
 
            if (blocking)
            {
                objectScheduler->waitForTermination();
                ARMARX_VERBOSE << "Blocking removal of ManagedIceObject " << objName << " finished";
            }
            else // only move to terminating list if delayed removal
            {
                ARMARX_VERBOSE << "Inserting ManagedIceObject into delayed removal list: "
                               << objName;
                std::scoped_lock lockTerm(terminatingObjectsMutex);
                terminatingObjects.push_back(objectScheduler);
            }
 
            return true;
        }
        catch (...)
        {
            ARMARX_ERROR << "Removing of object '" << objName << "' failed with an exception!\n"
                         << GetHandledExceptionString();
        }
 
        return false;
    }
 
    //bool ArmarXManager::removeObject(const ObjectSchedulerMap::iterator& iter, bool blocking)
    //{
    //    if (iter != managedObjects.end())
    //    {
    //        const std::string objName = iter->second->getObject()->getName();
 
    //        try
    //        {
 
    //            ARMARX_VERBOSE << "Removing ManagedIceObject " << objName << " - blocking = " << blocking;
 
 
    //            ArmarXObjectSchedulerPtr objectScheduler = iter->second;
 
 
    //            //            managedObjects.erase(iter);
    //            return removeObject(objectScheduler, blocking);
    //        }
    //        catch (...)
    //        {
    //            ARMARX_ERROR << "Removing of object '" << objName << "' failed with an exception!";
    //            handleExceptions();
    //        }
    //    }
 
    //    return false;
    //}
 
    ArmarXObjectSchedulerPtr
    ArmarXManager::findObjectScheduler(const std::string& objectName) const
    {
        std::scoped_lock lock(managedObjectsMutex);
        auto it = managedObjects.find(objectName);
        if (it != managedObjects.end())
        {
            return it->second;
        }
        else
        {
            return ArmarXObjectSchedulerPtr();
        }
    }
 
    void
    ArmarXManager::checkDependencies()
    {
 
 
        auto lock = acquireManagedObjectsMutex();
        if (!lock)
        {
            return;
        }
 
        try
        {
            ObjectSchedulerMap::const_iterator it = managedObjects.begin();
 
            for (; it != managedObjects.end(); it++)
            {
                const ArmarXObjectSchedulerPtr& scheduler = it->second;
 
                if (scheduler->getObjectState() == eManagedIceObjectStarted &&
                    !scheduler->checkDependenciesStatus())
                {
                    scheduler->disconnected(true);
                }
            }
        }
        catch (...)
        {
            throw;
        }
    }
 
    auto
    ArmarXManager::acquireManagedObjectsMutex() -> ScopedRecursiveLockPtr
    {
        // assure no state change until lock of managedObjectsMutex
        std::scoped_lock lock(managerStateMutex);
 
        if (managerState >= eShutdownInProgress)
        {
            return ScopedRecursiveLockPtr();
        }
 
        // lock access to managed objects
        ScopedRecursiveLockPtr lock2(
            new std::scoped_lock<std::recursive_mutex>(managedObjectsMutex));
 
        return lock2;
    }
 
    void
    ArmarXManager::installProcessFacet()
    {
        ApplicationPtr appInstance = Application::getInstance();
        Ice::CommunicatorPtr applicationCommunicator = getCommunicator();
        ARMARX_CHECK_EXPRESSION(applicationCommunicator);
        // remove default Ice::Process facet
        if (applicationCommunicator->findAdminFacet("Process"))
        {
            applicationCommunicator->removeAdminFacet("Process");
        }
        // create and register new Ice::Process facet
        Ice::ProcessPtr applicationProcessFacet = new ApplicationProcessFacet(*this);
        applicationCommunicator->addAdminFacet(applicationProcessFacet, "Process");
 
        IcePropertyChangeCallbackPtr propertyChangeCallback =
            new IcePropertyChangeCallback(appInstance);
 
        Ice::ObjectPtr obj = applicationCommunicator->findAdminFacet("Properties");
        Ice::NativePropertiesAdminPtr admin = Ice::NativePropertiesAdminPtr::dynamicCast(obj);
        if (admin)
        {
            admin->addUpdateCallback(propertyChangeCallback);
        }
        else
        {
            ARMARX_WARNING
                << "Could not get properties admin - online property changing will not work";
        }
 
        // activate new Ice::Process facet and PropertyChangeCallback
        applicationCommunicator->getAdmin();
    }
 
    std::mutex& FactoryCollectionBase_RegistrationListMutex();
 
    void
    ArmarXManager::RegisterKnownObjectFactoriesWithIce(const Ice::CommunicatorPtr& ic)
    {
        std::unique_lock lock(FactoryCollectionBase_RegistrationListMutex());
 
        for (const FactoryCollectionBasePtr& preregistration :
             FactoryCollectionBase::PreregistrationList())
        {
            //ARMARX_INFO_S << "looking for " << preregistration->getFactories();
            for (const auto& id2factory : preregistration->getFactories())
            {
                if (!ic->getValueFactoryManager()->find(id2factory.first))
                {
                    //ARMARX_IMPORTANT_S << "adding object factory for " << id2factory.first;
                    ic->getValueFactoryManager()->add(id2factory.second, id2factory.first);
                }
            }
        }
    }
 
    void
    ArmarXManager::registerKnownObjectFactoriesWithIce()
    {
        RegisterKnownObjectFactoriesWithIce(getIceManager()->getCommunicator());
    }
 
    const Ice::ObjectAdapterPtr&
    ArmarXManager::getAdapter() const
    {
        return armarxManagerAdapter;
    }
 
    const std::shared_ptr<SharedRemoteHandleState>&
    ArmarXManager::getSharedRemoteHandleState() const
    {
        return sharedRemoteHandleState;
    }
 
    void
    ArmarXManager::increaseSchedulers(int increaseBy)
    {
        std::scoped_lock lock(schedulerListMutex);
        for (int i = 0; i < increaseBy; ++i)
        {
            singleThreadedSchedulers.push_back(new ArmarXMultipleObjectsScheduler());
        }
    }
 
    bool
    LoadLibFromAbsolutePath(const std::string& path)
    {
        //these variables should be shared between ArmarXManager -> use static variables
        static std::map<std::string, DynamicLibraryPtr> loadedLibs;
        static std::mutex libsMutex;
        std::lock_guard<std::mutex> guard{libsMutex};
        if (loadedLibs.count(path))
        {
            return true;
        }
        DynamicLibraryPtr lib(new DynamicLibrary());
        try
        {
            lib->load(path);
        }
        catch (...)
        {
            handleExceptions();
            return false;
        }
 
        if (lib->isLibraryLoaded())
        {
            ARMARX_INFO << "Loaded library " << path;
            loadedLibs[path] = lib;
        }
        else
        {
            ARMARX_ERROR << "Could not load lib " + path + ": " + lib->getErrorMessage();
            return false;
        }
        return true;
    }
 
    //    bool ArmarXManager::loadLibFromPath(const std::string& path)
    //    {
    //        std::string absPath;
    //        if (ArmarXDataPath::getAbsolutePath(path, absPath))
    //        {
    //            return LoadLibFromAbsolutePath(absPath);
    //        }
    //        ARMARX_ERROR << "Could not find library " + path;
    //        return false;
    //    }
    //    bool ArmarXManager::loadLibFromPackage(const std::string& package, const std::string& libname)
    //    {
    //        CMakePackageFinder finder(package);
    //        if (!finder.packageFound())
    //        {
    //            ARMARX_ERROR << "Could not find package '" << package << "'";
    //            return false;
    //        }
    //
    //        for (auto libDirPath : Split(finder.getLibraryPaths(), ";"))
    //        {
    //            std::filesystem::path fullPath = libDirPath;
    //            fullPath /= "lib" + libname  + "." + DynamicLibrary::GetSharedLibraryFileExtension();
    //            if (!std::filesystem::exists(fullPath))
    //            {
    //                fullPath = libDirPath;
    //                fullPath /= libname;
    //                if (!std::filesystem::exists(fullPath))
    //                {
    //                    continue;
    //                }
    //            }
    //            if (LoadLibFromAbsolutePath(fullPath.string()))
    //            {
    //                return true;
    //            }
    //        }
    //        ARMARX_ERROR << "Could not find library " <<  libname << " in package " << package;
    //        return false;
    //    }
 
 
    StringVariantBaseMap
    armarx::ArmarXManager::getMetaInfo(const std::string& objectName, const Ice::Current&)
    {
        std::scoped_lock lock(managedObjectsMutex);
        StringStringDictionary propertyMap;
        ObjectSchedulerMap::iterator iter = managedObjects.find(objectName);
 
        if (iter == managedObjects.end())
        {
            return StringVariantBaseMap();
        }
 
        return iter->second->getObject()->getMetaInfoMap();
    }
} // namespace armarx