DiskIOController.cpp

Go to the documentation of this file.

/*
 * This file is part of ArmarX.
 *
 * Copyright (C) 2012-2025, High Performance Humanoid Technologies (H2T), Karlsruhe Institute of Technology (KIT), all rights reserved.
 *
 * ArmarX is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * ArmarX is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * @package    RobotStateComponent::
 * @author     Samet Soenmez (uewtt at student dot kit dot edu)
 * @date       2025t
 * @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
 *             GNU General Public License
 */
 
#include "DiskIOController.h"
 
#include <QThread>
#include <QTimer>
#include <QBoxLayout>
#include <QLabel>
 
#include <filesystem>
 
#include <SimoxUtility/algorithm/string/string_tools.h>
 
#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
 
#include <RobotAPI/libraries/armem/server/ltm/Memory.h>
#include <RobotAPI/libraries/armem/server/ltm/persistence/MemoryPersistenceStrategy.h>
#include <RobotAPI/libraries/armem/server/ltm/persistence/disk/DiskPersistence.h>
#include <RobotAPI/libraries/armem/server/query_proc/ltm/disk/ltm.h>
 
namespace armarx::armem::gui::controller
{
    DiskIOController::DiskIOController(
        MemoryViewerUIContext& uiContext,
        std::shared_ptr<armem::gui::model::MemoryViewerModel> model)
    {
        _model = std::move(model);
        _statusLabel = uiContext.statusLabel;
 
        if (uiContext.diskIOViewLayout)
        {
            _diskIOView = new armem::gui::view::DiskIOView();
            uiContext.diskIOViewLayout->addWidget(_diskIOView);
        }
 
        connect(_diskIOView,
                &armem::gui::view::DiskIOView::requestedStoreOnDisk,
                this,
                &This::onStoreOnDisk);
 
        connect(_diskIOView,
                &armem::gui::view::DiskIOView::requestedLoadFromDisk,
                this,
                &This::onLoadFromDisk);
    }
 
    void
    DiskIOController::onLoadFromDisk(QString directory)
    {
        QThread* thread = new QThread;
        QObject* worker = new QObject;
 
        worker->moveToThread(thread);
 
        connect(thread,
                &QThread::started,
                worker,
                [this, thread, worker, directory]()
                {
 
                    ARMARX_IMPORTANT << "Loading Snapshots from Disk";
 
                    loadFromDisk(directory.toStdString());
 
                    QTimer::singleShot(0, this, [this, thread, worker]()
                {
                    ARMARX_INFO << "After other class";
                    _diskIOView->loadFinished();
 
                    thread->quit();
                    thread->wait();
 
                    worker->deleteLater();
                    thread->deleteLater();
                });
                });
 
        thread->start();
    }
 
    void
    DiskIOController::onStoreOnDisk(QString directory)
    {
        TIMING_START(MemoryExport)
 
        std::string status;
        // only store selected Data
        auto memoryData = _model->memoryDataCopy();
        auto enabledMemories = _model->getEnabledMemories();
        for (auto it = memoryData.begin(); it != memoryData.end();)
        {
            if (std::find(enabledMemories.begin(), enabledMemories.end(), it->first) ==
                enabledMemories.end())
            {
                it = memoryData.erase(it);
            }
            else {
                it++;
            }
        }
 
        std::vector<wm::Memory> memoryDataVec = simox::alg::get_values(memoryData);
        storeOnDisk(directory, memoryDataVec, &status);
 
        _statusLabel->setText(QString::fromStdString(status));
        TIMING_END_STREAM(MemoryExport, ARMARX_VERBOSE)
    }
 
    static const std::string&
    handleSingular(int num, const std::string& singular, const std::string& plural)
    {
        return num == 1 ? singular : plural;
    }
 
    void
    DiskIOController::storeOnDisk(QString directory,
                               const std::vector<wm::Memory> memoryData,
                               std::string* outStatus)
    {
        std::filesystem::path path(directory.toUtf8().constData());
        ARMARX_CHECK_POSITIVE(path.string().size()); // An empty path indicates an error.
 
        std::stringstream status;
        if (std::filesystem::is_regular_file(path))
        {
            status << "Could not export memories contents to " << path
                   << ": Cannot overwrite existing file.";
        }
        else
        {
            int numStored = 0;
            for (const auto& data : memoryData)
            {
                std::string name = data.id().memoryName;
                if (std::filesystem::is_regular_file(path / name))
                {
                    status << "Could not export memory '" << name << "' to " << path
                           << ": Cannot overwrite existing file.\n";
                }
                else
                {
                    std::string defaultIdent = "DefaultDisk";
                    std::string exportName = "MemoryExport";
                    std::shared_ptr<armem::server::ltm::persistence::DiskPersistence>
                        diskPersistence =
                            std::make_shared<armem::server::ltm::persistence::DiskPersistence>(
                                defaultIdent, exportName, path);
 
                    armem::server::ltm::Memory memory(exportName, name);
 
                    if (memory.getPersistenceStrategy())
                    {
                        memory.getPersistenceStrategy()->addStrategy(diskPersistence);
                    }
                    else
                    {
                        std::shared_ptr<
                            armem::server::ltm::persistence::RedundantPersistenceStrategy>
                            redundantPersistence = std::make_shared<
                                armem::server::ltm::persistence::RedundantPersistenceStrategy>();
 
                        memory.setPersistenceStrategy(redundantPersistence);
                        memory.getPersistenceStrategy()->addStrategy(diskPersistence);
                    }
 
                    memory.directlyStore(data);
 
                    numStored++;
                }
            }
            status << "Exported " << numStored << " "
                   << handleSingular(numStored, "memory", "memories") << " to " << path << ".";
        }
 
        if (outStatus)
        {
            *outStatus = status.str();
        }
    }
 
    void
    DiskIOController::loadFromDisk(const std::string& directory)
    {
        ARMARX_INFO << "Loading in separate class";
 
        std::filesystem::path path(directory);
 
        std::map<std::filesystem::path, wm::Memory> memoryData;
 
        if (not std::filesystem::is_directory(path))
        {
            ARMARX_WARNING << "You are trying to load data from a path which does not exist";
            return;
        }
 
        ARMARX_INFO << "Starting to load from disk";
 
        // Find out whether this is a single memory or a collection of memories by searching
        // for a data.aron.* or metadata.aron.* file at depth 5 (if 6 then it is collection of memories)
        bool isSingleMemory = false;
        for (auto i = std::filesystem::recursive_directory_iterator(path);
            i != std::filesystem::recursive_directory_iterator();
            ++i)
        {
            if (i.depth() >
                armem::server::ltm::persistence::MemoryPersistenceStrategy::DEPTH_TO_DATA_FILES + 2)
            {
                // In case we do not find any memory data in here
                ARMARX_WARNING
                    << "You are trying to import a directory which does not contain memory data";
                return;
            }
 
            auto& dir = *i;
 
            // if one matches it is enough to check
            if (std::filesystem::is_regular_file(dir.path()) &&
                simox::alg::starts_with(dir.path().filename(), "data.aron"))
            {
                isSingleMemory =
                    (i.depth() ==
                    armem::server::ltm::persistence::MemoryPersistenceStrategy::DEPTH_TO_DATA_FILES);
                break;
            }
        }
 
        int numLoaded = 0;
        auto loadMemory = [&](const std::filesystem::path& p)
        {
            if (std::filesystem::is_directory(p))
            {
                std::string defaultIdent = "DefaultDisk";
                std::string exportFolderName = p.parent_path().filename();
                ARMARX_INFO << "Name of the folder trying to import: " << exportFolderName;
                std::string exportName = exportFolderName;
                std::shared_ptr<armem::server::ltm::persistence::DiskPersistence> diskPersistence =
                    std::make_shared<armem::server::ltm::persistence::DiskPersistence>(
                        defaultIdent, exportName, p.parent_path().parent_path());
 
                armem::server::ltm::Memory ltm(exportName, p.filename());
                if (ltm.getPersistenceStrategy())
                {
                    ltm.getPersistenceStrategy()->addStrategy(diskPersistence);
                }
                else
                {
                    std::shared_ptr<armem::server::ltm::persistence::RedundantPersistenceStrategy>
                        redundantPersistence = std::make_shared<
                            armem::server::ltm::persistence::RedundantPersistenceStrategy>();
 
                    ltm.setPersistenceStrategy(redundantPersistence);
                    ltm.getPersistenceStrategy()->addStrategy(diskPersistence);
                }
 
                armem::wm::Memory memory =
                    ltm.loadAllAndResolve(); // load list of references and load data
                memoryData[p] = std::move(memory);
 
                numLoaded++;
            }
        };
 
        if (isSingleMemory)
        {
            loadMemory(path);
        }
        else
        {
            // we have to load multiple memories (each subfolder)
            for (const auto& dir : std::filesystem::directory_iterator(path))
            {
                loadMemory(dir.path());
            }
        }
 
        for (auto& [path, memory] : memoryData)
        {
            std::string name = memory.id().memoryName;
 
            // TODO: check if equivalent!!
            if (auto writers = _model->memoryWritersCopy(); writers.count(name) > 0)
            {
                // we need to merge the current memory with the one from the disk.
                // Please note that this may ignores coresegments, if not already existent
                try
                {
                    auto commit = armem::toCommit(memory);
                    writers.at(name).commit(commit);
                }
                catch (const Ice::Exception& ex)
                {
                    ARMARX_ERROR << "Commit failed due to Ice Exception: " << ex.what();
                }
                catch (const std::exception& ex)
                {
                    ARMARX_ERROR << "Commit failed: " << ex.what();
                }
            }
            else
            {
                ARMARX_INFO << "No memory with name '" << name
                            << "' available for commit. Create new virtual memory.";
 
                // Please note: Here we assume that a memory server with the same name does not exist.
                // I think this assumption is ok, since nobody should use filepaths as memory name.
                // Nonetheless, we did not restrict the user to do so...
                std::string virtualMemoryName = name; // + " (at " + path.string() + ")";
                memory.id().memoryName = virtualMemoryName;
                memoryData[virtualMemoryName] = std::move(memory);
            }
        }
    }
}