DiskPersistence.cpp

Go to the documentation of this file.

#include "RobotAPI/libraries/armem/server/ltm/persistence/disk/DiskPersistence.h"
 
#include "RobotAPI/libraries/armem/server/ltm/detail/mixins/util/filesystem.h"
 
namespace armarx::armem::server::ltm::persistence
{
 
    std::vector<std::string>
    DiskPersistence::getContainerKeys(const armarx::armem::MemoryID& id)
    {
        ARMARX_DEBUG << "DiskPersistence: GetContainerKey, memID=" << id.str();
 
        std::vector<std::string> containers;
 
        if (!enabled_)
        {
            ARMARX_DEBUG << "Disk strategy " << identifier_ << " disabled. Aborting call.";
            return containers;
        }
 
        // If it has a nice form
        if (!id.hasEntityName() || id.hasTimestamp())
        {
            std::vector<std::filesystem::path> dirs = getAllDirectories(id);
 
 
            for (auto& path : dirs)
            {
                std::string container = path.filename().string();
 
                if (!container.empty())
                {
                    containers.emplace_back(container);
                }
            }
        }
        // If it has not a nice form
        else
        {
            std::vector<std::filesystem::path> dayDirs = getAllDirectories(id);
 
            for (std::filesystem::path& dayDir : dayDirs)
            {
                if (!util::fs::detail::isDateString(dayDir.filename()))
                {
                    ARMARX_WARNING << "Found a non-date folder inside an entity '" << id.str()
                                   << "' with name '" << dayDir.filename() << "'. "
                                   << "Ignoring this folder, however this is a bad situation.";
                    continue;
                }
 
                std::vector<std::filesystem::path> secondDirs = util::fs::getAllDirectories(dayDir);
 
                for (std::filesystem::path& secondDir : secondDirs)
                {
                    if (!util::fs::detail::isNumberString(secondDir.filename()))
                    {
                        ARMARX_WARNING << "Found a non-timestamp folder inside an entity '"
                                       << id.str() << "' hours folder with name '"
                                       << secondDir.filename() << "'. "
                                       << "Ignoring this folder, however this is a bad situation.";
                        continue;
                    }
 
                    std::vector<std::filesystem::path> timestampDirs =
                        util::fs::getAllDirectories(secondDir);
 
                    for (std::filesystem::path& timestampDir : timestampDirs)
                    {
                        if (!util::fs::detail::isNumberString(timestampDir.filename()))
                        {
                            ARMARX_WARNING
                                << "Found a non-timestamp folder inside an entity '" << id.str()
                                << "' seconds folder with name '" << timestampDir.filename()
                                << "'. "
                                << "Ignoring this folder, however this is a bad situation.";
                            continue;
                        }
 
                        std::string container = timestampDir.filename().string();
 
                        if (!container.empty())
                        {
                            containers.emplace_back(container);
                        }
                    }
                }
            }
        }
 
        return containers;
    }
 
    std::vector<std::string>
    DiskPersistence::getItemKeys(const armarx::armem::MemoryID& id)
    {
        if (!enabled_)
        {
            ARMARX_DEBUG << "Disk strategy " << identifier_ << " disabled. Aborting call.";
            return std::vector<std::string>();
        }
 
        std::vector<std::filesystem::path> files = getAllFiles(id);
        std::vector<std::string> filesStr;
 
        for (auto& path : files)
        {
            std::string item = path.filename().string();
 
            if (!item.empty())
            {
                filesStr.emplace_back(item);
            }
        }
 
        return filesStr;
    }
 
    bool
    DiskPersistence::containsContainer(const armarx::armem::MemoryID& id, std::string key)
    {
        ARMARX_DEBUG << "Contains container key=" << key << "? mem id=" << id.str();
 
        if (!enabled_)
        {
            ARMARX_DEBUG << "Disk strategy " << identifier_ << " disabled. Aborting call.";
            return false;
        }
 
        //from id get the directory for this memory item and append the key to it, then check if the directory exists
        auto path_to_id = getFullPath(id);
        auto correct_container_path = path_to_id / key;
        bool contains_container = util::fs::directoryExists(correct_container_path);
 
        ARMARX_DEBUG << "Contains container?: " << contains_container;
 
        return contains_container;
    }
 
    bool
    DiskPersistence::containsItem(const armarx::armem::MemoryID& id, std::string key)
    {
        if (!enabled_)
        {
            ARMARX_DEBUG << "Disk strategy " << identifier_ << " disabled. Aborting call.";
            return false;
        }
 
        return fileExists(id, key);
    }
 
    void
    DiskPersistence::storeItem(const armarx::armem::MemoryID& id,
                               std::string key,
                               std::vector<unsigned char>& data)
    {
        if (!enabled_)
        {
            ARMARX_DEBUG << "Disk strategy " << identifier_ << " disabled. Aborting call.";
            return;
        }
 
        ensureFullPathExists(id, true);
        ensureFileExists(id, key, true);
 
        if (enoughDiskSpaceLeft())
        {
            writeDataToFile(id, key, data);
        }
        else
        {
            ARMARX_DEBUG << "Not enough DiskSpace available for DiskPersistance-Strategy";
        }
    }
 
    std::vector<unsigned char>
    DiskPersistence::retrieveItem(const armarx::armem::MemoryID& id, std::string key)
    {
        if (!enabled_)
        {
            ARMARX_DEBUG << "Disk strategy " << identifier_ << " disabled. Aborting call.";
            return std::vector<unsigned char>();
        }
 
        if (fileExists(id, key))
        {
            return readDataFromFile(id, key);
        }
 
        return std::vector<unsigned char>();
    }
 
    std::filesystem::path
    DiskPersistence::getMemoryParentPath()
    {
        std::string p = memoryParentPath_.string();
 
        ArmarXDataPath::ReplaceEnvVars(p);
 
        return p;
    }
 
    std::filesystem::path
    DiskPersistence::getFullPath(const armarx::armem::MemoryID& id)
    {
        auto p = getMemoryParentPath() / getExportName();
 
        auto cleanID =
            id.cleanID(); //somehow, the iDs are jumbled when loading the LTM from disk, this solves it for now
 
        auto fullPath = util::fs::toPath(p, cleanID);
 
        //inform user about change:
        ARMARX_DEBUG << "Full path is " << fullPath;
 
        return fullPath;
    }
 
    bool
    DiskPersistence::fullPathExists(const armarx::armem::MemoryID& id)
    {
        auto p = getFullPath(id);
        return util::fs::directoryExists(p);
    }
 
    bool
    DiskPersistence::fileExists(const armarx::armem::MemoryID& id, const std::string& filename)
    {
        auto p = getFullPath(id) / filename;
        return util::fs::fileExists(p);
    }
 
    void
    DiskPersistence::ensureFullPathExists(const armarx::armem::MemoryID& id,
                                          bool createIfNotExistent)
    {
        auto p = getFullPath(id);
        util::fs::ensureDirectoryExists(p, createIfNotExistent);
    }
 
    void
    DiskPersistence::ensureFileExists(const armarx::armem::MemoryID& id,
                                      const std::string& filename,
                                      bool createIfNotExistent)
    {
        auto p = getFullPath(id) / filename;
        util::fs::ensureFileExists(p, createIfNotExistent);
    }
 
    void
    DiskPersistence::writeDataToFile(const armarx::armem::MemoryID& id,
                                     const std::string& filename,
                                     const std::vector<unsigned char>& data)
    {
        auto p = getFullPath(id) / filename;
        util::fs::writeDataToFile(p, data);
    }
 
    std::vector<unsigned char>
    DiskPersistence::readDataFromFile(const armarx::armem::MemoryID& id,
                                      const std::string& filename)
    {
        auto p = getFullPath(id) / filename;
        return util::fs::readDataFromFile(p);
    }
 
    std::vector<std::filesystem::path>
    DiskPersistence::getAllFiles(const armarx::armem::MemoryID& id)
    {
        if (fullPathExists(id))
        {
            auto p = getFullPath(id);
            return util::fs::getAllFiles(p);
        }
 
        return std::vector<std::filesystem::path>();
    }
 
    std::vector<std::filesystem::path>
    DiskPersistence::getAllDirectories(const armarx::armem::MemoryID& id)
    {
        if (fullPathExists(id))
        {
            auto p = getFullPath(id);
            return util::fs::getAllDirectories(p);
        }
 
        return std::vector<std::filesystem::path>();
    }
 
    bool
    DiskPersistence::enoughDiskSpaceLeft()
    {
        std::string path_to_disk = this->getMemoryParentPath();
        ARMARX_DEBUG << "Checking availability of disk space at " << path_to_disk;
 
        if (std::filesystem::exists(path_to_disk))
        {
            try
            {
                auto space_info = std::filesystem::space(path_to_disk);
                int const conversion_factor = 1024;
 
                auto available_space = space_info.available /
                                       (conversion_factor * conversion_factor * conversion_factor);
                ARMARX_DEBUG << "Capacity: "
                             << space_info.capacity /
                                    (conversion_factor * conversion_factor * conversion_factor)
                             << " GB\n";
                ARMARX_DEBUG << "Free space: "
                             << space_info.free /
                                    (conversion_factor * conversion_factor * conversion_factor)
                             << " GB\n";
                ARMARX_DEBUG << "Available space: "
                             << space_info.available /
                                    (conversion_factor * conversion_factor * conversion_factor)
                             << " GB\n";
 
                ARMARX_DEBUG << "Min disk space: " << this->minDiskSpace << " GB\n";
 
                return static_cast<bool>(available_space >= this->minDiskSpace);
            }
            catch (const std::filesystem::filesystem_error& e)
            {
                ARMARX_WARNING << "Error: " << e.what() << '\n';
                return false;
            }
            catch (...)
            {
                ARMARX_DEBUG << "Error while trying to get info on available disk space";
                return false;
            }
        }
        else
        {
            ARMARX_DEBUG << "Cannot find path to disk and thus cannot check if enough space is "
                            "still available";
            return true;
        }
    }
} // namespace armarx::armem::server::ltm::persistence