filesystem.cpp

Go to the documentation of this file.

#include "filesystem.h"
 
#include <errno.h>
 
#include <filesystem>
#include <fstream>
#include <iostream>
#include <stdexcept>
#include <string>
#include <system_error>
#include <vector>
 
#include <fcntl.h>
#include <sys/stat.h>
#include <unistd.h>
#include <zlib.h>
 
#include <SimoxUtility/algorithm/string.h>
 
#include <ArmarXCore/core/exceptions/local/ExpressionException.h>
 
#include <RobotAPI/libraries/armem/core/MemoryID.h>
#include <RobotAPI/libraries/armem/core/error.h>
 
namespace armarx::armem::server::ltm ::util::fs
{
 
    namespace detail
    {
        std::string
        escapeName(const std::string& segmentName)
        {
            std::string ret = segmentName;
            //simox::alg::replace_all(ret, Prefix, PrefixEscaped);
            for (const auto& [s, r] : EscapeTable)
            {
                ret = simox::alg::replace_all(ret, s, r);
            }
            return ret;
        }
 
        std::string
        unescapeName(const std::string& escapedName)
        {
            std::string ret = escapedName;
            for (
                const auto& [s, r] :
                EscapeTable) // Here we assume that noone uses the replaced char usually in the segment name... TODO
            {
                ret = simox::alg::replace_all(ret, r, s);
            }
            return ret;
        }
 
        std::string
        extractLastDirectoryFromPath(const std::string& path)
        {
            size_t pos = path.rfind('/');
 
            if (pos != std::string::npos)
            {
                return path.substr(pos + 1);
            }
            else
            {
                return path;
            }
        }
 
        std::string
        toDayString(const Time& t)
        {
            return t.toDateString();
        }
 
        std::string
        toSecondsString(const Time& t)
        {
            return std::to_string(t.toSecondsSinceEpoch());
        }
 
        bool
        isNumberString(const std::string& s)
        {
            for (char const& ch : s)
            {
                if (std::isdigit(ch) == 0)
                {
                    return false;
                }
            }
            return true;
        }
 
        bool
        isDateString(const std::string& s)
        {
            auto split = simox::alg::split(s, "-");
            if (split.size() != 3)
            {
                return false;
            }
 
            return isNumberString(split[0]) && isNumberString(split[1]) && isNumberString(split[2]);
        }
    } // namespace detail
 
    std::filesystem::path
    toPath(const std::filesystem::path& base, const armem::MemoryID& id)
    {
        ARMARX_CHECK(id.isWellDefined());
 
        std::filesystem::path p = base;
        if (id.hasMemoryName())
        {
            p /= detail::escapeName(id.memoryName);
        }
        if (id.hasCoreSegmentName())
        {
            p /= detail::escapeName(id.coreSegmentName);
        }
        if (id.hasProviderSegmentName())
        {
            p /= detail::escapeName(id.providerSegmentName);
        }
        if (id.hasEntityName())
        {
            p /= detail::escapeName(id.entityName);
        }
        if (id.hasTimestamp())
        {
            p /= detail::toDayString(id.timestamp);
            p /= detail::toSecondsString(id.timestamp);
            p /= id.timestampStr();
        }
        if (id.hasInstanceIndex())
        {
            p /= id.instanceIndexStr();
        }
 
        return p;
    }
 
    bool
    directoryExists(const std::filesystem::path& p)
    {
        return std::filesystem::exists(p) and std::filesystem::is_directory(p);
    }
 
    bool
    fileExists(const std::filesystem::path& p)
    {
        return std::filesystem::exists(p) && std::filesystem::is_regular_file(p);
    }
 
    void
    ensureDirectoryExists(const std::filesystem::path& p, bool createIfNotExistent)
    {
        if (createIfNotExistent)
        {
            // Use error_code version to avoid exceptions for already-exists case
            std::error_code ec;
            std::filesystem::create_directories(p, ec);
 
            // Only error if creation failed AND directory still doesn't exist
            if (ec && !std::filesystem::is_directory(p))
            {
                throw armarx::LocalException("Failed to create directory '" + p.string() +
                                            "': " + ec.message() + " (code: " +
                                            std::to_string(ec.value()) + ")");
            }
            // If directory exists now (either we created it or someone else did), success!
        }
        else if (!directoryExists(p))
        {
            throw armarx::LocalException("Directory existence cannot be ensured: " + p.string());
        }
    }
 
    void
    ensureFileExists(const std::filesystem::path& p, bool createIfNotExistent)
    {
        ensureDirectoryExists(p.parent_path(), createIfNotExistent);
 
        // Don't create the file here - just verify parent directory exists
        // The actual file will be created by writeDataToFile
 
        if (!createIfNotExistent && !fileExists(p))
        {
            throw armarx::LocalException("File existence cannot be ensured: " + p.string());
        }
    }
 
    void
    writeDataToFile(const std::filesystem::path& p,
                    const std::vector<unsigned char>& data,
                    WriteMode mode)
    {
        namespace fs = std::filesystem;
 
        if (mode == WriteMode::Plain)
        {
            std::ofstream dataofs;
            dataofs.open(p);
            if (!dataofs)
            {
                throw armarx::LocalException("Could not write data to filesystem file '" +
                                             p.string() + "'. Skipping this file.");
            }
            dataofs.write(reinterpret_cast<const char*>(data.data()), data.size());
            dataofs.close();
            return;
        }
 
        const fs::path dir = p.parent_path().empty() ? fs::current_path() : p.parent_path();
        const std::string filename = p.filename().string();
 
        // mkstemp template must end with XXXXXX and be mutable
        fs::path tmpl = dir / (filename + ".tmpXXXXXX");
        std::string tmpl_str = tmpl.string();
 
        // 1) Create a unique temp file in the same directory
        int fd = ::mkstemp(tmpl_str.data());
        if (fd == -1)
        {
            throw std::system_error(errno, std::generic_category(), "mkstemp failed");
        }
 
        // 2) Write the whole buffer
        const unsigned char* buf = data.data();
        size_t left = data.size();
        while (left > 0)
        {
            ssize_t n = ::write(fd, buf, left);
            if (n < 0)
            {
                int e = errno;
                ::close(fd);
                ::unlink(tmpl_str.c_str());
                throw std::system_error(e, std::generic_category(), "write failed");
            }
            buf += n;
            left -= static_cast<size_t>(n);
        }
 
        // 3) Flush file contents to disk. Skipped for WriteMode::Atomic: the rename below still
        // gives readers all-or-nothing contents, and a batch writer wants one directory sync at
        // the end rather than a synchronous flush per file.
        if (mode == WriteMode::AtomicDurable and ::fdatasync(fd) != 0)
        {
            int e = errno;
            ::close(fd);
            ::unlink(tmpl_str.c_str());
            throw std::system_error(e, std::generic_category(), "fdatasync failed");
        }
 
        if (::close(fd) != 0)
        {
            int e = errno;
            ::unlink(tmpl_str.c_str());
            throw std::system_error(e, std::generic_category(), "close failed");
        }
 
        // 4) Atomically replace the destination
        if (::rename(tmpl_str.c_str(), p.c_str()) != 0)
        {
            int e = errno;
            ::unlink(tmpl_str.c_str());
            throw std::system_error(e, std::generic_category(), "rename failed");
        }
 
        // 5) Make the rename durable by syncing the directory. The caller does this once for the
        // whole batch under WriteMode::Atomic.
        if (mode == WriteMode::AtomicDurable)
        {
            int dfd = ::open(dir.c_str(), O_DIRECTORY | O_RDONLY);
            if (dfd >= 0)
            {
                (void)::fsync(dfd);
                (void)::close(dfd);
            }
        }
    }
 
    std::vector<unsigned char>
    readDataFromFile(const std::filesystem::path& p)
    {
 
        if (!std::filesystem::exists(p))
        {
            throw std::runtime_error("File not found: " + p.string());
        }
 
        std::ifstream dataifs(p, std::ios::binary);
 
        if (!dataifs)
        {
            throw std::runtime_error("Could not open file: " + p.string());
        }
 
        std::vector<unsigned char> datafilecontent((std::istreambuf_iterator<char>(dataifs)),
                                                   (std::istreambuf_iterator<char>()));
 
 
        if (dataifs.bad())
        {
            throw std::runtime_error("Error reading file: " + p.string());
        }
 
        dataifs.close();
        return datafilecontent;
    }
 
    bool
    isGzip(const std::vector<unsigned char>& data)
    {
        return data.size() >= 2 and data[0] == 0x1f and data[1] == 0x8b;
    }
 
    namespace
    {
        /// windowBits 15 + 16 selects a gzip container (rather than raw deflate or zlib).
        constexpr int GZIP_WINDOW_BITS = 15 + 16;
        constexpr int GZIP_MEM_LEVEL = 8;
        constexpr std::size_t GZIP_CHUNK = 64 * 1024;
    } // namespace
 
    std::vector<unsigned char>
    gzipCompress(const std::vector<unsigned char>& data, int level)
    {
        z_stream zs{};
        int ret = ::deflateInit2(
            &zs, level, Z_DEFLATED, GZIP_WINDOW_BITS, GZIP_MEM_LEVEL, Z_DEFAULT_STRATEGY);
        if (ret != Z_OK)
        {
            throw std::runtime_error("gzipCompress: deflateInit2 failed with code " +
                                     std::to_string(ret));
        }
 
        std::vector<unsigned char> out;
        // deflateBound is an upper bound on the output size, so a single deflate() call suffices.
        out.resize(::deflateBound(&zs, static_cast<uLong>(data.size())));
 
        zs.next_in = const_cast<Bytef*>(data.data());
        zs.avail_in = static_cast<uInt>(data.size());
        zs.next_out = out.data();
        zs.avail_out = static_cast<uInt>(out.size());
 
        ret = ::deflate(&zs, Z_FINISH);
        const std::size_t written = zs.total_out;
        ::deflateEnd(&zs);
 
        if (ret != Z_STREAM_END)
        {
            throw std::runtime_error("gzipCompress: deflate did not finish, code " +
                                     std::to_string(ret));
        }
 
        out.resize(written);
        return out;
    }
 
    std::vector<unsigned char>
    gzipDecompress(const std::vector<unsigned char>& data)
    {
        z_stream zs{};
        int ret = ::inflateInit2(&zs, GZIP_WINDOW_BITS);
        if (ret != Z_OK)
        {
            throw std::runtime_error("gzipDecompress: inflateInit2 failed with code " +
                                     std::to_string(ret));
        }
 
        zs.next_in = const_cast<Bytef*>(data.data());
        zs.avail_in = static_cast<uInt>(data.size());
 
        std::vector<unsigned char> out;
        std::vector<unsigned char> chunk(GZIP_CHUNK);
 
        do
        {
            zs.next_out = chunk.data();
            zs.avail_out = static_cast<uInt>(chunk.size());
 
            ret = ::inflate(&zs, Z_NO_FLUSH);
 
            if (ret != Z_OK and ret != Z_STREAM_END and ret != Z_BUF_ERROR)
            {
                ::inflateEnd(&zs);
                throw std::runtime_error("gzipDecompress: inflate failed with code " +
                                         std::to_string(ret));
            }
 
            out.insert(out.end(), chunk.begin(), chunk.end() - zs.avail_out);
        // Keep going only while the output buffer was filled completely; if inflate left
        // room and did not report Z_STREAM_END, the input is truncated.
        } while (ret != Z_STREAM_END and zs.avail_out == 0);
 
        ::inflateEnd(&zs);
 
        if (ret != Z_STREAM_END)
        {
            throw std::runtime_error("gzipDecompress: truncated gzip stream");
        }
 
        return out;
    }
 
    std::vector<std::filesystem::path>
    getAllDirectories(const std::filesystem::path& p)
    {
        std::vector<std::filesystem::path> ret;
        for (const auto& subdir : std::filesystem::directory_iterator(p))
        {
            std::filesystem::path subdirPath = subdir.path();
            if (std::filesystem::is_directory(subdirPath))
            {
                ret.push_back(subdirPath);
            }
        }
        std::sort(ret.begin(),
                  ret.end(),
                  [](const std::filesystem::path& a, const std::filesystem::path& b) -> bool
                  { return a.string() < b.string(); });
        return ret;
    }
 
    std::vector<std::filesystem::path>
    getAllFiles(const std::filesystem::path& p)
    {
        std::vector<std::filesystem::path> ret;
        for (const auto& subdir : std::filesystem::directory_iterator(p))
        {
            std::filesystem::path subdirPath = subdir.path();
            if (std::filesystem::is_regular_file(subdirPath))
            {
                ret.push_back(subdirPath);
            }
        }
        std::sort(ret.begin(),
                  ret.end(),
                  [](const std::filesystem::path& a, const std::filesystem::path& b) -> bool
                  { return a.string() > b.string(); });
        return ret;
    }
 
    bool
    hasWritePermission(const std::filesystem::path& p)
    {
        namespace fs = std::filesystem;
 
        if (!fs::exists(p))
        {
            return false;  // Can't write to non-existent directory
        }
 
        if (!fs::is_directory(p))
        {
            return false;  // Not a directory
        }
 
        // Try to get permissions
        std::error_code ec;
        auto perms = fs::status(p, ec).permissions();
 
        if (ec)
        {
            return false;  // Can't determine permissions
        }
 
        // Check for owner write permission
        // Note: This is a basic check, doesn't account for group/other or ACLs
        using fs::perms;
        return (perms & perms::owner_write) != perms::none;
    }
 
    bool
    canCreateFiles(const std::filesystem::path& dir)
    {
        namespace fs = std::filesystem;
 
        if (!fs::exists(dir) || !fs::is_directory(dir))
        {
            return false;
        }
 
        // Try to create a temp file
        fs::path testFile = dir / ".ltm_write_test_XXXXXX";
        std::string testPath = testFile.string();
 
        int fd = ::mkstemp(testPath.data());
        if (fd == -1)
        {
            return false;  // Can't create files
        }
 
        ::close(fd);
        ::unlink(testPath.c_str());
        return true;
    }
} // namespace armarx::armem::server::ltm::util::fs