GridFileManager.cpp

Go to the documentation of this file.

/*
* This file is part of ArmarX.
*
* ArmarX is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* ArmarX is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package    MemoryX::Core
* @author     Alexey Kozlov ( kozlov at kit dot edu)
* @date       Sep 27, 2012
* @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
*             GNU General Public License
*/
 
#include <fstream>
#include <memory>
#include <filesystem>
#include <random>
#include <boost/regex.hpp>
 
#include <ArmarXCore/core/application/Application.h>
#include <ArmarXCore/core/system/ArmarXDataPath.h>
#include <ArmarXCore/util/CPPUtility/trace.h>
#include <MemoryX/core/entity/EntityAttribute.h>
#include <IceUtil/UUID.h>
#include "GridFileManager.h"
 
namespace memoryx
{
    namespace fs = std::filesystem;
 
    GridFileManager::GridFileManager(const CommonStorageInterfacePrx& databasePrx) :
        databasePrx(databasePrx)
    {
        std::string armarxCachePath = armarx::ArmarXDataPath::GetCachePath();
        if (!armarxCachePath.empty())
        {
            init(armarxCachePath);
        }
        else
        {
            static std::string cachePath = (std::filesystem::temp_directory_path() / std::to_string(std::random_device{}())).string();
            init(cachePath);
        }
    }
 
    GridFileManager::GridFileManager(const CommonStorageInterfacePrx& databasePrx, const std::string& cachePath) :
        databasePrx(databasePrx)
    {
        init(cachePath);
    }
 
 
 
    GridFileManager::~GridFileManager()
    {
    }
 
    void GridFileManager::init(std::string cachePath)
    {
        ARMARX_TRACE;
        armarx::ArmarXDataPath::ReplaceEnvVars(cachePath);
        ARMARX_DEBUG_S << "Cache path: " << cachePath;
 
        if (!fs::exists(cachePath))
        {
            fs::create_directory(cachePath);
        }
 
        fileCachePath = fs::path(cachePath) / fs::path("files");
 
        if (!fs::exists(fileCachePath))
        {
            fs::create_directory(fileCachePath);
        }
 
        // make path absolute to prevent errors with relative paths
        if (!fileCachePath.is_absolute())
        {
            ARMARX_TRACE;
            std::filesystem::path fullPath(std::filesystem::current_path());
            fileCachePath = fullPath / fileCachePath;
        }
    }
 
    std::string GridFileManager::getFileCachePath() const
    {
        return fileCachePath.string();
    }
 
    bool GridFileManager::ensureFileInCache(GridFileInterfacePrx& filePrx, std::string& cacheFileName,
                                            bool preserveOriginalName)
    {
        ARMARX_TRACE;
        if (!filePrx)
        {
            return false;
        }
 
        // files could be stored in local cache in two ways:
        // - preserving original file name (e.g. for textures used from .iv-file)
        // - under special names <mongo file id>_<MD5 file hash>. it's preferable since it allows
        //   to check the identity of local and remote files both simple and reliable
        fs::path filenameFromProxy(filePrx->getFilename());
        //ARMARX_INFO << "filenameFromProxy = " << filenameFromProxy;
        fs::path localFile = fileCachePath / filenameFromProxy.stem();
        //ARMARX_INFO << "localFile = " << localFile;
        if (!preserveOriginalName)
        {
            localFile            += fs::path(filePrx->getId() + "_" + filePrx->getMD5());
        }
        std::string localFileStr = localFile.string();
        std::string extensionStr = filenameFromProxy.extension().string();
        localFile = fs::path(localFileStr + extensionStr);
        // check if file was already cached:
        // 1) check file name (and so MD5 hash if not in preserveOriginalName mode)
        bool cached = fs::exists(localFile);
        if (cached)
        {
            // 2) check file size
            cached = cached && ((std::uintmax_t) filePrx->getFileSize() == fs::file_size(localFile));
            // 3) check file date (only needed in preserveOriginalName mode, otherwise MD5 match should suffice)
            ARMARX_TRACE;
            fs::file_time_type lwt = fs::last_write_time(localFile);
            auto sctp = std::chrono::time_point_cast<std::chrono::system_clock::duration>(lwt - fs::file_time_type::clock::now()
                        + std::chrono::system_clock::now());
            time_t time = std::chrono::system_clock::to_time_t(sctp);
            ARMARX_TRACE;
            cached = cached && (!preserveOriginalName || (time >= filePrx->getUploadDate() / 1000));
        }
 
        //    ARMARX_VERBOSE << "Local file time: " <<  fs::last_write_time(localFile) << " size: " << fs::file_size(localFile) <<  std::endl;
        //    ARMARX_VERBOSE << "Remote file time: " <<  filePrx->getUploadDate() << " size: " << filePrx->getFileSize() <<  std::endl;
 
        if (!cached)
        {
            ARMARX_TRACE;
            // assure directory is present
            std::filesystem::path filePath;
            filePath = localFile;
            filePath = filePath.parent_path();
            create_directories(filePath);
 
            ARMARX_VERBOSE << "Caching file to: " <<  localFile << std::flush;
            const std::string tmpLocalFile = localFile.string() + IceUtil::generateUUID() + ".part";
            std::fstream fsOut;
            fsOut.open(tmpLocalFile.c_str(), std::ios_base::out | std::ios_base::binary);
 
            memoryx::Blob buffer;
 
            ARMARX_TRACE;
            while (filePrx->getNextChunk(buffer))
            {
                fsOut.write((char*) &buffer[0], buffer.size());
            }
 
            fsOut.close();
            fs::rename(tmpLocalFile, localFile);
        }
 
        cacheFileName = localFile.string();
        return true;
    }
 
    bool GridFileManager::ensureFileInCache(const EntityAttributeBasePtr& fileAttr,
                                            std::string& cacheFileName, bool preserveOriginalName)
    {
        ARMARX_TRACE;
        bool result = false;
        GridFileInterfacePrx filePrx = getFileProxyFromAttr(fileAttr);
 
        if (!filePrx)
        {
            return false;
        }
 
        result = ensureFileInCache(filePrx, cacheFileName, preserveOriginalName);
        ARMARX_TRACE;
        databasePrx->releaseFileProxy(filePrx);
        return result;
    }
 
    bool GridFileManager::ensureFilesInCache(const EntityAttributeBasePtr& fileAttr, std::vector<std::string>& cacheFileNames,
            bool preserveOriginalNames /* = false */)
    {
        ARMARX_TRACE;
        bool result = false;
 
        GridFileList filePrxList = getFileProxiesFromAttr(fileAttr);
 
        if (filePrxList.empty())
        {
            return result;
        }
 
        std::string cacheFileName;
 
        for (GridFileList::iterator it = filePrxList.begin(); it != filePrxList.end(); ++it)
        {
            ARMARX_TRACE;
            result |= ensureFileInCache(*it, cacheFileName, preserveOriginalNames);
            cacheFileNames.push_back(cacheFileName);
            databasePrx->releaseFileProxy(*it);
        }
 
        return result;
    }
 
    bool GridFileManager::ensureFilesInCache(const EntityAttributeBasePtr& fileAttr,
            bool preserveOriginalNames /* = false */)
    {
        ARMARX_TRACE;
        std::vector<std::string> cacheFileNames;
        return ensureFilesInCache(fileAttr, cacheFileNames, preserveOriginalNames);
    }
 
    bool GridFileManager::getFileStream(GridFileInterfacePrx& filePrx, std::ifstream& fs)
    {
        ARMARX_TRACE;
        std::string cacheFileName;
 
        if (ensureFileInCache(filePrx, cacheFileName, false))
        {
            ARMARX_TRACE;
            fs.open(cacheFileName.c_str(), std::ios_base::in);
            return true;
        }
 
        return false;
    }
 
    bool GridFileManager::getFileStream(const EntityAttributeBasePtr& fileAttr, std::ifstream& fs)
    {
        ARMARX_TRACE;
        GridFileInterfacePrx filePrx = getFileProxyFromAttr(fileAttr);
        bool result = getFileStream(filePrx, fs);
        databasePrx->releaseFileProxy(filePrx);
        return result;
    }
 
    std::string GridFileManager::storeFileToAttr(const std::string& filesDBName,
            const std::string& localFileName, EntityAttributeBasePtr& fileAttr, const std::string& gridFSName /* = "" */)
    {
        ARMARX_TRACE;
        return addFileToAttr(filesDBName, localFileName, fileAttr, gridFSName);
    }
 
    std::string GridFileManager::addFileToAttr(const std::string& filesDBName,
            const std::string& localFileName, EntityAttributeBasePtr& fileAttr, const std::string& gridFSName /* = "" */)
    {
        ARMARX_TRACE;
        std::filesystem::path fname(localFileName);
        fname = std::filesystem::absolute(fname);
        std::string fileId;
        try
        {
            ARMARX_TRACE;
            fileId = databasePrx->storeFile(filesDBName, fname.string(), (gridFSName.empty() ? fname.filename().string() : gridFSName));
        }
        catch (const FileNotFoundException& e)
        {
            ARMARX_TRACE;
            // try again but now by reading file on this machine and sending it over ice
            std::ifstream fin(fname.string().c_str(), std::ios::binary);
            if (!fin.is_open())
            {
                throw FileNotFoundException("File not found: " +  fname.string(), fname.string());
            }
            std::stringstream contentStream;
            contentStream << fin.rdbuf();
            fileId = databasePrx->storeTextFile(filesDBName, contentStream.str(), (gridFSName.empty() ? fname.filename().string() : gridFSName));
        }
 
        MongoDBRefPtr fileRef = new MongoDBRef(filesDBName, fileId);
 
        const armarx::VariantPtr fileVar = new armarx::Variant(fileRef._ptr);
        fileAttr->addValue(fileVar);
 
        return fileId;
    }
 
    bool GridFileManager::storeDirectoryToAttr(const std::string& filesDBName,
            const std::string& localDirectoryName, EntityAttributeBasePtr& fileAttr, std::string excludeFilter)
    {
        ARMARX_TRACE;
        boost::regex exclude(excludeFilter.c_str());
 
        bool success = true;
        fileAttr->clear();
 
        std::filesystem::path relativePathBase = localDirectoryName;
        relativePathBase = relativePathBase.parent_path();
 
        // go through directory recursively
        for (std::filesystem::recursive_directory_iterator end, dir(localDirectoryName.c_str()); dir != end; ++dir)
        {
            // only add non directories
            if (dir->status().type() != std::filesystem::file_type::directory)
            {
                if (boost::regex_match(dir->path().filename().c_str(), exclude))
                {
                    dir.disable_recursion_pending();
                }
                else
                {
                    std::string fileName = makeRelativePath(dir->path(), relativePathBase);
                    ARMARX_VERBOSE_S << "Adding file '" <<  dir->path() << "' with name " << fileName;
                    const std::string fileId = addFileToAttr(filesDBName, dir->path().c_str(), fileAttr, fileName);
                    success &= !fileId.empty();
                }
            }
        }
 
        return success;
    }
 
 
    bool GridFileManager::storeFilesToAttr(const std::string& filesDBName,
                                           const std::string& localBaseDirectoryName,
                                           const std::vector<std::string>& localFiles,
                                           EntityAttributeBasePtr& fileAttr)
    {
        ARMARX_TRACE;
        bool success = true;
        fileAttr->clear();
 
        std::filesystem::path relativePathBase = localBaseDirectoryName;
        relativePathBase = relativePathBase.parent_path();
 
        for (size_t i = 0; i < localFiles.size(); i++)
        {
            std::filesystem::path f = localFiles[i];
            std::string fileName = makeRelativePath(f, relativePathBase);
            ARMARX_VERBOSE_S << "Adding file " <<  f << "with name " << fileName;
            const std::string fileId = addFileToAttr(filesDBName, f.c_str(), fileAttr, fileName);
            success &= !fileId.empty();
        }
 
        return success;
    }
 
    bool GridFileManager::removeAttrFile(const EntityAttributeBasePtr& fileAttr, unsigned int fileIndex)
    {
        ARMARX_TRACE;
        const MongoDBRefPtr fileRef = extractMongoDBRef(fileAttr->getValueAt(fileIndex));
 
        if (!fileRef)
        {
            return false;
        }
 
        return databasePrx->removeFileById(fileRef->dbName, fileRef->docId);
    }
 
 
    bool GridFileManager::removeAttrFiles(const EntityAttributeBasePtr& fileAttr)
    {
        ARMARX_TRACE;
        if (!fileAttr)
        {
            return false;
        }
 
        bool result = true;
 
        for (size_t i = 0; i < (size_t) fileAttr->size(); ++i)
        {
            result &= removeAttrFile(fileAttr, i);
        }
 
        return result;
    }
 
 
 
    GridFileInterfacePrx GridFileManager::getFileProxyFromAttr(const AttributeBasePtr& attr) const
    {
        ARMARX_TRACE;
        const MongoDBRefPtr fileRef = extractMongoDBRef(attr);
 
        if (fileRef)
        {
            return databasePrx->getFileProxyById(fileRef->dbName, fileRef->docId);
        }
        else
        {
            ARMARX_WARNING << "Could not get fileref for attribute " << attr->getName();
            return GridFileInterfacePrx();
        }
    }
 
    GridFileList GridFileManager::getFileProxiesFromAttr(const AttributeBasePtr& attr) const
    {
        ARMARX_TRACE;
        GridFileList result;
        const EntityAttributePtr entityAttr = EntityAttributePtr::dynamicCast(attr);
 
        if (!entityAttr)
        {
            return result;
        }
 
        for (size_t i = 0; i < (size_t) entityAttr->size(); ++i)
        {
            const MongoDBRefPtr fileRef = extractMongoDBRef(entityAttr->getValueAt(i));
 
            if (fileRef)
            {
                const GridFileInterfacePrx prx = databasePrx->getFileProxyById(fileRef->dbName, fileRef->docId);
                result.push_back(prx);
            }
        }
 
        return result;
    }
 
    MongoDBRefPtr GridFileManager::extractMongoDBRef(const AttributeBasePtr& attr) const
    {
        ARMARX_TRACE;
        if (attr)
        {
            ARMARX_TRACE;
            const EntityAttributePtr entityAttr = EntityAttributePtr::dynamicCast(attr);
 
            if (entityAttr)
            {
                return extractMongoDBRef(entityAttr->getValue());
            }
            else
            {
                ARMARX_WARNING << "Could not cast attribute " << attr->getName();
            }
        }
        else
        {
            ARMARX_WARNING << "NULL Attribute";
        }
 
        return MongoDBRefPtr();
    }
 
    MongoDBRefPtr GridFileManager::extractMongoDBRef(const armarx::VariantBasePtr& value) const
    {
        ARMARX_TRACE;
        const armarx::VariantPtr fileVar = armarx::VariantPtr::dynamicCast(value);
 
        if (fileVar)
        {
            return fileVar->getClass<MongoDBRef>();
        }
        else
        {
            return MongoDBRefPtr();
        }
    }
 
 
    std::string GridFileManager::makeRelativePath(const std::filesystem::path& directory, const std::filesystem::path& basePath)
    {
        ARMARX_TRACE;
        std::filesystem::path diffpath;
        std::filesystem::path tmppath = directory;
 
        while (tmppath != basePath)
        {
            diffpath = tmppath.filename() / diffpath;
            tmppath = tmppath.parent_path();
 
            if (tmppath.empty())
            {
                // no relative path found, take complete path
                diffpath = directory;
                break;
            }
        }
 
        return diffpath.string();
    }
}