GridFileManager.h

Go to the documentation of this file.

/*
* This file is part of ArmarX.
*
* ArmarX is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* ArmarX is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package    MemoryX::Core
* @author     Alexey Kozlov ( kozlov at kit dot edu)
* @date       Sep 27, 2012
* @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
*             GNU General Public License
*/
 
#pragma once
 
#include <filesystem>
#include <memory>
 
#include <ArmarXCore/core/logging/Logging.h>
 
#include <MemoryX/core/MongoDBRef.h>
#include <MemoryX/interface/components/CommonStorageInterface.h>
#include <MemoryX/interface/core/EntityBase.h>
#include <MemoryX/interface/core/GridFileInterface.h>
 
namespace memoryx
{
    /**
     * GridFileManager provides utility functions for working with files in Mongo GridFS and
     * links to them stored in entity attributes (s. memoryx::MongoDBRef)
     */
    class GridFileManager : public armarx::Logging
    {
    public:
        /**
         * Constructs new GridFileManager.
         *
         *  GridFileManager needs a MongoDB connection, so CommonStorage proxy needs to be specified.
         *  Cache path for files is retrived from the Application properties.
         *
         *  @param databasePrx CommonStorage proxy. It can be obtained either directly or via getCommonStorage()
         *  method of WorkingMemory and PriorKnowledge.
         */
        GridFileManager(const CommonStorageInterfacePrx& databasePrx);
 
        /**
         * Constructs new GridFileManager.
         *
         *  GridFileManager needs a MongoDB connection, so CommonStorage proxy needs to be specified. Use this constructor, if you
         *  want to provide an own cache path.
         *
         *  @param databasePrx CommonStorage proxy. It can be obtained either directly or via getCommonStorage()
         *  method of WorkingMemory and PriorKnowledge.
         *  @param cachePath Path where to store cache files. Path may contain environmenta variables, which get replaced by their value.
         */
        GridFileManager(const CommonStorageInterfacePrx& databasePrx, const std::string& cachePath);
 
        ~GridFileManager() override;
 
        /**
         * Retrieves a local path where files will be cached.
         */
        std::string getFileCachePath() const;
 
        /**
         * Caches the file locally and opens a filestream for it.
         *
         * @param filePrx  file proxy
         * @param fs       file stream to open.
         *
         * @return true on succcess, false otherwise
         */
        bool getFileStream(GridFileInterfacePrx& filePrx, std::ifstream& fs);
 
        /**
         * Caches the file locally and opens a filestream for it.
         *
         * @param fileAttr entity attribute containing a file reference
         * @param fs       file stream to open.
         *
         * @return true on succcess, false otherwise
         */
        bool getFileStream(const EntityAttributeBasePtr& fileAttr, std::ifstream& fs);
 
        /**
         * Caches the file locally and returns the filename.
         *
         * @param filePrx  file proxy
         * @param cacheFileName name of local file
         * @param preserveOriginalName whether file should be saved under its original GridFS name (true) or
         * or it could be given a special name to simplify modification check (false). Even though the latter option
         * is preferred, sometimes original names must be preserved because of links between files (e.g. textures in
         * .iv file)
         *
         * @return true on succcess, false otherwise
         */
        bool ensureFileInCache(GridFileInterfacePrx& filePrx,
                               std::string& cacheFileName,
                               bool preserveOriginalName = false);
 
        /**
         * Caches the file locally and returns the filename.
         *
         * @param fileAttr  entity attribute containing a file reference
         * @param cacheFileName name of local file
         * @param preserveOriginalName whether file should be saved under its original GridFS name (true) or
         * or it could be given a special name to simplify modification check (false). Even though the latter option
         * is preferred, sometimes original names must be preserved because of links between files (e.g. textures in
         * .iv file)
         *
         * @return true on succcess, false otherwise
         */
        bool ensureFileInCache(const EntityAttributeBasePtr& fileAttr,
                               std::string& cacheFileName,
                               bool preserveOriginalName = false);
 
        /**
         * Caches multiple files locally.
         *
         * @param fileAttr  entity attribute containing file references
         * @param cacheFileNames names of caches files
         * @param preserveOriginalName whether file should be saved under its original GridFS name (true) or
         * or it could be given a special name to simplify modification check (false). Even though the latter option
         * is preferred, sometimes original names must be preserved because of links between files (e.g. textures in
         * .iv file)
         *
         * @return true on succcess, false otherwise
         */
        bool ensureFilesInCache(const EntityAttributeBasePtr& fileAttr,
                                std::vector<std::string>& cacheFileNames,
                                bool preserveOriginalNames = false);
 
        /**
         * Caches multiple files locally. Version without returning the cache file names
         *
         * @param fileAttr  entity attribute containing file references
         * @param preserveOriginalName whether file should be saved under its original GridFS name (true) or
         * or it could be given a special name to simplify modification check (false). Even though the latter option
         * is preferred, sometimes original names must be preserved because of links between files (e.g. textures in
         * .iv file)
         *
         * @return true on succcess, false otherwise
         */
        bool ensureFilesInCache(const EntityAttributeBasePtr& fileAttr,
                                bool preserveOriginalNames = false);
 
        /**
         * Stores a file in GridFS and puts a reference to it into entity attribute. Overwrites any
         * previously stored attribute.
         *
         * @param  filesDBName      Mongo database name to store file into
         * @param  localFileName    name of a file to store
         * @param  fileAttr         entity attribute to store file reference into
         * @param  gridFSName       name of the file in GridFS. Default: localFileName WITHOUT path
         *
         * @return MongoDB id of the file object as returned by GridFS
         */
        std::string storeFileToAttr(const std::string& filesDBName,
                                    const std::string& localFileName,
                                    EntityAttributeBasePtr& fileAttr,
                                    const std::string& gridFSName = "");
 
        /**
         * Stores a file in GridFS and puts a reference to it into entity attribute. Allows
         * to add multiple files to an attribdatabasePrxute.
         *
         * @param  filesDBName      Mongo database name to store file into
         * @param  localFileName    name of a file to store (gets turned into an absolute path)
         * @param  fileAttr         entity attribute to store file reference into
         * @param  gridFSName       name of the file in GridFS. Default: localFileName WITHOUT path
         *
         * @return MongoDB id of the file object as returned by GridFS
         */
        std::string addFileToAttr(const std::string& filesDBName,
                                  const std::string& localFileName,
                                  EntityAttributeBasePtr& fileAttr,
                                  const std::string& gridFSName = "");
 
        /**
         * Stores a complete directory tree in GridFS and puts a reference to it into entity attribute
         * Overwrites any previously stored attribute.
         *
         * @param  filesDBName      Mongo database name to store file into
         * @param  localDirectoryName    name of a directory to store
         * @param  fileAttr         entity attribute to store file reference into
         *
         * @return true on succcess, false otherwise
         */
        bool storeDirectoryToAttr(const std::string& filesDBName,
                                  const std::string& localDirectoryName,
                                  EntityAttributeBasePtr& fileAttr,
                                  std::string excludeFilter = ".svn");
 
        /**
         * Stores a set of files with in GridFS and puts a reference to it into entity attribute.
         * The directory structure is preserved while localBaseDirectoryName and all preceding directories are removed from the filenames.
         * Overwrites any previously stored attribute.
         *
         * @param  filesDBName      Mongo database name to store file into
         * @param  localBaseDirectoryName    the base directory. All file names will be made realtive to this directory
         * @param  localFiles       The set of files. Absolute filenames.
         * @param  fileAttr         entity attribute to store file reference into
         *
         * @return true on succcess, false otherwise
         */
        bool storeFilesToAttr(const std::string& filesDBName,
                              const std::string& localBaseDirectoryName,
                              const std::vector<std::string>& localFiles,
                              EntityAttributeBasePtr& fileAttr);
 
        bool removeAttrFile(const EntityAttributeBasePtr& fileAttr, unsigned int fileIndex);
        /**
        * Removes all GridFS files referenced by entity attribute
        *
        * @param fileAttr  entity attribute containing file reference(s)
        *
        * @return true on succcess, false otherwise
        */
        bool removeAttrFiles(const EntityAttributeBasePtr& fileAttr);
 
 
        /**
         * Removes GridFS directory referenced by entity attribute
         *
         * @param fileAttr  entity attribute containing file reference(s)
         *
         * @return true on succcess, false otherwise
         */
        bool removeAttrDirectory(const EntityAttributeBasePtr& fileAttr);
 
        CommonStorageInterfacePrx
        getDBPrx() const
        {
            return databasePrx;
        }
 
    protected:
        void init(std::string cachePath);
 
    private:
        std::filesystem::path fileCachePath;
        CommonStorageInterfacePrx databasePrx;
 
        GridFileInterfacePrx getFileProxyFromAttr(const AttributeBasePtr& attr) const;
        GridFileList getFileProxiesFromAttr(const AttributeBasePtr& attr) const;
 
        MongoDBRefPtr extractMongoDBRef(const AttributeBasePtr& attr) const;
        MongoDBRefPtr extractMongoDBRef(const armarx::VariantBasePtr& value) const;
 
        std::string makeRelativePath(const std::filesystem::path& directory,
                                     const std::filesystem::path& basePath);
    };
 
    using GridFileManagerPtr = std::shared_ptr<GridFileManager>;
 
} // namespace memoryx