CommonStorage.h

Go to the documentation of this file.

/*
* This file is part of ArmarX.
*
* ArmarX is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* ArmarX is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package    MemoryX::CommonStorage
* @author     Alexey Kozlov ( kozlov at kit dot edu)
* @date       2012
* @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
*             GNU General Public License
*/
 
#pragma once
 
#include <memory>
#include <mutex>
 
#include <ArmarXCore/core/Component.h>
#include <ArmarXCore/core/services/tasks/PeriodicTask.h>
#include <ArmarXCore/core/system/ImportExportComponent.h>
 
#include "GridFileWrapper.h"
#include <MemoryX/interface/components/CommonStorageInterface.h>
 
namespace mongo
{
    class GridFS;
    class GridFile;
    class BSONObj;
    class DBClientConnection;
    class DBClientBase;
    class Query;
} // namespace mongo
 
namespace memoryx
{
 
    class GridFileWrapper;
    using GridFileWrapperPtr = IceInternal::Handle<GridFileWrapper>;
    using GridFSPtr = std::shared_ptr<mongo::GridFS>;
 
    class CommonStoragePropertyDefinitions : public armarx::ComponentPropertyDefinitions
    {
    public:
        CommonStoragePropertyDefinitions(std::string prefix) : ComponentPropertyDefinitions(prefix)
        {
            defineOptionalProperty<std::string>(
                "MongoHost", "localhost", "MongoDB hostname and optionally port number")
                .setCaseInsensitive(true);
 
            defineOptionalProperty<bool>(
                "MongoAuth", false, "Whether authentication should be used for MongoDB");
 
            defineRequiredProperty<std::string>("MongoUser", "MongoDB user name");
 
            defineRequiredProperty<std::string>("MongoPassword", "MongoDB password");
        }
    };
 
    /*!
     * \brief The CommonStorage class provides an interface to MongoDB.
     *
     * See \ref commonstorage "memoryx::CommonStorage"   - versatile storage backed up by MongoDB
     */
    class ARMARXCOMPONENT_IMPORT_EXPORT CommonStorage :
        virtual public CommonStorageInterface,
        virtual public armarx::Component
    {
    public:
        // inherited from Component
        std::string getDefaultName() const override;
        static std::string GetDefaultName();
        void onInitComponent() override;
        void onConnectComponent() override;
        void onExitComponent() override;
 
        /**
         * @see PropertyUser::createPropertyDefinitions()
         */
        armarx::PropertyDefinitionsPtr createPropertyDefinitions() override;
 
        // inherited from CommonStorageInterface
        std::string getMongoHostAndPort(const ::Ice::Current& c = Ice::emptyCurrent) override;
        NameList getDBNames(const ::Ice::Current& = Ice::emptyCurrent) override;
        NameList getCollectionNames(const ::std::string& dbName,
                                    const ::Ice::Current& = Ice::emptyCurrent) override;
        bool isConnected(const ::Ice::Current& c = Ice::emptyCurrent) override;
 
        bool reconnect(const ::std::string& hostAndPort,
                       const ::std::string& userName,
                       const ::std::string& password,
                       const ::Ice::Current& = Ice::emptyCurrent) override;
        bool authDB(const ::std::string& dbName,
                    const ::std::string& userName,
                    const ::std::string& password,
                    const ::Ice::Current& = Ice::emptyCurrent) override;
 
        DatabaseInterfacePrx requestDatabase(const ::std::string& dbName,
                                             const ::Ice::Current& = Ice::emptyCurrent) override;
        void releaseDatabase(const DatabaseInterfacePrx& db,
                             const ::Ice::Current& = Ice::emptyCurrent) override;
 
        CollectionInterfacePrx
        requestCollection(const std::string& collectionNS,
                          const ::Ice::Current& c = Ice::emptyCurrent) override;
        void releaseCollection(const CollectionInterfacePrx& coll,
                               const ::Ice::Current& c = Ice::emptyCurrent) override;
        void dropCollection(const std::string& collectionNS,
                            const ::Ice::Current& c = Ice::emptyCurrent) override;
 
        std::string storeFile(const ::std::string& dbName,
                              const ::std::string& fileName,
                              const ::std::string& gridFSName = "",
                              const ::Ice::Current& c = Ice::emptyCurrent) override;
        std::string storeTextFile(const ::std::string& dbName,
                                  const ::std::string& bufferToStore,
                                  const ::std::string& gridFSName = "",
                                  const ::Ice::Current& c = Ice::emptyCurrent) override;
        std::string storeBinaryFile(const ::std::string& dbName,
                                    const memoryx::Blob& bufferToStore,
                                    const ::std::string& gridFSName = "",
                                    const ::Ice::Current& c = Ice::emptyCurrent) override;
 
        bool getTextFileById(const ::std::string& dbName,
                             const ::std::string& fileId,
                             ::std::string& buffer,
                             const ::Ice::Current& c = Ice::emptyCurrent) override;
        bool getBinaryFileById(const ::std::string& dbName,
                               const ::std::string& fileId,
                               memoryx::Blob& buffer,
                               const ::Ice::Current& c = Ice::emptyCurrent) override;
        bool getTextFileByName(const ::std::string& dbName,
                               const ::std::string& gridFSName,
                               ::std::string& buffer,
                               const ::Ice::Current& c = Ice::emptyCurrent) override;
        bool getBinaryFileByName(const ::std::string& dbName,
                                 const ::std::string& gridFSName,
                                 memoryx::Blob& buffer,
                                 const ::Ice::Current& c = Ice::emptyCurrent) override;
 
        GridFileInterfacePrx getFileProxyById(const ::std::string& dbName,
                                              const ::std::string& fileId,
                                              const ::Ice::Current& c = Ice::emptyCurrent) override;
        GridFileInterfacePrx
        getFileProxyByName(const ::std::string& dbName,
                           const ::std::string& gridFSName,
                           const ::Ice::Current& c = Ice::emptyCurrent) override;
        void releaseFileProxy(const GridFileInterfacePrx& fileProxy,
                              const ::Ice::Current& c = Ice::emptyCurrent) override;
 
        bool removeFileById(const ::std::string& dbName,
                            const ::std::string& fileId,
                            const ::Ice::Current& c = Ice::emptyCurrent) override;
        bool removeFileByName(const ::std::string& dbName,
                              const ::std::string& gridFSName,
                              const ::Ice::Current& c = Ice::emptyCurrent) override;
 
        // methods called by Collection
        Ice::Int count(const std::string& ns);
 
        DBStorableData findByMongoId(const std::string& ns, const std::string& id);
        DBStorableDataList findByFieldValue(const std::string& ns,
                                            const std::string& fieldName,
                                            const ::std::string& fieldValue);
        DBStorableDataList findByFieldValueList(const std::string& ns,
                                                const std::string& fieldName,
                                                const NameList& fieldValueList);
        DBStorableData findOneByFieldValue(const std::string& ns,
                                           const std::string& fieldName,
                                           const ::std::string& fieldValue);
        DBStorableDataList
        findByQuery(const std::string& ns, const std::string& query, const std::string& where = "");
        DBStorableData findOneByQuery(const std::string& ns, const std::string& query);
        DBStorableDataList findAll(const std::string& ns);
        DBStorableData findAllUniqueByFieldName(const std::string& ns,
                                                const ::std::string& fieldName);
        EntityIdList findAllIds(const std::string& ns);
        NameList findAllFieldValues(const std::string& ns, const std::string& fieldName);
 
        std::string insert(const std::string& ns, const DBStorableData& obj, bool upsert = false);
        std::vector<std::string> insertList(const std::string& ns,
                                            const DBStorableDataList& objectList);
        bool update(const std::string& ns,
                    const DBStorableData& obj,
                    const std::string& keyField,
                    bool upsert = false);
        bool
        updateByQuery(const std::string& ns, const std::string& query, const mongo::BSONObj& obj);
 
        bool removeByMongoId(const std::string& ns, const std::string& id);
        bool removeByFieldValue(const std::string& ns,
                                const std::string& fieldName,
                                const std::string& fieldValue);
        bool removeByQuery(const std::string& ns, const std::string& query);
        bool clearCollection(const std::string& ns);
 
        bool ensureIndex(const std::string& ns, const std::string& fieldName, bool unique);
 
        void removeFileByQuery(const std::string& dbName, const mongo::BSONObj& fileQuery);
 
        NameList getFileNameList(const std::string& dbName,
                                 const Ice::Current& c = Ice::emptyCurrent) override;
        NameList getFileIdList(const std::string& dbName,
                               const Ice::Current& c = Ice::emptyCurrent) override;
 
    private:
        bool removeByMongoQuery(const std::string& ns, const mongo::Query& query);
        DBStorableDataList
        findByMongoQuery(const std::string& ns, const mongo::Query& query, bool justOne = false);
        NameList findFieldByMongoQuery(const std::string& ns,
                                       const mongo::Query& query,
                                       const std::string& fieldName);
        mongo::GridFile getFileByQuery(const std::string& dbName, const mongo::BSONObj& query);
        GridFileInterfacePrx createFileProxy(mongo::GridFile gridFile, const Ice::Current& c);
        std::string getDocumentId(const mongo::BSONObj& doc);
        std::string getDocumentField(const mongo::BSONObj& doc, const std::string& fieldName);
        GridFSPtr getGridFS(const std::string& dbName);
        bool readTextFile(mongo::GridFile& gridFile, std::string& buffer);
        bool readBinaryFile(mongo::GridFile& gridFile, memoryx::Blob& buffer);
 
        std::string createPasswordDigest(const std::string& username, const std::string& password);
        std::string extractDBNameFromNS(const std::string& ns);
        bool authenticateNS(const std::string& ns);
        bool authenticateDB(const std::string& dbName);
        bool forceAuthenticate(const std::string& dbName,
                               const std::string& userName,
                               const std::string& password);
 
        /**
         * @brief checkConnection is periodically called by memoryx::CommonStorage::connectionCheckerTask
         *
         * This method runs a query on the database and sets CommonStorage to state disconnected
         * if the query fails (usually because the database connection has been lost.
         */
        void checkConnection();
 
    private:
        mutable std::mutex serverSettingsMutex;
        std::string hostAndPort;
        bool useAuth;
        std::string userName;
        std::string pwdDigest;
        std::deque<std::shared_ptr<mongo::DBClientConnection>> pool;
 
        struct ConnectionWrapper
        {
            ConnectionWrapper(CommonStorage& storage,
                              std::shared_ptr<mongo::DBClientConnection> connPtr);
            ConnectionWrapper(ConnectionWrapper&&) = default;
            ~ConnectionWrapper();
 
            mongo::DBClientConnection& conn();
 
        private:
            std::shared_ptr<mongo::DBClientConnection> connPtr;
            const std::string hostAndPort;
            CommonStorage* const storage;
        };
 
        ConnectionWrapper getConnection();
 
        std::shared_ptr<mongo::DBClientConnection> conn;
        std::set<std::string> authDBs;
 
        std::map<Ice::Identity, DatabaseInterfacePtr> openedDatabases;
        mutable std::mutex openedDatabasesMutex;
        std::map<Ice::Identity, CollectionInterfacePtr> openedCollections;
        mutable std::mutex openedCollectionsMutex;
        std::map<Ice::Identity, GridFileWrapperPtr> openedFiles;
        mutable std::mutex openedFilesMutex;
        std::map<std::string, GridFSPtr> openedGridFS;
        mutable std::mutex openedGridFSMutex;
 
        mutable std::shared_ptr<std::mutex> accessGridFSFilesMutex;
 
        /**
         * @brief connectionCheckerTask periodically runs memoryx::CommonStorage::checkConnection()
         */
        armarx::PeriodicTask<CommonStorage>::pointer_type connectionCheckerTask;
 
        bool connect();
        bool keepOldFileIfEqual(const mongo::GridFile& oldFile,
                                const mongo::GridFile newFile,
                                const mongo::BSONObj& newFileDoc,
                                const std::string dbName,
                                std::string& oldId);
    };
 
    using CommonStoragePtr = IceUtil::Handle<CommonStorage>;
} // namespace memoryx