CommonStorage.h
Go to the documentation of this file.
1 /*
2 * This file is part of ArmarX.
3 *
4 * ArmarX is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * ArmarX is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 *
16 * @package MemoryX::CommonStorage
17 * @author Alexey Kozlov ( kozlov at kit dot edu)
18 * @date 2012
19 * @copyright http://www.gnu.org/licenses/gpl-2.0.txt
20 * GNU General Public License
21 */
22 
23 #pragma once
24 
25 #include <mutex>
26 #include <memory>
27 
28 #include "GridFileWrapper.h"
29 #include <ArmarXCore/core/Component.h>
30 #include <ArmarXCore/core/system/ImportExportComponent.h>
31 #include <ArmarXCore/core/services/tasks/PeriodicTask.h>
32 #include <MemoryX/interface/components/CommonStorageInterface.h>
33 
34 namespace mongo
35 {
36  class GridFS;
37  class GridFile;
38  class BSONObj;
39  class DBClientConnection;
40  class DBClientBase;
41  class Query;
42 }
43 
44 namespace memoryx
45 {
46 
47  class GridFileWrapper;
48  using GridFileWrapperPtr = IceInternal::Handle<GridFileWrapper>;
49  using GridFSPtr = std::shared_ptr<mongo::GridFS>;
50 
51  class CommonStoragePropertyDefinitions:
52  public armarx::ComponentPropertyDefinitions
53  {
54  public:
55  CommonStoragePropertyDefinitions(std::string prefix):
56  ComponentPropertyDefinitions(prefix)
57  {
58  defineOptionalProperty<std::string>("MongoHost", "localhost", "MongoDB hostname and optionally port number")
59  .setCaseInsensitive(true);
60 
61  defineOptionalProperty<bool>("MongoAuth", false, "Whether authentication should be used for MongoDB");
62 
63  defineRequiredProperty<std::string>("MongoUser", "MongoDB user name");
64 
65  defineRequiredProperty<std::string>("MongoPassword", "MongoDB password");
66  }
67  };
68 
69  /*!
70  * \brief The CommonStorage class provides an interface to MongoDB.
71  *
72  * See \ref commonstorage "memoryx::CommonStorage" - versatile storage backed up by MongoDB
73  */
74  class ARMARXCOMPONENT_IMPORT_EXPORT CommonStorage :
75  virtual public CommonStorageInterface,
76  virtual public armarx::Component
77  {
78  public:
79  // inherited from Component
80  std::string getDefaultName() const override;
81  void onInitComponent() override;
82  void onConnectComponent() override;
83  void onExitComponent() override;
84 
85  /**
86  * @see PropertyUser::createPropertyDefinitions()
87  */
88  armarx::PropertyDefinitionsPtr createPropertyDefinitions() override;
89 
90  // inherited from CommonStorageInterface
91  std::string getMongoHostAndPort(const ::Ice::Current& c = Ice::emptyCurrent) override;
92  NameList getDBNames(const ::Ice::Current& = Ice::emptyCurrent) override;
93  NameList getCollectionNames(const ::std::string& dbName, const ::Ice::Current& = Ice::emptyCurrent) override;
94  bool isConnected(const ::Ice::Current& c = Ice::emptyCurrent) override;
95 
96  bool reconnect(const ::std::string& hostAndPort, const ::std::string& userName, const ::std::string& password, const ::Ice::Current& = Ice::emptyCurrent) override;
97  bool authDB(const ::std::string& dbName, const ::std::string& userName, const ::std::string& password, const ::Ice::Current& = Ice::emptyCurrent) override;
98 
99  DatabaseInterfacePrx requestDatabase(const ::std::string& dbName, const ::Ice::Current& = Ice::emptyCurrent) override;
100  void releaseDatabase(const DatabaseInterfacePrx& db, const ::Ice::Current& = Ice::emptyCurrent) override;
101 
102  CollectionInterfacePrx requestCollection(const std::string& collectionNS, const ::Ice::Current& c = Ice::emptyCurrent) override;
103  void releaseCollection(const CollectionInterfacePrx& coll, const ::Ice::Current& c = Ice::emptyCurrent) override;
104  void dropCollection(const std::string& collectionNS, const ::Ice::Current& c = Ice::emptyCurrent) override;
105 
106  std::string storeFile(const ::std::string& dbName, const ::std::string& fileName, const ::std::string& gridFSName = "", const ::Ice::Current& c = Ice::emptyCurrent) override;
107  std::string storeTextFile(const ::std::string& dbName, const ::std::string& bufferToStore, const ::std::string& gridFSName = "", const ::Ice::Current& c = Ice::emptyCurrent) override;
108  std::string storeBinaryFile(const ::std::string& dbName, const memoryx::Blob& bufferToStore, const ::std::string& gridFSName = "", const ::Ice::Current& c = Ice::emptyCurrent) override;
109 
110  bool getTextFileById(const ::std::string& dbName, const ::std::string& fileId, ::std::string& buffer, const ::Ice::Current& c = Ice::emptyCurrent) override;
111  bool getBinaryFileById(const ::std::string& dbName, const ::std::string& fileId, memoryx::Blob& buffer, const ::Ice::Current& c = Ice::emptyCurrent) override;
112  bool getTextFileByName(const ::std::string& dbName, const ::std::string& gridFSName, ::std::string& buffer, const ::Ice::Current& c = Ice::emptyCurrent) override;
113  bool getBinaryFileByName(const ::std::string& dbName, const ::std::string& gridFSName, memoryx::Blob& buffer, const ::Ice::Current& c = Ice::emptyCurrent) override;
114 
115  GridFileInterfacePrx getFileProxyById(const ::std::string& dbName, const ::std::string& fileId, const ::Ice::Current& c = Ice::emptyCurrent) override;
116  GridFileInterfacePrx getFileProxyByName(const ::std::string& dbName, const ::std::string& gridFSName, const ::Ice::Current& c = Ice::emptyCurrent) override;
117  void releaseFileProxy(const GridFileInterfacePrx& fileProxy, const ::Ice::Current& c = Ice::emptyCurrent) override;
118 
119  bool removeFileById(const ::std::string& dbName, const ::std::string& fileId, const ::Ice::Current& c = Ice::emptyCurrent) override;
120  bool removeFileByName(const ::std::string& dbName, const ::std::string& gridFSName, const ::Ice::Current& c = Ice::emptyCurrent) override;
121 
122  // methods called by Collection
123  Ice::Int count(const std::string& ns);
124 
125  DBStorableData findByMongoId(const std::string& ns, const std::string& id);
126  DBStorableDataList findByFieldValue(const std::string& ns, const std::string& fieldName, const ::std::string& fieldValue);
127  DBStorableDataList findByFieldValueList(const std::string& ns, const std::string& fieldName, const NameList& fieldValueList);
128  DBStorableData findOneByFieldValue(const std::string& ns, const std::string& fieldName, const ::std::string& fieldValue);
129  DBStorableDataList findByQuery(const std::string& ns, const std::string& query, const std::string& where = "");
130  DBStorableData findOneByQuery(const std::string& ns, const std::string& query);
131  DBStorableDataList findAll(const std::string& ns);
132  DBStorableData findAllUniqueByFieldName(const std::string& ns, const ::std::string& fieldName);
133  EntityIdList findAllIds(const std::string& ns);
134  NameList findAllFieldValues(const std::string& ns, const std::string& fieldName);
135 
136  std::string insert(const std::string& ns, const DBStorableData& obj, bool upsert = false);
137  std::vector<std::string> insertList(const std::string& ns, const DBStorableDataList& objectList);
138  bool update(const std::string& ns, const DBStorableData& obj, const std::string& keyField, bool upsert = false);
139  bool updateByQuery(const std::string& ns, const std::string& query, const mongo::BSONObj& obj);
140 
141  bool removeByMongoId(const std::string& ns, const std::string& id);
142  bool removeByFieldValue(const std::string& ns, const std::string& fieldName, const std::string& fieldValue);
143  bool removeByQuery(const std::string& ns, const std::string& query);
144  bool clearCollection(const std::string& ns);
145 
146  bool ensureIndex(const std::string& ns, const std::string& fieldName, bool unique);
147 
148  void removeFileByQuery(const std::string& dbName, const mongo::BSONObj& fileQuery);
149 
150  NameList getFileNameList(const std::string& dbName, const Ice::Current& c = Ice::emptyCurrent) override;
151  NameList getFileIdList(const std::string& dbName, const Ice::Current& c = Ice::emptyCurrent) override;
152  private:
153  bool removeByMongoQuery(const std::string& ns, const mongo::Query& query);
154  DBStorableDataList findByMongoQuery(const std::string& ns, const mongo::Query& query, bool justOne = false);
155  NameList findFieldByMongoQuery(const std::string& ns, const mongo::Query& query, const std::string& fieldName);
156  mongo::GridFile getFileByQuery(const std::string& dbName, const mongo::BSONObj& query);
157  GridFileInterfacePrx createFileProxy(mongo::GridFile gridFile, const Ice::Current& c);
158  std::string getDocumentId(const mongo::BSONObj& doc);
159  std::string getDocumentField(const mongo::BSONObj& doc, const std::string& fieldName);
160  GridFSPtr getGridFS(const std::string& dbName);
161  bool readTextFile(mongo::GridFile& gridFile, std::string& buffer);
162  bool readBinaryFile(mongo::GridFile& gridFile, memoryx::Blob& buffer);
163 
164  std::string createPasswordDigest(const std::string& username, const std::string& password);
165  std::string extractDBNameFromNS(const std::string& ns);
166  bool authenticateNS(const std::string& ns);
167  bool authenticateDB(const std::string& dbName);
168  bool forceAuthenticate(const std::string& dbName, const std::string& userName, const std::string& password);
169 
170  /**
171  * @brief checkConnection is periodically called by memoryx::CommonStorage::connectionCheckerTask
172  *
173  * This method runs a query on the database and sets CommonStorage to state disconnected
174  * if the query fails (usually because the database connection has been lost.
175  */
176  void checkConnection();
177 
178  private:
179  mutable std::mutex serverSettingsMutex;
180  std::string hostAndPort;
181  bool useAuth;
182  std::string userName;
183  std::string pwdDigest;
184  std::deque<std::shared_ptr<mongo::DBClientConnection>> pool;
185  struct ConnectionWrapper
186  {
187  ConnectionWrapper(
188  CommonStorage& storage,
189  std::shared_ptr<mongo::DBClientConnection> connPtr
190  );
191  ConnectionWrapper(ConnectionWrapper&&) = default;
192  ~ConnectionWrapper();
193 
194  mongo::DBClientConnection& conn();
195 
196  private:
197  std::shared_ptr<mongo::DBClientConnection> connPtr;
198  const std::string hostAndPort;
199  CommonStorage* const storage;
200  };
201  ConnectionWrapper getConnection();
202 
203  std::shared_ptr<mongo::DBClientConnection> conn;
204  std::set<std::string> authDBs;
205 
206  std::map<Ice::Identity, DatabaseInterfacePtr> openedDatabases;
207  mutable std::mutex openedDatabasesMutex;
208  std::map<Ice::Identity, CollectionInterfacePtr> openedCollections;
209  mutable std::mutex openedCollectionsMutex;
210  std::map<Ice::Identity, GridFileWrapperPtr> openedFiles;
211  mutable std::mutex openedFilesMutex;
212  std::map<std::string, GridFSPtr> openedGridFS;
213  mutable std::mutex openedGridFSMutex;
214 
215  mutable std::shared_ptr<std::mutex> accessGridFSFilesMutex;
216 
217  /**
218  * @brief connectionCheckerTask periodically runs memoryx::CommonStorage::checkConnection()
219  */
220  armarx::PeriodicTask<CommonStorage>::pointer_type connectionCheckerTask;
221 
222  bool connect();
223  bool keepOldFileIfEqual(const mongo::GridFile& oldFile, const mongo::GridFile newFile, const mongo::BSONObj& newFileDoc, const std::string dbName, std::string& oldId);
224  };
225 
226  using CommonStoragePtr = IceUtil::Handle<CommonStorage>;
227 }
GridFileWrapper.h
memoryx::CommonStoragePropertyDefinitions
Definition: CommonStorage.h:51
armarx::PropertyDefinitionContainer::prefix
std::string prefix
Prefix of the properties such as namespace, domain, component name, etc.
Definition: PropertyDefinitionContainer.h:333
PeriodicTask.h
memoryx
VirtualRobot headers.
Definition: CommonPlacesTester.cpp:48
c
constexpr T c
Definition: UnscentedKalmanFilterTest.cpp:43
memoryx::CommonStoragePropertyDefinitions::CommonStoragePropertyDefinitions
CommonStoragePropertyDefinitions(std::string prefix)
Definition: CommonStorage.h:55
mongo
Definition: CommonStorage.h:34
IceInternal::Handle
Definition: forward_declarations.h:8
ARMARXCOMPONENT_IMPORT_EXPORT
#define ARMARXCOMPONENT_IMPORT_EXPORT
Definition: ImportExportComponent.h:38
armarx::armem::server::ltm::util::mongodb::detail::insert
bool insert(mongocxx::collection &coll, const nlohmann::json &value)
Definition: mongodb.cpp:43
armarx::armem::server::ltm::util::mongodb::detail::update
bool update(mongocxx::collection &coll, const nlohmann::json &query, const nlohmann::json &update)
Definition: mongodb.cpp:67
Component.h
armarx::Component
Baseclass for all ArmarX ManagedIceObjects requiring properties.
Definition: Component.h:95
armarx::ComponentPropertyDefinitions
Default component property definition container.
Definition: Component.h:70
IceUtil::Handle< class PropertyDefinitionContainer >
armarx::ComponentPropertyDefinitions::ComponentPropertyDefinitions
ComponentPropertyDefinitions(std::string prefix, bool hasObjectNameParameter=true)
Definition: Component.cpp:37
armarx::VariantType::Int
const VariantTypeId Int
Definition: Variant.h:916
memoryx::CommonStorage
The CommonStorage class provides an interface to MongoDB.
Definition: CommonStorage.h:74
memoryx::GridFSPtr
std::shared_ptr< mongo::GridFS > GridFSPtr
Definition: CommonStorage.h:49
ImportExportComponent.h