CommonStorage.h
Go to the documentation of this file.
1 /*
2 * This file is part of ArmarX.
3 *
4 * ArmarX is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * ArmarX is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 *
16 * @package MemoryX::CommonStorage
17 * @author Alexey Kozlov ( kozlov at kit dot edu)
18 * @date 2012
19 * @copyright http://www.gnu.org/licenses/gpl-2.0.txt
20 * GNU General Public License
21 */
22 
23 #pragma once
24 
25 #include <memory>
26 #include <mutex>
27 
28 #include <ArmarXCore/core/Component.h>
29 #include <ArmarXCore/core/services/tasks/PeriodicTask.h>
30 #include <ArmarXCore/core/system/ImportExportComponent.h>
31 
32 #include "GridFileWrapper.h"
33 #include <MemoryX/interface/components/CommonStorageInterface.h>
34 
35 namespace mongo
36 {
37  class GridFS;
38  class GridFile;
39  class BSONObj;
40  class DBClientConnection;
41  class DBClientBase;
42  class Query;
43 } // namespace mongo
44 
45 namespace memoryx
46 {
47 
48  class GridFileWrapper;
49  using GridFileWrapperPtr = IceInternal::Handle<GridFileWrapper>;
50  using GridFSPtr = std::shared_ptr<mongo::GridFS>;
51 
52  class CommonStoragePropertyDefinitions : public armarx::ComponentPropertyDefinitions
53  {
54  public:
55  CommonStoragePropertyDefinitions(std::string prefix) : ComponentPropertyDefinitions(prefix)
56  {
57  defineOptionalProperty<std::string>(
58  "MongoHost", "localhost", "MongoDB hostname and optionally port number")
59  .setCaseInsensitive(true);
60 
61  defineOptionalProperty<bool>(
62  "MongoAuth", false, "Whether authentication should be used for MongoDB");
63 
64  defineRequiredProperty<std::string>("MongoUser", "MongoDB user name");
65 
66  defineRequiredProperty<std::string>("MongoPassword", "MongoDB password");
67  }
68  };
69 
70  /*!
71  * \brief The CommonStorage class provides an interface to MongoDB.
72  *
73  * See \ref commonstorage "memoryx::CommonStorage" - versatile storage backed up by MongoDB
74  */
75  class ARMARXCOMPONENT_IMPORT_EXPORT CommonStorage :
76  virtual public CommonStorageInterface,
77  virtual public armarx::Component
78  {
79  public:
80  // inherited from Component
81  std::string getDefaultName() const override;
82  void onInitComponent() override;
83  void onConnectComponent() override;
84  void onExitComponent() override;
85 
86  /**
87  * @see PropertyUser::createPropertyDefinitions()
88  */
89  armarx::PropertyDefinitionsPtr createPropertyDefinitions() override;
90 
91  // inherited from CommonStorageInterface
92  std::string getMongoHostAndPort(const ::Ice::Current& c = Ice::emptyCurrent) override;
93  NameList getDBNames(const ::Ice::Current& = Ice::emptyCurrent) override;
94  NameList getCollectionNames(const ::std::string& dbName,
95  const ::Ice::Current& = Ice::emptyCurrent) override;
96  bool isConnected(const ::Ice::Current& c = Ice::emptyCurrent) override;
97 
98  bool reconnect(const ::std::string& hostAndPort,
99  const ::std::string& userName,
100  const ::std::string& password,
101  const ::Ice::Current& = Ice::emptyCurrent) override;
102  bool authDB(const ::std::string& dbName,
103  const ::std::string& userName,
104  const ::std::string& password,
105  const ::Ice::Current& = Ice::emptyCurrent) override;
106 
107  DatabaseInterfacePrx requestDatabase(const ::std::string& dbName,
108  const ::Ice::Current& = Ice::emptyCurrent) override;
109  void releaseDatabase(const DatabaseInterfacePrx& db,
110  const ::Ice::Current& = Ice::emptyCurrent) override;
111 
112  CollectionInterfacePrx
113  requestCollection(const std::string& collectionNS,
114  const ::Ice::Current& c = Ice::emptyCurrent) override;
115  void releaseCollection(const CollectionInterfacePrx& coll,
116  const ::Ice::Current& c = Ice::emptyCurrent) override;
117  void dropCollection(const std::string& collectionNS,
118  const ::Ice::Current& c = Ice::emptyCurrent) override;
119 
120  std::string storeFile(const ::std::string& dbName,
121  const ::std::string& fileName,
122  const ::std::string& gridFSName = "",
123  const ::Ice::Current& c = Ice::emptyCurrent) override;
124  std::string storeTextFile(const ::std::string& dbName,
125  const ::std::string& bufferToStore,
126  const ::std::string& gridFSName = "",
127  const ::Ice::Current& c = Ice::emptyCurrent) override;
128  std::string storeBinaryFile(const ::std::string& dbName,
129  const memoryx::Blob& bufferToStore,
130  const ::std::string& gridFSName = "",
131  const ::Ice::Current& c = Ice::emptyCurrent) override;
132 
133  bool getTextFileById(const ::std::string& dbName,
134  const ::std::string& fileId,
135  ::std::string& buffer,
136  const ::Ice::Current& c = Ice::emptyCurrent) override;
137  bool getBinaryFileById(const ::std::string& dbName,
138  const ::std::string& fileId,
139  memoryx::Blob& buffer,
140  const ::Ice::Current& c = Ice::emptyCurrent) override;
141  bool getTextFileByName(const ::std::string& dbName,
142  const ::std::string& gridFSName,
143  ::std::string& buffer,
144  const ::Ice::Current& c = Ice::emptyCurrent) override;
145  bool getBinaryFileByName(const ::std::string& dbName,
146  const ::std::string& gridFSName,
147  memoryx::Blob& buffer,
148  const ::Ice::Current& c = Ice::emptyCurrent) override;
149 
150  GridFileInterfacePrx getFileProxyById(const ::std::string& dbName,
151  const ::std::string& fileId,
152  const ::Ice::Current& c = Ice::emptyCurrent) override;
153  GridFileInterfacePrx
154  getFileProxyByName(const ::std::string& dbName,
155  const ::std::string& gridFSName,
156  const ::Ice::Current& c = Ice::emptyCurrent) override;
157  void releaseFileProxy(const GridFileInterfacePrx& fileProxy,
158  const ::Ice::Current& c = Ice::emptyCurrent) override;
159 
160  bool removeFileById(const ::std::string& dbName,
161  const ::std::string& fileId,
162  const ::Ice::Current& c = Ice::emptyCurrent) override;
163  bool removeFileByName(const ::std::string& dbName,
164  const ::std::string& gridFSName,
165  const ::Ice::Current& c = Ice::emptyCurrent) override;
166 
167  // methods called by Collection
168  Ice::Int count(const std::string& ns);
169 
170  DBStorableData findByMongoId(const std::string& ns, const std::string& id);
171  DBStorableDataList findByFieldValue(const std::string& ns,
172  const std::string& fieldName,
173  const ::std::string& fieldValue);
174  DBStorableDataList findByFieldValueList(const std::string& ns,
175  const std::string& fieldName,
176  const NameList& fieldValueList);
177  DBStorableData findOneByFieldValue(const std::string& ns,
178  const std::string& fieldName,
179  const ::std::string& fieldValue);
180  DBStorableDataList
181  findByQuery(const std::string& ns, const std::string& query, const std::string& where = "");
182  DBStorableData findOneByQuery(const std::string& ns, const std::string& query);
183  DBStorableDataList findAll(const std::string& ns);
184  DBStorableData findAllUniqueByFieldName(const std::string& ns,
185  const ::std::string& fieldName);
186  EntityIdList findAllIds(const std::string& ns);
187  NameList findAllFieldValues(const std::string& ns, const std::string& fieldName);
188 
189  std::string insert(const std::string& ns, const DBStorableData& obj, bool upsert = false);
190  std::vector<std::string> insertList(const std::string& ns,
191  const DBStorableDataList& objectList);
192  bool update(const std::string& ns,
193  const DBStorableData& obj,
194  const std::string& keyField,
195  bool upsert = false);
196  bool
197  updateByQuery(const std::string& ns, const std::string& query, const mongo::BSONObj& obj);
198 
199  bool removeByMongoId(const std::string& ns, const std::string& id);
200  bool removeByFieldValue(const std::string& ns,
201  const std::string& fieldName,
202  const std::string& fieldValue);
203  bool removeByQuery(const std::string& ns, const std::string& query);
204  bool clearCollection(const std::string& ns);
205 
206  bool ensureIndex(const std::string& ns, const std::string& fieldName, bool unique);
207 
208  void removeFileByQuery(const std::string& dbName, const mongo::BSONObj& fileQuery);
209 
210  NameList getFileNameList(const std::string& dbName,
211  const Ice::Current& c = Ice::emptyCurrent) override;
212  NameList getFileIdList(const std::string& dbName,
213  const Ice::Current& c = Ice::emptyCurrent) override;
214 
215  private:
216  bool removeByMongoQuery(const std::string& ns, const mongo::Query& query);
217  DBStorableDataList
218  findByMongoQuery(const std::string& ns, const mongo::Query& query, bool justOne = false);
219  NameList findFieldByMongoQuery(const std::string& ns,
220  const mongo::Query& query,
221  const std::string& fieldName);
222  mongo::GridFile getFileByQuery(const std::string& dbName, const mongo::BSONObj& query);
223  GridFileInterfacePrx createFileProxy(mongo::GridFile gridFile, const Ice::Current& c);
224  std::string getDocumentId(const mongo::BSONObj& doc);
225  std::string getDocumentField(const mongo::BSONObj& doc, const std::string& fieldName);
226  GridFSPtr getGridFS(const std::string& dbName);
227  bool readTextFile(mongo::GridFile& gridFile, std::string& buffer);
228  bool readBinaryFile(mongo::GridFile& gridFile, memoryx::Blob& buffer);
229 
230  std::string createPasswordDigest(const std::string& username, const std::string& password);
231  std::string extractDBNameFromNS(const std::string& ns);
232  bool authenticateNS(const std::string& ns);
233  bool authenticateDB(const std::string& dbName);
234  bool forceAuthenticate(const std::string& dbName,
235  const std::string& userName,
236  const std::string& password);
237 
238  /**
239  * @brief checkConnection is periodically called by memoryx::CommonStorage::connectionCheckerTask
240  *
241  * This method runs a query on the database and sets CommonStorage to state disconnected
242  * if the query fails (usually because the database connection has been lost.
243  */
244  void checkConnection();
245 
246  private:
247  mutable std::mutex serverSettingsMutex;
248  std::string hostAndPort;
249  bool useAuth;
250  std::string userName;
251  std::string pwdDigest;
252  std::deque<std::shared_ptr<mongo::DBClientConnection>> pool;
253 
254  struct ConnectionWrapper
255  {
256  ConnectionWrapper(CommonStorage& storage,
257  std::shared_ptr<mongo::DBClientConnection> connPtr);
258  ConnectionWrapper(ConnectionWrapper&&) = default;
259  ~ConnectionWrapper();
260 
261  mongo::DBClientConnection& conn();
262 
263  private:
264  std::shared_ptr<mongo::DBClientConnection> connPtr;
265  const std::string hostAndPort;
266  CommonStorage* const storage;
267  };
268 
269  ConnectionWrapper getConnection();
270 
271  std::shared_ptr<mongo::DBClientConnection> conn;
272  std::set<std::string> authDBs;
273 
274  std::map<Ice::Identity, DatabaseInterfacePtr> openedDatabases;
275  mutable std::mutex openedDatabasesMutex;
276  std::map<Ice::Identity, CollectionInterfacePtr> openedCollections;
277  mutable std::mutex openedCollectionsMutex;
278  std::map<Ice::Identity, GridFileWrapperPtr> openedFiles;
279  mutable std::mutex openedFilesMutex;
280  std::map<std::string, GridFSPtr> openedGridFS;
281  mutable std::mutex openedGridFSMutex;
282 
283  mutable std::shared_ptr<std::mutex> accessGridFSFilesMutex;
284 
285  /**
286  * @brief connectionCheckerTask periodically runs memoryx::CommonStorage::checkConnection()
287  */
288  armarx::PeriodicTask<CommonStorage>::pointer_type connectionCheckerTask;
289 
290  bool connect();
291  bool keepOldFileIfEqual(const mongo::GridFile& oldFile,
292  const mongo::GridFile newFile,
293  const mongo::BSONObj& newFileDoc,
294  const std::string dbName,
295  std::string& oldId);
296  };
297 
298  using CommonStoragePtr = IceUtil::Handle<CommonStorage>;
299 } // namespace memoryx
GridFileWrapper.h
memoryx::CommonStoragePropertyDefinitions
Definition: CommonStorage.h:52
armarx::PropertyDefinitionContainer::prefix
std::string prefix
Prefix of the properties such as namespace, domain, component name, etc.
Definition: PropertyDefinitionContainer.h:345
PeriodicTask.h
memoryx
VirtualRobot headers.
Definition: CommonPlacesTester.cpp:48
c
constexpr T c
Definition: UnscentedKalmanFilterTest.cpp:46
memoryx::CommonStoragePropertyDefinitions::CommonStoragePropertyDefinitions
CommonStoragePropertyDefinitions(std::string prefix)
Definition: CommonStorage.h:55
mongo
Definition: CommonStorage.h:35
IceInternal::Handle
Definition: forward_declarations.h:8
ARMARXCOMPONENT_IMPORT_EXPORT
#define ARMARXCOMPONENT_IMPORT_EXPORT
Definition: ImportExportComponent.h:38
armarx::armem::server::ltm::util::mongodb::detail::insert
bool insert(mongocxx::collection &coll, const nlohmann::json &value)
Definition: mongodb.cpp:44
armarx::armem::server::ltm::util::mongodb::detail::update
bool update(mongocxx::collection &coll, const nlohmann::json &query, const nlohmann::json &update)
Definition: mongodb.cpp:68
Component.h
armarx::Component
Baseclass for all ArmarX ManagedIceObjects requiring properties.
Definition: Component.h:91
armarx::ComponentPropertyDefinitions
Default component property definition container.
Definition: Component.h:69
IceUtil::Handle< class PropertyDefinitionContainer >
armarx::ComponentPropertyDefinitions::ComponentPropertyDefinitions
ComponentPropertyDefinitions(std::string prefix, bool hasObjectNameParameter=true)
Definition: Component.cpp:35
armarx::VariantType::Int
const VariantTypeId Int
Definition: Variant.h:917
memoryx::CommonStorage
The CommonStorage class provides an interface to MongoDB.
Definition: CommonStorage.h:75
memoryx::GridFSPtr
std::shared_ptr< mongo::GridFS > GridFSPtr
Definition: CommonStorage.h:50
ImportExportComponent.h