RobotUnitModuleLogging.h
Go to the documentation of this file.
1 /*
2  * This file is part of ArmarX.
3  *
4  * ArmarX is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  * ArmarX is distributed in the hope that it will be useful, but
9  * WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * along with this program. If not, see <http://www.gnu.org/licenses/>.
15  *
16  * @package RobotAPI::ArmarXObjects::RobotUnit
17  * @author Raphael Grimm ( raphael dot grimm at kit dot edu )
18  * @date 2018
19  * @copyright http://www.gnu.org/licenses/gpl-2.0.txt
20  * GNU General Public License
21  */
22 
23 #pragma once
24 
25 #include <atomic>
26 #include <deque>
27 #include <fstream>
28 #include <map>
29 #include <vector>
30 
32 
33 #include <RobotAPI/interface/units/RobotUnit/RobotUnitInterface.h>
34 
35 #include "../util/ControlThreadOutputBuffer.h"
36 #include "RobotUnitModuleBase.h"
37 
39 {
40  struct DoLoggingDurations;
41 }
42 
44 {
46  {
47  public:
49  {
50  defineOptionalProperty<std::size_t>(
51  "RTLogging_PeriodMs",
52  10,
53  "Period of the rt-logging thread in milliseconds. "
54  "A high period can cause spikes in disk activity if data is logged to disk. "
55  "A low period causes more computation overhead and memory consumption. "
56  "Must not be 0.");
57 
58  defineOptionalProperty<std::string>("RTLogging_DefaultLog",
59  "",
60  "If rt logging is active and a file path is given, "
61  "all data is logged to this file.");
62 
63  defineOptionalProperty<std::size_t>(
64  "RTLogging_MessageNumber",
65  1000,
66  "Number of messages that can be logged in the control thread");
67  defineOptionalProperty<std::size_t>(
68  "RTLogging_MessageBufferSize",
69  1024 * 1024,
70  "Number of bytes that can be logged in the control thread");
71  defineOptionalProperty<std::size_t>(
72  "RTLogging_MaxMessageNumber",
73  16000,
74  "Max number of messages that can be logged in the control thread");
75  defineOptionalProperty<std::size_t>(
76  "RTLogging_MaxMessageBufferSize",
77  16 * 1024 * 1024,
78  "Max number of bytes that can be logged in the control thread");
79 
80  defineOptionalProperty<bool>(
81  "RTLogging_EnableBacklog",
82  true,
83  "Enable/Disable the backlog (SensorValues, ControlTargets, Messages) that is kept"
84  "and can be dumped in case of an error.");
85  defineOptionalProperty<std::size_t>("RTLogging_KeepIterationsForMs",
86  60 * 1000,
87  "All logging data (SensorValues, ControlTargets, "
88  "Messages) is kept for this duration "
89  "and can be dumped in case of an error.");
90  defineOptionalProperty<std::size_t>(
91  "RTLogging_MaxBacklogSize",
92  5000,
93  "Maximum size of the backlog (SensorValues, ControlTargets, Messages) that is kept"
94  "and can be dumped in case of an error.");
95 
96  defineOptionalProperty<std::size_t>(
97  "RTLogging_StreamingDataMaxClientConnectionFailures",
98  10,
99  "If sending data to a client fails this often, the client is removed from the "
100  "list");
101 
102  defineOptionalProperty<int>(
103  "RTLogging_LogLastNMessagesOnly",
104  -1,
105  "If greater than 0, logs/streams only the last n timesteps that are in the entry "
106  "cue of the ControlThreadOutputBuffer");
107  }
108  };
109 
110  /**
111  * @ingroup Library-RobotUnit-Modules
112  * @brief This \ref ModuleBase "Module" manages logging of data.
113  *
114  * There are two types of data:
115  * - Messages: These are logged via \ref ARMARX_RT_LOGF macros and printed to the corresponding ArmarX streams
116  * - \ref SensorValueBase "SensorValues" / \ref ControlTargetBase "ControlTargets": These can be logged to CSV files
117  *
118  * @see ModuleBase
119  */
120  class Logging : virtual public ModuleBase, virtual public RobotUnitLoggingInterface
121  {
122  friend class ModuleBase;
123 
124  public:
125  /**
126  * @brief Returns the singleton instance of this class
127  * @return The singleton instance of this class
128  */
129  static Logging&
131  {
132  return ModuleBase::Instance<Logging>();
133  }
134 
135  // //////////////////////////////////////////////////////////////////////////////////////// //
136  // ///////////////////////////////// RobotUnitModule hooks //////////////////////////////// //
137  // //////////////////////////////////////////////////////////////////////////////////////// //
138  private:
139  /// @see ModuleBase::_postOnInitRobotUnit
140  void _postOnInitRobotUnit();
141  /// @see ModuleBase::_postFinishDeviceInitialization
142  void _postFinishDeviceInitialization();
143  /// @see ModuleBase::_preFinishControlThreadInitialization
144  void _preFinishControlThreadInitialization();
145  /// @see ModuleBase::_preFinishRunning
146  void _preFinishRunning();
147  // //////////////////////////////////////////////////////////////////////////////////////// //
148  // ///////////////////////////////////// ice interface //////////////////////////////////// //
149  // //////////////////////////////////////////////////////////////////////////////////////// //
150  public:
151  /**
152  * @brief Starts logging to a CSV file
153  * @param formatString The file to log to.
154  * @param loggingNames The data fields to log.
155  * @return A handle to the log. If it's last copy is deleted, logging is stopped.
156  */
157  SimpleRemoteReferenceCounterBasePtr
158  startRtLogging(const std::string& formatString,
159  const Ice::StringSeq& loggingNames,
160  const Ice::Current& = Ice::emptyCurrent) override;
161  /**
162  * @brief Starts logging to a CSV file
163  * @param formatString The file to log to.
164  * @param aliasNames Data fields to log. For an entry (key,value), the datafield key is logged and the table heading value is used.
165  * If value is empty, key is used as heading.
166  * @return A handle to the log. If it's last copy is deleted, logging is stopped.
167  */
168  SimpleRemoteReferenceCounterBasePtr
169  startRtLoggingWithAliasNames(const std::string& formatString,
170  const StringStringDictionary& aliasNames,
171  const Ice::Current& = Ice::emptyCurrent) override;
172 
173  /**
174  * @brief Stops logging to the given log.
175  * @param token The log to close.
176  */
177  void stopRtLogging(const armarx::SimpleRemoteReferenceCounterBasePtr& token,
178  const Ice::Current& = Ice::emptyCurrent) override;
179  /**
180  * @brief Adds a string to the log (it is added in a special column).
181  * @param token The log.
182  * @param marker The string to add.
183  */
184  void addMarkerToRtLog(const SimpleRemoteReferenceCounterBasePtr& token,
185  const std::string& marker,
186  const Ice::Current& = Ice::emptyCurrent) override;
187 
188  /**
189  * @brief Returns the names of all loggable data fields.
190  * @return The names of all loggable data fields.
191  */
192  Ice::StringSeq getLoggingNames(const Ice::Current& = Ice::emptyCurrent) const override;
193  /**
194  * @brief Dumps the backlog of all recent iterations to the given file.
195  * This helps debugging.
196  * @param formatString The file.
197  */
198  void writeRecentIterationsToFile(const std::string& formatString,
199  const Ice::Current& = Ice::emptyCurrent) const override;
200 
201  RobotUnitDataStreaming::DataStreamingDescription
202  startDataStreaming(const RobotUnitDataStreaming::ReceiverPrx& receiver,
203  const RobotUnitDataStreaming::Config& config,
204  const Ice::Current& = Ice::emptyCurrent) override;
205 
206  void stopDataStreaming(const RobotUnitDataStreaming::ReceiverPrx& receiver,
207  const Ice::Current& = Ice::emptyCurrent) override;
208  // //////////////////////////////////////////////////////////////////////////////////////// //
209  // //////////////////////////////////// implementation //////////////////////////////////// //
210  // //////////////////////////////////////////////////////////////////////////////////////// //
211  private:
212  /// @brief Executes the logging loop.
213  void doLogging();
214  void doLogging(details::DoLoggingDurations& durations,
215  const IceUtil::Time& now,
217  std::size_t i,
218  std::size_t num);
219  static bool MatchName(const std::string& pattern, const std::string& name);
220 
221  // //////////////////////////////////////////////////////////////////////////////////////// //
222  // ///////////////////////////////////////// Data ///////////////////////////////////////// //
223  // //////////////////////////////////////////////////////////////////////////////////////// //
224  private:
225  /// @brief Data structure holding information about an logging entry.
226  struct CSVLoggingEntry
227  {
228  /// @brief Whether logging should be stopped.
229  std::atomic_bool stopLogging{false};
230  /// @brief Bools whether a control device field should be logged.
231  std::vector<std::vector<std::vector<char>>> loggedControlDeviceValues;
232  /// @brief Bools whether a sensor device field should be logged.
233  std::vector<std::vector<char>> loggedSensorDeviceValues;
234  /// @brief The file stream for the csv file
235  std::ofstream stream;
236  /// @brief The file name
237  std::string filename;
238  /// @brief The marker to add in the next logging iteration. (protected by \ref rtLoggingMutex)
239  std::string marker;
240  };
241 
242  struct DataStreamingEntry
243  {
244  enum class ValueT
245  {
246  Bool,
247  Byte,
248  Short,
249  Int,
250  Long,
251  Float,
252  Double,
253  Skipped
254  };
255 
256  struct OutVal
257  {
258  std::size_t idx;
259  ValueT value = ValueT::Skipped;
260  };
261 
262  bool stopStreaming = false;
263 
264  std::size_t numBools = 0;
265  std::size_t numBytes = 0;
266  std::size_t numShorts = 0;
267  std::size_t numInts = 0;
268  std::size_t numLongs = 0;
269  std::size_t numFloats = 0;
270  std::size_t numDoubles = 0;
271 
272  bool onlyNewestFrame = false;
273  std::size_t connectionFailures = 0;
274  std::size_t rtStreamMaxClientErrors = 0;
275 
276  std::vector<std::vector<OutVal>> sensDevs;
277  std::vector<std::vector<std::vector<OutVal>>> ctrlDevs;
278 
279  RobotUnitDataStreaming::TimeStepSeq result;
280  std::deque<RobotUnitDataStreaming::TimeStep> entryBuffer;
281  std::deque<Ice::AsyncResultPtr> updateCalls;
282  void clearResult();
283  RobotUnitDataStreaming::TimeStep getResultElement();
284  RobotUnitDataStreaming::TimeStep allocateResultElement() const;
285 
286 
287  void processHeader(const ControlThreadOutputBuffer::Entry& e);
288  void processCtrl(const ControlTargetBase& val,
289  std::size_t didx,
290  std::size_t vidx,
291  std::size_t fidx);
292  void processSens(const SensorValueBase& val, std::size_t didx, std::size_t fidx);
293  void send(const RobotUnitDataStreaming::ReceiverPrx& r, uint64_t msgId);
294  };
295 
296  std::map<RobotUnitDataStreaming::ReceiverPrx, DataStreamingEntry> rtDataStreamingEntry;
297  std::uint64_t rtDataStreamingMsgID = 0;
298 
299  /// @brief The thread executing the logging functions.
300  std::thread rtLoggingTask;
301  std::atomic_bool stopRtLoggingTask{false};
302  /// @brief All entries for logging.
303  std::map<std::string, std::shared_ptr<CSVLoggingEntry>> rtLoggingEntries;
304  /// @brief Handle for the default log. (This log can be enabled via a property and logs everything)
305  SimpleRemoteReferenceCounterBasePtr defaultLogHandle;
306  /// @brief The stored backlog (it is dumped when \ref writeRecentIterationsToFile is called)
307  std::deque<::armarx::detail::ControlThreadOutputBufferEntry> backlog;
308  /// @brief The control threads unix thread id
309  Ice::Int controlThreadId{0};
310  /// @brief Mutex protecting the data structures of this class
311  mutable std::mutex rtLoggingMutex;
312 
313  struct ValueMetaData
314  {
316  {
317  std::string name;
318  const std::type_info* type;
319  };
320 
321  std::vector<FieldMetaData> fields;
322  };
323 
324  /// @brief Data field info for all \ref ControlTargetBase "ControlTargets" (dex x jctrl)
325  std::vector<std::vector<ValueMetaData>> controlDeviceValueMetaData;
326  /// @brief Data field info for all \ref SensorValueBase "SensorValues" (dev)
327  std::vector<ValueMetaData> sensorDeviceValueMetaData;
328 
329  /// @brief The initial size of the Message entry buffer
330  std::size_t messageBufferSize{0};
331  /// @brief The initial number of Message entries
332  std::size_t messageBufferNumberEntries{0};
333  /// @brief The maximal size of the Message entry buffer
334  std::size_t messageBufferMaxSize{0};
335  /// @brief The maximal number of Message entries
336  std::size_t messageBufferMaxNumberEntries{0};
337  /// @brief The logging thread's period
338  std::size_t rtLoggingTimestepMs{0};
339  /// @brief The time an entry should remain in the backlog.
340  IceUtil::Time rtLoggingBacklogRetentionTime;
341  /// @brief The maximum number of entries in the backlog.
342  std::size_t rtLoggingBacklogMaxSize;
343  /// @brief Enable/disable the backlog.
344  bool rtLoggingBacklogEnabled;
345 
346  /// @brief
347  int numberOfEntriesToLog{-1};
348 
349  friend void WriteTo(const auto& dentr,
350  const Logging::DataStreamingEntry::OutVal& out,
351  const auto& val,
352  std::size_t fidx,
353  auto& data);
354  };
355 } // namespace armarx::RobotUnitModule
armarx::RobotUnitModule::Logging::writeRecentIterationsToFile
void writeRecentIterationsToFile(const std::string &formatString, const Ice::Current &=Ice::emptyCurrent) const override
Dumps the backlog of all recent iterations to the given file.
Definition: RobotUnitModuleLogging.cpp:273
armarx::RobotUnitModule::Logging::ValueMetaData::FieldMetaData::type
const std::type_info * type
Definition: RobotUnitModuleLogging.h:318
armarx::SensorValueBase
The SensorValueBase class.
Definition: SensorValueBase.h:40
armarx::RobotUnitModule::Logging::ValueMetaData::FieldMetaData::name
std::string name
Definition: RobotUnitModuleLogging.h:317
armarx::ControlTargetBase
Brief description of class JointControlTargetBase.
Definition: ControlTargetBase.h:47
RobotUnitModuleBase.h
armarx::PropertyDefinitionContainer::prefix
std::string prefix
Prefix of the properties such as namespace, domain, component name, etc.
Definition: PropertyDefinitionContainer.h:333
armarx::RobotUnitModule::Logging
This Module manages logging of data.
Definition: RobotUnitModuleLogging.h:120
armarx::RobotUnitModule::Logging::DataStreamingEntry::OutVal::value
ValueT value
Definition: RobotUnitModuleLogging.h:259
armarx::RobotUnitModule::LoggingPropertyDefinitions::LoggingPropertyDefinitions
LoggingPropertyDefinitions(std::string prefix)
Definition: RobotUnitModuleLogging.h:48
armarx::detail::ControlThreadOutputBufferEntry
Definition: ControlThreadOutputBuffer.h:177
armarx::RobotUnitModule::Logging::WriteTo
friend void WriteTo(const auto &dentr, const Logging::DataStreamingEntry::OutVal &out, const auto &val, std::size_t fidx, auto &data)
Definition: RobotUnitModuleLogging.cpp:1071
armarx::RobotUnitModule::LoggingPropertyDefinitions
Definition: RobotUnitModuleLogging.h:45
armarx::RobotUnitModule::Logging::Instance
static Logging & Instance()
Returns the singleton instance of this class.
Definition: RobotUnitModuleLogging.h:130
armarx::RobotUnitModule::details
Definition: RobotUnitModuleLogging.cpp:39
data
uint8_t data[1]
Definition: EtherCATFrame.h:68
filename
std::string filename
Definition: VisualizationRobot.cpp:83
armarx::RobotUnitModule::Logging::startRtLogging
SimpleRemoteReferenceCounterBasePtr startRtLogging(const std::string &formatString, const Ice::StringSeq &loggingNames, const Ice::Current &=Ice::emptyCurrent) override
Starts logging to a CSV file.
Definition: RobotUnitModuleLogging.cpp:108
TaskUtil.h
armarx::armem::Time
armarx::core::time::DateTime Time
Definition: forward_declarations.h:13
armarx::RobotUnitModule::Logging::DataStreamingEntry::OutVal
Definition: RobotUnitModuleLogging.h:256
armarx::RobotUnitModule::Logging::getLoggingNames
Ice::StringSeq getLoggingNames(const Ice::Current &=Ice::emptyCurrent) const override
Returns the names of all loggable data fields.
Definition: RobotUnitModuleLogging.cpp:146
armarx::RobotUnitModule::Logging::DataStreamingEntry::OutVal::idx
std::size_t idx
Definition: RobotUnitModuleLogging.h:258
armarx::RobotUnitModule::Logging::addMarkerToRtLog
void addMarkerToRtLog(const SimpleRemoteReferenceCounterBasePtr &token, const std::string &marker, const Ice::Current &=Ice::emptyCurrent) override
Adds a string to the log (it is added in a special column).
Definition: RobotUnitModuleLogging.cpp:93
armarx::RobotUnitModule::Logging::startDataStreaming
RobotUnitDataStreaming::DataStreamingDescription startDataStreaming(const RobotUnitDataStreaming::ReceiverPrx &receiver, const RobotUnitDataStreaming::Config &config, const Ice::Current &=Ice::emptyCurrent) override
Definition: RobotUnitModuleLogging.cpp:381
armarx::RobotUnitModule::Logging::stopRtLogging
void stopRtLogging(const armarx::SimpleRemoteReferenceCounterBasePtr &token, const Ice::Current &=Ice::emptyCurrent) override
Stops logging to the given log.
Definition: RobotUnitModuleLogging.cpp:123
armarx::RobotUnitModule::details::DoLoggingDurations
Definition: RobotUnitModuleLogging.cpp:41
armarx::RobotUnitModule::Logging::ValueMetaData::FieldMetaData
Definition: RobotUnitModuleLogging.h:315
armarx::VariantType::Int
const VariantTypeId Int
Definition: Variant.h:916
armarx::RobotUnitModule
Definition: ControlDevice.h:34
armarx::RobotUnitModule::Logging::stopDataStreaming
void stopDataStreaming(const RobotUnitDataStreaming::ReceiverPrx &receiver, const Ice::Current &=Ice::emptyCurrent) override
Definition: RobotUnitModuleLogging.cpp:493
armarx::RobotUnitModule::ModuleBasePropertyDefinitions
Definition: RobotUnitModuleBase.h:107
armarx::RobotUnitModule::Logging::startRtLoggingWithAliasNames
SimpleRemoteReferenceCounterBasePtr startRtLoggingWithAliasNames(const std::string &formatString, const StringStringDictionary &aliasNames, const Ice::Current &=Ice::emptyCurrent) override
Starts logging to a CSV file.
Definition: RobotUnitModuleLogging.cpp:172
armarx::RobotUnitModule::ModuleBase
Base class for all RobotUnitModules.
Definition: RobotUnitModuleBase.h:183