RobotUnitModuleLogging.h

Go to the documentation of this file.

/*
 * This file is part of ArmarX.
 *
 * ArmarX is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * ArmarX is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * @package    RobotAPI::ArmarXObjects::RobotUnit
 * @author     Raphael Grimm ( raphael dot grimm at kit dot edu )
 * @date       2018
 * @copyright  http://www.gnu.org/licenses/gpl-2.0.txt
 *             GNU General Public License
 */
 
#pragma once
 
#include <atomic>
#include <deque>
#include <fstream>
#include <map>
#include <vector>
 
#include <ArmarXCore/core/services/tasks/TaskUtil.h>
 
#include <RobotAPI/interface/units/RobotUnit/RobotUnitInterface.h>
 
#include "../util/ControlThreadOutputBuffer.h"
#include "RobotUnitModuleBase.h"
 
namespace armarx::RobotUnitModule::details
{
    struct DoLoggingDurations;
}
 
namespace armarx::RobotUnitModule
{
    class LoggingPropertyDefinitions : public ModuleBasePropertyDefinitions
    {
    public:
        LoggingPropertyDefinitions(std::string prefix) : ModuleBasePropertyDefinitions(prefix)
        {
            defineOptionalProperty<std::size_t>(
                "RTLogging_PeriodMs",
                10,
                "Period of the rt-logging thread in milliseconds. "
                "A high period can cause spikes in disk activity if data is logged to disk. "
                "A low period causes more computation overhead and memory consumption. "
                "Must not be 0.");
 
            defineOptionalProperty<std::string>("RTLogging_DefaultLog",
                                                "",
                                                "If rt logging is active and a file path is given, "
                                                "all data is logged to this file.");
 
            defineOptionalProperty<std::size_t>(
                "RTLogging_MessageNumber",
                1000,
                "Number of messages that can be logged in the control thread");
            defineOptionalProperty<std::size_t>(
                "RTLogging_MessageBufferSize",
                1024 * 1024,
                "Number of bytes that can be logged in the control thread");
            defineOptionalProperty<std::size_t>(
                "RTLogging_MaxMessageNumber",
                16000,
                "Max number of messages that can be logged in the control thread");
            defineOptionalProperty<std::size_t>(
                "RTLogging_MaxMessageBufferSize",
                16 * 1024 * 1024,
                "Max number of bytes that can be logged in the control thread");
 
            defineOptionalProperty<bool>(
                "RTLogging_EnableBacklog",
                true,
                "Enable/Disable the backlog (SensorValues, ControlTargets, Messages) that is kept"
                "and can be dumped in case of an error.");
            defineOptionalProperty<std::size_t>("RTLogging_KeepIterationsForMs",
                                                60 * 1000,
                                                "All logging data (SensorValues, ControlTargets, "
                                                "Messages) is kept for this duration "
                                                "and can be dumped in case of an error.");
            defineOptionalProperty<std::size_t>(
                "RTLogging_MaxBacklogSize",
                5000,
                "Maximum size of the backlog (SensorValues, ControlTargets, Messages) that is kept"
                "and can be dumped in case of an error.");
 
            defineOptionalProperty<std::size_t>(
                "RTLogging_StreamingDataMaxClientConnectionFailures",
                10,
                "If sending data to a client fails this often, the client is removed from the "
                "list");
 
            defineOptionalProperty<int>(
                "RTLogging_LogLastNMessagesOnly",
                -1,
                "If greater than 0, logs/streams only the last n timesteps that are in the entry "
                "cue of the ControlThreadOutputBuffer");
        }
    };
 
    /**
     * @ingroup Library-RobotUnit-Modules
     * @brief This \ref ModuleBase "Module" manages logging of data.
     *
     * There are two types of data:
     *    - Messages: These are logged via \ref ARMARX_RT_LOGF macros and printed to the corresponding ArmarX streams
     *    - \ref SensorValueBase "SensorValues" / \ref ControlTargetBase "ControlTargets": These can be logged to CSV files
     *
     * @see ModuleBase
     */
    class Logging : virtual public ModuleBase, virtual public RobotUnitLoggingInterface
    {
        friend class ModuleBase;
 
    public:
        /**
         * @brief Returns the singleton instance of this class
         * @return The singleton instance of this class
         */
        static Logging&
        Instance()
        {
            return ModuleBase::Instance<Logging>();
        }
 
        // //////////////////////////////////////////////////////////////////////////////////////// //
        // ///////////////////////////////// RobotUnitModule hooks //////////////////////////////// //
        // //////////////////////////////////////////////////////////////////////////////////////// //
    private:
        /// @see ModuleBase::_postOnInitRobotUnit
        void _postOnInitRobotUnit();
        /// @see ModuleBase::_postFinishDeviceInitialization
        void _postFinishDeviceInitialization();
        /// @see ModuleBase::_preFinishControlThreadInitialization
        void _preFinishControlThreadInitialization();
        /// @see ModuleBase::_preFinishRunning
        void _preFinishRunning();
        // //////////////////////////////////////////////////////////////////////////////////////// //
        // ///////////////////////////////////// ice interface //////////////////////////////////// //
        // //////////////////////////////////////////////////////////////////////////////////////// //
    public:
        /**
         * @brief Starts logging to a CSV file
         * @param formatString The file to log to.
         * @param loggingNames The data fields to log.
         * @return A handle to the log. If it's last copy is deleted, logging is stopped.
         */
        SimpleRemoteReferenceCounterBasePtr
        startRtLogging(const std::string& formatString,
                       const Ice::StringSeq& loggingNames,
                       const Ice::Current& = Ice::emptyCurrent) override;
        /**
         * @brief Starts logging to a CSV file
         * @param formatString The file to log to.
         * @param aliasNames Data fields to log. For an entry (key,value), the datafield key is logged and the table heading value is used.
         * If value is empty, key is used as heading.
         * @return A handle to the log. If it's last copy is deleted, logging is stopped.
         */
        SimpleRemoteReferenceCounterBasePtr
        startRtLoggingWithAliasNames(const std::string& formatString,
                                     const StringStringDictionary& aliasNames,
                                     const Ice::Current& = Ice::emptyCurrent) override;
 
        /**
         * @brief Stops logging to the given log.
         * @param token The log to close.
         */
        void stopRtLogging(const armarx::SimpleRemoteReferenceCounterBasePtr& token,
                           const Ice::Current& = Ice::emptyCurrent) override;
        /**
         * @brief Adds a string to the log (it is added in a special column).
         * @param token The log.
         * @param marker The string to add.
         */
        void addMarkerToRtLog(const SimpleRemoteReferenceCounterBasePtr& token,
                              const std::string& marker,
                              const Ice::Current& = Ice::emptyCurrent) override;
 
        /**
         * @brief Returns the names of all loggable data fields.
         * @return The names of all loggable data fields.
         */
        Ice::StringSeq getLoggingNames(const Ice::Current& = Ice::emptyCurrent) const override;
        /**
          * @brief Dumps the backlog of all recent iterations to the given file.
          * This helps debugging.
          * @param formatString The file.
          */
        void writeRecentIterationsToFile(const std::string& formatString,
                                         const Ice::Current& = Ice::emptyCurrent) const override;
 
        RobotUnitDataStreaming::DataStreamingDescription
        startDataStreaming(const RobotUnitDataStreaming::ReceiverPrx& receiver,
                           const RobotUnitDataStreaming::Config& config,
                           const Ice::Current& = Ice::emptyCurrent) override;
 
        void stopDataStreaming(const RobotUnitDataStreaming::ReceiverPrx& receiver,
                               const Ice::Current& = Ice::emptyCurrent) override;
        // //////////////////////////////////////////////////////////////////////////////////////// //
        // //////////////////////////////////// implementation //////////////////////////////////// //
        // //////////////////////////////////////////////////////////////////////////////////////// //
    private:
        /// @brief Executes the logging loop.
        void doLogging();
        void doLogging(details::DoLoggingDurations& durations,
                       const IceUtil::Time& now,
                       const ControlThreadOutputBuffer::Entry& data,
                       std::size_t i,
                       std::size_t num);
        static bool MatchName(const std::string& pattern, const std::string& name);
 
        // //////////////////////////////////////////////////////////////////////////////////////// //
        // ///////////////////////////////////////// Data ///////////////////////////////////////// //
        // //////////////////////////////////////////////////////////////////////////////////////// //
    private:
        /// @brief Data structure holding information about an logging entry.
        struct CSVLoggingEntry
        {
            /// @brief Whether logging should be stopped.
            std::atomic_bool stopLogging{false};
            /// @brief Bools whether a control device field should be logged.
            std::vector<std::vector<std::vector<char>>> loggedControlDeviceValues;
            /// @brief Bools whether a sensor device field should be logged.
            std::vector<std::vector<char>> loggedSensorDeviceValues;
            /// @brief The file stream for the csv file
            std::ofstream stream;
            /// @brief The file name
            std::string filename;
            /// @brief The marker to add in the next logging iteration. (protected by \ref rtLoggingMutex)
            std::string marker;
        };
 
        struct DataStreamingEntry
        {
            enum class ValueT
            {
                Bool,
                Byte,
                Short,
                Int,
                Long,
                Float,
                Double,
                Skipped
            };
 
            struct OutVal
            {
                std::size_t idx;
                ValueT value = ValueT::Skipped;
            };
 
            bool stopStreaming = false;
 
            std::size_t numBools = 0;
            std::size_t numBytes = 0;
            std::size_t numShorts = 0;
            std::size_t numInts = 0;
            std::size_t numLongs = 0;
            std::size_t numFloats = 0;
            std::size_t numDoubles = 0;
 
            bool onlyNewestFrame = false;
            std::size_t connectionFailures = 0;
            std::size_t rtStreamMaxClientErrors = 0;
 
            std::vector<std::vector<OutVal>> sensDevs;
            std::vector<std::vector<std::vector<OutVal>>> ctrlDevs;
 
            RobotUnitDataStreaming::TimeStepSeq result;
            std::deque<RobotUnitDataStreaming::TimeStep> entryBuffer;
            std::deque<Ice::AsyncResultPtr> updateCalls;
            void clearResult();
            RobotUnitDataStreaming::TimeStep getResultElement();
            RobotUnitDataStreaming::TimeStep allocateResultElement() const;
 
 
            void processHeader(const ControlThreadOutputBuffer::Entry& e);
            void processCtrl(const ControlTargetBase& val,
                             std::size_t didx,
                             std::size_t vidx,
                             std::size_t fidx);
            void processSens(const SensorValueBase& val, std::size_t didx, std::size_t fidx);
            void send(const RobotUnitDataStreaming::ReceiverPrx& r, uint64_t msgId);
        };
 
        std::map<RobotUnitDataStreaming::ReceiverPrx, DataStreamingEntry> rtDataStreamingEntry;
        std::uint64_t rtDataStreamingMsgID = 0;
 
        /// @brief The thread executing the logging functions.
        std::thread rtLoggingTask;
        std::atomic_bool stopRtLoggingTask{false};
        /// @brief All entries for logging.
        std::map<std::string, std::shared_ptr<CSVLoggingEntry>> rtLoggingEntries;
        /// @brief Handle for the default log. (This log can be enabled via a property and logs everything)
        SimpleRemoteReferenceCounterBasePtr defaultLogHandle;
        /// @brief The stored backlog (it is dumped when \ref writeRecentIterationsToFile is called)
        std::deque<::armarx::detail::ControlThreadOutputBufferEntry> backlog;
        /// @brief The control threads unix thread id
        Ice::Int controlThreadId{0};
        /// @brief Mutex protecting the data structures of this class
        mutable std::mutex rtLoggingMutex;
 
        struct ValueMetaData
        {
            struct FieldMetaData
            {
                std::string name;
                const std::type_info* type;
            };
 
            std::vector<FieldMetaData> fields;
        };
 
        /// @brief Data field info for all \ref ControlTargetBase "ControlTargets" (dex x jctrl)
        std::vector<std::vector<ValueMetaData>> controlDeviceValueMetaData;
        /// @brief Data field info for all \ref SensorValueBase "SensorValues" (dev)
        std::vector<ValueMetaData> sensorDeviceValueMetaData;
 
        /// @brief The initial size of the Message entry buffer
        std::size_t messageBufferSize{0};
        /// @brief The initial number of Message entries
        std::size_t messageBufferNumberEntries{0};
        /// @brief The maximal size of the Message entry buffer
        std::size_t messageBufferMaxSize{0};
        /// @brief The maximal number of Message entries
        std::size_t messageBufferMaxNumberEntries{0};
        /// @brief The logging thread's period
        std::size_t rtLoggingTimestepMs{0};
        /// @brief The time an entry should remain in the backlog.
        IceUtil::Time rtLoggingBacklogRetentionTime;
        /// @brief The maximum number of entries in the backlog.
        std::size_t rtLoggingBacklogMaxSize;
        /// @brief Enable/disable the backlog.
        bool rtLoggingBacklogEnabled;
 
        /// @brief
        int numberOfEntriesToLog{-1};
 
        friend void WriteTo(const auto& dentr,
                            const Logging::DataStreamingEntry::OutVal& out,
                            const auto& val,
                            std::size_t fidx,
                            auto& data);
    };
} // namespace armarx::RobotUnitModule