ErrorReporting.cpp

Go to the documentation of this file.

#include "ErrorReporting.h"
 
#include <iomanip>
 
#include <boost/lockfree/policies.hpp>
#include <boost/lockfree/queue.hpp>
 
#include <RobotAPI/components/units/RobotUnit/util/RtLogging.h>
#include <RobotAPI/components/units/RobotUnit/util/RtTiming.h>
 
namespace armarx::control::ethercat::reporting
{
    std::ostream&
    operator<<(std::ostream& stream, const Type& rhs)
    {
        switch (rhs)
        {
            case Type::Bus:
                stream << "Bus";
                break;
            case Type::Slave:
                stream << "Slave";
                break;
            case Type::Device:
                stream << "Device";
                break;
            case Type::General:
                stream << "General";
                break;
        }
        return stream;
    }
 
    std::ostream&
    operator<<(std::ostream& stream, const Severity& rhs)
    {
        switch (rhs)
        {
            case Severity::Info:
                stream << "INFO";
                break;
            case Severity::Debug:
                stream << "DEBUG";
                break;
            case Severity::Warning:
                stream << "WARNING";
                break;
            case Severity::Error:
                stream << "ERROR";
                break;
            case Severity::Fatal:
                stream << "FATAL";
                break;
        }
        return stream;
    }
 
    struct Reporting::QueueImpl
    {
        static constexpr size_t queueSize = 100;
 
        boost::lockfree::queue<Entry, boost::lockfree::capacity<queueSize>> queue;
    };
 
    Reporting::Reporting(std::size_t maxErrorHistorySize) :
        pimpl(std::make_unique<QueueImpl>()), m_maxErrorHistorySize(maxErrorHistorySize)
    {
        setTag("ethercat::Reporting");
 
        m_timestampReportingStart = armarx::rtNow();
        m_errorProcessingThread = std::thread(&Reporting::errorProcessingLoop, this);
    }
 
    Reporting::~Reporting()
    {
        m_errorProcessingRunning.store(false);
        m_errorProcessingCondition.notify_all();
        m_errorProcessingThread.join();
    }
 
    Reporting&
    Reporting::getErrorReporting()
    {
        static Reporting _instance;
        return _instance;
    }
 
    Reporter
    Reporting::getErrorReporter()
    {
        return Reporter(this);
    }
 
    Reporter&
    Reporting::getGlobalErrorReporter()
    {
        static Reporter _instance = Reporter(&getErrorReporting());
        return _instance;
    }
 
    void
    Reporting::report(Entry&& error)
    {
        error.m_timestamp = armarx::rtNow();
        bool result = pimpl->queue.push(error);
        if (not result)
        {
            ARMARX_RT_LOGF_WARNING("Logging queue is full!").deactivateSpam(0.1);
        }
        m_errorProcessingCondition.notify_all();
    }
 
    void
    Reporting::setMaxErrorHistorySize(std::size_t maxErrorHistorySize)
    {
        m_maxErrorHistorySize = maxErrorHistorySize;
    }
 
    std::optional<std::filesystem::path>
    Reporting::dumpToFile(std::filesystem::path file, std::uint64_t lastNSeconds)
    {
        const auto endTime = armarx::rtNow();
        IceUtil::Time startTime =
            endTime - IceUtil::Time::seconds(static_cast<std::int64_t>(lastNSeconds));
        if (startTime < m_timestampReportingStart)
        {
            startTime = m_timestampReportingStart;
        }
        const std::string yearString = startTime.toString("%y-%m-%d");
        const std::string startTimeString = startTime.toString("%H-%M-%S");
        const std::string endTimeString = endTime.toString("%H-%M-%S");
 
        file.replace_filename(std::string(file.stem().c_str()) + "_" + yearString + "__" +
                              startTimeString + "_" + endTimeString);
        file.replace_extension(".errors");
 
        if (std::filesystem::exists(file))
        {
            file.replace_filename(file.filename().string() + ".new");
        }
 
        auto history = m_errorHistory;
        auto it = history.begin();
        for (const auto& [key, value] : history)
        {
            if (key.timestamp < startTime.toMicroSeconds())
            {
                it++;
            }
        }
        history.erase(history.begin(), it);
 
        if (history.size() > 0)
        {
            ARMARX_INFO << history.size() << " errors from " << startTimeString << " to "
                        << endTimeString << " will be saved to file " << file.string();
        }
        else
        {
            ARMARX_VERBOSE << "No errors happend from " << startTimeString << " to "
                           << endTimeString;
            return std::nullopt;
        }
 
 
        if (!std::filesystem::exists(file.parent_path()) &&
            !std::filesystem::create_directories(file.parent_path()))
        {
            throw std::runtime_error{"Could not create directories for '" + file.string() + "'"};
        }
 
        std::ofstream outFile{file.string(), std::ofstream::out};
        if (!outFile)
        {
            throw std::runtime_error{"dumpToFile could not open filestream for '" + file.string() +
                                     "'"};
        }
 
        for (const auto& [key, value] : history)
        {
 
            std::string timeStr = value.m_timestamp.toDateTime();
            timeStr = timeStr.substr(timeStr.find(' ') + 1);
 
            outFile << "[" << timeStr << "]";
            outFile << "[" << value.m_type << "]";
            outFile << "[" << value.m_severity << "] ";
            outFile << std::string(value.m_message, value.m_message_size) << "\n";
            if (value.m_sid.slaveIndex != -1)
            {
                outFile << value.m_sid.toString("          ") << "\n";
            }
        }
 
        return std::move(file);
    }
 
    void
    Reporting::errorProcessingLoop()
    {
        Entry e;
        auto handleEntries = [&, this]()
        {
            while (pimpl->queue.pop(e))
            {
                const std::int64_t timestamp = e.m_timestamp.toMicroSeconds();
                std::uint16_t i = 0;
 
                // Check for max size of history
                while (m_errorHistory.size() >= m_maxErrorHistorySize)
                {
                    m_errorHistory.erase(m_errorHistory.begin());
                }
 
                // Check if there is already an entry with the same timestamp and find correct subindex
                if (m_errorHistory.begin()->first.timestamp == timestamp)
                {
                    i = m_errorHistory.begin()->first.subid + 1;
                }
                while (m_errorHistory.count({timestamp, i}))
                {
                    i++;
                }
 
                m_errorHistory.insert({{timestamp, i}, e});
 
                auto logpointer =
                    (loghelper(e.m_metaInfo.file, e.m_metaInfo.line, e.m_metaInfo.function));
 
                std::string const identifier = e.getIdentifier();
 
                switch (e.m_severity)
                {
                    case reporting::Severity::Info:
                        *logpointer->setBacktrace(false)
                            << ::armarx::MessageTypeT::INFO
                            << deactivateSpam(e.m_deactivate_spam_seconds, identifier)
                            << e.toString();
                        return;
                    case reporting::Severity::Debug:
                        *logpointer->setBacktrace(false)
                            << ::armarx::MessageTypeT::DEBUG
                            << deactivateSpam(e.m_deactivate_spam_seconds, identifier)
                            << e.toString();
                        return;
                    case reporting::Severity::Warning:
                        *logpointer->setBacktrace(false)
                            << ::armarx::MessageTypeT::WARN
                            << deactivateSpam(e.m_deactivate_spam_seconds, identifier)
                            << e.toString();
                        return;
                    case reporting::Severity::Error:
                        *logpointer->setBacktrace(false)
                            << ::armarx::MessageTypeT::ERROR
                            << deactivateSpam(e.m_deactivate_spam_seconds, identifier)
                            << e.toString();
                        return;
                    case reporting::Severity::Fatal:
                        *logpointer << ::armarx::MessageTypeT::FATAL
                                    << deactivateSpam(e.m_deactivate_spam_seconds, identifier)
                                    << e.toString();
                        return;
                }
            }
        };
 
        while (m_errorProcessingRunning.load() == true)
        {
            std::unique_lock<std::mutex> lock{m_errorProcessingConditionMutex};
            m_errorProcessingCondition.wait(
                lock, [&]() { return !pimpl->queue.empty() || !m_errorProcessingRunning.load(); });
            handleEntries();
        }
 
        // Also handle leftover entries
        handleEntries();
    }
 
    Reporter::ReportWrapper
    Reporter::reportDebug(Entry& info)
    {
        info.m_severity = Severity::Debug;
        return Reporter::ReportWrapper{info, this};
    }
 
    Reporter::ReportWrapper
    Reporter::reportInfo(Entry& info)
    {
        info.m_severity = Severity::Info;
        return Reporter::ReportWrapper{info, this};
    }
 
    Reporter::ReportWrapper
    Reporter::reportWarning(Entry& error)
    {
        error.m_severity = Severity::Warning;
        m_errorCount++;
        return Reporter::ReportWrapper{error, this};
    }
 
    Reporter::ReportWrapper
    Reporter::reportError(Entry& error)
    {
        error.m_severity = Severity::Error;
        m_errorCount++;
        return Reporter::ReportWrapper{error, this};
    }
 
    void
    Reporter::reportErrorAndThrow(Entry& error)
    {
        error.m_severity = Severity::Fatal;
        m_errorCount++;
        m_errorReporting->report(std::move(error));
        throw std::runtime_error(error.toString());
    }
 
    bool
    Reporter::hasErrors() const
    {
        return m_errorCount > 0;
    }
 
    Reporter::Reporter(Reporting* errorReporting) : m_errorReporting(errorReporting)
    {
    }
 
    Entry&
    Entry::slaveIdentifier(SlaveIdentifier sid)
    {
        m_sid = sid;
        return *this;
    }
 
    Entry&
    Entry::errorType(Type type)
    {
        m_type = type;
        return *this;
    }
 
    Entry&
    Entry::metaInfo(const char* file, int line, const char* function)
    {
        snprintf(m_metaInfo.file, sizeof(m_metaInfo.file), "%s", file);
        m_metaInfo.line = line;
        snprintf(m_metaInfo.function, sizeof(m_metaInfo.function), "%s", function);
        return *this;
    }
 
    Entry&
    Entry::deviceName(const char* deviceName)
    {
        snprintf(m_device_name, sizeof(m_device_name), "%s", deviceName);
        return *this;
    }
 
    std::string
    Entry::toString() const
    {
        std::stringstream ss;
        ss << "[" << m_type << "] ";
        ss << std::string(m_message, m_message_size) << "\n";
        if (m_sid.slaveIndex != -1)
        {
            ss << m_sid.toString("    ");
        }
        if (m_device_name[0] != '\0')
        {
            ss << "    " << std::left << std::setw(16) << "Name: " << std::string(m_device_name);
        }
        if (m_bus_iteration_number != 0)
        {
            ss << "    " << std::left << std::setw(16)
               << "Bus Iteration: " << std::to_string(m_bus_iteration_number);
        }
        ss << "\n";
        return ss.str();
    }
 
    std::string
    Entry::getIdentifier() const
    {
        if (m_identifier_size > 0)
        {
            return m_identifier;
        }
        else
        {
            return std::string{m_metaInfo.file} + "/" + std::string{m_metaInfo.function} + "/" +
                   std::to_string(m_metaInfo.line);
        }
    }
 
    Entry&
    Entry::busIterationNumber(std::uint64_t iteration)
    {
        m_bus_iteration_number = iteration;
        return *this;
    }
 
    Reporter::ReportWrapper::ReportWrapper(Entry& error, Reporter* reporter) :
        m_error(error), m_reporter(reporter)
    {
    }
 
    Reporter::ReportWrapper::~ReportWrapper()
    {
        m_reporter->m_errorReporting->report(std::move(m_error));
    }
 
    Reporter::ReportWrapper&
    Reporter::ReportWrapper::deactivateSpam(float seconds)
    {
        m_error.m_deactivate_spam_seconds = std::fmax(0.f, seconds);
        return *this;
    }
 
} // namespace armarx::control::ethercat::reporting