DataView.cpp

Go to the documentation of this file.

 
#include "DataView.h"
 
#include <QApplication>
#include <QClipboard>
#include <QHBoxLayout>
#include <QHeaderView>
#include <QLabel>
#include <QSplitter>
#include <QTreeWidget>
 
#include <SimoxUtility/color/cmaps.h>
#include <SimoxUtility/math/SoftMinMax.h>
 
#include "RobotAPI/libraries/armem_gui/view/instance_group/instance/tree_visitors/TreeTypedJSONConverter.h"
#include <RobotAPI/libraries/armem/aron/MemoryID.aron.generated.h>
#include <RobotAPI/libraries/armem/core/aron_conversions.h>
#include <RobotAPI/libraries/armem_gui/util/gui_utils.h>
#include <RobotAPI/libraries/armem_gui/view/instance_group/instance/ImageView.h>
#include <RobotAPI/libraries/armem_gui/view/instance_group/instance/WidgetsWithToolbar.h>
#include <RobotAPI/libraries/armem_gui/view/instance_group/instance/sanitize_typename.h>
#include <RobotAPI/libraries/armem_gui/view/instance_group/instance/serialize_path.h>
#include <RobotAPI/libraries/armem_gui/view/instance_group/instance/tree_builders/DataTreeBuilder.h>
#include <RobotAPI/libraries/armem_gui/view/instance_group/instance/tree_builders/TypedDataTreeBuilder.h>
#include <RobotAPI/libraries/aron/converter/json/NLohmannJSONConverter.h>
#include <RobotAPI/libraries/aron/core/data/variant/complex/NDArray.h>
 
namespace armarx::armem::gui::instance
{
    DataView::DataView() :
        splitter(new QSplitter(Qt::Orientation::Vertical)), tree(new QTreeWidget(this))
    {
        Logging::setTag("DataView");
 
        QLayout* layout = new QVBoxLayout();
        this->setLayout(layout);
        int margin = 3;
        layout->setContentsMargins(margin, margin, margin, margin);
 
        layout->addWidget(splitter);
 
        splitter->addWidget(tree);
 
        QStringList columns;
        columns.insert(int(Columns::KEY), "Key");
        columns.insert(int(Columns::VALUE), "Value");
        columns.insert(int(Columns::TYPE), "Type");
        tree->setColumnCount(columns.size());
        tree->setHeaderLabels(columns);
 
        tree->header()->resizeSection(int(Columns::KEY), 250);
        tree->header()->resizeSection(int(Columns::VALUE), 250);
 
        treeItemData = new QTreeWidgetItem({"Data"});
        tree->addTopLevelItem(treeItemData);
        treeItemData->setExpanded(true);
        tree->setContextMenuPolicy(Qt::CustomContextMenu);
        connect(tree,
                &QTreeWidget::customContextMenuRequested,
                this,
                &DataView::prepareTreeContextMenu);
    }
 
    void
    DataView::setStatusLabel(QLabel* statusLabel)
    {
        this->statusLabel = statusLabel;
    }
 
    void
    DataView::setUseTypeInfo(bool enable)
    {
        this->useTypeInfo = enable;
        update();
        emit useTypeInfoChanged(enable);
    }
 
    void
    DataView::addDataView(DataView* dataView)
    {
        // ARMARX_IMPORTANT << "Adding instance view with toolbar for instance: " << instance.id();
        dataView->setStatusLabel(statusLabel);
        dataView->setUseTypeInfo(useTypeInfo);
 
        auto* child = new WidgetsWithToolbar();
        child->addWidget(dataView);
 
 
        splitter->addWidget(child);
 
        // Propagate these signals upwards.
        connect(dataView,
                &DataView::memoryIdResolutionRequested,
                this,
                &DataView::memoryIdResolutionRequested);
        connect(dataView, &DataView::actionsMenuRequested, this, &DataView::actionsMenuRequested);
        connect(this, &DataView::useTypeInfoChanged, dataView, &DataView::setUseTypeInfo);
    }
 
    void
    DataView::updateData(const aron::data::DictPtr& data, aron::type::ObjectPtr aronType)
    {
        if (!data)
        {
            treeItemData->setText(int(Columns::TYPE), QString::fromStdString(""));
 
            armarx::gui::clearItem(treeItemData);
            QTreeWidgetItem* item = new QTreeWidgetItem({"(No data.)"});
            treeItemData->addChild(item);
        }
        else if (useTypeInfo && aronType)
        {
            treeItemData->setText(
                int(Columns::TYPE),
                QString::fromStdString(sanitizeTypeName(aronType->getFullName())));
 
            TypedDataTreeBuilder builder;
            builder.setColumns(int(Columns::KEY), int(Columns::VALUE), int(Columns::TYPE));
            builder.updateTree(treeItemData, *aronType, *data);
        }
        else
        {
            treeItemData->setText(int(Columns::TYPE), QString::fromStdString(""));
 
            DataTreeBuilder builder;
            builder.setColumns(int(Columns::KEY), int(Columns::VALUE), int(Columns::TYPE));
            builder.updateTree(treeItemData, data);
        }
        treeItemData->setExpanded(true);
    }
 
    void
    DataView::showErrorMessage(const std::string& message)
    {
        if (statusLabel)
        {
            statusLabel->setText(QString::fromStdString(message));
        }
    }
 
    std::optional<aron::Path>
    DataView::getElementPath(const QTreeWidgetItem* item)
    {
        QStringList qpath = item->data(int(Columns::KEY), Qt::UserRole).toStringList();
        if (qpath.empty())
        {
            return std::nullopt;
        }
        else
        {
            aron::Path path = deserializePath(qpath);
            return path;
        }
    }
 
    std::optional<MemoryID>
    DataView::getElementMemoryID(const aron::Path& elementPath)
    {
        aron::data::DictPtr data = getData();
        if (!data)
        {
            showErrorMessage("Cannot get Memory ID for null element.");
            return std::nullopt;
        }
 
        aron::data::VariantPtr element;
        try
        {
            element = data->navigateAbsolute(elementPath);
        }
        // This can happen when the underlying entity structure changes (a new entity has been selected).
        catch (const aron::error::AronException&)
        {
            // showErrorMessage(e.what());
            return std::nullopt;
        }
        catch (const armarx::LocalException& e)
        {
            showErrorMessage(e.what());
            return std::nullopt;
        }
 
        std::stringstream couldNotParseMsg;
        couldNotParseMsg << "Element " << elementPath.toString()
                         << " could not be parsed as MemoryID.";
 
        auto dictElement = std::dynamic_pointer_cast<aron::data::Dict>(element);
        if (!dictElement)
        {
            showErrorMessage(couldNotParseMsg.str() + " (Failed to cast to DictNavigator.)");
            return std::nullopt;
        }
 
        try
        {
            arondto::MemoryID dto;
            dto.fromAron(dictElement);
 
            MemoryID id;
            armem::fromAron(dto, id);
            return id;
        }
        catch (const armarx::aron::error::AronException&)
        {
            showErrorMessage(couldNotParseMsg.str());
            return std::nullopt;
        }
    }
 
    QAction*
    DataView::makeActionResolveMemoryID(const MemoryID& id)
    {
        auto* action = new QAction("Resolve memory ID");
 
        if (not(id.hasEntityName() and id.isWellDefined()))
        {
            action->setDisabled(true);
            action->setText(action->text() + " (incomplete Memory ID)");
        }
        connect(action,
                &QAction::triggered,
                [this, id]()
                {
                    // ARMARX_IMPORTANT << "emit memoryIdResolutionRequested(id = " << id << ")";
                    emit memoryIdResolutionRequested(id);
                });
 
        return action;
    }
 
    QAction*
    DataView::makeActionCopyMemoryID(const MemoryID& id)
    {
        QAction* action = new QAction("Copy memory ID to clipboard");
 
        connect(action,
                &QAction::triggered,
                [/*this,*/ id]() // `this` for ARMARX_IMPORTANT
                {
                    const QString idStr = QString::fromStdString(id.str());
 
                    // ARMARX_IMPORTANT << "Copy '" << idStr.toStdString() << "' to clipboard.";
                    QClipboard* clipboard = QApplication::clipboard();
                    clipboard->setText(idStr);
                    QApplication::processEvents();
                });
 
        return action;
    }
 
    std::vector<QAction*>
    DataView::makeActionsCopyDataToClipboard()
    {
        auto data = getData();
        if (!data)
        {
            return {};
        }
        return makeCopyActions(data, currentAronType);
    }
 
    std::vector<QAction*>
    DataView::makeActionsCopyDataToClipboard(const aron::Path& path)
    {
        auto data = getData();
        if (!data)
        {
            return {};
        }
        try
        {
            aron::data::VariantPtr element = data->navigateAbsolute(path);
            aron::type::VariantPtr elementType = nullptr;
            if (currentAronType)
            {
                // There doesn't seem to be a way to check whether the path exists
                // without potentially throwing an exception.
                try
                {
                    elementType = currentAronType->navigateAbsolute(path);
                }
                catch (const aron::error::AronException& e)
                {
                    // No type available, elementType remains nullptr.
                }
            }
            return makeCopyActions(element, elementType);
        }
        catch (const aron::error::AronException& e)
        {
            ARMARX_WARNING << "Could not convert Aron data to JSON: " << e.getReason();
        }
        return {};
    }
 
    std::vector<QAction*>
    DataView::makeCopyActions(const aron::data::VariantPtr& element,
                              const aron::type::VariantPtr& elementType)
    {
        auto* easyJsonAction = new QAction("Copy data to clipboard as easy JSON");
        connect(easyJsonAction,
                &QAction::triggered,
                [this, element, elementType]()
                {
                    try
                    {
                        TreeTypedJSONConverter conv;
                        armarx::aron::data::visitRecursive(conv, element, elementType);
                        QClipboard* clipboard = QApplication::clipboard();
                        clipboard->setText(QString::fromStdString(conv.getJSON().dump(2)));
                        QApplication::processEvents();
                    }
                    catch (const aron::error::AronException& e)
                    {
                        ARMARX_WARNING << "Could not convert Aron data to JSON: " << e.getReason();
                    }
                });
 
        auto* aronJsonAction = new QAction("Copy data to clipboard as aron JSON");
        connect(aronJsonAction,
                &QAction::triggered,
                [this, element]()
                {
                    try
                    {
                        nlohmann::json json =
                            aron::data::converter::AronNlohmannJSONConverter::ConvertToNlohmannJSON(
                                element);
                        QClipboard* clipboard = QApplication::clipboard();
                        clipboard->setText(QString::fromStdString(json.dump(2)));
                        QApplication::processEvents();
                    }
                    catch (const aron::error::AronException& e)
                    {
                        ARMARX_WARNING << "Could not convert Aron data to JSON: " << e.getReason();
                    }
                });
 
        return {easyJsonAction, aronJsonAction};
    }
 
    QMenu*
    DataView::buildActionsMenu(const QPoint& pos)
    {
        QMenu* menu = new QMenu(this);
 
        const QTreeWidgetItem* item = tree->itemAt(pos);
        if (item == nullptr)
        {
            return menu; // Nothing was clicked on.
        }
 
        if (item == this->treeItemData && getData() != nullptr)
        {
            auto actions = makeActionsCopyDataToClipboard();
            for (const auto& action : actions)
            {
                if (action)
                {
                    menu->addAction(action);
                }
            }
        }
 
        aron::type::Descriptor type = static_cast<aron::type::Descriptor>(
            item->data(int(Columns::TYPE), Qt::UserRole).toInt());
        switch (type)
        {
            case aron::type::Descriptor::IMAGE:
            {
                if (const std::optional<aron::Path> path = getElementPath(item))
                {
                    QAction* viewAction = new QAction("Show image");
                    menu->addAction(viewAction);
                    connect(viewAction,
                            &QAction::triggered,
                            [this, path]() { this->showImageView(path.value()); });
 
                    try
                    {
                        aron::data::VariantPtr element =
                            getData() != nullptr ? getData()->navigateAbsolute(path.value())
                                                 : nullptr;
                        if (auto imageData = aron::data::NDArray::DynamicCast(element))
                        {
                            const std::vector<int> shape = imageData->getShape();
                            if (std::find(shape.begin(), shape.end(), 0) != shape.end())
                            {
                                viewAction->setText(viewAction->text() + " (image is empty)");
                                viewAction->setEnabled(false);
                            }
                        }
                    }
                    catch (const aron::error::AronException&)
                    {
                    }
                    catch (const armarx::LocalException&)
                    {
                    }
                }
            }
            break;
            default:
                break;
        }
 
        // Type name based actions
        const std::string typeName = item->text(int(Columns::TYPE)).toStdString();
        if (typeName == instance::sanitizedMemoryIDTypeName)
        {
            if (const std::optional<aron::Path> path = getElementPath(item))
            {
                if (std::optional<MemoryID> id = getElementMemoryID(path.value()))
                {
                    if (QAction* action = makeActionCopyMemoryID(id.value()))
                    {
                        menu->addAction(action);
                    }
                    if (QAction* action = makeActionResolveMemoryID(id.value()))
                    {
                        menu->addAction(action);
                    }
                }
            }
        }
 
        const std::optional<aron::Path> elementPath = getElementPath(item);
        if (elementPath)
        {
            auto actions = makeActionsCopyDataToClipboard(elementPath.value());
            for (const auto& action : actions)
            {
                if (action)
                {
                    menu->addAction(action);
                }
            }
        }
        return menu;
    }
 
    void
    DataView::prepareTreeContextMenu(const QPoint& pos)
    {
        auto* menu = buildActionsMenu(pos);
 
        if (menu->actions().isEmpty())
        {
            emit actionsMenuRequested(MemoryID(), this, tree->mapToGlobal(pos), nullptr);
        }
        else
        {
            emit actionsMenuRequested(MemoryID(), this, tree->mapToGlobal(pos), menu);
        }
    }
 
    void
    DataView::showImageView(const aron::Path& elementPath)
    {
        auto data = getData();
        if (!data)
        {
            return;
        }
        if (!imageView)
        {
            WidgetsWithToolbar* toolbar = new WidgetsWithToolbar();
 
            imageView = new ImageView();
            imageView->toolbar = toolbar;
            toolbar->addWidget(imageView);
 
            splitter->addWidget(toolbar);
 
            connect(toolbar, &WidgetsWithToolbar::closing, [this]() { imageView = nullptr; });
        }
        imageView->elementPath = elementPath;
        updateImageView(data);
    }
 
    void
    DataView::removeImageView()
    {
        imageView->toolbar->close();
        imageView = nullptr;
    }
 
    QImage
    DataView::ImageView::convertDepth32ToRGB32(const aron::data::NDArray& aron)
    {
        const std::vector<int> shape = aron.getShape();
        ARMARX_CHECK_EQUAL(shape.size(), 3);
        ARMARX_CHECK_EQUAL(shape.at(2), 4) << "Expected Depth32 image to have 4 bytes per pixel.";
 
        const int rows = shape.at(0);
        const int cols = shape.at(1);
 
        // Rendering seems to be optimized for RGB32
        // rows go along 0 = height, cols go along 1 = width
        QImage image(cols, rows, QImage::Format::Format_RGB32);
        const float* data = reinterpret_cast<float*>(aron.getData());
 
        auto updateLimits = [](float value, Limits& limits)
        {
            if (value > 0) // Exclude 0 from normalization (it may be only background)
            {
                limits.min = std::min(limits.min, value);
            }
            limits.max = std::max(limits.max, value);
        };
 
        // Find data range and adapt cmap.
        Limits limits;
        if (limitsHistory.empty())
        {
            const float* sourceRow = data;
            for (int row = 0; row < rows; ++row)
            {
                for (int col = 0; col < cols; ++col)
                {
                    float value = sourceRow[col];
                    updateLimits(value, limits);
                }
                sourceRow += cols;
            }
            cmap.set_vlimits(limits.min, limits.max);
        }
        // Only do it at the beginning and stop after enough samples were collected.
        else if (limitsHistory.size() < limitsHistoryMaxSize)
        {
            simox::math::SoftMinMax softMin(0.25, limitsHistory.size());
            simox::math::SoftMinMax softMax(0.25, limitsHistory.size());
 
            for (auto& l : limitsHistory)
            {
                softMin.add(l.min);
                softMax.add(l.max);
            }
 
            cmap.set_vlimits(softMin.getSoftMin(), softMax.getSoftMax());
        }
 
        // Update image
        {
            const float* sourceRow = data;
 
            const int bytesPerLine = image.bytesPerLine();
            uchar* targetRow = image.bits();
 
            for (int row = 0; row < rows; ++row)
            {
                for (int col = 0; col < cols; ++col)
                {
                    float value = sourceRow[col];
                    simox::Color color = value <= 0 ? simox::Color::white() : cmap(value);
                    targetRow[col * 4 + 0] = color.b;
                    targetRow[col * 4 + 1] = color.g;
                    targetRow[col * 4 + 2] = color.r;
                    targetRow[col * 4 + 3] = color.a;
 
                    updateLimits(value, limits);
                }
                sourceRow += cols;
                targetRow += bytesPerLine;
            }
        }
        if (limitsHistory.size() < limitsHistoryMaxSize)
        {
            limitsHistory.push_back(limits);
        }
 
        return image;
    }
 
    void
    DataView::updateImageView(const aron::data::DictPtr& data)
    {
        using aron::data::NDArray;
 
        if (not imageView)
        {
            return;
        }
        if (not data)
        {
            removeImageView();
            return;
        }
 
        aron::data::VariantPtr element;
        try
        {
            element = data->navigateAbsolute(imageView->elementPath);
        }
        // This can happen when the underlying entity structure changes (a new entity has been selected).
        // In this case, we disable the image view.
        catch (const aron::error::AronException&)
        {
            // showErrorMessage(e.what());
            removeImageView();
            return;
        }
        catch (const armarx::LocalException&)
        {
            // showErrorMessage(e.what());
            removeImageView();
            return;
        }
 
        NDArray::PointerType imageData = NDArray::DynamicCast(element);
        if (not imageData)
        {
            showErrorMessage("Expected NDArrayNavigator, but got: " +
                             simox::meta::get_type_name(element));
            return;
        }
 
        const std::vector<int> shape = imageData->getShape();
        if (shape.size() != 3)
        {
            showErrorMessage("Expected array shape with 3 dimensions, but got: " +
                             NDArray::DimensionsToString(shape));
            return;
        }
        const int rows = shape.at(0);
        const int cols = shape.at(1);
 
        using aron::type::image::PixelType;
        std::optional<PixelType> pixelType;
        try
        {
            // TODO We cannot know what the str in the pixeltype belongs to (e.g. coming from java, python, c++ it may contain different values!
            // pixelType = aron::type::Image::pixelTypeFromName(imageData->getType());
 
            // For now we assume it comes from c++ where '5' means CV_32FC1 (=5)
            pixelType = (imageData->getType() == "5" ? PixelType::DEPTH32 : PixelType::RGB24);
        }
        catch (const aron::error::AronException&)
        {
        }
 
        bool clearLimitsHistory = true;
        std::optional<QImage> image;
        if (pixelType)
        {
            switch (pixelType.value())
            {
                case PixelType::RGB24:
                    ARMARX_CHECK_EQUAL(shape.at(2), 3)
                        << "Expected Rgb24 image to have 3 bytes per pixel.";
                    image = QImage(imageData->getData(), cols, rows, QImage::Format::Format_RGB888);
                    break;
 
                case PixelType::DEPTH32:
                    image = imageView->convertDepth32ToRGB32(*imageData);
                    clearLimitsHistory = false;
                    break;
            }
        }
        else
        {
            QImage::Format format = QImage::Format_Invalid;
            switch (shape.at(2))
            {
                case 1:
                    format = QImage::Format::Format_Grayscale8;
                    break;
 
                case 3:
                    format = QImage::Format::Format_RGB888;
                    break;
 
                default:
                    showErrorMessage("Expected 1 or 3 elements in last dimension, but got shape: " +
                                     NDArray::DimensionsToString(shape));
                    return;
            }
            image = QImage(imageData->getData(), cols, rows, format);
        }
 
        ARMARX_CHECK(image.has_value());
 
        std::stringstream title;
        title << "Image element '" << imageView->elementPath.toString()
              << "'"; // of entity instance " << currentInstance->id();
        imageView->setTitle(QString::fromStdString(title.str()));
        imageView->view->setImage(image.value());
 
        if (clearLimitsHistory)
        {
            imageView->limitsHistory.clear();
        }
    }
 
    DataView::ImageView::ImageView() :
        cmap(simox::color::cmaps::plasma().reversed()), limitsHistoryMaxSize(32)
    {
        setLayout(new QHBoxLayout());
        int margin = 2;
        layout()->setContentsMargins(margin, margin, margin, margin);
        if (/* DISABLES CODE */ (false))
        {
            QFont font = this->font();
            font.setPointSizeF(font.pointSize() * 0.75);
            setFont(font);
        }
 
        view = new instance::ImageView();
        layout()->addWidget(view);
    }
 
} // namespace armarx::armem::gui::instance