27 #include <pcl/common/colors.h>
28 #include <pcl/filters/statistical_outlier_removal.h>
29 #include <pcl/surface/convex_hull.h>
31 #include <SimoxUtility/math/convert/mat4f_to_pos.h>
32 #include <SimoxUtility/math/convert/mat4f_to_rpy.h>
33 #include <SimoxUtility/math/convert/pos_rpy_to_mat4f.h>
34 #include <VirtualRobot/CollisionDetection/CollisionModel.h>
35 #include <VirtualRobot/MathTools.h>
36 #include <VirtualRobot/math/Helpers.h>
54 PointCloudFilterPropertyDefinitions::PointCloudFilterPropertyDefinitions(std::string prefix) :
57 defineOptionalProperty<std::string>(
58 "RobotStateComponentName",
"RobotStateComponent",
"Name of the robot state component.");
60 defineOptionalProperty<bool>(
61 "sourceFrameNameAuto",
63 "If enabled, try to get and use source frame name from point cloud provider.");
64 defineOptionalProperty<std::string>(
65 "sourceFrameName",
"DepthCamera",
"The source frame name.");
66 defineOptionalProperty<std::string>(
"targetFrameName",
"Global",
"The target frame name.");
68 defineOptionalProperty<std::string>(
71 "Format of the input and output point cloud (XYZRGBA, XYZL, XYZRGBL).");
73 defineOptionalProperty<bool>(
"EnableDownsampling",
true,
"Enable/disable downsampling.");
74 defineOptionalProperty<float>(
"leafSize", 5.0f,
"The voxel grid leaf size.");
77 defineOptionalProperty<bool>(
"StatisticalOutlierRemoval_enabled",
79 "Enable StatisticalOutlierRemoval (pre downsampling)");
80 defineOptionalProperty<int>(
81 "StatisticalOutlierRemoval_MeanK",
83 "(pcl parameter) number of nearest neighbors to use for mean distance estimation. ");
84 defineOptionalProperty<float>(
"StatisticalOutlierRemoval_StddevMulThresh",
86 "(pcl parameter) standard deviation multiplier for the "
87 "distance threshold calculation. ");
89 defineOptionalProperty<bool>(
"EnableCropping",
true,
"Enable/disable cropping.");
90 defineOptionalPropertyVector<Eigen::Vector3f>(
"minPoint",
91 Eigen::Vector3f(-1000.0f, -1000.0f, 0.0f),
92 "Minimal point of cropping box.",
94 defineOptionalPropertyVector<Eigen::Vector3f>(
"maxPoint",
95 Eigen::Vector3f(4800.0f, 11500.0f, 1800.0f),
96 "Maximal point of cropping box.",
98 defineOptionalPropertyVector<Eigen::Vector3f>(
101 "Roll, Pitch and Yaw applied to the points in the croping frame",
104 defineOptionalPropertyVector<Eigen::Vector3f>(
105 "ProviderFrameTranslation", Eigen::Vector3f(0.0f, 0.0f, 0.0f),
"",
',');
106 defineOptionalPropertyVector<Eigen::Vector3f>(
107 "ProviderFrameRotation", Eigen::Vector3f(0.0f, 0.0f, 0.0f),
"",
',');
109 defineOptionalProperty<std::string>(
110 "croppingFrame",
"Global",
"The coordinate frame in which cropping is applied.");
112 defineOptionalProperty<bool>(
"applyCollisionModelFilter",
114 "Discard points that appear to belong to the robot itself.");
116 defineOptionalProperty<std::string>(
117 "RemoteGuiName",
"RemoteGuiProvider",
"Name of the remote GUI provider");
119 defineOptionalProperty<bool>(
"reportCloudOutsideOfCroppingArea",
121 "If cropping is enabled, the points outside of the area are "
122 "reported in a separate cloud");
128 return "PointCloudFilter";
134 std::unique_lock lock(parametersMutex);
135 return parameters.targetFrameName;
140 const Vector3BasePtr&
max,
141 const std::string& frame,
144 std::unique_lock lock(parametersMutex);
145 parameters.croppingEnabled =
true;
146 parameters.croppingMinPoint = Vector3Ptr::dynamicCast(
min)->toEigen();
147 parameters.croppingMaxPoint = Vector3Ptr::dynamicCast(
max)->toEigen();
148 parameters.croppingFrameName = frame;
157 getProperty(parameters.sourceFrameName,
"sourceFrameName");
158 getProperty(parameters.targetFrameName,
"targetFrameName");
160 getProperty(parameters.croppingEnabled,
"EnableCropping");
161 getProperty(parameters.croppingFrameName,
"croppingFrame");
162 getProperty(parameters.croppingMinPoint,
"minPoint");
163 getProperty(parameters.croppingMaxPoint,
"maxPoint");
166 getProperty(parameters.downsamplingEnabled,
"EnableDownsampling");
169 getProperty(parameters.statisticalOutlierRemoval.enabled,
170 "StatisticalOutlierRemoval_enabled");
171 getProperty(parameters.statisticalOutlierRemoval.meanK,
"StatisticalOutlierRemoval_MeanK");
172 getProperty(parameters.statisticalOutlierRemoval.stddevMulThresh,
173 "StatisticalOutlierRemoval_StddevMulThresh");
176 parameters.providerFrameTransformation = simox::math::pos_rpy_to_mat4f(
177 getProperty<Eigen::Vector3f>(
"ProviderFrameTranslation").getValue(),
178 getProperty<Eigen::Vector3f>(
"ProviderFrameRotation").getValue());
181 std::string robotStateComponentName = getProperty<std::string>(
"RobotStateComponentName");
182 if (!robotStateComponentName.empty())
188 ARMARX_INFO <<
"No RobotStateComponent configured, not performing any point cloud "
192 getProperty(parameters.applyCollisionModelFilter,
"applyCollisionModelFilter");
193 getProperty(parameters.reportCloudOutsideOfCroppingArea,
194 "reportCloudOutsideOfCroppingArea");
195 if (parameters.applyCollisionModelFilter && robotStateComponentName.empty())
197 ARMARX_WARNING <<
"Collision model filter activated, but no RobotStateComponent "
198 "configured, deactivating collision model filtering";
199 parameters.applyCollisionModelFilter =
false;
210 const std::string robotStateComponentName =
211 getProperty<std::string>(
"RobotStateComponentName");
212 if (!robotStateComponentName.empty())
214 getProxy(robotStateComponent, robotStateComponentName);
215 localRobot = RemoteRobot::createLocalCloneFromFile(
216 robotStateComponent, VirtualRobot::RobotIO::RobotDescription::eStructure);
224 if (getProperty<bool>(
"sourceFrameNameAuto") && !providerSourceFrameName.empty())
226 ARMARX_INFO <<
"Using source frame '" << providerSourceFrameName
227 <<
"' from provider '" << providerNames.front() <<
"'.";
228 this->parameters.sourceFrameName = providerSourceFrameName;
232 if (pointCloudFormat ==
"XYZRGBA")
234 enableResultPointClouds<pcl::PointXYZRGBA>(
getName() +
"Result");
235 enableResultPointClouds<pcl::PointXYZRGBA>(
getName() +
"Outside");
237 else if (pointCloudFormat ==
"XYZL")
239 enableResultPointClouds<pcl::PointXYZL>(
getName() +
"Result");
240 enableResultPointClouds<pcl::PointXYZL>(
getName() +
"Outside");
242 else if (pointCloudFormat ==
"XYZRGBL")
244 enableResultPointClouds<pcl::PointXYZRGBL>(
getName() +
"Result");
245 enableResultPointClouds<pcl::PointXYZRGBL>(
getName() +
"Outside");
249 ARMARX_ERROR <<
"Could not initialize point cloud, because format '" << pointCloudFormat
251 <<
"\nKnown formats are: XYZL, XYZRGBA, XYZGBL.";
260 remoteGuiTask->start();
276 if (pointCloudFormat ==
"XYZRGBA")
278 processPointCloud<pcl::PointXYZRGBA>();
280 else if (pointCloudFormat ==
"XYZL")
282 processPointCloud<pcl::PointXYZL>();
284 else if (pointCloudFormat ==
"XYZRGBL")
286 processPointCloud<pcl::PointXYZRGBL>();
290 ARMARX_ERROR <<
"Could not process point cloud, because format '" << pointCloudFormat
292 <<
"\nKnown formats are: XYZL, XYZRGBA, XYZRGBL.";
301 remoteGuiTask->stop();
302 remoteGuiTask =
nullptr;
306 remoteGui->removeTab(
getName());
311 toUpper(
const std::string&
string)
314 std::transform(
string.begin(),
string.end(), std::back_inserter(upper), ::toupper);
318 template <
typename Po
intT>
327 std::unique_lock lock(parametersMutex);
328 params = this->parameters;
342 ARMARX_VERBOSE <<
"Timeout or error while waiting for point cloud data.";
346 const uint32_t originalWidth = inputCloud.
cloud->width;
347 const uint32_t originalHeight = inputCloud.
cloud->height;
350 if (robotStateComponent)
355 long timestampMicroseconds =
static_cast<long>(inputCloud.
cloud->header.stamp);
357 (void)timestampMicroseconds;
358 RemoteRobot::synchronizeLocalClone(localRobot, robotStateComponent);
361 if (params.statisticalOutlierRemoval.enabled)
365 pcl::StatisticalOutlierRemoval<PointT> sor;
366 sor.setMeanK(params.statisticalOutlierRemoval.meanK);
367 sor.setStddevMulThresh(params.statisticalOutlierRemoval.stddevMulThresh);
369 sor.setInputCloud(inputCloud.
cloud);
371 sor.filter(*tempCloud);
372 tempCloud.swap(inputCloud.
cloud);
375 if (params.downsamplingEnabled)
379 pcl::ApproximateVoxelGrid<PointT> grid;
380 grid.setLeafSize(params.gridLeafSize, params.gridLeafSize, params.gridLeafSize);
381 grid.setInputCloud(inputCloud.
cloud);
384 grid.filter(*tempCloud);
386 tempCloud.swap(inputCloud.
cloud);
391 ARMARX_DEBUG <<
"transformBy\n" << params.providerFrameTransformation;
392 inputCloud.
transformBy(params.providerFrameTransformation);
395 if (params.croppingEnabled)
400 changePointCloudFrame(inputCloud,
"cropping", params.croppingFrameName);
402 catch (
const armarx::LocalException& e)
405 <<
"Caught armarx::LocalException (stopping processing).\n"
415 outsideCloud.
cloud->clear();
418 VirtualRobot::MathTools::rpy2eigen3f(params.croppingRPY).inverse();
419 for (
const auto& p : inputCloud.
cloud->points)
421 const Eigen::Vector3f transfed =
transform * Eigen::Vector3f{p.x, p.y, p.z};
422 if (transfed(0) >= params.croppingMinPoint(0) &&
423 transfed(0) <= params.croppingMaxPoint(0) &&
424 transfed(1) >= params.croppingMinPoint(1) &&
425 transfed(1) <= params.croppingMaxPoint(1) &&
426 transfed(2) >= params.croppingMinPoint(2) &&
427 transfed(2) <= params.croppingMaxPoint(2))
429 tempCloud->push_back(p);
431 else if (params.reportCloudOutsideOfCroppingArea)
433 outsideCloud.
cloud->push_back(p);
436 tempCloud.swap(inputCloud.
cloud);
443 aabb.col(0) = params.croppingMinPoint;
444 aabb.col(1) = params.croppingMaxPoint;
448 VirtualRobot::MathTools::rpy2eigen4f(params.croppingRPY));
455 const bool collisionModelFilterEnabled =
456 params.applyCollisionModelFilter && robotStateComponent;
457 if (collisionModelFilterEnabled)
461 if (params.reportCloudOutsideOfCroppingArea)
468 PointCloudPtr collisionCloudPtr(
new pcl::PointCloud<PointT>());
470 for (VirtualRobot::CollisionModelPtr collisionModel : localRobot->getCollisionModels())
472 std::vector<Eigen::Vector3f> vertices = collisionModel->getModelVeticesGlobal();
474 pcl::PointCloud<PointT> hullCloud;
475 hullCloud.width =
static_cast<uint32_t
>(vertices.size());
476 hullCloud.height = 1;
477 hullCloud.points.resize(vertices.size());
478 for (
size_t i = 0; i < vertices.size(); ++i)
480 hullCloud.points[i].getVector3fMap() = vertices[i].cast<
float>();
483 (*collisionCloudPtr) += hullCloud;
486 pcl::ConvexHull<PointT> hull;
487 hull.setInputCloud(collisionCloudPtr);
488 hull.setDimension(3);
490 std::vector<pcl::Vertices> polygons;
492 hull.reconstruct(*surfaceHull, polygons);
494 pcl::CropHull<PointT> cropHull;
496 cropHull.setHullIndices(polygons);
497 cropHull.setHullCloud(surfaceHull);
498 cropHull.setCropOutside(
false);
499 cropHull.setInputCloud(inputCloud.
cloud);
502 cropHull.filter(*tempCloud);
504 tempCloud.swap(inputCloud.
cloud);
506 if (params.reportCloudOutsideOfCroppingArea)
508 cropHull.setInputCloud(outsideCloud.
cloud);
510 cropHull.filter(*tempCloud);
511 tempCloud.swap(outsideCloud.
cloud);
519 changePointCloudFrame(inputCloud,
"target", params.targetFrameName);
520 if (params.reportCloudOutsideOfCroppingArea)
522 changePointCloudFrame(
523 outsideCloud,
"collision model filter", params.targetFrameName);
526 catch (
const armarx::LocalException& e)
529 <<
"Caught armarx::LocalException (stopping processing).\n"
538 << originalHeight <<
" --> Filtered cloud " << resultCloud->width <<
" x "
539 << resultCloud->height;
546 PointCloudFilter::drawFrame(
const FramedPose& frame,
const std::string& role)
554 template <
typename Po
intT>
556 PointCloudFilter::changePointCloudFrame(FramedPointCloud<PointT>& pointCloud,
557 const std::string& role,
558 const std::string& frame)
560 ARMARX_DEBUG <<
"Transform from " << pointCloud.getFrame() <<
" to " << toUpper(role)
561 <<
" (" << frame <<
")";
562 pointCloud.changeFrame(frame, localRobot);
563 drawFrame(pointCloud.getFramedPose(localRobot), role);
574 PointCloudFilter::remoteGuiCreate()
578 const auto& p = parameters;
591 .
value(getProperty<bool>(
"sourceFrameNameAuto")))
594 .addOptions(localRobot->getRobotNodeNames())
595 .
value(p.sourceFrameName)))
600 .addOptions(localRobot->getRobotNodeNames())
601 .
value(p.targetFrameName))
606 .addChild(
makeCheckBox(
"StatisticalOutlierRemoval_enabled")
608 .
value(p.statisticalOutlierRemoval.enabled))
613 .
value(p.statisticalOutlierRemoval.meanK))
619 .
value(p.statisticalOutlierRemoval.stddevMulThresh)))
625 .
value(p.downsamplingEnabled))
631 .
value(p.gridLeafSize)))
636 .
value(p.applyCollisionModelFilter)));
641 const Eigen::Vector3f pos = simox::math::mat4f_to_pos(p.providerFrameTransformation);
642 const Eigen::Vector3f rpy = simox::math::mat4f_to_rpy(p.providerFrameTransformation);
671 makeCheckBox(
"CropEnabled").value(p.croppingEnabled).label(
"Enabled");
673 .value(p.reportCloudOutsideOfCroppingArea)
674 .label(
"Report points outside");
678 .addOptions(localRobot->getRobotNodeNames())
679 .
value(p.croppingFrameName);
685 .
value(p.croppingRPY)
701 rootLayout.addChild(
makeGroupBox(
"CropGroup").label(
"Cropping").child(cropGroupLayout));
709 makeCheckBox(
"VisuEnabled").label(
"Enabled").value(debugDrawer.enabled());
714 makeGroupBox(
"VisuGroup").label(
"Visualization").child(visuGroupLayout));
718 const std::string tabName =
getName();
719 remoteGui->createTab(tabName, rootLayout);
724 PointCloudFilter::remoteGuiRun()
726 int cycleDurationMs = 50;
728 while (!remoteGuiTask->isStopped())
732 remoteGuiUpdate(remoteGuiTab);
735 c.waitForCycleDuration();
745 std::unique_lock lock(parametersMutex);
750 if (tab.
getValue<
bool>(
"SourceFrameAutoEnabled").get())
752 parameters.sourceFrameName = this->providerSourceFrameName;
753 sourceFrame.
set(parameters.sourceFrameName);
758 parameters.sourceFrameName = sourceFrame.
get();
762 tab.
getValue(parameters.targetFrameName,
"TargetFrame");
765 tab.
getValue(parameters.downsamplingEnabled,
"DownsamplingEnabled");
766 tab.
getValue(parameters.gridLeafSize,
"DownsamplingGridLeafSize");
767 tab.
getValue(parameters.applyCollisionModelFilter,
"CollisionModelFilterEnabled");
768 tab.
getValue(parameters.reportCloudOutsideOfCroppingArea,
769 "reportCloudOutsideOfCroppingArea");
770 tab.
getValue(parameters.statisticalOutlierRemoval.enabled,
771 "StatisticalOutlierRemoval_enabled");
772 tab.
getValue(parameters.statisticalOutlierRemoval.meanK,
773 "StatisticalOutlierRemoval_MeanK");
774 tab.
getValue(parameters.statisticalOutlierRemoval.stddevMulThresh,
775 "StatisticalOutlierRemoval_StddevMulThresh");
778 tab.
getValue(parameters.croppingEnabled,
"CropEnabled");
779 tab.
getValue(parameters.croppingFrameName,
"CropFrame");
780 tab.
getValue(parameters.croppingMinPoint,
"CropMin");
781 tab.
getValue(parameters.croppingMaxPoint,
"CropMax");
782 tab.
getValue(parameters.croppingRPY,
"CropRPY");
784 parameters.providerFrameTransformation =
785 simox::math::pos_rpy_to_mat4f(tab.
getValue<Eigen::Vector3f>(
"transform.XYC").get(),
786 tab.
getValue<Eigen::Vector3f>(
"transform.RPY").get());
790 bool visuEnabled = tab.
getValue<
bool>(
"VisuEnabled").get();
794 debugDrawer.clearLayer(
false);
797 debugDrawer.setEnabled(visuEnabled);