27 #include <pcl/common/colors.h>
28 #include <pcl/surface/convex_hull.h>
29 #include <pcl/filters/statistical_outlier_removal.h>
31 #include <VirtualRobot/CollisionDetection/CollisionModel.h>
32 #include <VirtualRobot/math/Helpers.h>
33 #include <VirtualRobot/MathTools.h>
34 #include <SimoxUtility/math/convert/mat4f_to_rpy.h>
35 #include <SimoxUtility/math/convert/mat4f_to_pos.h>
36 #include <SimoxUtility/math/convert/pos_rpy_to_mat4f.h>
54 PointCloudFilterPropertyDefinitions::PointCloudFilterPropertyDefinitions(std::string prefix) :
57 defineOptionalProperty<std::string>(
"RobotStateComponentName",
"RobotStateComponent",
58 "Name of the robot state component.");
60 defineOptionalProperty<bool>(
"sourceFrameNameAuto",
true,
61 "If enabled, try to get and use source frame name from point cloud provider.");
62 defineOptionalProperty<std::string>(
"sourceFrameName",
"DepthCamera",
"The source frame name.");
63 defineOptionalProperty<std::string>(
"targetFrameName",
"Global",
"The target frame name.");
65 defineOptionalProperty<std::string>(
"PointCloudFormat",
"XYZRGBA",
"Format of the input and output point cloud (XYZRGBA, XYZL, XYZRGBL).");
67 defineOptionalProperty<bool>(
"EnableDownsampling",
true,
"Enable/disable downsampling.");
68 defineOptionalProperty<float>(
"leafSize", 5.0f,
"The voxel grid leaf size.");
71 defineOptionalProperty<bool>(
"StatisticalOutlierRemoval_enabled",
false,
"Enable StatisticalOutlierRemoval (pre downsampling)");
72 defineOptionalProperty<int>(
"StatisticalOutlierRemoval_MeanK", 50,
73 "(pcl parameter) number of nearest neighbors to use for mean distance estimation. ");
74 defineOptionalProperty<float>(
"StatisticalOutlierRemoval_StddevMulThresh", 1,
75 "(pcl parameter) standard deviation multiplier for the distance threshold calculation. ");
77 defineOptionalProperty<bool>(
"EnableCropping",
true,
"Enable/disable cropping.");
78 defineOptionalPropertyVector<Eigen::Vector3f>(
"minPoint", Eigen::Vector3f(-1000.0f, -1000.0f, 0.0f),
79 "Minimal point of cropping box.",
',');
80 defineOptionalPropertyVector<Eigen::Vector3f>(
"maxPoint", Eigen::Vector3f(4800.0f, 11500.0f, 1800.0f),
81 "Maximal point of cropping box.",
',');
82 defineOptionalPropertyVector<Eigen::Vector3f>(
"rpy", {0, 0, 0},
83 "Roll, Pitch and Yaw applied to the points in the croping frame",
',');
85 defineOptionalPropertyVector<Eigen::Vector3f>(
"ProviderFrameTranslation", Eigen::Vector3f(0.0f, 0.0f, 0.0f),
"",
',');
86 defineOptionalPropertyVector<Eigen::Vector3f>(
"ProviderFrameRotation", Eigen::Vector3f(0.0f, 0.0f, 0.0f),
"",
',');
88 defineOptionalProperty<std::string>(
"croppingFrame",
"Global",
"The coordinate frame in which cropping is applied.");
90 defineOptionalProperty<bool>(
"applyCollisionModelFilter",
true,
91 "Discard points that appear to belong to the robot itself.");
93 defineOptionalProperty<std::string>(
"RemoteGuiName",
"RemoteGuiProvider",
"Name of the remote GUI provider");
95 defineOptionalProperty<bool>(
"reportCloudOutsideOfCroppingArea",
false,
96 "If cropping is enabled, the points outside of the area are reported in a separate cloud");
102 return "PointCloudFilter";
107 std::unique_lock lock(parametersMutex);
108 return parameters.targetFrameName;
112 const Vector3BasePtr&
min,
const Vector3BasePtr&
max,
113 const std::string& frame,
const Ice::Current&)
115 std::unique_lock lock(parametersMutex);
116 parameters.croppingEnabled =
true;
117 parameters.croppingMinPoint = Vector3Ptr::dynamicCast(
min)->toEigen();
118 parameters.croppingMaxPoint = Vector3Ptr::dynamicCast(
max)->toEigen();
119 parameters.croppingFrameName = frame;
127 getProperty(parameters.sourceFrameName,
"sourceFrameName");
128 getProperty(parameters.targetFrameName,
"targetFrameName");
130 getProperty(parameters.croppingEnabled,
"EnableCropping");
131 getProperty(parameters.croppingFrameName,
"croppingFrame");
132 getProperty(parameters.croppingMinPoint,
"minPoint");
133 getProperty(parameters.croppingMaxPoint,
"maxPoint");
136 getProperty(parameters.downsamplingEnabled,
"EnableDownsampling");
139 getProperty(parameters.statisticalOutlierRemoval.enabled,
"StatisticalOutlierRemoval_enabled");
140 getProperty(parameters.statisticalOutlierRemoval.meanK,
"StatisticalOutlierRemoval_MeanK");
141 getProperty(parameters.statisticalOutlierRemoval.stddevMulThresh,
"StatisticalOutlierRemoval_StddevMulThresh");
144 parameters.providerFrameTransformation = simox::math::pos_rpy_to_mat4f(
145 getProperty<Eigen::Vector3f>(
"ProviderFrameTranslation").getValue(),
146 getProperty<Eigen::Vector3f>(
"ProviderFrameRotation").getValue()
150 std::string robotStateComponentName = getProperty<std::string>(
"RobotStateComponentName");
151 if (!robotStateComponentName.empty())
157 ARMARX_INFO <<
"No RobotStateComponent configured, not performing any point cloud transformations.";
160 getProperty(parameters.applyCollisionModelFilter,
"applyCollisionModelFilter");
161 getProperty(parameters.reportCloudOutsideOfCroppingArea,
"reportCloudOutsideOfCroppingArea");
162 if (parameters.applyCollisionModelFilter && robotStateComponentName.empty())
164 ARMARX_WARNING <<
"Collision model filter activated, but no RobotStateComponent configured, deactivating collision model filtering";
165 parameters.applyCollisionModelFilter =
false;
175 const std::string robotStateComponentName = getProperty<std::string>(
"RobotStateComponentName");
176 if (!robotStateComponentName.empty())
178 getProxy(robotStateComponent, robotStateComponentName);
179 localRobot = RemoteRobot::createLocalCloneFromFile(
180 robotStateComponent, VirtualRobot::RobotIO::RobotDescription::eStructure);
188 if (getProperty<bool>(
"sourceFrameNameAuto") && !providerSourceFrameName.empty())
190 ARMARX_INFO <<
"Using source frame '" << providerSourceFrameName <<
"' from provider '" << providerNames.front() <<
"'.";
191 this->parameters.sourceFrameName = providerSourceFrameName;
195 if (pointCloudFormat ==
"XYZRGBA")
197 enableResultPointClouds<pcl::PointXYZRGBA>(
getName() +
"Result");
198 enableResultPointClouds<pcl::PointXYZRGBA>(
getName() +
"Outside");
200 else if (pointCloudFormat ==
"XYZL")
202 enableResultPointClouds<pcl::PointXYZL>(
getName() +
"Result");
203 enableResultPointClouds<pcl::PointXYZL>(
getName() +
"Outside");
205 else if (pointCloudFormat ==
"XYZRGBL")
207 enableResultPointClouds<pcl::PointXYZRGBL>(
getName() +
"Result");
208 enableResultPointClouds<pcl::PointXYZRGBL>(
getName() +
"Outside");
212 ARMARX_ERROR <<
"Could not initialize point cloud, because format '" << pointCloudFormat <<
"' is unknown."
213 <<
"\nKnown formats are: XYZL, XYZRGBA, XYZGBL.";
223 this->remoteGuiRun();
225 remoteGuiTask->start();
240 if (pointCloudFormat ==
"XYZRGBA")
242 processPointCloud<pcl::PointXYZRGBA>();
244 else if (pointCloudFormat ==
"XYZL")
246 processPointCloud<pcl::PointXYZL>();
248 else if (pointCloudFormat ==
"XYZRGBL")
250 processPointCloud<pcl::PointXYZRGBL>();
254 ARMARX_ERROR <<
"Could not process point cloud, because format '" << pointCloudFormat <<
"' is unknown."
255 <<
"\nKnown formats are: XYZL, XYZRGBA, XYZRGBL.";
263 remoteGuiTask->stop();
264 remoteGuiTask =
nullptr;
268 remoteGui->removeTab(
getName());
273 static std::string toUpper(
const std::string&
string)
276 std::transform(
string.begin(),
string.end(), std::back_inserter(upper), ::toupper);
280 template <
typename Po
intT>
288 std::unique_lock lock(parametersMutex);
289 params = this->parameters;
303 ARMARX_VERBOSE <<
"Timeout or error while waiting for point cloud data.";
307 const uint32_t originalWidth = inputCloud.
cloud->width;
308 const uint32_t originalHeight = inputCloud.
cloud->height;
311 if (robotStateComponent)
316 long timestampMicroseconds =
static_cast<long>(inputCloud.
cloud->header.stamp);
318 (void) timestampMicroseconds;
319 RemoteRobot::synchronizeLocalClone(localRobot, robotStateComponent);
322 if (params.statisticalOutlierRemoval.enabled)
326 pcl::StatisticalOutlierRemoval<PointT> sor;
327 sor.setMeanK(params.statisticalOutlierRemoval.meanK);
328 sor.setStddevMulThresh(params.statisticalOutlierRemoval.stddevMulThresh);
330 sor.setInputCloud(inputCloud.
cloud);
332 sor.filter(*tempCloud);
333 tempCloud.swap(inputCloud.
cloud);
336 if (params.downsamplingEnabled)
340 pcl::ApproximateVoxelGrid<PointT> grid;
341 grid.setLeafSize(params.gridLeafSize, params.gridLeafSize, params.gridLeafSize);
342 grid.setInputCloud(inputCloud.
cloud);
345 grid.filter(*tempCloud);
347 tempCloud.swap(inputCloud.
cloud);
352 ARMARX_DEBUG <<
"transformBy\n" << params.providerFrameTransformation;
353 inputCloud.
transformBy(params.providerFrameTransformation);
356 if (params.croppingEnabled)
361 changePointCloudFrame(inputCloud,
"cropping", params.croppingFrameName);
363 catch (
const armarx::LocalException& e)
374 outsideCloud.
cloud->clear();
377 for (
const auto& p : inputCloud.
cloud->points)
379 const Eigen::Vector3f transfed =
transform * Eigen::Vector3f{p.x, p.y, p.z};
381 transfed(0) >= params.croppingMinPoint(0) && transfed(0) <= params.croppingMaxPoint(0) &&
382 transfed(1) >= params.croppingMinPoint(1) && transfed(1) <= params.croppingMaxPoint(1) &&
383 transfed(2) >= params.croppingMinPoint(2) && transfed(2) <= params.croppingMaxPoint(2))
385 tempCloud->push_back(p);
387 else if (params.reportCloudOutsideOfCroppingArea)
389 outsideCloud.
cloud->push_back(p);
392 tempCloud.swap(inputCloud.
cloud);
399 aabb.col(0) = params.croppingMinPoint;
400 aabb.col(1) = params.croppingMaxPoint;
403 inputCloud.
getGlobalPose(localRobot) * VirtualRobot::MathTools::rpy2eigen4f(params.croppingRPY));
410 const bool collisionModelFilterEnabled = params.applyCollisionModelFilter && robotStateComponent;
411 if (collisionModelFilterEnabled)
415 if (params.reportCloudOutsideOfCroppingArea)
422 PointCloudPtr collisionCloudPtr(
new pcl::PointCloud<PointT>());
424 for (VirtualRobot::CollisionModelPtr collisionModel : localRobot->getCollisionModels())
426 std::vector<Eigen::Vector3f> vertices = collisionModel->getModelVeticesGlobal();
428 pcl::PointCloud<PointT> hullCloud;
429 hullCloud.width =
static_cast<uint32_t
>(vertices.size());
430 hullCloud.height = 1;
431 hullCloud.points.resize(vertices.size());
432 for (
size_t i = 0; i < vertices.size(); ++i)
434 hullCloud.points[i].getVector3fMap() = vertices[i].cast<
float>();
437 (*collisionCloudPtr) += hullCloud;
440 pcl::ConvexHull<PointT> hull;
441 hull.setInputCloud(collisionCloudPtr);
442 hull.setDimension(3);
444 std::vector<pcl::Vertices> polygons;
446 hull.reconstruct(*surfaceHull, polygons);
448 pcl::CropHull<PointT> cropHull;
450 cropHull.setHullIndices(polygons);
451 cropHull.setHullCloud(surfaceHull);
452 cropHull.setCropOutside(
false);
453 cropHull.setInputCloud(inputCloud.
cloud);
456 cropHull.filter(*tempCloud);
458 tempCloud.swap(inputCloud.
cloud);
460 if (params.reportCloudOutsideOfCroppingArea)
462 cropHull.setInputCloud(outsideCloud.
cloud);
464 cropHull.filter(*tempCloud);
465 tempCloud.swap(outsideCloud.
cloud);
474 changePointCloudFrame(inputCloud,
"target", params.targetFrameName);
475 if (params.reportCloudOutsideOfCroppingArea)
477 changePointCloudFrame(outsideCloud,
"collision model filter", params.targetFrameName);
480 catch (
const armarx::LocalException& e)
490 <<
"Input cloud " << originalWidth <<
" x " << originalHeight
491 <<
" --> Filtered cloud " << resultCloud->width <<
" x " << resultCloud->height;
498 void PointCloudFilter::drawFrame(
const FramedPose& frame,
const std::string& role)
508 template<
typename Po
intT>
509 void PointCloudFilter::changePointCloudFrame(FramedPointCloud<PointT>& pointCloud,
510 const std::string& role,
const std::string& frame)
512 ARMARX_DEBUG <<
"Transform from " << pointCloud.getFrame()
513 <<
" to " << toUpper(role) <<
" (" << frame <<
")";
514 pointCloud.changeFrame(frame, localRobot);
515 drawFrame(pointCloud.getFramedPose(localRobot), role);
525 void PointCloudFilter::remoteGuiCreate()
529 const auto& p = parameters;
540 .label(
"Auto").
value(getProperty<bool>(
"sourceFrameNameAuto")))
543 .addOptions(localRobot->getRobotNodeNames())
544 .
value(p.sourceFrameName)))
549 .addOptions(localRobot->getRobotNodeNames())
550 .
value(p.targetFrameName))
554 .addChild(
makeCheckBox(
"StatisticalOutlierRemoval_enabled")
555 .label(
"Enabled").
value(p.statisticalOutlierRemoval.enabled))
558 .
min(1).
max(1000).
value(p.statisticalOutlierRemoval.meanK))
561 .
min(0.001f).
max(100.f).decimals(3).
value(p.statisticalOutlierRemoval.stddevMulThresh)))
566 .label(
"Enabled").
value(p.downsamplingEnabled))
569 .
min(0.1f).
max(1000.f).decimals(1).
value(p.gridLeafSize)))
573 .label(
"Enabled").
value(p.applyCollisionModelFilter))
579 const Eigen::Vector3f pos = simox::math::mat4f_to_pos(p.providerFrameTransformation);
580 const Eigen::Vector3f rpy = simox::math::mat4f_to_rpy(p.providerFrameTransformation);
602 WidgetPtr reportAllBox =
makeCheckBox(
"reportCloudOutsideOfCroppingArea").value(p.reportCloudOutsideOfCroppingArea).label(
"Report points outside");
606 .addOptions(localRobot->getRobotNodeNames())
607 .
value(p.croppingFrameName);
609 .
min(-100000).
max(100000);
611 .
min(-100000).
max(100000);
618 enabledCheckBox, reportAllBox,
619 frameLabel, frameLine,
620 minLabel, minVector3Spin,
621 maxLabel, maxVector3Spin,
622 rpyLabel, rpyVector3Spin
626 .label(
"Cropping").child(cropGroupLayout));
634 .label(
"Enabled").value(debugDrawer.enabled());
637 .addChild(enabledCheckBox));
640 .label(
"Visualization").child(visuGroupLayout));
644 const std::string tabName =
getName();
645 remoteGui->createTab(tabName, rootLayout);
650 void PointCloudFilter::remoteGuiRun()
652 int cycleDurationMs = 50;
654 while (!remoteGuiTask->isStopped())
658 remoteGuiUpdate(remoteGuiTab);
661 c.waitForCycleDuration();
671 std::unique_lock lock(parametersMutex);
676 if (tab.
getValue<
bool>(
"SourceFrameAutoEnabled").get())
678 parameters.sourceFrameName = this->providerSourceFrameName;
679 sourceFrame.
set(parameters.sourceFrameName);
684 parameters.sourceFrameName = sourceFrame.
get();
688 tab.
getValue(parameters.targetFrameName,
"TargetFrame");
691 tab.
getValue(parameters.downsamplingEnabled,
"DownsamplingEnabled");
692 tab.
getValue(parameters.gridLeafSize,
"DownsamplingGridLeafSize");
693 tab.
getValue(parameters.applyCollisionModelFilter,
"CollisionModelFilterEnabled");
694 tab.
getValue(parameters.reportCloudOutsideOfCroppingArea,
"reportCloudOutsideOfCroppingArea");
695 tab.
getValue(parameters.statisticalOutlierRemoval.enabled,
"StatisticalOutlierRemoval_enabled");
696 tab.
getValue(parameters.statisticalOutlierRemoval.meanK,
"StatisticalOutlierRemoval_MeanK");
697 tab.
getValue(parameters.statisticalOutlierRemoval.stddevMulThresh,
"StatisticalOutlierRemoval_StddevMulThresh");
700 tab.
getValue(parameters.croppingEnabled,
"CropEnabled");
701 tab.
getValue(parameters.croppingFrameName,
"CropFrame");
702 tab.
getValue(parameters.croppingMinPoint,
"CropMin");
703 tab.
getValue(parameters.croppingMaxPoint,
"CropMax");
704 tab.
getValue(parameters.croppingRPY,
"CropRPY");
706 parameters.providerFrameTransformation = simox::math::pos_rpy_to_mat4f(
707 tab.
getValue<Eigen::Vector3f>(
"transform.XYC").get(),
708 tab.
getValue<Eigen::Vector3f>(
"transform.RPY").get()
714 bool visuEnabled = tab.
getValue<
bool>(
"VisuEnabled").get();
718 debugDrawer.clearLayer(
false);
721 debugDrawer.setEnabled(visuEnabled);