25 #include <pcl/common/transforms.h>
26 #include <pcl/filters/approximate_voxel_grid.h>
27 #include <pcl/filters/frustum_culling.h>
28 #include <pcl/filters/passthrough.h>
29 #include <pcl/filters/uniform_sampling.h>
30 #include <pcl/filters/voxel_grid.h>
31 #include <pcl/io/pcd_io.h>
32 #include <pcl/point_types.h>
42 #include <VirtualRobot/CollisionDetection/CDManager.h>
43 #include <VirtualRobot/CollisionDetection/CollisionChecker.h>
44 #include <VirtualRobot/Visualization/CoinVisualization/CoinVisualizationFactory.h>
45 #include <VirtualRobot/Visualization/TriMeshUtils.h>
48 #include <VisionX/interface/components/Calibration.h>
50 #include <MotionPlanning/CSpace/CSpaceSampled.h>
51 #include <MotionPlanning/Planner/BiRrt.h>
52 #include <MotionPlanning/PostProcessing/ShortcutProcessor.h>
59 VoxelGridMappingProvider::VoxelGridMappingProvider()
64 VoxelGridMappingProvider::getDefaultName()
const
66 return "VoxelGridMappingProvider";
70 VoxelGridMappingProvider::createPropertyDefinitions()
77 VoxelGridMappingProvider::onInitPointCloudProcessor()
80 new IceReportSkipper(getProperty<float>(
"PointCloudStorageFrequency").getValue()));
81 cycleKeeper.reset(
new CycleUtil(1000.0f / getProperty<float>(
"UpdateRate").getValue()));
82 providerName = getProperty<std::string>(
"providerName").getValue();
83 sourceFrameName = getProperty<std::string>(
"sourceFrameName").getValue();
85 gridLeafSize = getProperty<float>(
"GridSize");
87 usingPointCloudProvider(providerName);
88 accumulatedPointCloud.reset(
new pcl::PointCloud<PointType>());
89 if (!getProperty<std::string>(
"PointCloudLoadFilepath").getValue().
empty())
92 << getProperty<std::string>(
"PointCloudLoadFilepath").getValue();
93 pcl::io::loadPCDFile(getProperty<std::string>(
"PointCloudLoadFilepath"),
94 *accumulatedPointCloud);
99 VoxelGridMappingProvider::onConnectPointCloudProcessor()
101 if (getProperty<std::string>(
"RobotStateComponentName").getValue() !=
"")
103 robotStateComponent = getProxy<RobotStateComponentInterfacePrx>(
104 getProperty<std::string>(
"RobotStateComponentName").getValue());
106 robotStateComponent, VirtualRobot::RobotIO::RobotDescription::eCollisionModel);
109 dd = getTopic<DebugDrawerInterfacePrx>(
"DebugDrawerUpdates");
110 collectingPointClouds = getProperty<bool>(
"ActivateOnStartUp");
111 octtree.reset(
new pcl::octree::OctreePointCloud<PointType>(gridLeafSize));
113 if (sourceFrameName.empty())
116 visionx::ReferenceFrameInterfacePrx proxy =
117 getProxy<visionx::ReferenceFrameInterfacePrx>(providerName);
120 sourceFrameName = proxy->getReferenceFrame();
121 ARMARX_INFO <<
"source frame name is " << sourceFrameName;
125 enableResultPointClouds<PointType>();
129 VoxelGridMappingProvider::onExitPointCloudProcessor()
134 VoxelGridMappingProvider::updateBoundaries()
136 auto limits =
Split(getProperty<std::string>(
"BoundingBox").getValue(),
";",
true);
139 for (
auto& pos : limits)
141 auto posVec = Split<float>(pos,
",",
true);
143 croppingMin(i) = posVec.at(0);
144 croppingMax(i) = posVec.at(1);
150 VoxelGridMappingProvider::process()
153 typename pcl::PointCloud<PointType>::Ptr inputCloudPtr(
new pcl::PointCloud<PointType>());
154 typename pcl::PointCloud<PointType>::Ptr downsampledTransformedInputCloudPtr(
155 new pcl::PointCloud<PointType>());
156 typename pcl::PointCloud<PointType>::Ptr outputCloudPtr(
new pcl::PointCloud<PointType>());
158 if (!collectingPointClouds)
164 if (!waitForPointClouds(providerName, 10000))
166 ARMARX_INFO <<
"Timeout or error while waiting for point cloud data" <<
flush;
171 getPointClouds(providerName, inputCloudPtr);
175 ARMARX_INFO <<
"Got cloud of size: " << inputCloudPtr->size();
177 visionx::MetaPointCloudFormatPtr format = getPointCloudFormat(providerName);
179 << IceUtil::Time::microSeconds(format->timeProvided).toDateTime();
180 if (robotStateComponent)
183 localRobot, robotStateComponent, format->timeProvided))
186 <<
"failed to sync robot to timestamp! conversion to globalpose "
187 "will fail! aborting";
194 if (robotStateComponent && sourceFrameName !=
"Global")
200 cam2robot << 0, 0, 1, 0, 0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1;
202 pcl::FrustumCulling<PointType> fc;
203 fc.setInputCloud(inputCloudPtr);
204 fc.setNegative(
false);
205 fc.setVerticalFOV(35);
206 fc.setHorizontalFOV(45);
207 fc.setNearPlaneDistance(0.0);
208 fc.setFarPlaneDistance(3500);
210 fc.setCameraPose(cam2robot);
212 fc.filter(*outputCloudPtr);
213 ARMARX_INFO <<
"size before frustrum culling: " << inputCloudPtr->size()
214 <<
" size after: " << outputCloudPtr->size();
215 inputCloudPtr.swap(outputCloudPtr);
224 pcl::transformPointCloud(*inputCloudPtr, *outputCloudPtr,
transform);
225 outputCloudPtr.swap(inputCloudPtr);
230 pcl::PassThrough<PointType> pass;
233 pass.setInputCloud(inputCloudPtr);
234 pass.setFilterFieldName(
"z");
235 pass.setFilterLimits(croppingMin(2), croppingMax(2));
236 pass.filter(*outputCloudPtr);
237 outputCloudPtr.swap(inputCloudPtr);
239 pass.setInputCloud(inputCloudPtr);
240 pass.setFilterFieldName(
"x");
241 pass.setFilterLimits(croppingMin(0), croppingMax(0));
242 pass.filter(*outputCloudPtr);
243 outputCloudPtr.swap(inputCloudPtr);
245 pass.setInputCloud(inputCloudPtr);
246 pass.setFilterFieldName(
"y");
247 pass.setFilterLimits(croppingMin(1), croppingMax(1));
248 pass.filter(*outputCloudPtr);
249 outputCloudPtr.swap(inputCloudPtr);
254 gridLeafSize = getProperty<float>(
"GridSize");
255 pcl::VoxelGrid<PointType> grid;
256 grid.setMinimumPointsNumberPerVoxel(
257 getProperty<int>(
"MinimumPointNumberPerVoxel").getValue());
259 grid.setLeafSize(gridLeafSize, gridLeafSize, gridLeafSize);
260 grid.setInputCloud(inputCloudPtr);
261 grid.filter(*outputCloudPtr);
263 <<
" after downsampling: " << outputCloudPtr->size();
264 outputCloudPtr.swap(inputCloudPtr);
269 lastPointCloud.reset(
new Cloud());
298 typename pcl::PointCloud<PointType>::Ptr occludedPoints(
new pcl::PointCloud<PointType>());
299 typename pcl::PointCloud<PointType>::Ptr oldPointsInView(
new pcl::PointCloud<PointType>());
302 std::unique_lock lock(dataMutex);
304 cam2robot << 0, 0, 1, 0, 0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1;
306 pcl::FrustumCulling<PointType> fc;
307 fc.setInputCloud(accumulatedPointCloud);
308 fc.setNegative(
false);
309 fc.setVerticalFOV(40);
310 fc.setHorizontalFOV(50);
311 fc.setNearPlaneDistance(0.0);
312 fc.setFarPlaneDistance(2500);
316 fc.filter(*oldPointsInView);
319 fc.setNegative(
true);
320 fc.filter(*outputCloudPtr);
321 outputCloudPtr.swap(accumulatedPointCloud);
326 Eigen::Vector3f cameraPos =
transform.block<3, 1>(0, 3);
327 for (
auto& p : *oldPointsInView)
329 Eigen::Vector3f currentPoint(
330 p._PointXYZRGBA::x, p._PointXYZRGBA::y, p._PointXYZRGBA::z);
331 VirtualRobot::MathTools::BaseLine<Eigen::Vector3f> line(
transform.block<3, 1>(0, 3),
332 currentPoint - cameraPos);
333 float distP1 = (currentPoint - cameraPos).
norm();
334 for (
auto& p2pcl : *inputCloudPtr)
337 p2pcl._PointXYZRGBA::x, p2pcl._PointXYZRGBA::y, p2pcl._PointXYZRGBA::z);
338 float distP2 = (cameraPos - p2).
norm();
341 auto distance = VirtualRobot::MathTools::distPointLine(line, p2);
349 occludedPoints->push_back(newP);
358 inputCloudPtr = removeRobotVoxels(inputCloudPtr,
localRobot, gridLeafSize);
359 ARMARX_VERBOSE <<
"Size after removal of robot voxels: " << inputCloudPtr->size();
368 *accumulatedPointCloud += *inputCloudPtr;
369 *accumulatedPointCloud += *occludedPoints;
372 *accumulatedPointCloud += *occludedPoints;
373 ARMARX_VERBOSE <<
"Accumulated pc size: " << accumulatedPointCloud->size()
374 <<
" new point count: " << inputCloudPtr->size()
375 <<
" number of occluded points: " << occludedPoints->size();
380 pcl::VoxelGrid<PointType> grid2;
381 grid2.setLeafSize(gridLeafSize, gridLeafSize, gridLeafSize);
382 grid2.setInputCloud(accumulatedPointCloud);
383 grid2.filter(*outputCloudPtr);
384 outputCloudPtr.swap(accumulatedPointCloud);
386 << accumulatedPointCloud->size() <<
" " <<
VAROUT(gridLeafSize);
390 provideResultPointClouds(accumulatedPointCloud);
395 std::vector<VirtualRobot::VisualizationFactory::Color> gridColors;
396 gridPositions.reserve(accumulatedPointCloud->size());
397 gridColors.reserve(accumulatedPointCloud->size());
399 DebugDrawerTriMesh ddTriMesh;
401 this->gridPositions.clear();
402 this->gridPositions.reserve(accumulatedPointCloud->size());
405 for (
auto& p : *accumulatedPointCloud)
407 if (!std::isnan(p._PointXYZRGBA::x) && !std::isnan(p._PointXYZRGBA::y) &&
408 !std::isnan(p._PointXYZRGBA::z))
411 this->gridPositions.push_back(
412 {p._PointXYZRGBA::x, p._PointXYZRGBA::y, p._PointXYZRGBA::z});
413 gridColors.push_back(
415 p._PointXYZRGBA::g / 255.f,
416 p._PointXYZRGBA::b / 255.f,
424 +[](
const Vector3f& p) -> Eigen::Vector3f {
425 return {p.e0, p.e1, p.e2};
433 gridMesh = VirtualRobot::TriMeshUtils::CreateSparseBoxGrid(
439 VirtualRobot::VisualizationFactory::Color::Blue(),
448 dd->setTriMeshVisu(
"VoxelGrid",
"PointCloudCollisionGrid", ddTriMesh);
453 if (accumulatedPointCloud->size() > 0 && skipper->checkFrequency(
"SavingPCD") &&
454 !getProperty<std::string>(
"PointCloudStoreFilepath").getValue().empty())
456 std::unique_lock lock(dataMutex);
458 << getProperty<std::string>(
"PointCloudStoreFilepath").getValue();
459 pcl::io::savePCDFile(getProperty<std::string>(
"PointCloudStoreFilepath").getValue(),
460 *accumulatedPointCloud);
465 cycleKeeper->waitForCycleDuration();
471 float distanceThreshold)
474 VirtualRobot::CDManager cdm;
475 auto colModels = robot->getCollisionModels();
476 auto collisionChecker = robot->getCollisionChecker();
477 for (
auto& p : *cloud)
479 bool collisionFound =
false;
480 for (
auto& colModel : colModels)
482 Eigen::Vector3f pEigen(p._PointXYZRGBA::x, p._PointXYZRGBA::y, p._PointXYZRGBA::z);
483 if (collisionChecker->checkCollision(colModel, pEigen, distanceThreshold))
485 collisionFound =
true;
491 result->push_back(p);
498 VoxelGridMappingProvider::getFilledGridPositions(
const Ice::Current&)
const
500 std::unique_lock lock(dataMutex);
501 return gridPositions;
505 VoxelGridMappingProvider::getGridSize(
const Ice::Current&)
const
511 VoxelGridMappingProvider::startCollectingPointClouds(
const Ice::Current&)
513 collectingPointClouds =
true;
517 VoxelGridMappingProvider::stopCollectingPointClouds(
const Ice::Current&)
519 collectingPointClouds =
false;
523 VoxelGridMappingProvider::reset(
const Ice::Current&)
525 std::unique_lock lock(dataMutex);
527 accumulatedPointCloud->clear();
531 VoxelGridMappingProvider::componentPropertiesUpdated(
532 const std::set<std::string>& changedProperties)
534 if (changedProperties.count(
"BoundingBox"))