31 #include <pcl/point_types.h>
32 #include <pcl/common/transforms.h>
33 #include <pcl/filters/approximate_voxel_grid.h>
34 #include <pcl/filters/voxel_grid.h>
35 #include <pcl/filters/uniform_sampling.h>
36 #include <pcl/filters/passthrough.h>
37 #include <pcl/filters/frustum_culling.h>
38 #include <pcl/io/pcd_io.h>
41 #include <VirtualRobot/CollisionDetection/CDManager.h>
42 #include <MotionPlanning/CSpace/CSpaceSampled.h>
43 #include <MotionPlanning/Planner/BiRrt.h>
44 #include <MotionPlanning/PostProcessing/ShortcutProcessor.h>
45 #include <VirtualRobot/Visualization/CoinVisualization/CoinVisualizationFactory.h>
47 #include <VisionX/interface/components/Calibration.h>
50 #include <VirtualRobot/Visualization/TriMeshUtils.h>
56 VoxelGridMappingProvider::VoxelGridMappingProvider()
61 std::string VoxelGridMappingProvider::getDefaultName()
const
63 return "VoxelGridMappingProvider";
69 getConfigIdentifier()));
76 void VoxelGridMappingProvider::onInitPointCloudProcessor()
78 skipper.reset(
new IceReportSkipper(getProperty<float>(
"PointCloudStorageFrequency").getValue()));
79 cycleKeeper.reset(
new CycleUtil(1000.0f / getProperty<float>(
"UpdateRate").getValue()));
80 providerName = getProperty<std::string>(
"providerName").getValue();
81 sourceFrameName = getProperty<std::string>(
"sourceFrameName").getValue();
83 gridLeafSize = getProperty<float>(
"GridSize");
85 usingPointCloudProvider(providerName);
86 accumulatedPointCloud.reset(
new pcl::PointCloud<PointType>());
87 if (!getProperty<std::string>(
"PointCloudLoadFilepath").getValue().
empty())
89 ARMARX_INFO <<
"Loading pointcloud from " << getProperty<std::string>(
"PointCloudLoadFilepath").getValue();
90 pcl::io::loadPCDFile(getProperty<std::string>(
"PointCloudLoadFilepath"), *accumulatedPointCloud);
94 void VoxelGridMappingProvider::onConnectPointCloudProcessor()
96 if (getProperty<std::string>(
"RobotStateComponentName").getValue() !=
"")
98 robotStateComponent = getProxy<RobotStateComponentInterfacePrx>(getProperty<std::string>(
"RobotStateComponentName").getValue());
100 robotPool.reset(
new RobotPool(localRobot, 3));
102 dd = getTopic<DebugDrawerInterfacePrx>(
"DebugDrawerUpdates");
103 collectingPointClouds = getProperty<bool>(
"ActivateOnStartUp");
104 octtree.reset(
new pcl::octree::OctreePointCloud<PointType>(gridLeafSize));
106 if (sourceFrameName.empty())
109 visionx::ReferenceFrameInterfacePrx proxy = getProxy<visionx::ReferenceFrameInterfacePrx>(providerName);
112 sourceFrameName = proxy->getReferenceFrame();
113 ARMARX_INFO <<
"source frame name is " << sourceFrameName;
117 enableResultPointClouds<PointType>();
120 void VoxelGridMappingProvider::onExitPointCloudProcessor()
127 void VoxelGridMappingProvider::updateBoundaries()
129 auto limits =
Split(getProperty<std::string>(
"BoundingBox").getValue(),
";",
true);
132 for (
auto& pos : limits)
134 auto posVec = Split<float>(pos,
",",
true);
136 croppingMin(i) = posVec.at(0);
137 croppingMax(i) = posVec.at(1);
142 void VoxelGridMappingProvider::process()
145 typename pcl::PointCloud<PointType>::Ptr inputCloudPtr(
new pcl::PointCloud<PointType>());
146 typename pcl::PointCloud<PointType>::Ptr downsampledTransformedInputCloudPtr(
new pcl::PointCloud<PointType>());
147 typename pcl::PointCloud<PointType>::Ptr outputCloudPtr(
new pcl::PointCloud<PointType>());
149 if (!collectingPointClouds)
155 if (!waitForPointClouds(providerName, 10000))
157 ARMARX_INFO <<
"Timeout or error while waiting for point cloud data" <<
flush;
162 getPointClouds(providerName, inputCloudPtr);
166 ARMARX_INFO <<
"Got cloud of size: " << inputCloudPtr->size();
167 auto localRobot = robotPool->getRobot();
168 visionx::MetaPointCloudFormatPtr format = getPointCloudFormat(providerName);
170 if (robotStateComponent)
181 if (robotStateComponent && sourceFrameName !=
"Global")
183 transform = localRobot->getRobotNode(sourceFrameName)->getGlobalPose();
187 cam2robot << 0, 0, 1, 0,
192 pcl::FrustumCulling<PointType> fc;
193 fc.setInputCloud(inputCloudPtr);
194 fc.setNegative(
false);
195 fc.setVerticalFOV(35);
196 fc.setHorizontalFOV(45);
197 fc.setNearPlaneDistance(0.0);
198 fc.setFarPlaneDistance(3500);
200 fc.setCameraPose(cam2robot);
202 fc.filter(*outputCloudPtr);
203 ARMARX_INFO <<
"size before frustrum culling: " << inputCloudPtr->size() <<
" size after: " << outputCloudPtr->size();
204 inputCloudPtr.swap(outputCloudPtr);
213 pcl::transformPointCloud(*inputCloudPtr, *outputCloudPtr,
transform);
214 outputCloudPtr.swap(inputCloudPtr);
219 pcl::PassThrough<PointType> pass;
222 pass.setInputCloud(inputCloudPtr);
223 pass.setFilterFieldName(
"z");
224 pass.setFilterLimits(croppingMin(2), croppingMax(2));
225 pass.filter(*outputCloudPtr);
226 outputCloudPtr.swap(inputCloudPtr);
228 pass.setInputCloud(inputCloudPtr);
229 pass.setFilterFieldName(
"x");
230 pass.setFilterLimits(croppingMin(0), croppingMax(0));
231 pass.filter(*outputCloudPtr);
232 outputCloudPtr.swap(inputCloudPtr);
234 pass.setInputCloud(inputCloudPtr);
235 pass.setFilterFieldName(
"y");
236 pass.setFilterLimits(croppingMin(1), croppingMax(1));
237 pass.filter(*outputCloudPtr);
238 outputCloudPtr.swap(inputCloudPtr);
243 gridLeafSize = getProperty<float>(
"GridSize");
244 pcl::VoxelGrid<PointType> grid;
245 grid.setMinimumPointsNumberPerVoxel(getProperty<int>(
"MinimumPointNumberPerVoxel").getValue());
247 grid.setLeafSize(gridLeafSize, gridLeafSize, gridLeafSize);
248 grid.setInputCloud(inputCloudPtr);
249 grid.filter(*outputCloudPtr);
250 ARMARX_VERBOSE <<
"Got cloud of size: " << inputCloudPtr->size() <<
" after downsampling: " << outputCloudPtr->size();
251 outputCloudPtr.swap(inputCloudPtr);
256 lastPointCloud.reset(
new Cloud());
287 typename pcl::PointCloud<PointType>::Ptr occludedPoints(
new pcl::PointCloud<PointType>());
288 typename pcl::PointCloud<PointType>::Ptr oldPointsInView(
new pcl::PointCloud<PointType>());
291 std::unique_lock lock(dataMutex);
293 cam2robot << 0, 0, 1, 0,
298 pcl::FrustumCulling<PointType> fc;
299 fc.setInputCloud(accumulatedPointCloud);
300 fc.setNegative(
false);
301 fc.setVerticalFOV(40);
302 fc.setHorizontalFOV(50);
303 fc.setNearPlaneDistance(0.0);
304 fc.setFarPlaneDistance(2500);
308 fc.filter(*oldPointsInView);
311 fc.setNegative(
true);
312 fc.filter(*outputCloudPtr);
313 outputCloudPtr.swap(accumulatedPointCloud);
320 Eigen::Vector3f cameraPos =
transform.block<3, 1>(0, 3);
321 for (
auto& p : *oldPointsInView)
323 Eigen::Vector3f currentPoint(p._PointXYZRGBA::x,
326 VirtualRobot::MathTools::BaseLine<Eigen::Vector3f> line(
transform.block<3, 1>(0, 3), currentPoint - cameraPos);
327 float distP1 = (currentPoint - cameraPos).
norm();
328 for (
auto& p2pcl : *inputCloudPtr)
330 Eigen::Vector3f p2(p2pcl._PointXYZRGBA::x,
331 p2pcl._PointXYZRGBA::y,
332 p2pcl._PointXYZRGBA::z);
333 float distP2 = (cameraPos - p2).
norm();
336 auto distance = VirtualRobot::MathTools::distPointLine(line, p2);
344 occludedPoints->push_back(newP);
353 inputCloudPtr = removeRobotVoxels(inputCloudPtr, localRobot, gridLeafSize);
354 ARMARX_VERBOSE <<
"Size after removal of robot voxels: " << inputCloudPtr->size();
363 *accumulatedPointCloud += *inputCloudPtr;
364 *accumulatedPointCloud += *occludedPoints;
367 *accumulatedPointCloud += *occludedPoints;
368 ARMARX_VERBOSE <<
"Accumulated pc size: " << accumulatedPointCloud->size() <<
" new point count: " << inputCloudPtr->size() <<
" number of occluded points: " << occludedPoints->size();
373 pcl::VoxelGrid<PointType> grid2;
374 grid2.setLeafSize(gridLeafSize, gridLeafSize, gridLeafSize);
375 grid2.setInputCloud(accumulatedPointCloud);
376 grid2.filter(*outputCloudPtr);
377 outputCloudPtr.swap(accumulatedPointCloud);
378 ARMARX_VERBOSE <<
"Accumulated pc size after grid filtering: " << accumulatedPointCloud->size() <<
" " <<
VAROUT(gridLeafSize);
382 provideResultPointClouds(accumulatedPointCloud);
387 std::vector<VirtualRobot::VisualizationFactory::Color> gridColors;
388 gridPositions.reserve(accumulatedPointCloud->size());
389 gridColors.reserve(accumulatedPointCloud->size());
391 DebugDrawerTriMesh ddTriMesh;
393 this->gridPositions.clear();
394 this->gridPositions.reserve(accumulatedPointCloud->size());
397 for (
auto& p : *accumulatedPointCloud)
399 if (!std::isnan(p._PointXYZRGBA::x) && !std::isnan(p._PointXYZRGBA::y) && !std::isnan(p._PointXYZRGBA::z))
402 this->gridPositions.push_back({p._PointXYZRGBA::x,
407 p._PointXYZRGBA::g / 255.f,
408 p._PointXYZRGBA::b / 255.f,
414 std::vector<Eigen::Vector3f> gridPositionsEigen =
armarx::transform(this->gridPositions, +[](
const Vector3f& p) -> Eigen::Vector3f
416 return {p.e0, p.e1, p.e2};
425 gridLeafSize, gridLeafSize, gridLeafSize,
426 VirtualRobot::VisualizationFactory::Color::Blue(),
435 dd->setTriMeshVisu(
"VoxelGrid",
"PointCloudCollisionGrid", ddTriMesh);
442 if (accumulatedPointCloud->size() > 0 && skipper->checkFrequency(
"SavingPCD")
443 && !getProperty<std::string>(
"PointCloudStoreFilepath").getValue().empty())
445 std::unique_lock lock(dataMutex);
446 ARMARX_INFO <<
"Saving point cloud to " << getProperty<std::string>(
"PointCloudStoreFilepath").getValue();
447 pcl::io::savePCDFile(getProperty<std::string>(
"PointCloudStoreFilepath").getValue(), *accumulatedPointCloud);
452 cycleKeeper->waitForCycleDuration();
462 VirtualRobot::CDManager cdm;
463 auto colModels = robot->getCollisionModels();
464 auto collisionChecker = robot->getCollisionChecker();
465 for (
auto& p : *cloud)
467 bool collisionFound =
false;
468 for (
auto& colModel : colModels)
470 Eigen::Vector3f pEigen(p._PointXYZRGBA::x, p._PointXYZRGBA::y, p._PointXYZRGBA::z);
471 if (collisionChecker->checkCollision(colModel, pEigen, distanceThreshold))
473 collisionFound =
true;
479 result->push_back(p);
486 Vector3fSeq VoxelGridMappingProvider::getFilledGridPositions(
const Ice::Current&)
const
488 std::unique_lock lock(dataMutex);
489 return gridPositions;
492 Ice::Float VoxelGridMappingProvider::getGridSize(
const Ice::Current&)
const
498 void VoxelGridMappingProvider::startCollectingPointClouds(
const Ice::Current&)
500 collectingPointClouds =
true;
503 void VoxelGridMappingProvider::stopCollectingPointClouds(
const Ice::Current&)
505 collectingPointClouds =
false;
508 void VoxelGridMappingProvider::reset(
const Ice::Current&)
510 std::unique_lock lock(dataMutex);
512 accumulatedPointCloud->clear();
517 void VoxelGridMappingProvider::componentPropertiesUpdated(
const std::set<std::string>& changedProperties)
519 if (changedProperties.count(
"BoundingBox"))