19#include <Eigen/Geometry>
22#include <IceUtil/Time.h>
25#include <pcl/common/transforms.h>
26#include <pcl/io/pcd_io.h>
27#include <pcl/point_cloud.h>
28#include <pcl/point_types.h>
31#include <opencv2/core/mat.hpp>
32#include <opencv2/imgcodecs.hpp>
35#include <SimoxUtility/algorithm/string/string_tools.h>
36#include <SimoxUtility/json/json.hpp>
37#include <SimoxUtility/math.h>
49#include <cartographer/common/configuration_file_resolver.h>
50#include <cartographer/common/time.h>
51#include <cartographer/io/file_writer.h>
52#include <cartographer/io/points_processor_pipeline_builder.h>
53#include <cartographer/io/proto_stream.h>
54#include <cartographer/mapping/2d/grid_2d.h>
55#include <cartographer/mapping/2d/probability_grid.h>
56#include <cartographer/mapping/2d/submap_2d.h>
57#include <cartographer/mapping/internal/2d/tsdf_2d.h>
58#include <cartographer/mapping/map_builder.h>
59#include <cartographer/mapping/map_builder_interface.h>
60#include <cartographer/mapping/pose_graph_interface.h>
61#include <cartographer/mapping/submaps.h>
62#include <cartographer/mapping/trajectory_builder_interface.h>
63#include <cartographer/mapping/trajectory_node.h>
64#include <cartographer/sensor/point_cloud.h>
65#include <cartographer/sensor/rangefinder_point.h>
66#include <cartographer/transform/rigid_transform.h>
67#include <cartographer/transform/transform.h>
83namespace fs = ::std::filesystem;
91 struct tm* timeinfo{};
92 std::array<char, 4 + 1 + 2 + 1 + 2 + 1 + 2 + 1 + 2 + 1> buffer;
95 timeinfo = localtime(&rawtime);
97 strftime(buffer.data(), buffer.size(),
"%Y-%m-%d-%H-%M", timeinfo);
111 mapBuilder->FinishTrajectory(trajectoryId);
120 const std::filesystem::path& mapPath,
121 const std::filesystem::path& configPath,
124 const std::optional<std::filesystem::path>& mapToLoad,
125 const std::optional<Eigen::Isometry3f>& map_T_robot_prior,
127 config(config), mapPath(mapPath), slamDataCallable(slamDataCallable)
130 const std::string mapBuilderLuaCode = R
"text(
131 include "map_builder.lua"
132 return MAP_BUILDER)text";
134 const std::string trajectoryBuilderLuaCode = R
"text(
135 include "trajectory_builder.lua"
136 return TRAJECTORY_BUILDER)text";
139 auto mapBuilderOptions =
140 carto::mapping::CreateMapBuilderOptions(mapBuilderParameters.get());
143 if (config.global_localization_min_score > 0)
145 ARMARX_INFO <<
"Setting `global_localization_min_score` to "
146 << config.global_localization_min_score;
147 mapBuilderOptions.mutable_pose_graph_options()
148 ->mutable_constraint_builder_options()
149 ->set_global_localization_min_score(config.global_localization_min_score);
152 if (config.min_score > 0)
154 ARMARX_INFO <<
"Setting `min_score` to " << config.min_score;
155 mapBuilderOptions.mutable_pose_graph_options()
156 ->mutable_constraint_builder_options()
157 ->set_min_score(config.min_score);
160 const auto trajectoryBuilderParameters =
162 trajectoryBuilderOptions =
163 carto::mapping::CreateTrajectoryBuilderOptions(trajectoryBuilderParameters.get());
166 mapBuilder = carto::mapping::CreateMapBuilder(mapBuilderOptions);
168 if (config.useLaserScanner)
170 sensorSet.insert(sensorIdLaser);
173 if (config.useOdometry)
175 sensorSet.insert(sensorIdOdom);
180 sensorSet.insert(sensorIdIMU);
183 std::vector<std::string> sensorSetNames;
184 std::transform(sensorSet.begin(),
186 std::back_inserter(sensorSetNames),
187 [](
const auto& sensorId) { return sensorId.id; });
191 if (config.useLaserScanner)
194 << std::vector<std::string>(config.laserScanners.begin(),
195 config.laserScanners.end());
197 std::for_each(config.laserScanners.begin(),
198 config.laserScanners.end(),
199 [&](
const auto& laserScannerName)
201 const SensorId sensorId{.type = SensorId::SensorType::RANGE,
202 .id = laserScannerName};
203 laserSensorMap.insert(
205 LaserScannerSensor(sensorId, Eigen::Isometry3f::Identity())});
214 carto::io::ProtoStreamReader reader(mapToLoad->string());
215 mapBuilder->LoadState(&reader,
true );
216 mapBuilder->pose_graph()->RunFinalOptimization();
219 if (map_T_robot_prior)
221 ARMARX_INFO <<
"Using pose prior \n" << map_T_robot_prior->matrix();
223 ::cartographer::mapping::proto::InitialTrajectoryPose* initial_trajectory_pose =
224 new ::cartographer::mapping::proto::InitialTrajectoryPose;
226 initial_trajectory_pose->set_allocated_relative_pose(
227 new ::cartographer::transform::proto::Rigid3d(
228 ::cartographer::transform::ToProto(
fromEigen(*map_T_robot_prior))));
230 trajectoryBuilderOptions.set_allocated_initial_trajectory_pose(
231 std::move(initial_trajectory_pose));
236 trajectoryId = mapBuilder->AddTrajectoryBuilder(
238 trajectoryBuilderOptions,
239 [
this](
auto&&... args)
240 {
return getLocalSlamResultCallback(std::forward<
decltype(args)>(args)...); });
243 const auto dt =
static_cast<int64_t
>(1'000'000 / config.frequency);
244 const float dtHistoryLength = 1'000'000;
249 std::make_unique<ApproximateTimeQueue>(
dt,
251 config.laserScanners,
253 config.useOdometryBasedLaserCorrection);
259 cartographerInputDataReporterLaserScanner.emplace(
260 debugObserver,
"CartographerMappingAndLocalization_input_laser");
261 cartographerInputDataReporterOdometry.emplace(
262 debugObserver,
"CartographerMappingAndLocalization_input_odom");
263 if (config.enableDebugTiming)
265 debugObserverHelper.emplace(
266 "CartographerMappingAndLocalization_CartographerAdapter", debugObserver,
true);
275 pushInOdomData(
data.odomPose);
276 pushInLaserData(
data.laserData);
277 if (debugObserverHelper)
280 std::unique_lock g(debugObserverHelperMtx);
281 debugObserverHelper->sendDebugObserverBatch();
286 CartographerAdapter::pushInOdomData(
290 const DebugObserverScopedTimer timer(
this,
"pushInOdomData | Entire Method");
291 if (lastOdomTimestamp.has_value())
294 if (odomPose.
timestamp <= lastOdomTimestamp.value())
297 <<
"Requested to add old data into odom queue. Not willing to do this. It is "
298 << (odomPose.
timestamp - lastOdomTimestamp.value()) <<
"ms in the past.";
308 auto isOdomSensor = [](
const SensorId& sensorId)
309 {
return sensorId.type == SensorId::SensorType::ODOMETRY; };
311 if (not std::any_of(sensorSet.begin(), sensorSet.end(), isOdomSensor))
314 ARMARX_WARNING <<
"No odometry sensor is registered. Will not use odometry data.";
322 const Eigen::Vector3f p = odomPose.
pose.translation();
324 carto::sensor::OdometryData odomData;
326 carto::transform::Rigid3d::FromArrays(std::array<double, 4>{
q.w(),
q.x(),
q.y(),
q.z()},
327 std::array<double, 3>{p.x(), p.y(), p.z()});
328 odomData.time = util::toCarto(
timestamp);
334 const DebugObserverScopedTimer localTimer(
this,
"pushInOdomData | Inserting Odom Data");
335 std::lock_guard guard{cartoInsertMtx};
337 mapBuilder->GetTrajectoryBuilder(trajectoryId)
338 ->AddSensorData(sensorIdOdom.id, odomData);
341 if (cartographerInputDataReporterOdometry.has_value())
344 cartographerInputDataReporterOdometry->add(
timestamp);
351 CartographerAdapter::pushInLaserData(
const LaserMessages& laserMessages)
355 if (not config.useLaserScanner)
361 const DebugObserverScopedTimer timer(
this,
"pushInLaserData | Entire Method");
362 carto::sensor::TimedPointCloudData timedPointCloudData;
363 timedPointCloudData.origin = Eigen::Vector3f::Zero();
367 const DebugObserverScopedTimer laserMessageTimer(
368 this,
"pushInLaserData | Iterate over laser messages");
369 for (
const auto& message : laserMessages)
371 const auto&
data = message.data;
374 << message.data.frame;
379 auto& lastTimestamp = lastSensorDataTimestamp[
data.frame];
381 if (
data.timestamp <= lastTimestamp)
383 ARMARX_DEBUG <<
"Requested to add old data with timestamp "
384 <<
data.timestamp <<
" into laser queue for sensor "
385 <<
data.frame <<
". Not willing to do this. It is "
386 << (
data.timestamp - lastTimestamp) <<
"ms in the past.";
390 lastTimestamp =
data.timestamp;
394 Eigen::Isometry3f robot_T_sensor;
397 const LaserScannerSensor sensor = lookupSensor(
data.frame);
399 << sensor.pose.translation();
401 robot_T_sensor = sensor.pose;
403 catch (
const armarx::LocalException&)
405 ARMARX_INFO <<
"Sensor `" <<
data.frame <<
"` not found in sensor set.";
412 const DebugObserverScopedTimer localTimer(
413 this,
"pushInLaserData | Reserve TimedPointCloudData");
414 timedPointCloudData.time =
415 util::toCarto(
data.timestamp);
416 timedPointCloudData.ranges.reserve(timedPointCloudData.ranges.size() +
422 const DebugObserverScopedTimer localTimer(
423 this,
"pushInLaserData | Transform to TimedPointCloudData");
424 std::transform(
data.scan.begin(),
426 std::back_inserter(timedPointCloudData.ranges),
427 [&](
const auto& scanStep) -> carto::sensor::TimedRangefinderPoint
430 toCartesian<Eigen::Vector3f>(scanStep);
433 carto::sensor::TimedRangefinderPoint point;
435 point.position = robot_T_sensor * rangePoint;
443 auto cloud =
toPCL(timedPointCloudData.ranges);
445 const DebugObserverScopedTimer localTimer(
this,
"pushInLaserData | onLaserSensorData");
447 slamDataCallable.onLaserSensorData({util::fromCarto(timedPointCloudData.time), cloud});
451 const DebugObserverScopedTimer localTimer(
this,
452 "pushInLaserData | Inserting Laser Data");
453 std::lock_guard guard{cartoInsertMtx};
455 auto* trajBuilder = mapBuilder->GetTrajectoryBuilder(trajectoryId);
456 if (trajBuilder ==
nullptr)
462 const std::int64_t laserScannerInsertCartoTimestamp =
463 timedPointCloudData.time.time_since_epoch().count();
465 ARMARX_DEBUG <<
"Inserting " << sensorIdLaser.id <<
" with timestamp "
466 << laserScannerInsertCartoTimestamp;
468 if (lastLaserScannerInsertCartoTimestamp.has_value())
470 if (laserScannerInsertCartoTimestamp <= lastLaserScannerInsertCartoTimestamp)
472 ARMARX_INFO <<
"Received laser scanner data out of order. Will not handle it.";
477 lastLaserScannerInsertCartoTimestamp = laserScannerInsertCartoTimestamp;
480 const DebugObserverScopedTimer addSensorDataTimer(
481 this,
"pushInLaserData | Add Sensor Data to TrajectoryBuilder");
483 trajBuilder->AddSensorData(sensorIdLaser.id, timedPointCloudData);
486 if (cartographerInputDataReporterLaserScanner)
488 cartographerInputDataReporterLaserScanner->add(
501 const DebugObserverScopedTimer timer(
this,
"processOdometryPose | Entire Method");
505 odomPose.
pose.setIdentity();
509 approxTimeQueue->insertOdomData(std::move(odomPose));
516 const DebugObserverScopedTimer timer(
this,
"processSensorValues | Entire Method");
535 approxTimeQueue->insertLaserData(std::move(
data));
539 CartographerAdapter::getLocalSlamResultCallback(
540 const int trajectoryId,
541 const ::cartographer::common::Time time,
542 const ::cartographer::transform::Rigid3d localPose,
543 ::cartographer::sensor::RangeData rangeDataInLocal,
544 const std::unique_ptr<
545 const ::cartographer::mapping::TrajectoryBuilderInterface::InsertionResult>
549 const DebugObserverScopedTimer timer(
this,
"getLocalSlamResultCallback | Entire Method");
552 ARMARX_DEBUG <<
"Callback (getLocalSlamResultCallback)";
575 const DebugObserverScopedTimer localTimer(
576 this,
"getLocalSlamResultCallback | Prepare Local Slam Data");
580 << IceUtil::Time::microSeconds(timestampInMicroSeconds);
583 localSlamData.
misses =
toPCL(rangeDataInLocal.misses.points());
585 localSlamData.
timestamp = timestampInMicroSeconds;
587 toEigen(mapBuilder->pose_graph()->GetLocalToGlobalTransform(trajectoryId));
589 ARMARX_DEBUG <<
"Callbacks will be triggered (getLocalSlamResultCallback)";
592 const DebugObserverScopedTimer localTimer(
593 this,
"getLocalSlamResultCallback | On Local Slam Data");
594 slamDataCallable.onLocalSlamData(localSlamData);
600 const auto nConstraints = numConstraints();
602 if (debugObserverHelper.has_value())
604 const std::unique_lock lock(debugObserverHelperMtx);
605 debugObserverHelper->setDebugObserverDatafield(
"Number of constraints",
610 ARMARX_DEBUG <<
"Callback done (getLocalSlamResultCallback)";
623 const DebugObserverScopedTimer timer(
this,
"hasReceivedSensorData | Entire Method");
624 std::lock_guard g{cartoInsertMtx};
626 const auto trajectoryNodes = mapBuilder->pose_graph()->GetTrajectoryNodes();
628 auto matchesTrajectoryId = [&](
const auto& node) ->
bool
629 {
return node.id.trajectory_id == trajectoryId; };
632 return std::any_of(trajectoryNodes.begin(), trajectoryNodes.end(), matchesTrajectoryId);
639 std::ofstream ofs(jsonFile, std::ios::out);
649 ARMARX_INFO <<
"Creating dummy registration file `" << jsonFile <<
"`";
659 const std::string filenameNoExtension =
timestamp();
661 const fs::path mapFilename =
662 std::filesystem::path(mapOutputPath.
toSystemPath()) / (filenameNoExtension +
".carto");
663 ARMARX_DEBUG <<
"The concrete map file is " << mapFilename.string();
664 assert(!fs::exists(map_filename));
669 const int trajectoryIdToBeOptimized = trajectoryId;
671 std::lock_guard g{cartoInsertMtx};
673 trajectoryId = mapBuilder->AddTrajectoryBuilder(
675 trajectoryBuilderOptions,
677 {
return getLocalSlamResultCallback(std::forward<
decltype(args)>(args)...); });
680 mapBuilder->FinishTrajectory(trajectoryIdToBeOptimized);
681 mapBuilder->pose_graph()->RunFinalOptimization();
684 carto::io::ProtoStreamWriter writer(mapFilename);
685 mapBuilder->SerializeState(
true, &writer);
690 const fs::path registrationFilename = mapPath / (filenameNoExtension +
".json");
691 ARMARX_INFO <<
"The default registration data will be stored in " << registrationFilename;
701 CartographerAdapter::lookupSensor(
const std::string& device)
const
704 const DebugObserverScopedTimer timer(
this,
"lookupSensor");
705 auto laserSensorIt = laserSensorMap.find(device);
706 if (laserSensorIt == laserSensorMap.end())
708 throw armarx::LocalException(
"No sensor with id '" + device +
"' found.");
710 return laserSensorIt->second;
717 auto laserSensorIt = laserSensorMap.find(sensorId);
718 if (laserSensorIt == laserSensorMap.end())
720 throw armarx::LocalException(
"No sensor with id '" + sensorId +
"' found.");
722 laserSensorIt->second.pose = pose;
725 std::vector<std::string>
732 CartographerAdapter::DebugObserverScopedTimer::~DebugObserverScopedTimer()
735 if (parent->debugObserverHelper)
738 const std::unique_lock lock(parent->debugObserverHelperMtx);
739 parent->debugObserverHelper->setDebugObserverDatafield(name +
" [ms]",
744 CartographerAdapter::DebugObserverScopedTimer::DebugObserverScopedTimer(
745 const CartographerAdapter* parent,
746 const std::string& name) :
754 const auto constraints = mapBuilder->pose_graph()->constraints();
755 return std::count_if(
758 [trajectoryId = this->trajectoryId](
759 const ::cartographer::mapping::PoseGraphInterface::Constraint& constraint)
760 { return constraint.node_id.trajectory_id == trajectoryId; });
763 ::cartographer::mapping::MapBuilderInterface&
constexpr int SCALE_M_TO_MM
SpamFilterDataPtr deactivateSpam(SpamFilterDataPtr const &spamFilter, float deactivationDurationSec, const std::string &identifier, bool deactivate)
static Duration MicroSeconds(std::int64_t microSeconds)
Constructs a duration in microseconds.
static std::filesystem::path toSystemPath(const data::PackagePath &pp)
Represents a point in time.
std::int64_t toMicroSeconds() const
Returns the amount of microseconds.
void processSensorValues(LaserScannerMessage data)
void processOdometryPose(OdomData odomData)
CartographerAdapter(const std::filesystem::path &mapPath, const std::filesystem::path &configPath, SlamDataCallable &slamDataCallable, const Config &config, const std::optional< std::filesystem::path > &mapToLoad=std::nullopt, const std::optional< Eigen::Isometry3f > &map_T_robot_prior=std::nullopt, const DebugObserverInterfacePrx &debugObserver=nullptr)
std::vector< std::string > laserSensorIds() const
void setSensorPose(const std::string &sensorId, const Eigen::Isometry3f &pose)
::cartographer::mapping::MapBuilderInterface & getMapBuilder()
unsigned int numConstraints() const
virtual ~CartographerAdapter()
bool hasReceivedSensorData() const noexcept
void onTimedDataAvailable(const TimedData &data) override
bool createMap(const armarx::PackagePath &mapOutputPath)
#define ARMARX_CHECK(expression)
Shortcut for ARMARX_CHECK_EXPRESSION.
#define ARMARX_CHECK_NOT_NULL(ptr)
This macro evaluates whether ptr is not null and if it turns out to be false it will throw an Express...
#define ARMARX_INFO
The normal logging level.
#define ARMARX_IMPORTANT
The logging level for always important information, but expected behaviour (in contrast to ARMARX_WAR...
#define ARMARX_DEBUG
The logging level for output that is only interesting while debugging.
#define ARMARX_WARNING
The logging level for unexpected behaviour, but not a serious problem.
#define ARMARX_VERBOSE
The logging level for verbose information.
Quaternion< float, 0 > Quaternionf
pcl::PointXYZ toPCL(const Eigen::Vector3f &v)
int64_t fromCarto(::cartographer::common::Time time)
Convert cartographer time to unix time in [µs].
std::unique_ptr<::cartographer::common::LuaParameterDictionary > resolveLuaParameters(const std::string &luaCode, const std::filesystem::path &configPath)
Helper function to create Lua parameter object from string.
This file is part of ArmarX.
bool storeDefaultRegistrationFile(const fs::path &jsonFile)
::pcl::PointCloud<::pcl::PointXYZ > toPCL(const std::vector<::cartographer::sensor::TimedRangefinderPoint > &points)
::cartographer::transform::Rigid3d fromEigen(const Eigen::Isometry3f &pose)
This file offers overloads of toIce() and fromIce() functions for STL container types.
::IceInternal::ProxyHandle<::IceProxy::armarx::DebugObserverInterface > DebugObserverInterfacePrx
void getMapKeys(const MapType &map, OutputIteratorType it)
Eigen::Vector3f toEigen(const pcl::PointXYZ &pt)
Eigen::Quaternionf orientation
int64_t timestamp
timestamp in [ms]
EIGEN_MAKE_ALIGNED_OPERATOR_NEW Eigen::Vector3f position
Eigen::Isometry3f trajectory_pose
pcl::PointCloud< pcl::PointXYZ > misses
Eigen::Isometry3f local_pose
pcl::PointCloud< pcl::PointXYZ > local_points