19#include <Eigen/Geometry>
22#include <IceUtil/Time.h>
25#include <pcl/common/transforms.h>
26#include <pcl/io/pcd_io.h>
27#include <pcl/point_cloud.h>
28#include <pcl/point_types.h>
31#include <opencv2/core/mat.hpp>
32#include <opencv2/imgcodecs.hpp>
35#include <SimoxUtility/algorithm/string/string_tools.h>
36#include <SimoxUtility/json/json.hpp>
37#include <SimoxUtility/math.h>
49#include <cartographer/common/configuration_file_resolver.h>
50#include <cartographer/common/time.h>
51#include <cartographer/io/file_writer.h>
52#include <cartographer/io/points_processor_pipeline_builder.h>
53#include <cartographer/io/proto_stream.h>
54#include <cartographer/mapping/2d/grid_2d.h>
55#include <cartographer/mapping/2d/probability_grid.h>
56#include <cartographer/mapping/2d/submap_2d.h>
57#include <cartographer/mapping/internal/2d/tsdf_2d.h>
58#include <cartographer/mapping/map_builder.h>
59#include <cartographer/mapping/map_builder_interface.h>
60#include <cartographer/mapping/pose_graph_interface.h>
61#include <cartographer/mapping/submaps.h>
62#include <cartographer/mapping/trajectory_builder_interface.h>
63#include <cartographer/mapping/trajectory_node.h>
64#include <cartographer/sensor/point_cloud.h>
65#include <cartographer/sensor/rangefinder_point.h>
66#include <cartographer/transform/rigid_transform.h>
67#include <cartographer/transform/transform.h>
83namespace fs = ::std::filesystem;
91 struct tm* timeinfo{};
92 std::array<char, 4 + 1 + 2 + 1 + 2 + 1 + 2 + 1 + 2 + 1> buffer;
95 timeinfo = localtime(&rawtime);
97 strftime(buffer.data(), buffer.size(),
"%Y-%m-%d-%H-%M", timeinfo);
111 approxTimeQueue.reset();
114 mapBuilder->FinishTrajectory(trajectoryId);
123 const std::filesystem::path& mapPath,
124 const std::filesystem::path& configPath,
127 const std::optional<std::filesystem::path>& mapToLoad,
128 const std::optional<Eigen::Isometry3f>& map_T_robot_prior,
130 config(config), mapPath(mapPath), slamDataCallable(slamDataCallable)
133 const std::string mapBuilderLuaCode = R
"text(
134 include "map_builder.lua"
135 return MAP_BUILDER)text";
137 const std::string trajectoryBuilderLuaCode = R
"text(
138 include "trajectory_builder.lua"
139 return TRAJECTORY_BUILDER)text";
142 auto mapBuilderOptions =
143 carto::mapping::CreateMapBuilderOptions(mapBuilderParameters.get());
146 if (config.global_localization_min_score > 0)
148 ARMARX_INFO <<
"Setting `global_localization_min_score` to "
149 << config.global_localization_min_score;
150 mapBuilderOptions.mutable_pose_graph_options()
151 ->mutable_constraint_builder_options()
152 ->set_global_localization_min_score(config.global_localization_min_score);
155 if (config.min_score > 0)
157 ARMARX_INFO <<
"Setting `min_score` to " << config.min_score;
158 mapBuilderOptions.mutable_pose_graph_options()
159 ->mutable_constraint_builder_options()
160 ->set_min_score(config.min_score);
163 const auto trajectoryBuilderParameters =
165 trajectoryBuilderOptions =
166 carto::mapping::CreateTrajectoryBuilderOptions(trajectoryBuilderParameters.get());
169 mapBuilder = carto::mapping::CreateMapBuilder(mapBuilderOptions);
171 if (config.useLaserScanner)
173 sensorSet.insert(sensorIdLaser);
176 if (config.useOdometry)
178 sensorSet.insert(sensorIdOdom);
183 sensorSet.insert(sensorIdIMU);
186 std::vector<std::string> sensorSetNames;
187 std::transform(sensorSet.begin(),
189 std::back_inserter(sensorSetNames),
190 [](
const auto& sensorId) { return sensorId.id; });
194 if (config.useLaserScanner)
197 << std::vector<std::string>(config.laserScanners.begin(),
198 config.laserScanners.end());
200 std::for_each(config.laserScanners.begin(),
201 config.laserScanners.end(),
202 [&](
const auto& laserScannerName)
204 const SensorId sensorId{.type = SensorId::SensorType::RANGE,
205 .id = laserScannerName};
206 laserSensorMap.insert(
208 LaserScannerSensor(sensorId, Eigen::Isometry3f::Identity())});
217 carto::io::ProtoStreamReader reader(mapToLoad->string());
218 mapBuilder->LoadState(&reader,
true );
219 mapBuilder->pose_graph()->RunFinalOptimization();
222 if (map_T_robot_prior)
224 ARMARX_INFO <<
"Using pose prior \n" << map_T_robot_prior->matrix();
226 ::cartographer::mapping::proto::InitialTrajectoryPose* initial_trajectory_pose =
227 new ::cartographer::mapping::proto::InitialTrajectoryPose;
229 initial_trajectory_pose->set_allocated_relative_pose(
230 new ::cartographer::transform::proto::Rigid3d(
231 ::cartographer::transform::ToProto(
fromEigen(*map_T_robot_prior))));
233 trajectoryBuilderOptions.set_allocated_initial_trajectory_pose(
234 std::move(initial_trajectory_pose));
239 trajectoryId = mapBuilder->AddTrajectoryBuilder(
241 trajectoryBuilderOptions,
242 [
this](
auto&&... args)
243 {
return getLocalSlamResultCallback(std::forward<
decltype(args)>(args)...); });
246 const auto dt =
static_cast<int64_t
>(1'000'000 / config.frequency);
247 const float dtHistoryLength = 1'000'000;
252 std::make_unique<ApproximateTimeQueue>(
dt,
254 config.laserScanners,
256 config.useOdometryBasedLaserCorrection);
262 cartographerInputDataReporterLaserScanner.emplace(
263 debugObserver,
"CartographerMappingAndLocalization_input_laser");
264 cartographerInputDataReporterOdometry.emplace(
265 debugObserver,
"CartographerMappingAndLocalization_input_odom");
266 if (config.enableDebugTiming)
268 debugObserverHelper.emplace(
269 "CartographerMappingAndLocalization_CartographerAdapter", debugObserver,
true);
278 pushInOdomData(
data.odomPose);
279 pushInLaserData(
data.laserData);
280 if (debugObserverHelper)
283 std::unique_lock g(debugObserverHelperMtx);
284 debugObserverHelper->sendDebugObserverBatch();
289 CartographerAdapter::pushInOdomData(
293 const DebugObserverScopedTimer timer(
this,
"pushInOdomData | Entire Method");
294 if (lastOdomTimestamp.has_value())
297 if (odomPose.
timestamp <= lastOdomTimestamp.value())
300 <<
"Requested to add old data into odom queue. Not willing to do this. It is "
301 << (odomPose.
timestamp - lastOdomTimestamp.value()) <<
"ms in the past.";
311 auto isOdomSensor = [](
const SensorId& sensorId)
312 {
return sensorId.type == SensorId::SensorType::ODOMETRY; };
314 if (not std::any_of(sensorSet.begin(), sensorSet.end(), isOdomSensor))
317 ARMARX_WARNING <<
"No odometry sensor is registered. Will not use odometry data.";
325 const Eigen::Vector3f p = odomPose.
pose.translation();
327 carto::sensor::OdometryData odomData;
329 carto::transform::Rigid3d::FromArrays(std::array<double, 4>{
q.w(),
q.x(),
q.y(),
q.z()},
330 std::array<double, 3>{p.x(), p.y(), p.z()});
331 odomData.time = util::toCarto(
timestamp);
337 const DebugObserverScopedTimer localTimer(
this,
"pushInOdomData | Inserting Odom Data");
338 std::lock_guard guard{cartoInsertMtx};
340 mapBuilder->GetTrajectoryBuilder(trajectoryId)
341 ->AddSensorData(sensorIdOdom.id, odomData);
344 if (cartographerInputDataReporterOdometry.has_value())
347 cartographerInputDataReporterOdometry->add(
timestamp);
354 CartographerAdapter::pushInLaserData(
const LaserMessages& laserMessages)
358 if (not config.useLaserScanner)
364 const DebugObserverScopedTimer timer(
this,
"pushInLaserData | Entire Method");
365 carto::sensor::TimedPointCloudData timedPointCloudData;
366 timedPointCloudData.origin = Eigen::Vector3f::Zero();
370 const DebugObserverScopedTimer laserMessageTimer(
371 this,
"pushInLaserData | Iterate over laser messages");
372 for (
const auto& message : laserMessages)
374 const auto&
data = message.data;
377 << message.data.frame;
382 auto& lastTimestamp = lastSensorDataTimestamp[
data.frame];
384 if (
data.timestamp <= lastTimestamp)
386 ARMARX_DEBUG <<
"Requested to add old data with timestamp "
387 <<
data.timestamp <<
" into laser queue for sensor "
388 <<
data.frame <<
". Not willing to do this. It is "
389 << (
data.timestamp - lastTimestamp) <<
"ms in the past.";
393 lastTimestamp =
data.timestamp;
397 Eigen::Isometry3f robot_T_sensor;
400 const LaserScannerSensor sensor = lookupSensor(
data.frame);
402 << sensor.pose.translation();
404 robot_T_sensor = sensor.pose;
406 catch (
const armarx::LocalException&)
408 ARMARX_INFO <<
"Sensor `" <<
data.frame <<
"` not found in sensor set.";
415 const DebugObserverScopedTimer localTimer(
416 this,
"pushInLaserData | Reserve TimedPointCloudData");
417 timedPointCloudData.time =
418 util::toCarto(
data.timestamp);
419 timedPointCloudData.ranges.reserve(timedPointCloudData.ranges.size() +
425 const DebugObserverScopedTimer localTimer(
426 this,
"pushInLaserData | Transform to TimedPointCloudData");
427 std::transform(
data.scan.begin(),
429 std::back_inserter(timedPointCloudData.ranges),
430 [&](
const auto& scanStep) -> carto::sensor::TimedRangefinderPoint
433 toCartesian<Eigen::Vector3f>(scanStep);
436 carto::sensor::TimedRangefinderPoint point;
438 point.position = robot_T_sensor * rangePoint;
446 auto cloud =
toPCL(timedPointCloudData.ranges);
448 const DebugObserverScopedTimer localTimer(
this,
"pushInLaserData | onLaserSensorData");
450 slamDataCallable.onLaserSensorData({util::fromCarto(timedPointCloudData.time), cloud});
454 const DebugObserverScopedTimer localTimer(
this,
455 "pushInLaserData | Inserting Laser Data");
456 std::lock_guard guard{cartoInsertMtx};
458 auto* trajBuilder = mapBuilder->GetTrajectoryBuilder(trajectoryId);
459 if (trajBuilder ==
nullptr)
465 const std::int64_t laserScannerInsertCartoTimestamp =
466 timedPointCloudData.time.time_since_epoch().count();
468 ARMARX_DEBUG <<
"Inserting " << sensorIdLaser.id <<
" with timestamp "
469 << laserScannerInsertCartoTimestamp;
471 if (lastLaserScannerInsertCartoTimestamp.has_value())
473 if (laserScannerInsertCartoTimestamp <= lastLaserScannerInsertCartoTimestamp)
475 ARMARX_INFO <<
"Received laser scanner data out of order. Will not handle it.";
480 lastLaserScannerInsertCartoTimestamp = laserScannerInsertCartoTimestamp;
483 const DebugObserverScopedTimer addSensorDataTimer(
484 this,
"pushInLaserData | Add Sensor Data to TrajectoryBuilder");
486 trajBuilder->AddSensorData(sensorIdLaser.id, timedPointCloudData);
489 if (cartographerInputDataReporterLaserScanner)
491 cartographerInputDataReporterLaserScanner->add(
504 const DebugObserverScopedTimer timer(
this,
"processOdometryPose | Entire Method");
508 odomPose.
pose.setIdentity();
512 approxTimeQueue->insertOdomData(std::move(odomPose));
519 const DebugObserverScopedTimer timer(
this,
"processSensorValues | Entire Method");
538 approxTimeQueue->insertLaserData(std::move(
data));
542 CartographerAdapter::getLocalSlamResultCallback(
543 const int trajectoryId,
544 const ::cartographer::common::Time time,
545 const ::cartographer::transform::Rigid3d localPose,
546 ::cartographer::sensor::RangeData rangeDataInLocal,
547 const std::unique_ptr<
548 const ::cartographer::mapping::TrajectoryBuilderInterface::InsertionResult>
552 const DebugObserverScopedTimer timer(
this,
"getLocalSlamResultCallback | Entire Method");
555 ARMARX_DEBUG <<
"Callback (getLocalSlamResultCallback)";
578 const DebugObserverScopedTimer localTimer(
579 this,
"getLocalSlamResultCallback | Prepare Local Slam Data");
583 << IceUtil::Time::microSeconds(timestampInMicroSeconds);
586 localSlamData.
misses =
toPCL(rangeDataInLocal.misses.points());
588 localSlamData.
timestamp = timestampInMicroSeconds;
590 toEigen(mapBuilder->pose_graph()->GetLocalToGlobalTransform(trajectoryId));
592 ARMARX_DEBUG <<
"Callbacks will be triggered (getLocalSlamResultCallback)";
595 const DebugObserverScopedTimer localTimer(
596 this,
"getLocalSlamResultCallback | On Local Slam Data");
597 slamDataCallable.onLocalSlamData(localSlamData);
603 const auto nConstraints = numConstraints();
605 if (debugObserverHelper.has_value())
607 const std::unique_lock lock(debugObserverHelperMtx);
608 debugObserverHelper->setDebugObserverDatafield(
"Number of constraints",
613 ARMARX_DEBUG <<
"Callback done (getLocalSlamResultCallback)";
626 const DebugObserverScopedTimer timer(
this,
"hasReceivedSensorData | Entire Method");
627 std::lock_guard g{cartoInsertMtx};
629 const auto trajectoryNodes = mapBuilder->pose_graph()->GetTrajectoryNodes();
631 auto matchesTrajectoryId = [&](
const auto& node) ->
bool
632 {
return node.id.trajectory_id == trajectoryId; };
635 return std::any_of(trajectoryNodes.begin(), trajectoryNodes.end(), matchesTrajectoryId);
642 std::ofstream ofs(jsonFile, std::ios::out);
652 ARMARX_INFO <<
"Creating dummy registration file `" << jsonFile <<
"`";
662 const std::string filenameNoExtension =
timestamp();
664 const fs::path mapFilename =
665 std::filesystem::path(mapOutputPath.
toSystemPath()) / (filenameNoExtension +
".carto");
666 ARMARX_DEBUG <<
"The concrete map file is " << mapFilename.string();
667 assert(!fs::exists(map_filename));
672 const int trajectoryIdToBeOptimized = trajectoryId;
674 std::lock_guard g{cartoInsertMtx};
676 trajectoryId = mapBuilder->AddTrajectoryBuilder(
678 trajectoryBuilderOptions,
680 {
return getLocalSlamResultCallback(std::forward<
decltype(args)>(args)...); });
683 mapBuilder->FinishTrajectory(trajectoryIdToBeOptimized);
684 mapBuilder->pose_graph()->RunFinalOptimization();
687 carto::io::ProtoStreamWriter writer(mapFilename);
688 mapBuilder->SerializeState(
true, &writer);
693 const fs::path registrationFilename = mapPath / (filenameNoExtension +
".json");
694 ARMARX_INFO <<
"The default registration data will be stored in " << registrationFilename;
704 CartographerAdapter::lookupSensor(
const std::string& device)
const
707 const DebugObserverScopedTimer timer(
this,
"lookupSensor");
708 auto laserSensorIt = laserSensorMap.find(device);
709 if (laserSensorIt == laserSensorMap.end())
711 throw armarx::LocalException(
"No sensor with id '" + device +
"' found.");
713 return laserSensorIt->second;
720 auto laserSensorIt = laserSensorMap.find(sensorId);
721 if (laserSensorIt == laserSensorMap.end())
723 throw armarx::LocalException(
"No sensor with id '" + sensorId +
"' found.");
725 laserSensorIt->second.pose = pose;
728 std::vector<std::string>
735 CartographerAdapter::DebugObserverScopedTimer::~DebugObserverScopedTimer()
738 if (parent->debugObserverHelper)
741 const std::unique_lock lock(parent->debugObserverHelperMtx);
742 parent->debugObserverHelper->setDebugObserverDatafield(name +
" [ms]",
747 CartographerAdapter::DebugObserverScopedTimer::DebugObserverScopedTimer(
748 const CartographerAdapter* parent,
749 const std::string& name) :
757 const auto constraints = mapBuilder->pose_graph()->constraints();
758 return std::count_if(
761 [trajectoryId = this->trajectoryId](
762 const ::cartographer::mapping::PoseGraphInterface::Constraint& constraint)
763 { return constraint.node_id.trajectory_id == trajectoryId; });
766 ::cartographer::mapping::MapBuilderInterface&
constexpr int SCALE_M_TO_MM
SpamFilterDataPtr deactivateSpam(SpamFilterDataPtr const &spamFilter, float deactivationDurationSec, const std::string &identifier, bool deactivate)
static Duration MicroSeconds(std::int64_t microSeconds)
Constructs a duration in microseconds.
static std::filesystem::path toSystemPath(const data::PackagePath &pp)
Represents a point in time.
std::int64_t toMicroSeconds() const
Returns the amount of microseconds.
void processSensorValues(LaserScannerMessage data)
void processOdometryPose(OdomData odomData)
CartographerAdapter(const std::filesystem::path &mapPath, const std::filesystem::path &configPath, SlamDataCallable &slamDataCallable, const Config &config, const std::optional< std::filesystem::path > &mapToLoad=std::nullopt, const std::optional< Eigen::Isometry3f > &map_T_robot_prior=std::nullopt, const DebugObserverInterfacePrx &debugObserver=nullptr)
std::vector< std::string > laserSensorIds() const
void setSensorPose(const std::string &sensorId, const Eigen::Isometry3f &pose)
::cartographer::mapping::MapBuilderInterface & getMapBuilder()
unsigned int numConstraints() const
virtual ~CartographerAdapter()
bool hasReceivedSensorData() const noexcept
void onTimedDataAvailable(const TimedData &data) override
bool createMap(const armarx::PackagePath &mapOutputPath)
#define ARMARX_CHECK(expression)
Shortcut for ARMARX_CHECK_EXPRESSION.
#define ARMARX_CHECK_NOT_NULL(ptr)
This macro evaluates whether ptr is not null and if it turns out to be false it will throw an Express...
#define ARMARX_INFO
The normal logging level.
#define ARMARX_IMPORTANT
The logging level for always important information, but expected behaviour (in contrast to ARMARX_WAR...
#define ARMARX_DEBUG
The logging level for output that is only interesting while debugging.
#define ARMARX_WARNING
The logging level for unexpected behaviour, but not a serious problem.
#define ARMARX_VERBOSE
The logging level for verbose information.
Quaternion< float, 0 > Quaternionf
pcl::PointXYZ toPCL(const Eigen::Vector3f &v)
int64_t fromCarto(::cartographer::common::Time time)
Convert cartographer time to unix time in [µs].
std::unique_ptr<::cartographer::common::LuaParameterDictionary > resolveLuaParameters(const std::string &luaCode, const std::filesystem::path &configPath)
Helper function to create Lua parameter object from string.
This file is part of ArmarX.
bool storeDefaultRegistrationFile(const fs::path &jsonFile)
::pcl::PointCloud<::pcl::PointXYZ > toPCL(const std::vector<::cartographer::sensor::TimedRangefinderPoint > &points)
::cartographer::transform::Rigid3d fromEigen(const Eigen::Isometry3f &pose)
This file offers overloads of toIce() and fromIce() functions for STL container types.
::IceInternal::ProxyHandle<::IceProxy::armarx::DebugObserverInterface > DebugObserverInterfacePrx
void getMapKeys(const MapType &map, OutputIteratorType it)
Eigen::Vector3f toEigen(const pcl::PointXYZ &pt)
Eigen::Quaternionf orientation
int64_t timestamp
timestamp in [ms]
EIGEN_MAKE_ALIGNED_OPERATOR_NEW Eigen::Vector3f position
Eigen::Isometry3f trajectory_pose
pcl::PointCloud< pcl::PointXYZ > misses
Eigen::Isometry3f local_pose
pcl::PointCloud< pcl::PointXYZ > local_points