ObjectLocalizerProcessor.h
Go to the documentation of this file.
1 /*
2  * This file is part of ArmarX.
3  *
4  * Copyright (C) 2011-2016, High Performance Humanoid Technologies (H2T), Karlsruhe Institute of Technology (KIT), all rights reserved.
5  *
6  * ArmarX is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  *
10  * ArmarX is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program. If not, see <http://www.gnu.org/licenses/>.
17  *
18  * @package VisionX::Component
19  * @author Kai Welke (welke at kit dot edu)
20  * @date 2013
21  * @copyright http://www.gnu.org/licenses/gpl-2.0.txt
22  * GNU General Public License
23  */
24 
25 #pragma once
26 
28 
29 // VisionX
32 
33 // VisionXInterface
34 #include <VisionX/interface/units/ObjectRecognitionUnit.h>
35 #include <VisionX/interface/components/ObjectLocalizerInterfaces.h>
36 
37 // MemoryX
38 #include <MemoryX/interface/components/PriorKnowledgeInterface.h>
41 
42 // IVT
43 #include <Structs/ObjectDefinitions.h>
44 #include <Calibration/StereoCalibration.h>
45 
46 #include <Eigen/Core>
47 
48 #include <boost/ptr_container/ptr_vector.hpp>
49 #include <string>
50 
51 
52 // forward declarations
53 class CByteImage;
54 class CTexturedRecognition;
55 
56 namespace memoryx
57 {
60 }
61 
62 
63 
64 namespace visionx
65 {
66 
67  static Eigen::Vector3i extractColorValue(std::string propertyValue)
68  {
69  int number = (int) strtol(&propertyValue.c_str()[1], NULL, 16);
70 
71  int r = number >> 16;
72  int g = number >> 8 & 0xFF;
73  int b = number & 0xFF;
74 
75  return Eigen::Vector3i(r, g, b);
76  }
77 
80  {
81  public:
82 
85  {
86  defineOptionalProperty<std::string>("PriorKnowledgeProxyName", "PriorKnowledge", "name of prior memory proxy");
87  defineOptionalProperty<std::string>("DataBaseObjectCollectionName", "memdb.Prior_Objects", "name of collection from database to use for object classes");
88  defineOptionalProperty<std::string>("ImageProviderName", "ImageProvider", "name of the image provider to use");
89  defineRequiredProperty<std::string>("AgentName", "Name of the agent for which the sensor values are provided");
90  defineOptionalProperty<std::string>("CalibrationUpdateTopicName", "StereoCalibrationInterface", "Topic name of the stereo calibration provider");
91  defineOptionalProperty<float>("2DLocalizationNoise", 4.0, "2D localization noise of object recognition. Used in order to calculate the 3D world localization noise.");
92  defineOptionalProperty<bool>("EnableResultImages", true, "Enable the calculation of resultimages which are then provided using an ImageProvider");
93  defineOptionalProperty<bool>("useResultImageMask", false, "Use a color mask for the result images if set. Otherwise the camera images are used");
95  defineOptionalProperty<Eigen::Vector3i>("colorMask", extractColorValue("#FFFFFF"), "image color that should be used as alpha channel").setFactory(f);
96 
97  defineOptionalProperty<std::string>("DebugObserverName", "DebugObserver", "Name of the topic the DebugObserver listens on");
98  }
99 
100  };
101 
102 
103  /**
104  * ObjectLocalizerProcessor
105  *
106  */
108  virtual public ImageProcessor,
109  virtual public ObjectLocalizerImageInterface
110  {
112 
113  public:
115 
116  /**
117  * @see PropertyUser::createPropertyDefinitions()
118  */
120  {
122  }
123 
124  /**
125  * Called from framework. Sets up the required proxies and properties from config file.
126  * Calls the onInitObjectLocalizerProcessor() hook.
127  *
128  * @see Component::onInitComponent()
129  */
130  void onInitImageProcessor() override;
131 
132  /**
133  * Called from framework. Establishes the database connection, retrieves image provider settings and stereo calibration, initializes visualization
134  * if enabled and reads the object classes from the MemoryX database.
135  *
136  * Calls the onConnectObjectLocalizerProcessor() hook.
137  *
138  * @see Component::onConnectComponent()
139  */
140  void onConnectImageProcessor() override;
141 
142  /**
143  * Called from framework.
144  *
145  * Calls the onDisconnectObjectLocalizerProcessor() hook.
146  *
147  * @see Component::onExitComponent()
148  */
149  void onDisconnectComponent() override
150  {
151  // Prevent deadlock: Cancel job before stopping the other threads
152  job.exit();
155  }
156 
157  /**
158  * Called from framework.
159  *
160  * Calls the onExitObjectLocalizerProcessor() hook.
161  *
162  * @see Component::onExitComponent()
163  */
164  void onExitImageProcessor() override
165  {
167  }
168 
169  /**
170  * The process method is inherited from ImageProcessor. The process method is called by the framework
171  * in a separate thread and. The loading of the database and the triggering of recognition is performed
172  * in this thread.
173  *
174  * @see ImageProcessor::process()
175  */
176  void process() override;
177 
178  /**
179  * The process method is inherited from the ObjectLocalizationProcessorInterface and is called by the WorkingMemoryUpdater.
180  * It should be considered final. Implement ObjectLocalizerProcessor::localizeObjectClass(const std::string& objecClassName, CByteImage** cameraImages, CByteIamge** resultImages)
181  * in your subclass instead.
182  */
183  memoryx::ObjectLocalizationResultList localizeObjectClasses(const memoryx::ObjectClassNameList& objectClassNames, const Ice::Current& c = Ice::emptyCurrent) override;
184 
185  protected:
186  /**
187  * ObjectLocalizerProcessor interface: The localizeObjectClass method needs to be implemented by any ObjectLocalizer.
188  * Based on the object class name and the camera images it generates a list of localization results which correspond
189  * to found instances of objects belonging to that class.
190  *
191  * This method is called by an @ObjectLocalizerProcessorJob.
192  *
193  * @param objectClassNames names of the class to localize
194  * @param cameraImages the two input images
195  * @param resultImages the two result images. are provided if result images are enabled.
196  *
197  * @return list of object instances
198  */
199  virtual memoryx::ObjectLocalizationResultList localizeObjectClasses(const std::vector<std::string>& objectClassNames, CByteImage** cameraImages, armarx::MetaInfoSizeBasePtr imageMetaInfo, CByteImage** resultImages) = 0;
200 
201  /**
202  * ObjectLocalizerProcessor interface: The initRecognizer method needs to be implemented by any ObjectLocalizer.
203  * the method is called in the same thread as addObjectClass and localizeObjectClass. Initialize your
204  * recognition method here.
205  *
206  * @return success of adding this entity to the recognition method
207  */
208  virtual bool initRecognizer() = 0;
209 
210  /**
211  * ObjectLocalizerProcessor interface: The addObjectClass method needs to be implemented by any ObjectLocalizer.
212  * Add object class to localizer. Called by the ObjectLocalizerProcessor for each object class in prior knowledge, that
213  * has the RecognitionMethod attribute corresponding to the default name of this component (see Component::getDefaultName())
214  *
215  * @param objectClassEntity entity containing all information available for the object class
216  * @param fileManager GridFileManager required to read files associated to prior knowledgge from the database. Usually accessed via an AbstractEntityWrapper.
217  *
218  * @return success of adding this entity to the recognition method
219  */
220  virtual bool addObjectClass(const memoryx::EntityPtr& objectClassEntity, const memoryx::GridFileManagerPtr& fileManager) = 0;
221 
222  /**
223  * ObjectLocalizerProcessor interface: subclass hook
224  *
225  * @see ObjectLocalizerProcessor::onInitImageProcessor()
226  */
228 
229  /**
230  * ObjectLocalizerProcessor interface: subclass hook
231  *
232  * @see ObjectLocalizerProcessor::onConnectImageProcessor()
233  */
235 
236  /**
237  * ObjectLocalizerProcessor interface: subclass hook
238  *
239  * @see ObjectLocalizerProcessor::onDisconnectImageProcessor()
240  */
242 
243  /**
244  * ObjectLocalizerProcessor interface: subclass hook
245  *
246  * @see ObjectLocalizerProcessor::onExitImageProcessor()
247  */
249 
250  /**
251  * Calculate 3D uncertainty from two 2d points in left and right camera. Uses the
252  * property "2DLocalizationNoise" for estimating the 3D noise. Uses unscented
253  * transform through epipolar geometry to derive an approximation
254  * The reference frame is always the left camera.
255  *
256  * @param left_point point in left camera image
257  * @param right_point point in right camera image
258  * @return 3D localization uncertainty
259  */
261 
262  /**
263  * Calculate localization uncertainty for a 3d point. First projects the point
264  * to the camera images and then applies calculateLocalizationUncertainty(Vec2d left_point, Vec2d right_point).
265  * The reference frame is always the left camera.
266  *
267  * @param position 3d position of the point
268  * @return 3D localization uncertainty
269  */
271 
272  void reportStereoCalibrationChanged(const StereoCalibration& stereoCalibration, bool imagesAreUndistorted, const std::string& referenceFrame, const Ice::Current& c = Ice::emptyCurrent) override
273  {
274  this->stereoCalibration.reset(visionx::tools::convert(stereoCalibration));
275  this->imagesAreUndistorted = imagesAreUndistorted;
276  this->referenceFrameName = referenceFrame;
277  }
278 
279  /**
280  * Retrieve whether result images are enabled
281  *
282  * @return result images enables
283  */
285  {
286  return resultImagesEnabled;
287  }
288 
289  /**
290  * Retrieve whether result images are enabled
291  *
292  * @return result images enables
293  */
294  std::string getReferenceFrameName() const
295  {
296  return referenceFrameName;
297  }
298 
299  /**
300  * Retrieve format of input images
301  *
302  * @return iamge format
303  */
304  ImageFormatInfo getImageFormat() const
305  {
306  return imageFormat;
307  }
308 
309  /**
310  * Retrieve whether images are undistorted
311  *
312  * @return whether images are undistorted
313  */
315  {
316  return imagesAreUndistorted;
317  }
318 
320  {
321  return useResultImageMask;
322  }
323 
324  Eigen::Vector3i getColorMask() const
325  {
326  return colorMask;
327  }
328 
329  /**
330  * Retrieve stereo calibration corresponding to image provider
331  *
332  * @return stereo calibration
333  */
334  CStereoCalibration* getStereoCalibration() const
335  {
336  return stereoCalibration.get();
337  }
338 
339  private:
340  // read object classes from prior memory
341  void initObjectClasses();
342  // setup image objects for a specific size
343  void setupImages(int width, int height);
344 
345 
346  Eigen::Vector3i colorMask;
347  bool useResultImageMask;
348  bool resultImagesEnabled;
349 
350  // proxies
351  ImageProviderInterfacePrx imageProviderPrx;
352  memoryx::PriorKnowledgeInterfacePrx priorKnowledgePrx;
353  memoryx::PersistentObjectClassSegmentBasePrx classesSegmentPrx;
354  memoryx::CommonStorageInterfacePrx databasePrx;
355 
356  memoryx::GridFileManagerPtr fileManager;
357 
358 
359  // stereo information
360  std::unique_ptr<CStereoCalibration> stereoCalibration;
361  bool imagesAreUndistorted;
362  ImageFormatInfo imageFormat;
363 
364  protected:
365  // settings
366  std::string imageProviderName;
368  std::string referenceFrameName;
369 
370  // images
371  CByteImage cameraImagesData[2];
372  CByteImage* cameraImages[2];
373  boost::ptr_vector<boost::nullable<CByteImage>> resultImagesData;
374  CByteImage** resultImages;
375  armarx::MetaInfoSizeBasePtr imageMetaInfo;
377 
378 
379  private:
380  // noise
381  Eigen::Vector2d imageNoiseLeft;
382  Eigen::Vector2d imageNoiseRight;
383 
384  // threading
386 
387  public:
388  EIGEN_MAKE_ALIGNED_OPERATOR_NEW
389 
390 
391  };
392 
393 }
394 
visionx::ObjectLocalizerProcessor::createPropertyDefinitions
armarx::PropertyDefinitionsPtr createPropertyDefinitions() override
Definition: ObjectLocalizerProcessor.h:119
visionx::ObjectLocalizerProcessor::onConnectObjectLocalizerProcessor
virtual void onConnectObjectLocalizerProcessor()
ObjectLocalizerProcessor interface: subclass hook.
Definition: ObjectLocalizerProcessor.h:234
visionx::ObjectLocalizerProcessor::resultImages
CByteImage ** resultImages
Definition: ObjectLocalizerProcessor.h:374
visionx::ObjectLocalizerProcessor::reportStereoCalibrationChanged
void reportStereoCalibrationChanged(const StereoCalibration &stereoCalibration, bool imagesAreUndistorted, const std::string &referenceFrame, const Ice::Current &c=Ice::emptyCurrent) override
Definition: ObjectLocalizerProcessor.h:272
ObjectLocalizerProcessorJob.h
visionx::ObjectLocalizerProcessor::localizeObjectClasses
memoryx::ObjectLocalizationResultList localizeObjectClasses(const memoryx::ObjectClassNameList &objectClassNames, const Ice::Current &c=Ice::emptyCurrent) override
The process method is inherited from the ObjectLocalizationProcessorInterface and is called by the Wo...
Definition: ObjectLocalizerProcessor.cpp:176
visionx::ObjectLocalizerProcessor::referenceFrameName
std::string referenceFrameName
Definition: ObjectLocalizerProcessor.h:368
visionx
ArmarX headers.
Definition: OpenPoseStressTest.h:38
visionx::ObjectLocalizerProcessor::getImageFormat
ImageFormatInfo getImageFormat() const
Retrieve format of input images.
Definition: ObjectLocalizerProcessor.h:304
visionx::ImageProcessor
The ImageProcessor class provides an interface for access to ImageProviders via Ice and shared memory...
Definition: ImageProcessor.h:87
visionx::ObjectLocalizerProcessor::getResultImagesEnabled
bool getResultImagesEnabled() const
Retrieve whether result images are enabled.
Definition: ObjectLocalizerProcessor.h:284
armarx::PropertyDefinitionContainer::prefix
std::string prefix
Prefix of the properties such as namespace, domain, component name, etc.
Definition: PropertyDefinitionContainer.h:333
GridFileManager.h
visionx::ObjectLocalizerProcessor::onDisconnectObjectLocalizerProcessor
virtual void onDisconnectObjectLocalizerProcessor()
ObjectLocalizerProcessor interface: subclass hook.
Definition: ObjectLocalizerProcessor.h:241
visionx::ObjectLocalizerProcessor::initRecognizer
virtual bool initRecognizer()=0
ObjectLocalizerProcessor interface: The initRecognizer method needs to be implemented by any ObjectLo...
GfxTL::Vec2d
VectorXD< 2, double > Vec2d
Definition: VectorXD.h:694
memoryx
VirtualRobot headers.
Definition: CommonPlacesTester.cpp:48
c
constexpr T c
Definition: UnscentedKalmanFilterTest.cpp:43
visionx::ObjectLocalizerProcessorJob::exit
void exit()
Requests exit to the job.
Definition: ObjectLocalizerProcessorJob.cpp:67
visionx::ObjectLocalizerProcessorPropertyDefinitions::ObjectLocalizerProcessorPropertyDefinitions
ObjectLocalizerProcessorPropertyDefinitions(std::string prefix)
Definition: ObjectLocalizerProcessor.h:83
visionx::ObjectLocalizerProcessor::addObjectClass
virtual bool addObjectClass(const memoryx::EntityPtr &objectClassEntity, const memoryx::GridFileManagerPtr &fileManager)=0
ObjectLocalizerProcessor interface: The addObjectClass method needs to be implemented by any ObjectLo...
visionx::ObjectLocalizerProcessor::imageMetaInfo
armarx::MetaInfoSizeBasePtr imageMetaInfo
Definition: ObjectLocalizerProcessor.h:375
IceInternal::Handle
Definition: forward_declarations.h:8
visionx::tools::convert
CByteImage::ImageType convert(const ImageType visionxImageType)
Converts a VisionX image type into an image type of IVT's ByteImage.
Definition: TypeMapping.cpp:95
armarx::extractColorValue
Eigen::Vector3i extractColorValue(std::string propertyValue)
Definition: ResultImageFuser.h:50
visionx::ObjectLocalizerProcessor::process
void process() override
The process method is inherited from ImageProcessor.
Definition: ObjectLocalizerProcessor.cpp:196
visionx::ObjectLocalizerProcessor::getImagesAreUndistorted
bool getImagesAreUndistorted() const
Retrieve whether images are undistorted.
Definition: ObjectLocalizerProcessor.h:314
ImageProcessor.h
visionx::ObjectLocalizerProcessor::cameraImages
CByteImage * cameraImages[2]
Definition: ObjectLocalizerProcessor.h:372
visionx::ObjectLocalizerProcessor::isResultImageMaskEnabled
bool isResultImageMaskEnabled() const
Definition: ObjectLocalizerProcessor.h:319
visionx::ObjectLocalizerProcessor::onExitImageProcessor
void onExitImageProcessor() override
Called from framework.
Definition: ObjectLocalizerProcessor.h:164
visionx::ObjectLocalizerProcessor::priorKnowledgeProxyName
std::string priorKnowledgeProxyName
Definition: ObjectLocalizerProcessor.h:367
visionx::ObjectLocalizerProcessor::onConnectImageProcessor
void onConnectImageProcessor() override
Called from framework.
Definition: ObjectLocalizerProcessor.cpp:92
visionx::ObjectLocalizerProcessor::imageProviderName
std::string imageProviderName
Definition: ObjectLocalizerProcessor.h:366
visionx::ObjectLocalizerProcessor::onDisconnectComponent
void onDisconnectComponent() override
Called from framework.
Definition: ObjectLocalizerProcessor.h:149
visionx::ObjectLocalizerProcessor::onExitObjectLocalizerProcessor
virtual void onExitObjectLocalizerProcessor()
ObjectLocalizerProcessor interface: subclass hook.
Definition: ObjectLocalizerProcessor.h:248
visionx::ObjectLocalizerProcessor::resultImagesData
boost::ptr_vector< boost::nullable< CByteImage > > resultImagesData
Definition: ObjectLocalizerProcessor.h:373
memoryx::MultivariateNormalDistributionPtr
IceInternal::Handle< MultivariateNormalDistribution > MultivariateNormalDistributionPtr
Definition: ProbabilityMeasures.h:231
visionx::ObjectLocalizerProcessor::onInitObjectLocalizerProcessor
virtual void onInitObjectLocalizerProcessor()
ObjectLocalizerProcessor interface: subclass hook.
Definition: ObjectLocalizerProcessor.h:227
armarx::Component::getConfigIdentifier
std::string getConfigIdentifier()
Retrieve config identifier for this component as set in constructor.
Definition: Component.cpp:74
visionx::ObjectLocalizerProcessor::getStereoCalibration
CStereoCalibration * getStereoCalibration() const
Retrieve stereo calibration corresponding to image provider.
Definition: ObjectLocalizerProcessor.h:334
visionx::ObjectLocalizerProcessorJob
ObjectLocalizerProcessorJob encapsules the object localization job.
Definition: ObjectLocalizerProcessorJob.h:43
memoryx::GridFileManagerPtr
std::shared_ptr< GridFileManager > GridFileManagerPtr
Definition: AbstractEntityWrapper.h:32
visionx::ObjectLocalizerProcessor::getReferenceFrameName
std::string getReferenceFrameName() const
Retrieve whether result images are enabled.
Definition: ObjectLocalizerProcessor.h:294
Entity.h
IceUtil::Handle< class PropertyDefinitionContainer >
visionx::ImageProcessorPropertyDefinitions
Definition: ImageProcessor.h:61
visionx::ObjectLocalizerProcessorPropertyDefinitions
Definition: ObjectLocalizerProcessor.h:78
visionx::ObjectLocalizerProcessor::getColorMask
Eigen::Vector3i getColorMask() const
Definition: ObjectLocalizerProcessor.h:324
visionx::ObjectLocalizerProcessor::onInitImageProcessor
void onInitImageProcessor() override
Called from framework.
Definition: ObjectLocalizerProcessor.cpp:61
TypeMapping.h
visionx::ObjectLocalizerProcessor::calculateLocalizationUncertainty
memoryx::MultivariateNormalDistributionPtr calculateLocalizationUncertainty(Vec2d left_point, Vec2d right_point)
Calculate 3D uncertainty from two 2d points in left and right camera.
Definition: ObjectLocalizerProcessor.cpp:201
armarx::PropertyDefinitionsPtr
IceUtil::Handle< class PropertyDefinitionContainer > PropertyDefinitionsPtr
PropertyDefinitions smart pointer type.
Definition: forward_declarations.h:34
visionx::ObjectLocalizerProcessor::cameraImagesData
CByteImage cameraImagesData[2]
Definition: ObjectLocalizerProcessor.h:371
armarx::PropertyDefinition::PropertyFactoryFunction
std::function< PropertyType(std::string)> PropertyFactoryFunction
Definition: PropertyDefinition.h:114
visionx::ImageProcessor::onDisconnectComponent
void onDisconnectComponent() override
Definition: ImageProcessor.cpp:83
visionx::ObjectLocalizerProcessor::numberOfResultImages
int numberOfResultImages
Definition: ObjectLocalizerProcessor.h:376
armarx::VariantType::MultivariateNormalDistribution
const armarx::VariantTypeId MultivariateNormalDistribution
Definition: ProbabilityMeasures.h:36
visionx::ObjectLocalizerProcessor::ObjectLocalizerProcessor
ObjectLocalizerProcessor()
Definition: ObjectLocalizerProcessor.cpp:51
visionx::ObjectLocalizerProcessor
ObjectLocalizerProcessor.
Definition: ObjectLocalizerProcessor.h:107