Main MRPT website > C++ reference
MRPT logo
vision/include/mrpt/vision/types.h
Go to the documentation of this file.
1 /* +---------------------------------------------------------------------------+
2  | The Mobile Robot Programming Toolkit (MRPT) C++ library |
3  | |
4  | http://www.mrpt.org/ |
5  | |
6  | Copyright (C) 2005-2012 University of Malaga |
7  | |
8  | This software was written by the Machine Perception and Intelligent |
9  | Robotics Lab, University of Malaga (Spain). |
10  | Contact: Jose-Luis Blanco <jlblanco@ctima.uma.es> |
11  | |
12  | This file is part of the MRPT project. |
13  | |
14  | MRPT is free software: you can redistribute it and/or modify |
15  | it under the terms of the GNU General Public License as published by |
16  | the Free Software Foundation, either version 3 of the License, or |
17  | (at your option) any later version. |
18  | |
19  | MRPT is distributed in the hope that it will be useful, |
20  | but WITHOUT ANY WARRANTY; without even the implied warranty of |
21  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
22  | GNU General Public License for more details. |
23  | |
24  | You should have received a copy of the GNU General Public License |
25  | along with MRPT. If not, see <http://www.gnu.org/licenses/>. |
26  | |
27  +---------------------------------------------------------------------------+ */
28 
29 #ifndef mrpt_vision_types_H
30 #define mrpt_vision_types_H
31 
32 #include <mrpt/utils/CImage.h>
33 #include <mrpt/utils/CLoadableOptions.h>
34 #include <mrpt/utils/TMatchingPair.h>
35 
36 #include <mrpt/vision/link_pragmas.h>
37 
38 namespace mrpt
39 {
40  namespace vision
41  {
42  /** \addtogroup mrpt_vision_grp
43  * @{ */
44 
45  using std::vector;
46  //using namespace mrpt::slam;
47  using namespace mrpt::math;
48  using namespace mrpt::utils;
49 
50 
51  typedef uint64_t TFeatureID; //!< Definition of a feature ID
52 
53  typedef uint64_t TLandmarkID; //!< Unique IDs for landmarks
54  typedef uint64_t TCameraPoseID; //!< Unique IDs for camera frames (poses)
55 
56  typedef mrpt::aligned_containers<TCameraPoseID,CPose3D>::map_t TFramePosesMap; //!< A list of camera frames (6D poses) indexed by unique IDs.
57  typedef mrpt::aligned_containers<CPose3D>::vector_t TFramePosesVec; //!< A list of camera frames (6D poses), which assumes indexes are unique, consecutive IDs.
58 
59  typedef std::map<TLandmarkID,TPoint3D> TLandmarkLocationsMap; //!< A list of landmarks (3D points) indexed by unique IDs.
60  typedef std::vector<TPoint3D> TLandmarkLocationsVec; //!< A list of landmarks (3D points), which assumes indexes are unique, consecutive IDs.
61 
62 
63  /** Types of features - This means that the point has been detected with this algorithm, which is independent of additional descriptors a feature may also have
64  */
65  enum TFeatureType
66  {
67  featNotDefined = -1, //!< Non-defined feature (also used for Occupancy features)
68  featKLT = 0, //!< Kanade-Lucas-Tomasi feature [SHI'94]
69  featHarris, //!< Harris border and corner detector [HARRIS]
70  featBCD, //!< Binary corder detector
71  featSIFT, //!< Scale Invariant Feature Transform [LOWE'04]
72  featSURF, //!< Speeded Up Robust Feature [BAY'06]
73  featBeacon, //!< A especial case: this is not an image feature, but a 2D/3D beacon (used for range-only SLAM from mrpt::slam::CLandmark)
74  featFAST, //!< FAST feature detector, OpenCV's implementation ("Faster and better: A machine learning approach to corner detection", E. Rosten, R. Porter and T. Drummond, PAMI, 2009).
75  featFASTER9, //!< FASTER-9 detector, Edward Rosten's libcvd implementation optimized for SSE2.
76  featFASTER10, //!< FASTER-9 detector, Edward Rosten's libcvd implementation optimized for SSE2.
77  featFASTER12 //!< FASTER-9 detector, Edward Rosten's libcvd implementation optimized for SSE2.
78  };
79 
80  /** The bitwise OR combination of values of TDescriptorType are used in CFeatureExtraction::computeDescriptors to indicate which descriptors are to be computed for features.
81  */
82  enum TDescriptorType
83  {
84  descAny = 0, //!< Used in some methods to mean "any of the present descriptors"
85  descSIFT = 1, //!< SIFT descriptors
86  descSURF = 2, //!< SURF descriptors
87  descSpinImages = 4, //!< Intensity-domain spin image descriptors
88  descPolarImages = 8, //!< Polar image descriptor
89  descLogPolarImages = 16 //!< Log-Polar image descriptor
90  };
91 
92  enum TFeatureTrackStatus
93  {
94  // Init value
95  status_IDLE = 0, //!< Inactive (right after detection, and before being tried to track)
96 
97  // Ok:
98  status_TRACKED = 5, //!< Feature correctly tracked
99 
100  // Bad:
101  status_OOB = 1, //!< Feature felt Out Of Bounds
102  status_LOST = 10, //!< Unable to track this feature
103 
104  // KLT specific:
105  statusKLT_IDLE = 0, //!< Inactive
106  statusKLT_OOB = 1, //!< Out Of Bounds (Value identical to status_OOB)
107  statusKLT_SMALL_DET = 2, //!< Determinant of the matrix too small
108  statusKLT_LARGE_RESIDUE = 3, //!< Error too big
109  statusKLT_MAX_RESIDUE = 4,
110  statusKLT_TRACKED = 5, //!< Feature correctly tracked (Value identical to status_TRACKED)
111  statusKLT_MAX_ITERATIONS = 6 //!< Iteration maximum reached
112  };
113 
114  typedef TFeatureTrackStatus TKLTFeatureStatus; //!< For backward compatibility
115 
116 
117  /** One feature observation entry, used within sequences with TSequenceFeatureObservations */
118  struct VISION_IMPEXP TFeatureObservation
119  {
120  inline TFeatureObservation() { }
121  inline TFeatureObservation(const TLandmarkID _id_feature, const TCameraPoseID _id_frame, const TPixelCoordf _px) : id_feature(_id_feature), id_frame(_id_frame), px(_px) { }
122 
123  TLandmarkID id_feature; //!< A unique ID of this feature
124  TCameraPoseID id_frame; //!< A unique ID of a "frame" (camera position) from where the feature was observed.
125  TPixelCoordf px; //!< The pixel coordinates of the observed feature
126  };
127 
128  /** One relative feature observation entry, used with some relative bundle-adjustment functions.
129  */
130  struct TRelativeFeaturePos
131  {
132  inline TRelativeFeaturePos() { }
133  inline TRelativeFeaturePos(const mrpt::vision::TCameraPoseID _id_frame_base, const mrpt::math::TPoint3D _pos) : id_frame_base(_id_frame_base), pos(_pos) { }
134 
135  mrpt::vision::TCameraPoseID id_frame_base; //!< The ID of the camera frame which is the coordinate reference of \a pos
136  mrpt::math::TPoint3D pos; //!< The (x,y,z) location of the feature, wrt to the camera frame \a id_frame_base
137  };
138 
139  /** An index of feature IDs and their relative locations */
140  typedef std::map<mrpt::vision::TFeatureID, TRelativeFeaturePos> TRelativeFeaturePosMap;
141 
142  /** A complete sequence of observations of features from different camera frames (poses).
143  * This structure is the input to some (Bundle-adjustment) methods in mrpt::vision
144  * \note Pixel coordinates can be either "raw" or "undistorted". Read the doc of functions handling this structure to see what they expect.
145  * \sa mrpt::vision::bundle_adj_full
146  */
147  struct VISION_IMPEXP TSequenceFeatureObservations : public std::vector<TFeatureObservation>
148  {
149  typedef std::vector<TFeatureObservation> BASE;
150 
151  inline TSequenceFeatureObservations() {}
152  inline TSequenceFeatureObservations(size_t size) : BASE(size) {}
153  inline TSequenceFeatureObservations(const TSequenceFeatureObservations& o) : BASE(o) {}
154 
155  /** Saves all entries to a text file, with each line having this format: #FRAME_ID #FEAT_ID #PIXEL_X #PIXEL_Y
156  * The file is self-descripting, since the first line contains a comment line (starting with '%') explaining the format.
157  * Generated files can be loaded from MATLAB.
158  * \sa loadFromTextFile \exception std::exception On I/O error */
159  void saveToTextFile(const std::string &filName, bool skipFirstCommentLine = false) const;
160 
161  /** Load from a text file, in the format described in \a saveToTextFile \exception std::exception On I/O or format error */
162  void loadFromTextFile(const std::string &filName);
163 
164  /** Save the list of observations + the point locations + the camera frame poses to a pair of files in the format
165  * used by the Sparse Bundle Adjustment (SBA) C++ library.
166  *
167  * Point file lines: X Y Z nframes frame0 x0 y0 frame1 x1 y1 ...
168  *
169  * Camera file lines: qr qx qy qz x y z (Pose as a quaternion)
170  * \return false on any error
171  */
172  bool saveAsSBAFiles(
173  const TLandmarkLocationsVec &pts,
174  const std::string &pts_file,
175  const TFramePosesVec &cams,
176  const std::string &cams_file) const;
177 
178 
179  /** Remove all those features that don't have a minimum number of observations from different camera frame IDs.
180  * \return the number of erased entries.
181  * \sa After calling this you may want to call \a compressIDs */
182  size_t removeFewObservedFeatures(size_t minNumObservations = 3);
183 
184  /** Remove all but one out of \a decimate_ratio camera frame IDs from the list (eg: from N camera pose IDs at return there will be just N/decimate_ratio)
185  * The algorithm first builds a sorted list of frame IDs, then keep the lowest ID, remove the next "decimate_ratio-1", and so on.
186  * \sa After calling this you may want to call \a compressIDs */
187  void decimateCameraFrames(const size_t decimate_ratio);
188 
189  /** Rearrange frame and feature IDs such as they start at 0 and there are no gaps.
190  * \param old2new_camIDs If provided, the mapping from old to new IDs is stored here.
191  * \param old2new_lmIDs If provided, the mapping from old to new IDs is stored here. */
192  void compressIDs(
193  std::map<TCameraPoseID,TCameraPoseID> *old2new_camIDs=NULL,
194  std::map<TLandmarkID,TLandmarkID> *old2new_lmIDs=NULL );
195 
196  };
197 
198  /** Parameters associated to a stereo system
199  */
200  struct VISION_IMPEXP TStereoSystemParams : public mrpt::utils::CLoadableOptions
201  {
202  /** Initilization of default parameters
203  */
204  TStereoSystemParams( );
205 
206  /** See utils::CLoadableOptions
207  */
208  void loadFromConfigFile(
209  const mrpt::utils::CConfigFileBase &source,
210  const std::string &section);
211 
212  /** See utils::CLoadableOptions
213  */
214  void dumpToTextStream(CStream &out) const;
215 
216  /** Method for propagating the feature's image coordinate uncertainty into 3D space. Default value: Prop_Linear
217  */
218  enum TUnc_Prop_Method
219  {
220  /** Linear propagation of the uncertainty
221  */
222  Prop_Linear = -1,
223  /** Uncertainty propagation through the Unscented Transformation
224  */
225  Prop_UT,
226  /** Uncertainty propagation through the Scaled Unscented Transformation
227  */
228  Prop_SUT
229  };
230 
231  TUnc_Prop_Method uncPropagation;
232 
233  /** Stereo Fundamental matrix */
234  CMatrixDouble33 F;
235 
236  /** Intrinsic parameters
237  */
238  CMatrixDouble33 K;
239  /** Baseline. Default value: baseline = 0.119f; [Bumblebee]
240  */
241  float baseline;
242  /** Standard deviation of the error in feature detection. Default value: stdPixel = 1
243  */
244  float stdPixel;
245  /** Standard deviation of the error in disparity computation. Default value: stdDisp = 1
246  */
247  float stdDisp;
248  /** Maximum allowed distance. Default value: maxZ = 20.0f
249  */
250  float maxZ;
251  /** Maximum allowed distance. Default value: minZ = 0.5f
252  */
253  float minZ;
254  /** Maximum allowed height. Default value: maxY = 3.0f
255  */
256  float maxY;
257  /** K factor for the UT. Default value: k = 1.5f
258  */
259  float factor_k;
260  /** Alpha factor for SUT. Default value: a = 1e-3
261  */
262  float factor_a;
263  /** Beta factor for the SUT. Default value: b = 2.0f
264  */
265  float factor_b;
266 
267  /** Parameters initialization
268  */
269  //TStereoSystemParams();
270 
271  }; // End struct TStereoSystemParams
272 
273  /** A structure for storing a 3D ROI
274  */
275  struct VISION_IMPEXP TROI
276  {
277  // Constructors
278  TROI();
279  TROI(float x1, float x2, float y1, float y2, float z1, float z2);
280 
281  // Members
282  float xMin;
283  float xMax;
284  float yMin;
285  float yMax;
286  float zMin;
287  float zMax;
288  }; // end struct TROI
289 
290  /** A structure for defining a ROI within an image
291  */
292  struct VISION_IMPEXP TImageROI
293  {
294  // Constructors
295  TImageROI();
296  TImageROI( float x1, float x2, float y1, float y2 );
297 
298  // Members
299  /** X coordinate limits [0,imageWidth)
300  */
301  float xMin, xMax;
302  /** Y coordinate limits [0,imageHeight)
303  */
304  float yMin, yMax;
305  }; // end struct TImageROI
306 
307  /** A structure containing options for the matching
308  */
309  struct VISION_IMPEXP TMatchingOptions : public mrpt::utils::CLoadableOptions
310  {
311 
312  /** Method for propagating the feature's image coordinate uncertainty into 3D space. Default value: Prop_Linear
313  */
314  enum TMatchingMethod
315  {
316  /** Matching by cross correlation of the image patches
317  */
318  mmCorrelation = 0,
319  /** Matching by Euclidean distance between SIFT descriptors
320  */
321  mmDescriptorSIFT,
322  /** Matching by Euclidean distance between SURF descriptors
323  */
324  mmDescriptorSURF,
325  /** Matching by sum of absolute differences of the image patches
326  */
327  mmSAD
328  };
329 
330  // For determining
331  bool useEpipolarRestriction; //!< Whether or not take into account the epipolar restriction for finding correspondences
332  bool hasFundamentalMatrix; //!< Whether or not there is a fundamental matrix
333  bool parallelOpticalAxis; //!< Whether or not the stereo rig has the optical axes parallel
334  bool useXRestriction; //!< Whether or not employ the x-coord restriction for finding correspondences (bumblebee camera, for example)
335  bool addMatches; //!< Whether or not to add the matches found into the input matched list (if false the input list will be cleared before being filled with the new matches)
336 
337  CMatrixDouble33 F;
338 
339  // General
340  TMatchingMethod matching_method; //!< Matching method
341  float epipolar_TH; //!< Epipolar constraint (rows of pixels)
342 
343  // SIFT
344  float maxEDD_TH; //!< Maximum Euclidean Distance Between SIFT Descriptors
345  float EDD_RATIO; //!< Boundary Ratio between the two lowest EDD
346 
347  // KLT
348  float minCC_TH; //!< Minimum Value of the Cross Correlation
349  float minDCC_TH; //!< Minimum Difference Between the Maximum Cross Correlation Values
350  float rCC_TH; //!< Maximum Ratio Between the two highest CC values
351 
352  // SURF
353  float maxEDSD_TH; //!< Maximum Euclidean Distance Between SURF Descriptors
354  float EDSD_RATIO; //!< Boundary Ratio between the two lowest SURF EDSD
355 
356  // SAD
357  double maxSAD_TH; //!< Minimum Euclidean Distance Between Sum of Absolute Differences
358  double SAD_RATIO; //!< Boundary Ratio between the two highest SAD
359 
360 // // To estimate depth
361  bool estimateDepth; //!< Whether or not estimate the 3D position of the real features for the matches (only with parallelOpticalAxis by now).
362  double maxDepthThreshold; //!< The maximum allowed depth for the matching. If its computed depth is larger than this, the match won't be considered.
363 // double fx,cx,cy,baseline; //!< Intrinsic parameters of the stereo rig
364 
365  /** Constructor
366  */
367  TMatchingOptions( );
368 
369  /** See utils::CLoadableOptions
370  */
371  void loadFromConfigFile(
372  const mrpt::utils::CConfigFileBase &source,
373  const std::string &section);
374 
375  /** See utils::CLoadableOptions
376  */
377  void dumpToTextStream(CStream &out) const;
378 
379  }; // end struct TMatchingOptions
380 
381  /** Struct containing the output after matching multi-resolution SIFT-like descriptors
382  */
383  struct VISION_IMPEXP TMultiResMatchingOutput
384  {
385  int nMatches;
386 
387  std::vector<int> firstListCorrespondences; //!< Contains the indexes within the second list corresponding to the first one.
388  std::vector<int> secondListCorrespondences; //!< Contains the indexes within the first list corresponding to the second one.
389  std::vector<int> firstListFoundScales; //!< Contains the scales of the first list where the correspondence was found.
390  std::vector<double> firstListDistance; //!< Contains the distances between the descriptors.
391 
392  TMultiResMatchingOutput() : nMatches(0),
393  firstListCorrespondences(), secondListCorrespondences(),
394  firstListFoundScales(), firstListDistance() {}
395 
396  }; // end struct TMultiResMatchingOutput
397 
398  /** Struct containing the options when matching multi-resolution SIFT-like descriptors
399  */
400  struct VISION_IMPEXP TMultiResDescMatchOptions : public mrpt::utils::CLoadableOptions
401  {
402  bool useOriFilter; //!< Whether or not use the filter based on orientation test
403  double oriThreshold; //!< The threshold for the orientation test
404 
405  bool useDepthFilter; //!< Whether or not use the filter based on the depth test
406 
407  double matchingThreshold; //!< The absolute threshold in descriptor distance for considering a match
408  double matchingRatioThreshold; //!< The ratio between the two lowest distances threshold for considering a match
409  unsigned int lowScl1, lowScl2; //!< The lowest scales in the two features to be taken into account in the matching process
410  unsigned int highScl1, highScl2; //!< The highest scales in the two features to be taken into account in the matching process
411 
412  int searchAreaSize; //!< Size of the squared area where to search for a match.
413  int lastSeenThreshold; //!< The allowed number of frames since a certain feature was seen for the last time.
414  int timesSeenThreshold; //!< The minimum number of frames for a certain feature to be considered stable.
415 
416  int minFeaturesToFind; //!< The minimum number of features allowed in the system. If current number is below this value, more features will be found.
417  int minFeaturesToBeLost; //!< The minimum number of features allowed in the system to not be considered to be lost.
418 
419  /** Default constructor
420  */
421  TMultiResDescMatchOptions() :
422  useOriFilter( true ), oriThreshold( 0.2 ),
423  useDepthFilter( true ), matchingThreshold( 1e4 ), matchingRatioThreshold( 0.5 ),
424  lowScl1(0), lowScl2(0), highScl1(6), highScl2(6), searchAreaSize(20), lastSeenThreshold(10), timesSeenThreshold(5),
425  minFeaturesToFind(30), minFeaturesToBeLost(5) {}
426 
427  TMultiResDescMatchOptions(
428  const bool &_useOriFilter, const double &_oriThreshold, const bool &_useDepthFilter,
429  const double &_th, const double &_th2, const unsigned int &_lwscl1, const unsigned int &_lwscl2,
430  const unsigned int &_hwscl1, const unsigned int &_hwscl2, const int &_searchAreaSize, const int &_lsth, const int &_tsth,
431  const int &_minFeaturesToFind, const int &_minFeaturesToBeLost ) :
432  useOriFilter( _useOriFilter ), oriThreshold( _oriThreshold ), useDepthFilter( _useDepthFilter ),
433  matchingThreshold ( _th ), matchingRatioThreshold ( _th2 ), lowScl1( _lwscl1 ), lowScl2( _lwscl2 ),
434  highScl1( _hwscl1 ), highScl2( _hwscl2 ), searchAreaSize( _searchAreaSize ), lastSeenThreshold( _lsth ), timesSeenThreshold( _tsth ),
435  minFeaturesToFind( _minFeaturesToFind ), minFeaturesToBeLost(_minFeaturesToBeLost) {}
436 
437  void loadFromConfigFile( const mrpt::utils::CConfigFileBase &cfg, const std::string &section );
438  void saveToConfigFile( mrpt::utils::CConfigFileBase &cfg, const std::string &section );
439  void dumpToTextStream( mrpt::utils::CStream &out) const;
440 
441  }; // end TMultiResDescMatchOptions
442 
443  /** Struct containing the options when computing the multi-resolution SIFT-like descriptors
444  */
445  struct VISION_IMPEXP TMultiResDescOptions : public mrpt::utils::CLoadableOptions
446  {
447  unsigned int basePSize; //!< The size of the base patch
448  vector<double> scales; //!< The set of scales relatives to the base patch
449  unsigned int comLScl, comHScl; //!< The subset of scales for which to compute the descriptors
450  double sg1, sg2, sg3; //!< The sigmas for the Gaussian kernels
451  bool computeDepth; //!< Whether or not to compute the depth of the feature
452  bool blurImage; //!< Whether or not to blur the image previously to compute the descriptors
453  double fx,cx,cy,baseline; //!< Intrinsic stereo pair parameters for computing the depth of the feature
454  bool computeHashCoeffs; //!< Whether or not compute the coefficients for mantaining a HASH table of descriptors (for relocalization)
455 
456  double cropValue; //!< The SIFT-like descriptor is cropped at this value during normalization
457 
458  /** Default constructor
459  */
460  TMultiResDescOptions() :
461  basePSize(23), sg1 (0.5), sg2(7.5), sg3(8.0), computeDepth(true), blurImage(true), fx(0.0), cx(0.0), cy(0.0), baseline(0.0), computeHashCoeffs(false), cropValue(0.2)
462  {
463  scales.resize(7);
464  scales[0] = 0.5;
465  scales[1] = 0.8;
466  scales[2] = 1.0;
467  scales[3] = 1.2;
468  scales[4] = 1.5;
469  scales[5] = 1.8;
470  scales[6] = 2.0;
471  comLScl = 0;
472  comHScl = 6;
473  }
474 
475  TMultiResDescOptions( const unsigned int &_basePSize, const vector<double> &_scales,
476  const unsigned int &_comLScl, const unsigned int &_comHScl,
477  const double &_sg1, const double &_sg2, const double &_sg3,
478  const bool &_computeDepth, const bool _blurImage, const double &_fx, const double &_cx, const double &_cy, const double &_baseline, const bool &_computeHashCoeffs, const double &_cropValue ):
479  basePSize( _basePSize ), comLScl( _comLScl ), comHScl( _comHScl ),
480  sg1( _sg1 ), sg2( _sg2 ), sg3( _sg3 ),
481  computeDepth( _computeDepth ), blurImage( _blurImage ), fx( _fx ), cx( _cx ), cy( _cy ), baseline( _baseline ), computeHashCoeffs( _computeHashCoeffs), cropValue( _cropValue )
482  {
483  scales.resize( _scales.size() );
484  for(unsigned int k = 0; k < _scales.size(); ++k)
485  scales[k] = _scales[k];
486  }
487 
488  void loadFromConfigFile( const mrpt::utils::CConfigFileBase &cfg, const std::string &section );
489  void saveToConfigFile( mrpt::utils::CConfigFileBase &cfg, const std::string &section );
490  void dumpToTextStream( mrpt::utils::CStream &out) const;
491 
492  }; // end TMultiResDescOptions
493 
494 
495  /** @} */ // end of grouping
496 
497  }
498 }
499 
500 
501 #endif


Page generated by Doxygen 1.8.3 for MRPT 0.9.6 SVN: at Fri Feb 15 22:05:02 EST 2013