Main MRPT website > C++ reference
MRPT logo
CFeatureExtraction.h
Go to the documentation of this file.
1 /* +---------------------------------------------------------------------------+
2  | The Mobile Robot Programming Toolkit (MRPT) C++ library |
3  | |
4  | http://www.mrpt.org/ |
5  | |
6  | Copyright (C) 2005-2012 University of Malaga |
7  | |
8  | This software was written by the Machine Perception and Intelligent |
9  | Robotics Lab, University of Malaga (Spain). |
10  | Contact: Jose-Luis Blanco <jlblanco@ctima.uma.es> |
11  | |
12  | This file is part of the MRPT project. |
13  | |
14  | MRPT is free software: you can redistribute it and/or modify |
15  | it under the terms of the GNU General Public License as published by |
16  | the Free Software Foundation, either version 3 of the License, or |
17  | (at your option) any later version. |
18  | |
19  | MRPT is distributed in the hope that it will be useful, |
20  | but WITHOUT ANY WARRANTY; without even the implied warranty of |
21  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
22  | GNU General Public License for more details. |
23  | |
24  | You should have received a copy of the GNU General Public License |
25  | along with MRPT. If not, see <http://www.gnu.org/licenses/>. |
26  | |
27  +---------------------------------------------------------------------------+ */
28 #ifndef CFeatureExtraction_H
29 #define CFeatureExtraction_H
30 
31 #include <mrpt/utils/CImage.h>
32 #include <mrpt/utils/CTicTac.h>
33 #include <mrpt/vision/utils.h>
34 #include <mrpt/vision/CFeature.h>
35 #include <mrpt/vision/TSimpleFeature.h>
36 
37 namespace mrpt
38 {
39  namespace vision
40  {
41  /** The central class from which images can be analyzed in search of different kinds of interest points and descriptors computed for them.
42  * To extract features from an image, create an instance of CFeatureExtraction,
43  * fill out its CFeatureExtraction::options field, including the algorithm to use (see
44  * CFeatureExtraction::TOptions::featsType), and call CFeatureExtraction::detectFeatures.
45  * This will return a set of features of the class mrpt::vision::CFeature, which include
46  * details for each interest point as well as the desired descriptors and/or patches.
47  *
48  * By default, a 21x21 patch is extracted for each detected feature. If the patch is not needed,
49  * set patchSize to 0 in CFeatureExtraction::options
50  *
51  * The implemented <b>detection</b> algorithms are (see CFeatureExtraction::TOptions::featsType):
52  * - KLT (Kanade-Lucas-Tomasi): A detector (no descriptor vector).
53  * - Harris: A detector (no descriptor vector).
54  * - BCD (Binary Corner Detector): A detector (no descriptor vector) (Not implemented yet).
55  * - SIFT: An implementation of the SIFT detector and descriptor. The implemention may be selected with CFeatureExtraction::TOptions::SIFTOptions::implementation.
56  * - SURF: OpenCV's implementation of SURF detector and descriptor.
57  * - The FAST feature detector (OpenCV's implementation)
58  * - The FASTER (9,10,12) detectors (Edward Rosten's libcvd implementation optimized for SSE2).
59  *
60  * Additionally, given a list of interest points onto an image, the following
61  * <b>descriptors</b> can be computed for each point by calling CFeatureExtraction::computeDescriptors :
62  * - SIFT descriptor (Lowe's descriptors).
63  * - SURF descriptor (OpenCV's implementation - Requires OpenCV 1.1.0 from SVN or later).
64  * - Intensity-domain spin images (SpinImage): Creates a vector descriptor with the 2D histogram as a single row.
65  * - A circular patch in polar coordinates (Polar images): The matrix descriptor is a 2D polar image centered at the interest point.
66  * - A log-polar image patch (Log-polar images): The matrix descriptor is the 2D log-polar image centered at the interest point.
67  *
68  *
69  * Apart from the normal entry point \a detectFeatures(), these other low-level static methods are provided for convenience:
70  * - CFeatureExtraction::detectFeatures_SSE2_FASTER9()
71  * - CFeatureExtraction::detectFeatures_SSE2_FASTER10()
72  * - CFeatureExtraction::detectFeatures_SSE2_FASTER12()
73  *
74  * \note The descriptor "Intensity-domain spin images" is described in "A sparse texture representation using affine-invariant regions", S Lazebnik, C Schmid, J Ponce, 2003 IEEE Computer Society Conference on Computer Vision.
75  * \sa mrpt::vision::CFeature
76  * \ingroup mrptvision_features
77  */
78  class VISION_IMPEXP CFeatureExtraction
79  {
80  public:
81  enum TSIFTImplementation
82  {
83  LoweBinary = 0,
84  CSBinary,
85  VedaldiBinary,
86  Hess,
87  OpenCV
88  };
89 
90  /** The set of parameters for all the detectors & descriptor algorithms */
91  struct VISION_IMPEXP TOptions : public utils::CLoadableOptions
92  {
93  /** Initalizer
94  */
95  TOptions(const TFeatureType featsType = featKLT);
96 
97  /** See utils::CLoadableOptions
98  */
99  void loadFromConfigFile(
100  const mrpt::utils::CConfigFileBase &source,
101  const std::string &section);
102 
103  /** See utils::CLoadableOptions
104  */
105  void dumpToTextStream(CStream &out) const;
106 
107  /** Type of the extracted features
108  */
109  TFeatureType featsType;
110 
111  /** Size of the patch to extract, or 0 if no patch is desired (default=21).
112  */
113  unsigned int patchSize;
114 
115  /** Whether to use a mask for determining the regions where not to look for keypoints (default=false).
116  */
117  bool useMask;
118 
119  /** Whether to add the found features to the input feature list or clear it before adding them (default=false).
120  */
121  bool addNewFeatures;
122 
123  /** Indicates if subpixel accuracy is desired for the extracted points (only applicable to KLT and Harris features)
124  */
125  bool FIND_SUBPIXEL;
126 
127  /** KLT Options */
128  struct VISION_IMPEXP TKLTOptions
129  {
130  int radius; // size of the block of pixels used
131  float threshold; // (default=0.1) for rejecting weak local maxima (with min_eig < threshold*max(eig_image))
132  float min_distance; // minimum distance between features
133  bool tile_image; // splits the image into 8 tiles and search for the best points in all of them (distribute the features over all the image)
134  } KLTOptions;
135 
136  /** Harris Options */
137  struct VISION_IMPEXP THarrisOptions
138  {
139  float threshold; // (default=0.005) for rejecting weak local maxima (with min_eig < threshold*max(eig_image))
140  float k; // k factor for the Harris algorithm
141  float sigma; // standard deviation for the gaussian smoothing function
142  int radius; // size of the block of pixels used
143  float min_distance; // minimum distance between features
144  bool tile_image; // splits the image into 8 tiles and search for the best points in all of them (distribute the features over all the image)
145  } harrisOptions;
146 
147  /** BCD Options */
148  struct VISION_IMPEXP TBCDOptions
149  {
150  } BCDOptions;
151 
152  /** FAST and FASTER Options */
153  struct VISION_IMPEXP TFASTOptions
154  {
155  int threshold; //!< default= 20
156  bool nonmax_suppression; //!< Default = true
157  float min_distance; //!< (default=5) minimum distance between features (in pixels)
158  bool use_KLT_response; //!< (default=false) If true, use CImage::KLT_response to compute the response at each point instead of the FAST "standard response".
159  } FASTOptions;
160 
161  /** SIFT Options */
162  struct VISION_IMPEXP TSIFTOptions
163  {
164  TSIFTOptions() : threshold(0.04), edgeThreshold(10) { }
165 
166  TSIFTImplementation implementation;
167  double threshold; //!< default= 0.04
168  double edgeThreshold; //!< default= 10
169  } SIFTOptions;
170 
171  struct VISION_IMPEXP TSURFOptions
172  {
173  TSURFOptions() : rotation_invariant(true),hessianThreshold(600), nOctaves(2), nLayersPerOctave(4) { }
174 
175  /** SURF Options
176  */
177  bool rotation_invariant; //!< Compute the rotation invariant SURF (dim=128) if set to true (default), or the smaller uSURF otherwise (dim=64)
178  int hessianThreshold; //!< Default: 600
179  int nOctaves; //!< Default: 2
180  int nLayersPerOctave; //!< Default: 4
181  } SURFOptions;
182 
183  struct VISION_IMPEXP TSpinImagesOptions
184  {
185  /** SpinImages Options
186  */
187  unsigned int hist_size_intensity; //!< Number of bins in the "intensity" axis of the 2D histogram (default=10).
188  unsigned int hist_size_distance; //!< Number of bins in the "distance" axis of the 2D histogram (default=10).
189  float std_dist; //!< Standard deviation in "distance", used for the "soft histogram" (default=0.4 pixels)
190  float std_intensity; //!< Standard deviation in "intensity", used for the "soft histogram" (default=20 units [0,255])
191  unsigned int radius; //!< Maximum radius of the area of which the histogram is built, in pixel units (default=20 pixels)
192  } SpinImagesOptions;
193 
194  /** PolarImagesOptions Options
195  */
196  struct VISION_IMPEXP TPolarImagesOptions
197  {
198  unsigned int bins_angle; //!< Number of bins in the "angular" axis of the polar image (default=8).
199  unsigned int bins_distance; //!< Number of bins in the "distance" axis of the polar image (default=6).
200  unsigned int radius; //!< Maximum radius of the area of which the polar image is built, in pixel units (default=20 pixels)
201  } PolarImagesOptions;
202 
203  /** LogPolarImagesOptions Options
204  */
205  struct VISION_IMPEXP TLogPolarImagesOptions
206  {
207  unsigned int radius; //!< Maximum radius of the area of which the log polar image is built, in pixel units (default=30 pixels)
208  unsigned int num_angles; //!< (default=16) Log-Polar image patch will have dimensions WxH, with: W=num_angles, H= rho_scale * log(radius)
209  double rho_scale; //!< (default=5) Log-Polar image patch will have dimensions WxH, with: W=num_angles, H= rho_scale * log(radius)
210  } LogPolarImagesOptions;
211 
212  };
213 
214  TOptions options; //!< Set all the parameters of the desired method here before calling "detectFeatures"
215 
216  /** Constructor
217  */
218  CFeatureExtraction();
219 
220  /** Virtual destructor.
221  */
222  virtual ~CFeatureExtraction();
223 
224  /** Extract features from the image based on the method defined in TOptions.
225  * \param img (input) The image from where to extract the images.
226  * \param feats (output) A complete list of features (containing a patch for each one of them if options.patchsize > 0).
227  * \param nDesiredFeatures (op. input) Number of features to be extracted. Default: all possible.
228  * \param ROI (op. input) Region of Interest. Default: The whole image.
229  *
230  * \sa computeDescriptors
231  */
232  void detectFeatures( const CImage & img,
233  CFeatureList & feats,
234  const unsigned int init_ID = 0,
235  const unsigned int nDesiredFeatures = 0,
236  const TImageROI & ROI = TImageROI(),
237  const CMatrixBool * mask = NULL ) const; // Important: This was a const ref. in mrpt <0.9.4, but the instantiation of a default value
238  // for CMatrixBool being a template generated duplicated linking errors for MSVC, thus it was changed to a pointer.
239 
240  /** Compute one (or more) descriptors for the given set of interest points onto the image, which may have been filled out manually or from \a detectFeatures
241  * \param in_img (input) The image from where to compute the descriptors.
242  * \param inout_features (input/output) The list of features whose descriptors are going to be computed.
243  * \param in_descriptor_list (input) The bitwise OR of one or several descriptors defined in TDescriptorType.
244  *
245  * Each value in "in_descriptor_list" represents one descriptor to be computed, for example:
246  * \code
247  * // This call will compute both, SIFT and Spin-Image descriptors for a list of feature points lstFeats.
248  * fext.computeDescriptors(img, lstFeats, descSIFT | descSpinImages );
249  * \endcode
250  *
251  * \note The SIFT descriptors for already located features can only be computed through the Hess and
252  * CSBinary implementations which may be specified in CFeatureExtraction::TOptions::SIFTOptions.
253  *
254  * \note This call will also use additional parameters from \a options
255  */
256  void computeDescriptors(
257  const CImage &in_img,
258  CFeatureList &inout_features,
259  TDescriptorType in_descriptor_list) const;
260 
261  /** Extract more features from the image (apart from the provided ones) based on the method defined in TOptions.
262  * \param img (input) The image from where to extract the images.
263  * \param inList (input) The actual features in the image.
264  * \param outList (output) The list of new features (containing a patch for each one of them if options.patchsize > 0).
265  * \param nDesiredFeatures (op. input) Number of features to be extracted. Default: all possible.
266  *
267  * \sa The more powerful class: mrpt::vision::CGenericFeatureTracker
268  */
269  void findMoreFeatures( const CImage &img,
270  const CFeatureList &inList,
271  CFeatureList &outList,
272  unsigned int nDesiredFeats = 0) const;
273 
274 
275  /** @name Static methods with low-level detector functionality
276  @{ */
277 
278  /** A SSE2-optimized implementation of FASTER-9 (requires img to be grayscale). If SSE2 is not available, it gratefully falls back to a non-optimized version.
279  *
280  * Only the pt.{x,y} fields are filled out for each feature: the rest of fields are left <b>uninitialized</b> and their content is <b>undefined</b>.
281  * Note that (x,y) are already scaled to the 0-level image coordinates if octave>0, by means of:
282  *
283  * \code
284  * pt.x = detected.x << octave;
285  * pt.y = detected.y << octave;
286  * \endcode
287  *
288  * If \a append_to_list is true, the \a corners list is not cleared before adding the newly detected feats.
289  *
290  * If a valid pointer is provided for \a out_feats_index_by_row, upon return you will find a vector with
291  * as many entries as rows in the image (the real number of rows, disregarding the value of \a octave).
292  * The number in each entry is the 0-based index (in \a corners) of
293  * the first feature that falls in that line of the image. This index can be used to fasten looking for correspondences.
294  *
295  * \ingroup mrptvision_features
296  */
297  static void detectFeatures_SSE2_FASTER9(
298  const CImage &img,
299  TSimpleFeatureList & corners,
300  const int threshold = 20,
301  bool append_to_list = false,
302  uint8_t octave = 0,
303  std::vector<size_t> * out_feats_index_by_row = NULL );
304 
305  /** Just like \a detectFeatures_SSE2_FASTER9() for another version of the detector.
306  * \ingroup mrptvision_features */
307  static void detectFeatures_SSE2_FASTER10(
308  const CImage &img,
309  TSimpleFeatureList & corners,
310  const int threshold = 20,
311  bool append_to_list = false,
312  uint8_t octave = 0,
313  std::vector<size_t> * out_feats_index_by_row = NULL );
314 
315  /** Just like \a detectFeatures_SSE2_FASTER9() for another version of the detector.
316  * \ingroup mrptvision_features */
317  static void detectFeatures_SSE2_FASTER12(
318  const CImage &img,
319  TSimpleFeatureList & corners,
320  const int threshold = 20,
321  bool append_to_list = false,
322  uint8_t octave = 0,
323  std::vector<size_t> * out_feats_index_by_row = NULL );
324 
325  /** @} */
326 
327  private:
328  /** Compute the SIFT descriptor of the provided features into the input image
329  * \param in_img (input) The image from where to compute the descriptors.
330  * \param in_features (input/output) The list of features whose descriptors are going to be computed.
331  *
332  * \note The SIFT descriptors for already located features can only be computed through the Hess and
333  CSBinary implementations which may be specified in CFeatureExtraction::TOptions::SIFTOptions.
334  */
335  void internal_computeSiftDescriptors( const CImage &in_img,
336  CFeatureList &in_features) const;
337 
338 
339  /** Compute the SURF descriptor of the provided features into the input image
340  * \param in_img (input) The image from where to compute the descriptors.
341  * \param in_features (input/output) The list of features whose descriptors are going to be computed.
342  */
343  void internal_computeSurfDescriptors( const CImage &in_img,
344  CFeatureList &in_features) const;
345 
346  /** Compute the intensity-domain spin images descriptor of the provided features into the input image
347  * \param in_img (input) The image from where to compute the descriptors.
348  * \param in_features (input/output) The list of features whose descriptors are going to be computed.
349  *
350  * \note Additional parameters from CFeatureExtraction::TOptions::SpinImagesOptions are used in this method.
351  */
352  void internal_computeSpinImageDescriptors( const CImage &in_img,
353  CFeatureList &in_features) const;
354 
355  /** Compute a polar-image descriptor of the provided features into the input image
356  * \param in_img (input) The image from where to compute the descriptors.
357  * \param in_features (input/output) The list of features whose descriptors are going to be computed.
358  *
359  * \note Additional parameters from CFeatureExtraction::TOptions::PolarImagesOptions are used in this method.
360  */
361  void internal_computePolarImageDescriptors( const CImage &in_img,
362  CFeatureList &in_features) const;
363 
364  /** Compute a log-polar image descriptor of the provided features into the input image
365  * \param in_img (input) The image from where to compute the descriptors.
366  * \param in_features (input/output) The list of features whose descriptors are going to be computed.
367  *
368  * \note Additional parameters from CFeatureExtraction::TOptions::LogPolarImagesOptions are used in this method.
369  */
370  void internal_computeLogPolarImageDescriptors( const CImage &in_img,
371  CFeatureList &in_features) const;
372 
373  /** Select good features using the openCV implementation of the KLT method.
374  * \param img (input) The image from where to select extract the images.
375  * \param feats (output) A complete list of features (containing a patch for each one of them if options.patchsize > 0).
376  * \param nDesiredFeatures (op. input) Number of features to be extracted. Default: all possible.
377  * \param omitPixels (op. input) A mask for determining the ROI. (0: do not omit this pixel, 1: omit this pixel)
378  */
379  void selectGoodFeaturesKLT(
380  const CImage &inImg,
381  CFeatureList &feats,
382  unsigned int init_ID = 0,
383  unsigned int nDesiredFeatures = 0,
384  void *mask_ = NULL) const;
385 
386  /** Extract features from the image based on the KLT method.
387  * \param img The image from where to extract the images.
388  * \param feats The list of extracted features.
389  * \param nDesiredFeatures Number of features to be extracted. Default: authomatic.
390  * \param ROI (op. input) Region of Interest. Default: All the image.
391  */
392  void extractFeaturesKLT(
393  const CImage &img,
394  CFeatureList &feats,
395  unsigned int init_ID = 0,
396  unsigned int nDesiredFeatures = 0,
397  const TImageROI &ROI = TImageROI()) const;
398 
399  // ------------------------------------------------------------------------------------
400  // BCD
401  // ------------------------------------------------------------------------------------
402  /** Extract features from the image based on the BCD method.
403  * \param img The image from where to extract the images.
404  * \param feats The list of extracted features.
405  * \param nDesiredFeatures Number of features to be extracted. Default: authomatic.
406  * \param ROI (op. input) Region of Interest. Default: All the image.
407  */
408  void extractFeaturesBCD(
409  const CImage &img,
410  CFeatureList &feats,
411  unsigned int init_ID = 0,
412  unsigned int nDesiredFeatures = 0,
413  const TImageROI &ROI = TImageROI()) const;
414 
415  // ------------------------------------------------------------------------------------
416  // SIFT
417  // ------------------------------------------------------------------------------------
418  /** Extract features from the image based on the SIFT method.
419  * \param img The image from where to extract the images.
420  * \param feats The list of extracted features.
421  * \param nDesiredFeatures Number of features to be extracted. Default: authomatic.
422  * \param ROI (op. input) Region of Interest. Default: All the image.
423  */
424  void extractFeaturesSIFT(
425  const CImage &img,
426  CFeatureList &feats,
427  unsigned int init_ID = 0,
428  unsigned int nDesiredFeatures = 0,
429  const TImageROI &ROI = TImageROI()) const;
430 
431  // ------------------------------------------------------------------------------------
432  // SURF
433  // ------------------------------------------------------------------------------------
434  /** Extract features from the image based on the SURF method.
435  * \param img The image from where to extract the images.
436  * \param feats The list of extracted features.
437  * \param nDesiredFeatures Number of features to be extracted. Default: authomatic.
438  * \param ROI (op. input) Region of Interest. Default: All the image.
439  */
440  void extractFeaturesSURF(
441  const CImage &img,
442  CFeatureList &feats,
443  unsigned int init_ID = 0,
444  unsigned int nDesiredFeatures = 0,
445  const TImageROI &ROI = TImageROI()) const;
446 
447  // ------------------------------------------------------------------------------------
448  // FAST
449  // ------------------------------------------------------------------------------------
450  /** Extract features from the image based on the FAST method.
451  * \param img The image from where to extract the images.
452  * \param feats The list of extracted features.
453  * \param nDesiredFeatures Number of features to be extracted. Default: authomatic.
454  * \param ROI (op. input) Region of Interest. Default: All the image.
455  */
456  void extractFeaturesFAST(
457  const CImage &img,
458  CFeatureList &feats,
459  unsigned int init_ID = 0,
460  unsigned int nDesiredFeatures = 0,
461  const TImageROI &ROI = TImageROI(),
462  const CMatrixBool * mask = NULL ) const; // Important: This was a const ref. in mrpt <0.9.4, but the instantiation of a default value
463  // for CMatrixBool being a template generated duplicated linking errors for MSVC, thus it was changed to a pointer.
464 
465  /** Edward's "FASTER & Better" detector, N=9,10,12 */
466  void extractFeaturesFASTER_N(
467  const int N,
468  const CImage &img,
469  CFeatureList &feats,
470  unsigned int init_ID = 0,
471  unsigned int nDesiredFeatures = 0,
472  const TImageROI &ROI = TImageROI()) const;
473 
474 
475  // ------------------------------------------------------------------------------------
476  // my_scale_space_extrema
477  // ------------------------------------------------------------------------------------
478  /** Computes extrema in the scale space.
479  * \param dog_pyr Pyramid of images.
480  * \param octvs Number of considered octaves.
481  * \param intvls Number of intervales in octaves.
482  */
483  void* my_scale_space_extrema(
484  CFeatureList &featList, void* dog_pyr,
485  int octvs, int intvls, double contr_thr, int curv_thr,
486  void* storage ) const;
487 
488  /** Adjust scale if the image was initially doubled.
489  * \param features The sequence of features.
490  */
491  void my_adjust_for_img_dbl( void* features ) const;
492 
493  /** Gets the number of times that a point in the image is higher or lower than the surroundings in the image-scale space
494  * \param dog_pyr Pyramid of images.
495  * \param octvs Number of considered octaves.
496  * \param intvls Number of intervales in octaves.
497  * \param row The row of the feature in the original image.
498  * \param col The column of the feature in the original image.
499  * \param nMin [out]: Times that the feature is lower than the surroundings.
500  * \param nMax [out]: Times that the feature is higher than the surroundings.
501  */
502  void getTimesExtrema( void* dog_pyr, int octvs, int intvls, float row, float col, unsigned int &nMin, unsigned int &nMax ) const;
503 
504  /** Computes the Laplacian value of the feature in the corresponing image in the pyramid.
505  * \param dog_pyr Pyramid of images.
506  * \param octvs Number of considered octaves.
507  * \param intvls Number of intervales in octaves.
508  * \param row The row of the feature in the original image.
509  * \param col The column of the feature in the original image.
510  */
511  double getLaplacianValue( void* dog_pyr, int octvs, int intvls, float row, float col ) const;
512 
513  /** Append a sequence of openCV features into an MRPT feature list.
514  * \param features The sequence of features.
515  * \param list [in-out] The list of MRPT features.
516  * \param init_ID [in] The initial ID for the new features.
517  */
518  void insertCvSeqInCFeatureList( void* features, CFeatureList &list, unsigned int init_ID = 0 ) const;
519 
520  /** Converts a sequence of openCV features into an MRPT feature list.
521  * \param features The sequence of features.
522  * \param list [in-out] The list of MRPT features.
523  * \param init_ID [in][optional] The initial ID for the features (default = 0).
524  * \param ROI [in][optional] The initial ID for the features (default = empty ROI -> not used).
525  */
526  void convertCvSeqInCFeatureList( void* features, CFeatureList &list, unsigned int init_ID = 0, const TImageROI &ROI = TImageROI() ) const;
527 
528  }; // end of class
529  } // end of namespace
530 } // end of namespace
531 #endif


Page generated by Doxygen 1.8.3 for MRPT 0.9.6 SVN: at Fri Feb 15 22:05:02 EST 2013