Main MRPT website > C++ reference
MRPT logo
tracking.h
Go to the documentation of this file.
1 /* +---------------------------------------------------------------------------+
2  | The Mobile Robot Programming Toolkit (MRPT) C++ library |
3  | |
4  | http://www.mrpt.org/ |
5  | |
6  | Copyright (C) 2005-2012 University of Malaga |
7  | |
8  | This software was written by the Machine Perception and Intelligent |
9  | Robotics Lab, University of Malaga (Spain). |
10  | Contact: Jose-Luis Blanco <jlblanco@ctima.uma.es> |
11  | |
12  | This file is part of the MRPT project. |
13  | |
14  | MRPT is free software: you can redistribute it and/or modify |
15  | it under the terms of the GNU General Public License as published by |
16  | the Free Software Foundation, either version 3 of the License, or |
17  | (at your option) any later version. |
18  | |
19  | MRPT is distributed in the hope that it will be useful, |
20  | but WITHOUT ANY WARRANTY; without even the implied warranty of |
21  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
22  | GNU General Public License for more details. |
23  | |
24  | You should have received a copy of the GNU General Public License |
25  | along with MRPT. If not, see <http://www.gnu.org/licenses/>. |
26  | |
27  +---------------------------------------------------------------------------+ */
28 
29 #ifndef mrpt_vision_tracking_H
30 #define mrpt_vision_tracking_H
31 
32 #include <mrpt/vision/types.h>
33 #include <mrpt/vision/link_pragmas.h>
34 
35 #include <mrpt/vision/CFeature.h>
36 #include <mrpt/vision/TSimpleFeature.h>
37 #include <mrpt/utils/CImage.h>
38 #include <mrpt/utils/CTimeLogger.h>
39 #include <mrpt/utils/TParameters.h>
40 
41 #include <mrpt/utils/metaprogramming.h>
42 
43 #include <memory> // for auto_ptr
44 
45 namespace mrpt
46 {
47  namespace vision
48  {
49  using namespace mrpt::math;
50  using namespace mrpt::utils;
51 
52  /** \addtogroup vision_tracking Feature detection and tracking
53  * \ingroup mrpt_vision_grp
54  * @{ */
55 
56  /** A virtual interface for all feature trackers, implementing the part of feature tracking that is common to any specific tracker implementation.
57  * This class provides a quite robust tracking of features, avoiding as many outliers as possible but not all of them:
58  * more robust tracking would require application-specific information and could be done in a number of very different approaches,
59  * so this class will not try to do any kind of RANSAC or any other advanced outlier rejection; instead, it should
60  * be done by the users or the classes that employ this class.
61  *
62  * The basic usage of this class is as follows:
63  * \code
64  * CFeatureTracker_KL tracker; // Note: CFeatureTracker_KL is the most robust implementation for now.
65  * tracker.extra_params["add_new_features"] = 1; // Enable detection of new features, not only tracking
66  * tracker.extra_params[...] = ...
67  * // ....
68  * CFeatureList theFeats; // The list of features
69  * mrpt::utils::CImage previous_img, current_img;
70  *
71  * while (true) {
72  * current_img = ... // Grab new image.
73  * if ( previous_img_is_ok )
74  * tracker.trackFeatures(previous_img, current_img, theFeats);
75  * previous_img = current_img;
76  * }
77  * \endcode
78  *
79  * Below follows the list of optional parameters for "extra_params" which can be set
80  * and will be understood by this base class for any specific tracker implementation.
81  * Note that all parameters are double's, but boolean flags are emulated by the values 0.0 (false) and 1.0 (true).
82  *
83  * List of parameters:
84  * <table border="1" >
85  * <tr><td align="center" > <b>Parameter name</b> </td> <td align="center" > <b>Default value</b> </td> <td align="center" > <b>Comments</b> </td> </tr>
86  * <tr><td align="center" > add_new_features </td> <td align="center" > 0 </td>
87  * <td> If set to "1", the class will not only track existing features, but will also perform (after doing the actual tracking) an efficient
88  * search for new features with the FAST detector, and will add them to the passed "CFeatureList" if they fulfill a set of restrictions,
89  * as stablished by the other parameters (see <i>add_new_feat_min_separation</i>,<i>add_new_feat_max_features</i>,<i>minimum_KLT_response_to_add</i>).
90  * </td> </tr>
91  * <tr><td align="center" > add_new_feat_min_separation </td> <td align="center" > 15 </td>
92  * <td> If <i>add_new_features</i>==1, this is the minimum separation (in pixels) to any other (old, or new) feature for it
93  * being considered a candidate to be added.
94  * </td> </tr>
95  * <tr><td align="center" > desired_num_features_adapt </td> <td align="center" > (img_width*img_height)/512 </td>
96  * <td> If <i>add_new_features</i>==1, the threshold of the FAST(ER) feature detector is dynamically adapted such as the number of
97  * raw FAST keypoints is around this number. This number should be much higher than the real desired numbre of features, since this
98  * one includes many features concentrated in space which are later discarded for the minimum distance.
99  * </td> </tr>
100  * <tr><td align="center" > desired_num_features </td> <td align="center" > 100 </td>
101  * <td> If <i>add_new_features</i>==1, the target number of the patch associated to each feature will be updated with every N'th frame. </td> </tr>
102  * <tr><td align="center" > add_new_feat_patch_size </td> <td align="center" > 11 </td>
103  * <td> If <i>add_new_features</i>==1, for each new added feature, this is the size of the patch to be extracted around the keypoint (set to 0 if patches are not required at all).
104  * </td> </tr>
105  * <tr><td align="center" > minimum_KLT_response_to_add </td> <td align="center" > 10 </td>
106  * <td> If <i>add_new_features</i>==1, this sets the minimum KLT response of candidate FAST features to be added in each frame, if they also fulfil the other restrictions (e.g. min.distance).
107  * </td> </tr>
108  * <tr><td align="center" > check_KLT_response_every </td> <td align="center" > 0 </td>
109  * <td> If >0, it will compute the KLT response at each feature point every <i>N</i> frames
110  * and those below <i>minimum_KLT_response</i> will be marked as "lost" in their "track_status" field.
111  * </td> </tr>
112  * <tr><td align="center" > minimum_KLT_response </td> <td align="center" > 5 </td>
113  * <td> See explanation of <i>check_KLT_response_every</i>.
114  * </td> </tr>
115  * <tr><td align="center" > KLT_response_half_win </td> <td align="center" > 4 </td>
116  * <td> When computing the KLT response of features (see <i>minimum_KLT_response</i> and <i>minimum_KLT_response_to_add</i>),
117  * the window centered at the point for its estimation will be of size (2*W+1)x(2*W+1), with <i>W</i> being this parameter value.
118  * </td> </tr>
119  * <tr><td align="center" > update_patches_every </td> <td align="center" > 0 </td>
120  * <td> If !=0, the patch associated to each feature will be updated with every N'th frame. </td> </tr>
121  * <tr><td align="center" > remove_lost_features </td> <td align="center" > 0 </td>
122  * <td> If !=0, out-of-bound features or those lost while tracking, will be automatically removed from the list of features.
123  * Otherwise, the user will have to manually remove them by checking the track_status field. </td> </tr>
124  * </table>
125  *
126  * This class also offers a time profiler, disabled by default (see getProfiler and enableTimeLogger).
127  *
128  * \sa CFeatureTracker_KL, the example application "track-video-features".
129  */
130  struct VISION_IMPEXP CGenericFeatureTracker
131  {
132  /** Optional list of extra parameters to the algorithm. */
133  mrpt::utils::TParametersDouble extra_params;
134 
135  /** Default ctor */
136  inline CGenericFeatureTracker() : m_timlog(false), m_update_patches_counter(0),m_check_KLT_counter(0),m_detector_adaptive_thres(10)
137  { }
138  /** Ctor with extra parameters */
139  inline CGenericFeatureTracker(mrpt::utils::TParametersDouble extraParams) : extra_params(extraParams), m_timlog(false), m_update_patches_counter(0),m_check_KLT_counter(0),m_detector_adaptive_thres(10)
140  { }
141  /** Dtor */
142  virtual ~CGenericFeatureTracker()
143  { }
144 
145  /** Perform feature tracking from "old_img" to "new_img", with a (possibly empty) list of previously tracked features "inout_featureList".
146  * This is a list of parameters (in "extraParams") accepted by ALL implementations of feature tracker (see each derived class for more specific parameters).
147  * - "add_new_features" (Default=0). If set to "1", new features will be also added to the existing ones in areas of the image poor of features.
148  * This method does:
149  * - Convert old and new images to grayscale, if they're in color.
150  * - Call the pure virtual "trackFeatures_impl" method.
151  * - Implement the optional detection of new features if "add_new_features"!=0.
152  */
153  void trackFeatures(const CImage &old_img,const CImage &new_img,TSimpleFeatureList &inout_featureList );
154 
155  /** \overload with subpixel precision */
156  void trackFeatures(const CImage &old_img,const CImage &new_img,TSimpleFeaturefList &inout_featureList );
157 
158  /** \overload This overload version uses the old (and much slower) CFeatureList */
159  void trackFeatures(const CImage &old_img,const CImage &new_img,CFeatureList &inout_featureList );
160 
161  /** A wrapper around the basic trackFeatures() method, but keeping the original list of features unmodified and returns the tracked ones in a new list. */
162  inline void trackFeaturesNewList(
163  const CImage &old_img,
164  const CImage &new_img,
165  const vision::CFeatureList &in_featureList,
166  vision::CFeatureList &out_featureList
167  )
168  {
169  out_featureList = in_featureList;
170  std::for_each(
171  out_featureList.begin(),out_featureList.end(),
172  mrpt::utils::metaprogramming::ObjectMakeUnique() );
173  this->trackFeatures(old_img, new_img, out_featureList);
174  }
175 
176  /** Returns a read-only reference to the internal time logger */
177  inline const mrpt::utils::CTimeLogger & getProfiler() const { return m_timlog; }
178  /** Returns a reference to the internal time logger */
179  inline mrpt::utils::CTimeLogger & getProfiler() { return m_timlog; }
180 
181  /** Returns a read-only reference to the internal time logger */
182  inline void enableTimeLogger(bool enable=true) { m_timlog.enable(enable); }
183 
184  /** Returns the current adaptive threshold used by the FAST(ER) detector to find out new features in empty areas */
185  inline int getDetectorAdaptiveThreshold() const { return m_detector_adaptive_thres; }
186 
187  struct VISION_IMPEXP TExtraOutputInfo
188  {
189  size_t raw_FAST_feats_detected; //!< In the new_img with the last adaptive threshold
190  size_t num_deleted_feats; //!< The number of features which were deleted due to OOB, bad tracking, etc... (only if "remove_lost_features" is enabled)
191  };
192 
193  TExtraOutputInfo last_execution_extra_info; //!< Updated with each call to trackFeatures()
194 
195  protected:
196  /** The tracking method implementation, to be implemented in children classes. */
197  virtual void trackFeatures_impl(const CImage &old_img,const CImage &new_img,TSimpleFeaturefList &inout_featureList );
198 
199  /** The tracking method implementation, to be implemented in children classes. */
200  virtual void trackFeatures_impl(const CImage &old_img,const CImage &new_img,TSimpleFeatureList &inout_featureList ) = 0;
201 
202  /** This version falls back to the version with TSimpleFeatureList if the derived class does not implement it. */
203  virtual void trackFeatures_impl(const CImage &old_img,const CImage &new_img,CFeatureList &inout_featureList ) = 0;
204 
205  mrpt::utils::CTimeLogger m_timlog; //!< the internal time logger, disabled by default.
206 
207  /** This field is clared by \a trackFeatures() before calling \a trackFeatures_impl(), and
208  * can be filled out with newly defected FAST(ER) features in the latter.
209  * If it's not the case, feats will be computed anyway if the user enabled the "add_new_features" option.
210  */
211  mrpt::vision::TSimpleFeatureList m_newly_detected_feats;
212 
213  /** Adapts the threshold \a m_detector_adaptive_thres according to the real and desired number of features just detected */
214  void updateAdaptiveNewFeatsThreshold(
215  const size_t nNewlyDetectedFeats,
216  const size_t desired_num_features);
217 
218  private:
219  size_t m_update_patches_counter; //!< for use when "update_patches_every">=1
220  size_t m_check_KLT_counter; //!< For use when "check_KLT_response_every">=1
221  int m_detector_adaptive_thres; //!< For use in "add_new_features" == true
222 
223  template <typename FEATLIST>
224  void internal_trackFeatures(
225  const CImage &old_img,
226  const CImage &new_img,
227  FEATLIST &inout_featureList );
228  };
229 
230  typedef std::auto_ptr<CGenericFeatureTracker> CGenericFeatureTrackerAutoPtr;
231 
232 
233  /** Track a set of features from old_img -> new_img using sparse optimal flow (classic KL method).
234  *
235  * See CGenericFeatureTracker for a more detailed explanation on how to use this class.
236  *
237  * List of additional parameters in "extra_params" (apart from those in CGenericFeatureTracker) accepted by this class:
238  * - "window_width" (Default=15)
239  * - "window_height" (Default=15)
240  * - "LK_levels" (Default=3) Number of pyramids to build for LK tracking (this parameter only has effects when tracking with CImage's, not with CImagePyramid's).
241  * - "LK_max_iters" (Default=10) Max. number of iterations in LK tracking.
242  * - "LK_epsilon" (Default=0.1) Minimum epsilon step in interations of LK_tracking.
243  * - "LK_max_tracking_error" (Default=150.0) The maximum "tracking error" of LK tracking such as a feature is marked as "lost".
244  *
245  * \sa OpenCV's method cvCalcOpticalFlowPyrLK
246  */
247  struct VISION_IMPEXP CFeatureTracker_KL : public CGenericFeatureTracker
248  {
249  /** Default ctor */
250  inline CFeatureTracker_KL() { }
251  /** Ctor with extra parameters */
252  inline CFeatureTracker_KL(mrpt::utils::TParametersDouble extraParams) : CGenericFeatureTracker(extraParams) { }
253 
254  protected:
255  virtual void trackFeatures_impl(const CImage &old_img,const CImage &new_img,vision::CFeatureList &inout_featureList );
256  virtual void trackFeatures_impl(const CImage &old_img,const CImage &new_img,TSimpleFeatureList &inout_featureList );
257  virtual void trackFeatures_impl(const CImage &old_img,const CImage &new_img,TSimpleFeaturefList &inout_featureList );
258 
259  private:
260  template <typename FEATLIST>
261  void trackFeatures_impl_templ(
262  const CImage &old_img,
263  const CImage &new_img,
264  FEATLIST &inout_featureList );
265 
266  };
267 
268  /** Track a set of features from old_img -> new_img by patch correlation over the closest FAST features, using a KD-tree for looking closest correspondences.
269  * See CGenericFeatureTracker for a more detailed explanation on how to use this class.
270  *
271  * List of additional parameters in "extra_params" (apart from those in CGenericFeatureTracker) accepted by this class:
272  * - "window_width" (Default=15)
273  * - "window_height" (Default=15)
274  *
275  */
276  struct VISION_IMPEXP CFeatureTracker_FAST : public CGenericFeatureTracker
277  {
278  /** Ctor */
279  CFeatureTracker_FAST(const mrpt::utils::TParametersDouble & extraParams = mrpt::utils::TParametersDouble() );
280 
281  protected:
282  virtual void trackFeatures_impl(
283  const CImage &old_img,
284  const CImage &new_img,
285  vision::CFeatureList &inout_featureList );
286 
287  virtual void trackFeatures_impl(
288  const CImage &old_img,
289  const CImage &new_img,
290  TSimpleFeatureList &inout_featureList );
291 
292  };
293 
294 
295  /** Track a set of features from old_img -> new_img by patch matching over a fixed window centered at each feature's previous location.
296  * See CGenericFeatureTracker for a more detailed explanation on how to use this class.
297  *
298  * List of additional parameters in "extra_params" (apart from those in CGenericFeatureTracker) accepted by this class:
299  * - "window_width" (Default=15)
300  * - "window_height" (Default=15)
301  *
302  */
303  struct VISION_IMPEXP CFeatureTracker_PatchMatch : public CGenericFeatureTracker
304  {
305  /** Ctor */
306  CFeatureTracker_PatchMatch(const mrpt::utils::TParametersDouble & extraParams = mrpt::utils::TParametersDouble() );
307 
308  protected:
309  virtual void trackFeatures_impl(
310  const CImage &old_img,
311  const CImage &new_img,
312  vision::CFeatureList &inout_featureList );
313  };
314 
315  /** Search for correspondences which are not in the same row and deletes them
316  * ...
317  */
318  void VISION_IMPEXP checkTrackedFeatures( CFeatureList &leftList,
319  CFeatureList &rightList,
320  vision::TMatchingOptions options);
321 
322 
323  /** Filter bad correspondences by distance
324  * ...
325  */
326  void VISION_IMPEXP filterBadCorrsByDistance( mrpt::utils::TMatchingPairList &list, // The list of correspondences
327  unsigned int numberOfSigmas ); // Threshold
328 
329 
330 
331  /** @} */ // end of grouping
332  }
333 }
334 
335 
336 #endif


Page generated by Doxygen 1.8.3 for MRPT 0.9.6 SVN: at Fri Feb 15 22:05:02 EST 2013