TaskRelevanceMap.H

Go to the documentation of this file.
00001 /*!@file Neuro/TaskRelevanceMap.H Class declarations for task-relevance map class */
00002 
00003 // //////////////////////////////////////////////////////////////////// //
00004 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2001 by the //
00005 // University of Southern California (USC) and the iLab at USC.         //
00006 // See http://iLab.usc.edu for information about this project.          //
00007 // //////////////////////////////////////////////////////////////////// //
00008 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00009 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00010 // in Visual Environments, and Applications'' by Christof Koch and      //
00011 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00012 // pending; application number 09/912,225 filed July 23, 2001; see      //
00013 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00014 // //////////////////////////////////////////////////////////////////// //
00015 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00016 //                                                                      //
00017 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00018 // redistribute it and/or modify it under the terms of the GNU General  //
00019 // Public License as published by the Free Software Foundation; either  //
00020 // version 2 of the License, or (at your option) any later version.     //
00021 //                                                                      //
00022 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00023 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00024 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00025 // PURPOSE.  See the GNU General Public License for more details.       //
00026 //                                                                      //
00027 // You should have received a copy of the GNU General Public License    //
00028 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00029 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00030 // Boston, MA 02111-1307 USA.                                           //
00031 // //////////////////////////////////////////////////////////////////// //
00032 //
00033 // Primary maintainer for this file: Laurent Itti <itti@usc.edu>
00034 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/Neuro/TaskRelevanceMap.H $
00035 // $Id: TaskRelevanceMap.H 13648 2010-07-07 01:32:32Z jshen $
00036 //
00037 
00038 #ifndef TASKRELEVANCEMAP_H_DEFINED
00039 #define TASKRELEVANCEMAP_H_DEFINED
00040 
00041 #include "Component/ModelComponent.H"
00042 #include "Component/ModelParam.H"
00043 #include "Channels/InputFrame.H"
00044 #include "Image/LevelSpec.H"
00045 #include "Image/ImageCache.H"
00046 #include "Neuro/NeuroSimEvents.H"
00047 #include "Simulation/SimModule.H"
00048 #include "Simulation/SimEvents.H"
00049 #include "Media/TestImages.H"
00050 // add class definitions for Object Recognition
00051 //class TestImages;
00052 
00053 typedef TestImages::SceneData Scene;
00054 typedef TestImages::ObjData Object;
00055 
00056 // ######################################################################
00057 //! The task-relevance map base class
00058 // ######################################################################
00059 /*! This is a 2D task-relevance map. It is just a base class with
00060   virtual function definitions. Various implementations are available
00061   below.  The TRM is a topographic map where a value of zero indicates
00062   that a location is not relevant to the task at hand, while a value
00063   of 1.0 indicates neutral relevance and values higher than 1.0
00064   indicate high relevance. */
00065 class TaskRelevanceMap : public SimModule
00066 {
00067 public:
00068   // ######################################################################
00069   //! @name Constructor, destructor, and reset
00070   //@{
00071 
00072   //! Ininitialized constructor
00073   /*! The map will be resized and initialized the first time input() is
00074     called */
00075   TaskRelevanceMap(OptionManager& mgr,
00076                    const std::string& descrName = "Task-Relevance Map",
00077                    const std::string& tagName = "TaskRelevanceMap");
00078 
00079   //! Destructor
00080   virtual ~TaskRelevanceMap();
00081 
00082   //@}
00083 
00084 private:
00085   // forbid assignment and copy-construction:
00086   TaskRelevanceMap& operator=(const TaskRelevanceMap& sm);
00087   TaskRelevanceMap(const TaskRelevanceMap& sm);
00088 };
00089 
00090 // ######################################################################
00091 //! TaskRelevanceMap configurator
00092 // ######################################################################
00093 /*! This will export the --trm-type=XX command-line option and will
00094   instantiate a TRM of the desired type as the option gets assigned a
00095   value. As this happens, new options may become available in the
00096   command-line. To see them, use --help AFTER you have chosen the type
00097   to use. The current TRM may be retrieved using getTRM(). */
00098 class TaskRelevanceMapConfigurator : public ModelComponent
00099 {
00100 public:
00101   //! Constructor
00102   TaskRelevanceMapConfigurator(OptionManager& mgr,
00103                                const std::string& descrName =
00104                                "Task-Relevance Map Configurator",
00105                                const std::string& tagName =
00106                                "TaskRelevanceMapConfigurator");
00107 
00108   //! destructor
00109   virtual ~TaskRelevanceMapConfigurator();
00110 
00111   //! Get the chosen TRM
00112   /*! You should call this during start() of the ModelComponent that
00113     needs the TRM. CAUTION: this may be a rutz::shared_ptr(NULL) if no
00114     TRM type has been chosen. */
00115   nub::ref<TaskRelevanceMap> getTRM() const;
00116 
00117 protected:
00118   OModelParam<std::string> itsTRMtype; //!< type of map
00119 
00120   //! Intercept people changing our ModelParam
00121   /*! See ModelComponent.H; as parsing the command-line or reading a
00122     config file sets our name, we'll also here instantiate a
00123     controller of the proper type (and export its options) */
00124   virtual void paramChanged(ModelParamBase* const param,
00125                             const bool valueChanged,
00126                             ParamClient::ChangeStatus* status);
00127 
00128 private:
00129   nub::ref<TaskRelevanceMap> itsTRM; // the map
00130 };
00131 
00132 
00133 // ######################################################################
00134 //! Stub implementation of TaskRelevanceMap
00135 // ######################################################################
00136 class TaskRelevanceMapStub : public TaskRelevanceMap
00137 {
00138 public:
00139   //! Constructor
00140   TaskRelevanceMapStub(OptionManager& mgr,
00141                        const std::string& descrName = "Task-Relevance Map Stub",
00142                        const std::string& tagName = "TaskRelevanceMapStub");
00143 
00144   //! Destructor
00145   virtual ~TaskRelevanceMapStub();
00146 };
00147 
00148 // ######################################################################
00149 //! Partial implementation of TaskRelevanceMap for further derivation
00150 // ######################################################################
00151 class TaskRelevanceMapAdapter : public TaskRelevanceMap
00152 {
00153 public:
00154   //! Constructor
00155   TaskRelevanceMapAdapter(OptionManager& mgr,
00156                           const std::string& descrName,
00157                           const std::string& tagName);
00158 
00159   //! Destructor
00160   virtual ~TaskRelevanceMapAdapter();
00161 
00162 protected:
00163   //! Callback for when a new input frame is available from the retina
00164   SIMCALLBACK_DECLARE(TaskRelevanceMapAdapter, SimEventRetinaImage);
00165 
00166   //! Callback for when a new eye activity is going on
00167   SIMCALLBACK_DECLARE(TaskRelevanceMapAdapter, SimEventSaccadeStatusEye);
00168 
00169   //! Callback for every clock tick, to run our diff equations in integrate()
00170   SIMCALLBACK_DECLARE(TaskRelevanceMapAdapter, SimEventClockTick);
00171 
00172   //! Callback for every time we should save our outputs
00173   SIMCALLBACK_DECLARE(TaskRelevanceMapAdapter, SimEventSaveOutput);
00174 
00175   //! LevelSpec for scale of saliency map
00176   OModelParam<LevelSpec> itsLevelSpec;
00177 
00178   //! Save our internals when saveResults() is called?
00179   OModelParam<bool> itsSaveResults;
00180 
00181   //! Save our internals
00182   /*! This implementation just saves the map with a file name prefix
00183     of "TRM". */
00184   virtual void save1(const ModelComponentSaveInfo& sinfo);
00185 
00186   //! Get reset. See ModelComponent
00187   /*! In the adapter, we just free itsMap. */
00188   virtual void reset1();
00189 
00190   // ### Derived classes will need to implement those:
00191 
00192   //! Called when a new input frame is available from the Retina
00193   virtual void inputFrame(const InputFrame& f) = 0;
00194 
00195   //! Some TRM implementations will reset their memory upon saccade
00196   virtual void saccadicSuppression(const bool on) = 0;
00197 
00198   //! Integrate for one time step
00199   virtual void integrate(SimEventQueue& q) = 0;
00200 
00201   //! This map contains the relevance values.
00202   /*! The TaskRelevanceMap adapter will initialize this to the correct
00203     size and all-ones as soon as an input frame is received. As long
00204     as itsMap is not initialized, derived classes should just do
00205     nothing. Derived classes should update itsMap with the new
00206     computed relevance values, as this is the one which will get
00207     saved, displayed, posted, etc. */
00208   Image<float> itsMap;
00209 };
00210 
00211 // ######################################################################
00212 //! The standard task-relevance map
00213 // ######################################################################
00214 /*! This is our current standard TRM implementation. It is very
00215   incomplete as there is no good way to actually change relevance of
00216   anything. It will need in future work to be connected to an agent
00217   that computes the relevance of fixated locations and feeds that to
00218   the TRM. */
00219 class TaskRelevanceMapStd : public TaskRelevanceMapAdapter
00220 {
00221 public:
00222   //! Uninitialized constructor
00223   TaskRelevanceMapStd(OptionManager& mgr, const std::string& descrName =
00224                       "Task-Relevance Map Std",
00225                       const std::string& tagName =
00226                       "TaskRelevanceMapStd");
00227 
00228   //! Destructor
00229   virtual ~TaskRelevanceMapStd();
00230 
00231 protected:
00232   //! Learn the TRM for the given task by inspecting several scenes
00233   OModelParam<bool> itsLearnTRM;
00234 
00235   //! Learn and update the TRM as we inspecting several scenes
00236   OModelParam<bool> itsLearnUpdateTRM;
00237 
00238   //! If an image is specified with this option, then use it to bias the TRM
00239   OModelParam<std::string> itsBiasTRM;
00240 
00241   //! Called when a new input frame is available from the Retina
00242   virtual void inputFrame(const InputFrame& f);
00243 
00244   //! Some TRM implementations will reset their memory upon saccade
00245   virtual void saccadicSuppression(const bool on);
00246 
00247   //! Integrate for one time step
00248   virtual void integrate(SimEventQueue& q);
00249 };
00250 
00251 // ######################################################################
00252 //! A killStatic task-relevance map
00253 // ######################################################################
00254 /*! This is a very simple TRM where image locations that do not change
00255   much over time see theri relevance progressively decrease. */
00256 class TaskRelevanceMapKillStatic : public TaskRelevanceMapAdapter
00257 {
00258 public:
00259   //! Uninitialized constructor
00260   TaskRelevanceMapKillStatic(OptionManager& mgr, const std::string& descrName =
00261                       "Task-Relevance Map KillStatic",
00262                       const std::string& tagName =
00263                       "TaskRelevanceMapKillStatic");
00264 
00265   //! Destructor
00266   virtual ~TaskRelevanceMapKillStatic();
00267 
00268 protected:
00269   /*! Anything in the difference between current and cumulated
00270     previous frames (range 0..255) less than this will be considered
00271     static */
00272   OModelParam<float> itsKillStaticThresh;
00273 
00274   /*! Max killing achievable for a purely static location (should be
00275     in 0.0 .. 1.0 range) */
00276   OModelParam<float> itsKillStaticCoeff;
00277 
00278   //! Called when a new input frame is available from the Retina
00279   virtual void inputFrame(const InputFrame& f);
00280 
00281   //! Some TRM implementations will reset their memory upon saccade
00282   virtual void saccadicSuppression(const bool on);
00283 
00284   //! Integrate for one time step
00285   virtual void integrate(SimEventQueue& q);
00286 
00287   //! Save our internals
00288   /*! Call the TaskRelevanceMapAdapter version, plus save our static buf. */
00289   virtual void save1(const ModelComponentSaveInfo& sinfo);
00290 
00291   //! Get reset. See ModelComponent
00292   virtual void reset1();
00293 
00294 private:
00295   // cumulative buffer of the luminance image, at the resolution of
00296   // the saliency map. This is a sliding weighted average of the
00297   // downscaled luminance of the past few input frames, and it is used
00298   // to determine how different the current input is from that
00299   // accumulated average, and in turn to set the task relevance map so
00300   // that more static regions are less relevant.
00301   Image<float> itsStaticBuff;
00302 };
00303 
00304 // ######################################################################
00305 //! A KillN task-relevance map
00306 // ######################################################################
00307 /*! This is an alternative to IOR, in which we set the relevance of
00308   the last N attended locations to zero. */
00309 class TaskRelevanceMapKillN : public TaskRelevanceMapAdapter
00310 {
00311 public:
00312   //! Uninitialized constructor
00313   TaskRelevanceMapKillN(OptionManager& mgr, const std::string& descrName =
00314                       "Task-Relevance Map KillN",
00315                       const std::string& tagName =
00316                       "TaskRelevanceMapKillN");
00317 
00318   //! Destructor
00319   virtual ~TaskRelevanceMapKillN();
00320 
00321 protected:
00322   //! Number of inputs to compute over
00323   OModelParam<int> itsN;
00324 
00325   //! Called when a new input frame is available from the Retina
00326   virtual void inputFrame(const InputFrame& f);
00327 
00328   //! Some TRM implementations will reset their memory upon saccade
00329   virtual void saccadicSuppression(const bool on);
00330 
00331   //! Integrate for one time step
00332   virtual void integrate(SimEventQueue& q);
00333 
00334   //! Get started. See ModelComponent.
00335   virtual void start1();
00336 
00337   //! Get reset. See ModelComponent
00338   virtual void reset1();
00339 
00340 private:
00341   ImageCacheMinMax<float> itsCache;
00342 };
00343 
00344 // ######################################################################
00345 //! A GistClassify task-relevance map
00346 // ######################################################################
00347 /*! This is an alternative to KillStatic method, we use the gist vector to
00348  classify the frames into different categories and assign each frame a
00349  pre-defined TP map*/
00350 class TaskRelevanceMapGistClassify : public TaskRelevanceMapAdapter
00351 {
00352 public:
00353   //! Uninitialized constructor
00354   TaskRelevanceMapGistClassify(OptionManager& mgr, const std::string& descrName =
00355                       "Task-Relevance Map GistClassify",
00356                       const std::string& tagName =
00357                       "TaskRelevanceMapGistClassify");
00358 
00359   //! Destructor
00360   virtual ~TaskRelevanceMapGistClassify();
00361 
00362 protected:
00363   //! Number of inputs to compute over
00364   OModelParam<std::string> itsClusterCenterFileName; // the clustered center of the gist
00365   OModelParam<std::string> itsTemplateDir; // the dir of the pre-defined TD map
00366   OModelParam<int> itsPCADims;
00367   OModelParam<std::string> itsPCAMatrixName;
00368   OModelParam<int> itsCacheSize;
00369   OModelParam<int> itsUpdatePeriod;
00370 
00371   //! Called when a new input frame is available from the Retina
00372   virtual void inputFrame(const InputFrame& f);
00373 
00374   //! Some TRM implementations will reset their memory upon saccade
00375   virtual void saccadicSuppression(const bool on);
00376 
00377   //! Integrate for one time step
00378   virtual void integrate(SimEventQueue& q);
00379 
00380   // classify the current frame's gist to one category
00381   virtual void gistmatch(Image<float> gist);
00382 
00383   // get the PCA Matrix
00384   virtual void getPCAMatrix();
00385 
00386   // compute the distance between current gist and gist center
00387   virtual Image<float> computeGistDist(Image<float> currGist);
00388 
00389   // compute the TD map according to the gistDist
00390   virtual Image<float> getTDMap(Image<float> gistDist);
00391 
00392   //! Get reset. See ModelComponent
00393   virtual void reset1();
00394 
00395 private:
00396   Image<float> itsGist;
00397   Image<float> itsPCAMatrix;
00398   int itsMatchCategory;
00399   Image<float> itsCurrentTDMap;
00400   Image<float> itsTmpTDMap;
00401   std::string itsTDMapName;
00402   int itsNumOfCategory;
00403   int itsUsedDims;
00404   bool itsMapComputedForCurrentFrame;
00405 
00406   ImageCacheAvg<float> itsTDMap;
00407   int itsFrame;
00408 
00409 };
00410 
00411 // ######################################################################
00412 //!  A Tigs task-relevance map
00413 // ######################################################################
00414 /*! This is an alternative to GistClassify method, we use the gist vector to
00415  get the TD map from a learned Tigs Matrix (see Rob's CVPR 2007 paper) */
00416 
00417 class TaskRelevanceMapTigs : public TaskRelevanceMapAdapter
00418 {
00419 public:
00420   //! Uninitialized constructor
00421   TaskRelevanceMapTigs(OptionManager& mgr, const std::string& descrName =
00422                       "Task-Relevance Map Tigs",
00423                       const std::string& tagName =
00424                       "TaskRelevanceMapTigs");
00425 
00426   //! Destructor
00427   virtual ~TaskRelevanceMapTigs();
00428 
00429 protected:
00430   //! Number of inputs to compute over
00431   OModelParam<std::string> itsTigsMatrixName; // the trained Tigs Matrix file Name
00432   OModelParam<int> itsPCADims;
00433   OModelParam<std::string> itsPCAMatrixName;
00434   OModelParam<int> itsCacheSize;
00435   OModelParam<int> itsUpdatePeriod;
00436 
00437   //! Called when a new input frame is available from the Retina
00438   virtual void inputFrame(const InputFrame& f);
00439 
00440   //! Some TRM implementations will reset their memory upon saccade
00441   virtual void saccadicSuppression(const bool on);
00442 
00443   //! Integrate for one time step
00444   virtual void integrate(SimEventQueue& q);
00445 
00446   // get the PCA Matrix
00447   virtual void getPCAMatrix();
00448 
00449   // compute the TD map according to the gistDist
00450   virtual Image<float> getTDMap(Image<float> gistDist);
00451 
00452   // get the Trained Tigs Matrix
00453   virtual void getTigsMatrix();
00454 
00455   // decode the 1x300 vector to a 2D image
00456   virtual Image<float> decode(Image<float> tmp);
00457 
00458   //! Save our internals
00459   /*! Call the TaskRelevanceMapAdapter version, plus save our static buf. */
00460   virtual void save1(const ModelComponentSaveInfo& sinfo);
00461 
00462   //! Get reset. See ModelComponent
00463   virtual void reset1();
00464 
00465 private:
00466   Image<float> itsGist;
00467   Image<float> itsPCAMatrix;
00468   Image<float> itsTigsMatrix;
00469   Image<float> itsCurrentTDMap;
00470   Image<float> itsTmpTDMap;
00471   int itsUsedDims;
00472   bool itsMapComputedForCurrentFrame;
00473 
00474   ImageCacheAvg<float> itsTDMap;
00475   int itsFrame;
00476 };
00477 
00478 
00479 // ######################################################################
00480 //! A Tigs task-relevance map A Tigs combined image PCA task-relevance map
00481 // ######################################################################
00482 /*! This is an alternative to GistClassify method, we use the gist vector
00483   and pca image information as input to compute the TDmap from a learned
00484   TD map Matrix*/
00485 class TaskRelevanceMapTigs2 : public TaskRelevanceMapAdapter
00486 {
00487 public:
00488   //! Uninitialized constructor
00489   TaskRelevanceMapTigs2(OptionManager& mgr, const std::string& descrName =
00490                         "Task-Relevance Map Tigs2",
00491                         const std::string& tagName =
00492                         "TaskRelevanceMapTigs2");
00493 
00494   //! Destructor
00495   virtual ~TaskRelevanceMapTigs2();
00496 
00497 protected:
00498   //! Number of inputs to compute over
00499   OModelParam<std::string> itsTigsMatrixName; // the trained Tigs Matrix file Name
00500   OModelParam<int> itsGistPCADims;
00501   OModelParam<int> itsImgPCADims;
00502   OModelParam<std::string> itsGistPCAMatrixName;
00503   OModelParam<std::string> itsImgPCAMatrixName;
00504   OModelParam<int> itsCacheSize;
00505   OModelParam<int> itsUpdatePeriod;
00506 
00507   //! Called when a new input frame is available from the Retina
00508   virtual void inputFrame(const InputFrame& f);
00509 
00510   //! Some TRM implementations will reset their memory upon saccade
00511   virtual void saccadicSuppression(const bool on);
00512 
00513   //! Integrate for one time step
00514   virtual void integrate(SimEventQueue& q);
00515 
00516   // get the Gist PCA Matrix
00517   virtual void getGistPCAMatrix();
00518 
00519   // get the image PCA Matrix
00520   virtual void getImgPCAMatrix();
00521 
00522   // compute the TD map according to the gistDist
00523   virtual Image<float> getTDMap(Image<float> gistDist);
00524 
00525   // get the Trained Tigs Matrix
00526   virtual void getTigsMatrix();
00527 
00528   // decode the 1x300 vector to a 2D image
00529   virtual Image<float> decode(Image<float> tmp);
00530 
00531   //! Save our internals
00532   /*! Call the TaskRelevanceMapAdapter version, plus save our static buf. */
00533   virtual void save1(const ModelComponentSaveInfo& sinfo);
00534 
00535   //! Get reset. See ModelComponent
00536   virtual void reset1();
00537 
00538 private:
00539   Image<float> itsGist;
00540   Image<float> itsGistPCAMatrix;
00541   Image<float> itsImgPCAMatrix;
00542   Image<float> itsTigsMatrix;
00543   Image<float> itsCurrentTDMap;
00544   Image<float> itsTmpTDMap;
00545   Image<float> itsCurrFrameVec;
00546   int itsUsedDims;
00547   bool itsMapComputedForCurrentFrame;
00548 
00549   ImageCacheAvg<float> itsTDMap;
00550   int itsFrame;
00551 };
00552 
00553 ///// Task Relevance Map based on VLM input /////
00554 class TaskRelevanceMapSocial : public TaskRelevanceMapAdapter
00555 {
00556 public:
00557   //! Uninitialized constructor
00558   TaskRelevanceMapSocial(OptionManager& mgr, const std::string& descrName =
00559                       "Task-Relevance Map Social",
00560                       const std::string& tagName =
00561                       "TaskRelevanceMapSocial");
00562 
00563   //! Destructor
00564   virtual ~TaskRelevanceMapSocial();
00565 
00566 protected:
00567   //! Number of inputs to compute over
00568   //OModelParam<int> itsN;
00569 
00570   //! Called when a new input frame is available from the Retina
00571   virtual void inputFrame(const InputFrame& f);
00572 
00573   //! Some TRM implementations will reset their memory upon saccade
00574   virtual void saccadicSuppression(const bool on);
00575 
00576   //! Integrate for one time step
00577   virtual void integrate(SimEventQueue& q);
00578 
00579   //! Get started. See ModelComponent.
00580   virtual void start1();
00581 
00582   //! Get reset. See ModelComponent
00583   virtual void reset1();
00584 
00585 private:
00586   OModelParam<std::string> itsXMLFName;
00587   rutz::shared_ptr<TestImages> itsObjectsInfo;
00588   std::vector<std::string> itsObjectsNames;
00589   uint itsNumObjects;
00590 
00591   uint itsFrame;
00592 };
00593 
00594 #endif
00595 
00596 // ######################################################################
00597 /* So things look consistent in everyone's emacs... */
00598 /* Local Variables: */
00599 /* indent-tabs-mode: nil */
00600 /* End: */
Generated on Sun May 8 08:41:05 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3