EnvSaliencyMap.C

Go to the documentation of this file.
00001 /*!@file Neuro/EnvSaliencyMap.C */
00002 
00003 // //////////////////////////////////////////////////////////////////// //
00004 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2000-2005   //
00005 // by the University of Southern California (USC) and the iLab at USC.  //
00006 // See http://iLab.usc.edu for information about this project.          //
00007 // //////////////////////////////////////////////////////////////////// //
00008 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00009 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00010 // in Visual Environments, and Applications'' by Christof Koch and      //
00011 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00012 // pending; application number 09/912,225 filed July 23, 2001; see      //
00013 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00014 // //////////////////////////////////////////////////////////////////// //
00015 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00016 //                                                                      //
00017 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00018 // redistribute it and/or modify it under the terms of the GNU General  //
00019 // Public License as published by the Free Software Foundation; either  //
00020 // version 2 of the License, or (at your option) any later version.     //
00021 //                                                                      //
00022 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00023 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00024 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00025 // PURPOSE.  See the GNU General Public License for more details.       //
00026 //                                                                      //
00027 // You should have received a copy of the GNU General Public License    //
00028 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00029 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00030 // Boston, MA 02111-1307 USA.                                           //
00031 // //////////////////////////////////////////////////////////////////// //
00032 //
00033 // Primary maintainer for this file: Rob Peters <rjpeters at usc dot edu>
00034 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/Neuro/EnvSaliencyMap.C $
00035 // $Id: EnvSaliencyMap.C 11389 2009-06-30 03:08:01Z lior $
00036 //
00037 
00038 #ifndef NEURO_ENVSALIENCYMAP_C_DEFINED
00039 #define NEURO_ENVSALIENCYMAP_C_DEFINED
00040 
00041 #include "Neuro/EnvSaliencyMap.H"
00042 
00043 #include "Component/GlobalOpts.H" // for OPT_TextLogFile
00044 #include "Component/ModelOptionDef.H"
00045 #include "Image/MathOps.H"
00046 #include "Image/DrawOps.H"
00047 #include "Image/ShapeOps.H"
00048 #include "Neuro/EnvOpts.H"
00049 #include "Util/MathFunctions.H" // for clampValue()
00050 #include "Util/TextLog.H"
00051 #include "Util/log.H"
00052 
00053 static const ModelOptionCateg MOC_ESM = {
00054   MOC_SORTPRI_3, "EnvSaliencyMap-related Options" };
00055 
00056 // Used by: EnvSaliencyMap
00057 const ModelOptionDef OPT_EsmUseFixed =
00058   { MODOPT_FLAG, "EsmUseFixed", &MOC_ESM, OPTEXP_CORE,
00059     "Whether to use a fixed center-of-attention given by the user",
00060     "esm-use-fixed", '\0', "", "false" };
00061 
00062 // Used by: EnvSaliencyMap
00063 const ModelOptionDef OPT_EsmFixedX =
00064   { MODOPT_ARG(int), "EsmFixedX", &MOC_ESM, OPTEXP_CORE,
00065     "X coordinate of fixed center-of-atteniton location",
00066     "esm-fixed-x", '\0', "int", "20" };
00067 
00068 // Used by: EnvSaliencyMap
00069 const ModelOptionDef OPT_EsmFixedY =
00070   { MODOPT_ARG(int), "EsmFixedY", &MOC_ESM, OPTEXP_CORE,
00071     "Y coordinate of fixed center-of-atteniton location",
00072     "esm-fixed-y", '\0', "int", "15" };
00073 
00074 // Used by: EnvSaliencyMap
00075 const ModelOptionDef OPT_GetNMostSalientLoc =
00076   { MODOPT_ARG(int), "GetNMostSalientLoc", &MOC_ESM, OPTEXP_CORE,
00077     "Return a vector with the stop n most salient locations",
00078     "get-nmost-salient-loc", '\0', "int", "1" };
00079 
00080 // Used by: EnvSaliencyMap
00081 const ModelOptionDef OPT_InternalIORRadius =
00082   { MODOPT_ARG(int), "InternalIORRadius", &MOC_ESM, OPTEXP_CORE,
00083     "The size of the internal IOR in pixels used for retrieving the N Most salient locations. "
00084     " This options will not change the actual saliency map.",
00085     "internal-ior-radius", '\0', "int", "10" };
00086 
00087 // Used by: EnvSaliencyMap
00088 const ModelOptionDef OPT_EsmIorHalfLife =
00089   { MODOPT_ARG(double), "EsmIorHalfLife", &MOC_ESM, OPTEXP_CORE,
00090     "Number of frames in which the IOR map decays by half",
00091     "esm-ior-halflife", '\0', "<double>", "6.5" };
00092 
00093 // Used by: EnvSaliencyMap
00094 const ModelOptionDef OPT_EsmIorStrength =
00095   { MODOPT_ARG(double), "EsmIorStrength", &MOC_ESM, OPTEXP_CORE,
00096     "Magnitude of IOR (useful range 0..255)",
00097     "esm-ior-strength", '\0', "<double>", "16.0" };
00098 
00099 // Used by: EnvSaliencyMap
00100 const ModelOptionDef OPT_EsmIorRadius =
00101   { MODOPT_ARG(double), "EsmIorRadius", &MOC_ESM, OPTEXP_CORE,
00102     "Radius of IOR",
00103     "esm-ior-radius", '\0', "<double>", "32.0" };
00104 
00105 // Used by: EnvSaliencyMap
00106 const ModelOptionDef OPT_EsmInertiaRadius =
00107   { MODOPT_ARG(double), "EsmInertiaRadius", &MOC_ESM, OPTEXP_CORE,
00108     "Radius of inertia blob",
00109     "esm-inertia-radius", '\0', "<double>", "32.0" };
00110 
00111 // Used by: EnvSaliencyMap
00112 const ModelOptionDef OPT_EsmInertiaStrength =
00113   { MODOPT_ARG(double), "EsmInertiaStrength", &MOC_ESM, OPTEXP_CORE,
00114     "Initial strength of inertia blob",
00115     "esm-inertia-strength", '\0', "<double>", "100.0" };
00116 
00117 // Used by: EnvSaliencyMap
00118 const ModelOptionDef OPT_EsmInertiaHalfLife =
00119   { MODOPT_ARG(double), "EsmInertiaHalfLife", &MOC_ESM, OPTEXP_CORE,
00120     "Number of frames in which the inertia blob decays by half",
00121     "esm-inertia-halflife", '\0', "<double>", "6.5" };
00122 
00123 // Used by: EnvSaliencyMap
00124 const ModelOptionDef OPT_EsmInertiaShiftThresh =
00125   { MODOPT_ARG(double), "EsmInertiaShiftThresh", &MOC_ESM, OPTEXP_CORE,
00126     "Distance threshold for inertia shift",
00127     "esm-inertia-thresh", '\0', "<double>", "5.0" };
00128 
00129 static const ModelOptionDef OPT_DynamicFeedback =
00130   { MODOPT_ARG(double), "DynamicFeedback", &MOC_ESM, OPTEXP_CORE,
00131     "How strongly the amount of temporal change in the visual cortex "
00132     "output causes suppression of IOR and inertia. A large value means "
00133     "that IOR and inertia will be totally suppressed by a small amount "
00134     "of temporal change in the visual cortex output, while a value of "
00135     "0.0 means that IOR and inertia will not be suppressed at all, "
00136     "regardless of the amount of temporal change in the visual cortex "
00137     "output.",
00138     "dynamic-feedback", '\0', "<double>", "1.5" };
00139 
00140 // ######################################################################
00141 static double meanAbsDiff(const Image<float>* x,
00142                           const Image<byte>* y)
00143 {
00144   ASSERT(x->getDims() == y->getDims());
00145 
00146   double sum = 0.0;
00147 
00148   const size_t sz = x->getSize();
00149 
00150   const Image<float>::const_iterator xptr = x->begin();
00151   const Image<byte>::const_iterator yptr = y->begin();
00152 
00153   for (size_t i = 0; i < sz; ++i)
00154     sum += fabs(xptr[i] - yptr[i]);
00155 
00156   return sz > 0 ? sum/sz : 0.0;
00157 }
00158 
00159 // ######################################################################
00160 static double sigmoid(double v)
00161 {
00162   return 1.0 / (1.0 + exp(-v));
00163 }
00164 
00165 // ######################################################################
00166 EnvSaliencyMap::EnvSaliencyMap(OptionManager& mgr)
00167   :
00168   ModelComponent(mgr, "Embeddable Saliency Map", "EnvSaliencyMap"),
00169   itsInertiaLoc(-1,-1),
00170   itsCurrentInertiaFactor(1.0),
00171   itsInertiaMap(),
00172   itsInhibMap(),
00173   itsUseFixed(&OPT_EsmUseFixed, this, ALLOW_ONLINE_CHANGES),
00174   itsFixedX(&OPT_EsmFixedX, this, ALLOW_ONLINE_CHANGES),
00175   itsFixedY(&OPT_EsmFixedY, this, ALLOW_ONLINE_CHANGES),
00176   itsGetNMostSalientLoc(&OPT_GetNMostSalientLoc, this, ALLOW_ONLINE_CHANGES),
00177   itsInternalIORRadius(&OPT_InternalIORRadius, this, ALLOW_ONLINE_CHANGES),
00178   itsDynamicFeedback(&OPT_DynamicFeedback, this, ALLOW_ONLINE_CHANGES),
00179   itsInertiaHalfLife(&OPT_EsmInertiaHalfLife, this, ALLOW_ONLINE_CHANGES),
00180   itsInertiaStrength(&OPT_EsmInertiaStrength, this, ALLOW_ONLINE_CHANGES),
00181   itsInertiaRadius(&OPT_EsmInertiaRadius, this, ALLOW_ONLINE_CHANGES),
00182   itsInertiaShiftThresh(&OPT_EsmInertiaShiftThresh, this, ALLOW_ONLINE_CHANGES),
00183   itsIorHalfLife(&OPT_EsmIorHalfLife, this, ALLOW_ONLINE_CHANGES),
00184   itsIorStrength(&OPT_EsmIorStrength, this, ALLOW_ONLINE_CHANGES),
00185   itsIorRadius(&OPT_EsmIorRadius, this, ALLOW_ONLINE_CHANGES),
00186   itsLevelSpec(&OPT_EnvLevelSpec, this),
00187   itsTextLogFile(&OPT_TextLogFile, this),
00188   itsDynamicFactor(1.0),
00189   itsVcxMovingAvg(),
00190   itsVcxFlicker(0.0),
00191   itsVcxMeanDiffCenter(4.0),
00192   itsVcxFlickerMin(sigmoid(0.5*(-itsVcxMeanDiffCenter)))
00193 {}
00194 
00195 // ######################################################################
00196 EnvSaliencyMap::~EnvSaliencyMap()
00197 {}
00198 
00199 // ######################################################################
00200 void EnvSaliencyMap::paramChanged(ModelParamBase* const param,
00201                                   const bool valueChanged,
00202                                   ParamClient::ChangeStatus* status)
00203 {
00204   if (param == &itsUseFixed)
00205     {
00206       itsFixedX.setInactive(!itsUseFixed.getVal());
00207       itsFixedY.setInactive(!itsUseFixed.getVal());
00208     }
00209 }
00210 
00211 // ######################################################################
00212 EnvSaliencyMap::State
00213 EnvSaliencyMap::getSalmap(const Image<byte>& vcxmap,
00214                           const Point2D<int>& forceWinnerFullres)
00215 {
00216   State result;
00217 
00218   result.salmap = vcxmap;
00219 
00220 //  if (itsInertiaMap.initialized() && itsInhibMap.initialized())
00221 //    result.salmap += (itsInertiaMap - itsInhibMap);
00222 //  else if (itsInertiaMap.initialized())
00223 //    result.salmap += itsInertiaMap;
00224 //  else if (itsInhibMap.initialized())
00225 //    result.salmap -= itsInertiaMap;
00226 
00227   const int zoom = (1 << itsLevelSpec.getVal().mapLevel());
00228 
00229   //Apply the bias if we have one
00230   if (itsBiasImg.initialized())
00231   {
00232     if (itsBiasImg.getDims() != result.salmap.getDims())
00233       itsBiasImg = rescale(itsBiasImg, result.salmap.getDims());
00234     result.salmap *= itsBiasImg;
00235   }
00236 
00237   if (itsUseFixed.getVal())
00238     {
00239       const Point2D<int> forceWinner
00240         (clampValue(itsFixedX.getVal(), 0, result.salmap.getWidth()-1),
00241          clampValue(itsFixedY.getVal(), 0, result.salmap.getHeight()-1));
00242 
00243       ASSERT(result.salmap.coordsOk(forceWinner));
00244       result.lowres_maxpos = forceWinner;
00245       result.maxval = result.salmap.getVal(result.lowres_maxpos);
00246     }
00247   else if (forceWinnerFullres.isValid())
00248     {
00249       const Point2D<int> forceWinner
00250         (clampValue(forceWinnerFullres.i / zoom, 0, result.salmap.getWidth()-1),
00251          clampValue(forceWinnerFullres.j / zoom, 0, result.salmap.getHeight()-1));
00252 
00253       ASSERT(result.salmap.coordsOk(forceWinner));
00254       result.lowres_maxpos = forceWinner;
00255       result.maxval = result.salmap.getVal(result.lowres_maxpos);
00256     }
00257   else
00258     {
00259       findMax(result.salmap, result.lowres_maxpos, result.maxval);
00260     }
00261 
00262 
00263   //Get the N most salient location from a temporary saliency map
00264   //The first location is retrieved from the result structure itself
00265   Image<byte> tmpSmap = vcxmap;
00266 
00267 
00268   Point2D<int> currentMaxLoc = result.lowres_maxpos;
00269   LocInfo locInfo;
00270   locInfo.lowres_maxpos = result.lowres_maxpos;
00271   locInfo.fullres_maxpos = locInfo.lowres_maxpos * zoom + zoom/2;
00272   locInfo.maxval = result.maxval;
00273   result.nMostSalientLoc.push_back(locInfo);
00274 
00275   for(int i=1; i<itsGetNMostSalientLoc.getVal(); i++)
00276   {
00277     //Apply the IOR to the tmp smap
00278     drawDisk(tmpSmap, currentMaxLoc, itsInternalIORRadius.getVal(), (byte)0);
00279 
00280     //Get the next most salient location
00281     findMax(tmpSmap, locInfo.lowres_maxpos, locInfo.maxval);
00282     locInfo.fullres_maxpos = locInfo.lowres_maxpos * zoom + zoom/2;
00283     result.nMostSalientLoc.push_back(locInfo);
00284     currentMaxLoc = locInfo.lowres_maxpos;
00285   }
00286 
00287   result.fullres_maxpos = result.lowres_maxpos * zoom + zoom/2;
00288 
00289   if (!itsVcxMovingAvg.initialized())
00290     itsVcxMovingAvg = vcxmap;
00291 
00292   const double vcxmeandiff = meanAbsDiff(&itsVcxMovingAvg, &vcxmap);
00293   itsVcxFlicker =
00294     (sigmoid(0.5*(vcxmeandiff-itsVcxMeanDiffCenter)) - itsVcxFlickerMin)
00295     /
00296     (1.0 - itsVcxFlickerMin);
00297 
00298   itsDynamicFactor =
00299     pow(1.0 - itsVcxFlicker,
00300         std::max(0.0, itsDynamicFeedback.getVal()));
00301 
00302   // Update inertia map. The inertia center always follows the current
00303   // saliency map peak. If the new peak is within a threshold distance
00304   // from the previous peak, then the inertia strength decays by a
00305   // factor according to its half-life -- this behavior allow an
00306   // inertia blob to track a moving object while decaying at the same
00307   // time. On the other hand if the new peak is distant from the
00308   // previous peak, then the inertia strength is reset to the max
00309   // value so that we essentially begin a new inertia sequence at the
00310   // new peak location.
00311 
00312   if (itsInertiaLoc.i < 0 || itsInertiaLoc.j < 0
00313       || (result.lowres_maxpos.squdist(itsInertiaLoc)
00314           > (itsInertiaShiftThresh.getVal()
00315              * itsInertiaShiftThresh.getVal())))
00316     {
00317       itsCurrentInertiaFactor = 1.0;
00318       LDEBUG("inertia shift to (%d,%d)",
00319              result.lowres_maxpos.i, result.lowres_maxpos.j);
00320     }
00321   else
00322     {
00323       const float factor =
00324         (itsDynamicFactor * itsInertiaHalfLife.getVal()) > 0
00325         ? pow(0.5, 1.0/(itsDynamicFactor * itsInertiaHalfLife.getVal()))
00326         : 0.0f;
00327       itsCurrentInertiaFactor *= factor;
00328     }
00329 
00330   {
00331     if (itsInertiaMap.getDims() != result.salmap.getDims())
00332       itsInertiaMap = Image<byte>(result.salmap.getDims(), ZEROS);
00333 
00334     itsInertiaLoc = result.lowres_maxpos;
00335 
00336     Image<float>::iterator iptr = itsInertiaMap.beginw();
00337 
00338     const double s =
00339       itsInertiaStrength.getVal()
00340       * itsDynamicFactor
00341       * itsCurrentInertiaFactor;
00342 
00343     const double r_inv =
00344       itsInertiaRadius.getVal() > 0.0
00345       ? (1.0 / itsInertiaRadius.getVal())
00346       : 0.0;
00347 
00348     Point2D<int> p;
00349     for (p.j = 0; p.j < itsInertiaMap.getHeight(); ++p.j)
00350       for (p.i = 0; p.i < itsInertiaMap.getWidth(); ++p.i)
00351         {
00352           const int dsq =
00353             (itsInertiaLoc.i - p.i) * (itsInertiaLoc.i - p.i)
00354             + (itsInertiaLoc.j - p.j) * (itsInertiaLoc.j - p.j);
00355 
00356           *iptr++ = s * exp(-dsq * r_inv);
00357         }
00358   }
00359 
00360   if (itsDynamicFactor * itsIorStrength.getVal() > 0.0)
00361     {
00362       if (itsInhibMap.getDims() != result.salmap.getDims())
00363         itsInhibMap = Image<byte>(result.salmap.getDims(), ZEROS);
00364 
00365       const float factor =
00366         (itsDynamicFactor * itsIorHalfLife.getVal()) > 0
00367         ? pow(0.5, 1.0/(itsDynamicFactor * itsIorHalfLife.getVal()))
00368         : 0.0f;
00369 
00370       Image<byte>::iterator iptr = itsInhibMap.beginw();
00371 
00372       Point2D<int> p;
00373       for (p.j = 0; p.j < itsInhibMap.getHeight(); ++p.j)
00374         for (p.i = 0; p.i < itsInhibMap.getWidth(); ++p.i)
00375           {
00376             const int dsq =
00377               (result.lowres_maxpos.i - p.i) * (result.lowres_maxpos.i - p.i)
00378               + (result.lowres_maxpos.j - p.j) * (result.lowres_maxpos.j - p.j);
00379 
00380             const int newval =
00381               int(*iptr * factor
00382                   + (itsDynamicFactor * itsIorStrength.getVal()
00383                      * exp(- dsq / itsIorRadius.getVal())));
00384 
00385             *iptr++ =
00386               newval < 0
00387               ? 0
00388               : newval > 255
00389               ? 255
00390               : byte(newval);
00391           }
00392     }
00393   else
00394     itsInhibMap.clear(0);
00395 
00396   itsVcxMovingAvg += vcxmap;
00397   itsVcxMovingAvg *= 0.5;
00398 
00399   textLog(itsTextLogFile.getVal(), "FOAcenter",
00400           convertToString(result.fullres_maxpos));
00401 
00402   return result;
00403 }
00404 
00405 // ######################################################################
00406 /* So things look consistent in everyone's emacs... */
00407 /* Local Variables: */
00408 /* mode: c++ */
00409 /* indent-tabs-mode: nil */
00410 /* End: */
00411 
00412 #endif // NEURO_ENVSALIENCYMAP_C_DEFINED
Generated on Sun May 8 08:41:02 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3