ImageInfo.C

Go to the documentation of this file.
00001 /*!@file ImageInfo.C A class to maintain chip state */
00002 
00003 // //////////////////////////////////////////////////////////////////// //
00004 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2000-2005   //
00005 // by the University of Southern California (USC) and the iLab at USC.  //
00006 // See http://iLab.usc.edu for information about this project.          //
00007 // //////////////////////////////////////////////////////////////////// //
00008 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00009 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00010 // in Visual Environments, and Applications'' by Christof Koch and      //
00011 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00012 // pending; application number 09/912,225 filed July 23, 2001; see      //
00013 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00014 // //////////////////////////////////////////////////////////////////// //
00015 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00016 //                                                                      //
00017 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00018 // redistribute it and/or modify it under the terms of the GNU General  //
00019 // Public License as published by the Free Software Foundation; either  //
00020 // version 2 of the License, or (at your option) any later version.     //
00021 //                                                                      //
00022 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00023 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00024 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00025 // PURPOSE.  See the GNU General Public License for more details.       //
00026 //                                                                      //
00027 // You should have received a copy of the GNU General Public License    //
00028 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00029 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00030 // Boston, MA 02111-1307 USA.                                           //
00031 // //////////////////////////////////////////////////////////////////// //
00032 //
00033 // Primary maintainer for this file: Lior Elazary <elazary@usc.edu>
00034 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/Apps/BorderWatch/ImageInfo.C $
00035 // $Id: ImageInfo.C 12962 2010-03-06 02:13:53Z irock $
00036 //
00037 
00038 #include "Apps/BorderWatch/ImageInfo.H"
00039 #include "Image/DrawOps.H"
00040 #include "GUI/DebugWin.H"
00041 #include "Util/StringUtil.H"
00042 #include "Channels/ChannelMaps.H"
00043 #include "Channels/ChannelOpts.H"
00044 #include "Neuro/NeuroSimEvents.H"
00045 #include "Simulation/SimEventQueue.H"
00046 #include "Transport/FrameInfo.H"
00047 #include <cmath>
00048 
00049 const ModelOptionCateg MOC_ImageInfo = {
00050 MOC_SORTPRI_3, "Image Info-related options" };
00051 
00052 const ModelOptionDef OPT_RandGain =
00053 { MODOPT_ARG(float), "RandGain", &MOC_ImageInfo, OPTEXP_CORE,
00054   "The multiplier for the rand when computing the chip score",
00055   "k-rand", '\0', "<float>", "0.0" };
00056 
00057 const ModelOptionDef OPT_EntropyGain =
00058 { MODOPT_ARG(float), "EntropyGain", &MOC_ImageInfo, OPTEXP_CORE,
00059   "The multiplier for the entropy when computing the chip score",
00060   "k-entropy", '\0', "<float>", "0.0" };
00061 
00062 const ModelOptionDef OPT_SaliencyGain =
00063 { MODOPT_ARG(float), "SaliencyGain", &MOC_ImageInfo, OPTEXP_CORE,
00064   "The multiplier for the saliency when computing the chip score",
00065   "k-saliency", '\0', "<float>", "0.0" };
00066 
00067 const ModelOptionDef OPT_UniquenessGain =
00068 { MODOPT_ARG(float), "UniquenessGain", &MOC_ImageInfo, OPTEXP_CORE,
00069   "The multiplier for the uniqueness when computing the chip score",
00070   "k-uniqueness", '\0', "<float>", "0.0" };
00071 
00072 const ModelOptionDef OPT_MSDSurpriseGain =
00073 { MODOPT_ARG(float), "MSDSurpriseGain", &MOC_ImageInfo, OPTEXP_CORE,
00074   "The multiplier for the msd surprise when computing the chip score",
00075   "k-msdsurprise", '\0', "<float>", "0.0" };
00076 
00077 const ModelOptionDef OPT_KLSurpriseGain =
00078 { MODOPT_ARG(float), "KLSurpriseGain", &MOC_ImageInfo, OPTEXP_CORE,
00079   "The multiplier for the kl surprise when computing the chip score",
00080   "k-klsurprise", '\0', "<float>", "1.0" };
00081 
00082 
00083 // ######################################################################
00084 ImageInfo::ImageInfo(OptionManager& mgr, const std::string& descrName, const std::string& tagName) :
00085   SimModule(mgr, descrName, tagName),
00086   itsVcc(new VisualCortexConfigurator(mgr)),
00087   itsRandGain(&OPT_RandGain, this, 0),
00088   itsEntropyGain(&OPT_EntropyGain, this, 0),
00089   itsSaliencyGain(&OPT_SaliencyGain, this, 0),
00090   itsUniquenessGain(&OPT_UniquenessGain, this, 0),
00091   itsMSDSurpriseGain(&OPT_MSDSurpriseGain, this, 0),
00092   itsKLSurpriseGain(&OPT_KLSurpriseGain, this, 0)
00093 {
00094   addSubComponent(itsVcc);
00095 }
00096 
00097 // ######################################################################
00098 ImageInfo::~ImageInfo()
00099 { }
00100 
00101 // ######################################################################
00102 ImageInfo::ImageStats ImageInfo::update(nub::ref<SimEventQueue>& q,
00103                                         const Image<PixRGB<byte> >& img, const int frameID)
00104 {
00105   ImageStats stats;
00106 
00107   // Post the image to the queue:
00108   q->post(rutz::make_shared(new SimEventRetinaImage(this, InputFrame(InputFrame::fromRgb(&img, q->now())),
00109                                                     Rectangle(Point2D<int>(0,0), img.getDims()),
00110                                                     Point2D<int>(0,0))));
00111   // Get the visual cortex output:
00112   if (SeC<SimEventVisualCortexOutput> e = q->check<SimEventVisualCortexOutput>(this, SEQ_ANY))
00113     stats.smap = e->vco(1.0F);
00114   else LFATAL("Can not get the Visual cortex output");
00115 
00116   // Find the most salient point in the saliency map
00117   findMax(stats.smap, stats.salpoint, stats.saliency);
00118 
00119   // Compute the 'energy' (sum of saliency) of the saliency map
00120   if (itsUniquenessGain.getVal() != 0.0F || itsRandGain.getVal() != 0.0F || itsEntropyGain.getVal() != 0.0F)
00121     stats.energy = sum(stats.smap); // needed by uniqueness, rand, and entropy
00122   else stats.energy = 0.0F;
00123 
00124   // Find the uniqueness of the salient point: Which is the difference
00125   //    between the most salient value, and the average of the rest of the values:
00126   const uint sz = stats.smap.getSize();
00127   if (itsUniquenessGain.getVal() != 0.0F)
00128     stats.uniqueness = stats.saliency - ( (stats.energy - stats.saliency) / (sz - 1) );
00129   else stats.uniqueness = 0.0F;
00130 
00131   // Compute the entropy and the 'rand' (an unnormalized version of the entropy which somehow
00132   //    seems to give much better ROC results)
00133   if (itsRandGain.getVal() != 0.0F || itsEntropyGain.getVal() != 0.0F) {
00134     stats.entropy = 0.0F; stats.rand = 0.0F;
00135     for (uint i = 0; i < sz; ++i)
00136       if (stats.smap[i] > 0.0F)
00137         {
00138           stats.entropy += (stats.smap[i] / stats.energy) * logf(stats.smap[i] / stats.energy);
00139           stats.rand += stats.smap[i] * logf(stats.smap[i]) / sz;
00140         }
00141   } else { stats.entropy = 0.0F; stats.rand = 0.0F; }
00142 
00143   // Compute the surprise using Lior's Kalman Filter method
00144   if (itsKLSurpriseGain.getVal() != 0.0F) {
00145     //const float smean = float(mean(smap)) + 1.0e-20F;
00146     (void)integrateData(stats.smap,
00147                         5, //smean * 0.001F,
00148                         0.1, //smean * 0.01F,
00149                         itsBelief1Mu, itsBelief1Sig);
00150     //const float smean2 = float(mean(itsBelief1Mu)) + 1.0e-20F;
00151     stats.KLsurprise = integrateData(itsBelief1Mu,
00152                                      5, //smean2 * 0.001F,
00153                                      0.01, //smean2 * 0.01F,
00154                                      itsBelief2Mu, itsBelief2Sig);
00155     stats.belief1 = itsBelief1Mu;
00156     stats.belief2 = itsBelief2Mu;
00157   } else stats.KLsurprise = 0.0F; // belief1 and belief2 have a default constructor and are ok
00158 
00159   // Compute the surprise using Bruce's Mean Square Difference method
00160   if (itsMSDSurpriseGain.getVal() != 0.0F) stats.MSDsurprise = updateMSDiff(img, stats.smap);
00161   else stats.MSDsurprise = 0.0F;
00162 
00163   // compute the final score:
00164   stats.score = itsRandGain.getVal() * stats.rand + itsEntropyGain.getVal() * stats.entropy +
00165     itsSaliencyGain.getVal() * stats.saliency + itsUniquenessGain.getVal() * stats.uniqueness +
00166     itsMSDSurpriseGain.getVal() * stats.MSDsurprise + itsKLSurpriseGain.getVal() * stats.KLsurprise;
00167 
00168   return stats;
00169 }
00170 
00171 // ######################################################################
00172 // Update surprise value based on a simple mean squared difference of
00173 // the current and previous saliency maps.  Both saliency maps are
00174 // normalized to have unity integrals.
00175 float ImageInfo::updateMSDiff(const Image<PixRGB<byte> >& img, Image<float> salMap)
00176 {
00177   // Just an excess of caution -- this condition should never be true.
00178   if (!salMap.initialized()) { LINFO("Current smap not set"); return 0.0; }
00179 
00180   // Always exit here on the first frame, since there is no previous map.
00181   if (!itsPrevSmap.initialized()) { itsPrevSmap = salMap; return 0.0; }
00182 
00183   // Compute the normalizing constant for both maps.
00184   const float sCurr = sum(salMap);
00185   const float sPrev = sum(itsPrevSmap);
00186 
00187   // Compute the sum of the squared differences of each map.
00188   float diffSum = 0.0F;
00189   for (int i = 0; i < salMap.getSize(); ++i)
00190     {
00191       const float cVal = salMap.getVal(i);
00192       const float pVal = itsPrevSmap.getVal(i);
00193       const float diff = cVal / sCurr - pVal / sPrev;
00194       diffSum += diff * diff;
00195     }
00196 
00197   // Surprise is sum squared normalized by pixel count
00198   float surprise = diffSum / float(salMap.getSize());
00199 
00200   // Use logistic fit to map surprise to (0,1) interval
00201   const float m = 2.8900e-05F, s = 9.6030e-06F;
00202   surprise = 1.0F / (1.0F + expf(-(surprise - m) / s));
00203 
00204   // Update prior map
00205   itsPrevSmap = salMap;
00206 
00207   return surprise;
00208 }
00209 
00210 // ######################################################################
00211 float ImageInfo::integrateData(const Image<float> &data, const float R, const float Q,
00212                               Image<float>& bel_mu, Image<float>& bel_sig)
00213 {
00214   if (!bel_mu.initialized()) { bel_mu = data; bel_sig.resize(bel_mu.getDims(), true); }
00215 
00216   Image<float>::const_iterator inPtr = data.begin(), inStop = data.end();
00217   Image<float>::iterator muPtr = bel_mu.beginw(), sigPtr = bel_sig.beginw();
00218   float surprise = 0.0F;
00219 
00220   // Kalman filtering for each pixel:
00221   while (inPtr != inStop)
00222     {
00223       // Predict
00224       const float mu_hat = *muPtr;
00225       const float sig_hat = *sigPtr + Q;
00226 
00227       // update
00228       const float K = sig_hat / (sig_hat + R);
00229       *muPtr = mu_hat + K * (*inPtr - mu_hat);
00230       *sigPtr = (1.0F - K) * sig_hat;
00231 
00232       // Calculate surprise KL(P(M|D),P(M))
00233       // P(M|D) = N(*muPtr, *sigPtr);
00234       // P(M) = N(mu_hat, sig_hat);
00235       //float localSurprise = (((*muPtr - mu_hat)*(*muPtr - mu_hat)) + (*sigPtr * *sigPtr) + (sig_hat * sig_hat));
00236       //localSurprise = localSurprise / (2.0F * sig_hat * sig_hat);
00237       //localSurprise += log(sig_hat / *sigPtr);
00238 
00239       float localSurprise = fabs(*muPtr - mu_hat);
00240       surprise += localSurprise;
00241 
00242       ++inPtr; ++muPtr; ++sigPtr;
00243     }
00244 
00245   return surprise;
00246 }
00247 
Generated on Sun May 8 08:40:11 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3