00001 /*!@file Neuro/SimulationViewerSurpCont.C entry interface between INVT and ASAC */ 00002 00003 // //////////////////////////////////////////////////////////////////// // 00004 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2000-2003 // 00005 // by the University of Southern California (USC) and the iLab at USC. // 00006 // See http://iLab.usc.edu for information about this project. // 00007 // //////////////////////////////////////////////////////////////////// // 00008 // Major portions of the iLab Neuromorphic Vision Toolkit are protected // 00009 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency // 00010 // in Visual Environments, and Applications'' by Christof Koch and // 00011 // Laurent Itti, California Institute of Technology, 2001 (patent // 00012 // pending; application number 09/912,225 filed July 23, 2001; see // 00013 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status). // 00014 // //////////////////////////////////////////////////////////////////// // 00015 // This file is part of the iLab Neuromorphic Vision C++ Toolkit. // 00016 // // 00017 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can // 00018 // redistribute it and/or modify it under the terms of the GNU General // 00019 // Public License as published by the Free Software Foundation; either // 00020 // version 2 of the License, or (at your option) any later version. // 00021 // // 00022 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope // 00023 // that it will be useful, but WITHOUT ANY WARRANTY; without even the // 00024 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR // 00025 // PURPOSE. See the GNU General Public License for more details. // 00026 // // 00027 // You should have received a copy of the GNU General Public License // 00028 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write // 00029 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, // 00030 // Boston, MA 02111-1307 USA. // 00031 // //////////////////////////////////////////////////////////////////// // 00032 // 00033 // Primary maintainer for this file: T. Nathan Mundhenk <mundhenk@usc.edu> 00034 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/Neuro/SimulationViewerSurpCont.C $ 00035 // $Id: SimulationViewerSurpCont.C 13065 2010-03-28 00:01:00Z itti $ 00036 // 00037 00038 #include "Neuro/SimulationViewerSurpCont.H" 00039 00040 #include "Neuro/SimulationViewerStd.H" 00041 #include "Channels/ChannelOpts.H" 00042 #include "Image/ColorOps.H" // for normalizeC() 00043 #include "Image/CutPaste.H" // for concatX() 00044 #include "Image/DrawOps.H" 00045 #include "Image/MathOps.H" // for takeMax() 00046 #include "Image/PyramidOps.H" 00047 #include "Image/ShapeOps.H" // for crop() 00048 #include "Image/Transforms.H" // for contour2D() 00049 #include "Image/LevelSpec.H" 00050 #include "Neuro/AttentionGuidanceMap.H" 00051 #include "Neuro/NeuroOpts.H" 00052 #include "Neuro/SaliencyMap.H" 00053 #include "Neuro/TaskRelevanceMap.H" 00054 #include "Simulation/SimEventQueue.H" 00055 #include "Transport/FrameInfo.H" 00056 #include "Transport/FrameOstream.H" 00057 00058 00059 00060 // ###################################################################### 00061 SimulationViewerSurpCont::SimulationViewerSurpCont(OptionManager& mgr, 00062 const std::string& descrName, 00063 const std::string& tagName) : 00064 SimulationViewer(mgr, descrName, tagName), 00065 SIMCALLBACK_INIT(SimEventInputFrame), 00066 SIMCALLBACK_INIT(SimEventClockTick), 00067 SIMCALLBACK_INIT(SimEventSaveOutput), 00068 itsMetrics(new SpatialMetrics(mgr)), 00069 itsDrawDiffParts(&OPT_ASACdrawDiffParts, this), 00070 itsDrawBetaParts(&OPT_ASACdrawBetaParts, this), 00071 itsDrawBiasParts(&OPT_ASACdrawBiasParts, this), 00072 itsDrawSeperableParts(&OPT_ASACdrawSeperableParts, this), 00073 itsConfigFile(&OPT_ASACconfigFile, this), 00074 itsLevelSpec(&OPT_LevelSpec, this) 00075 { 00076 this->addSubComponent(itsMetrics); 00077 itsInit = false; 00078 } 00079 00080 // ###################################################################### 00081 SimulationViewerSurpCont::~SimulationViewerSurpCont() 00082 { } 00083 00084 // ###################################################################### 00085 void SimulationViewerSurpCont::init(const ushort baseSizeX, 00086 const ushort baseSizeY) 00087 { 00088 std::string configFile = itsConfigFile.getVal(); 00089 00090 itsScaleSurpriseControl.SSCreadConfig(configFile); 00091 itsScaleSurpriseControl.SSCinit(baseSizeX,baseSizeY); 00092 00093 itsInit = true; 00094 } 00095 00096 // ###################################################################### 00097 void SimulationViewerSurpCont::setBrain(Brain* brain) 00098 { 00099 LevelSpec ls = itsLevelSpec.getVal(); 00100 00101 const uint lmin = ls.levMin(); 00102 const uint lmax = ls.levMax(); 00103 const uint dmin = ls.delMin(); 00104 const uint dmax = ls.delMax(); 00105 const uint mlev = ls.mapLevel(); 00106 const uint midx = ls.maxIndex(); 00107 const uint mdep = ls.maxDepth(); 00108 00109 itsScaleSurpriseControl.SSCsetLevelSpecInfo(lmin,lmax,dmin,dmax, 00110 mlev,midx,mdep); 00111 00112 } 00113 00114 // ###################################################################### 00115 void SimulationViewerSurpCont:: 00116 onSimEventInputFrame(SimEventQueue& q, rutz::shared_ptr<SimEventInputFrame>& e) 00117 { 00118 LINFO("Inputing to Surprise Control raw image"); 00119 // keep a copy of the image 00120 itsInput = e->frame().asRgb(); 00121 00122 if (!itsInit) init(itsInput.getWidth(), itsInput.getHeight()); 00123 00124 // we have a new input; this will force redrawing various things on 00125 // the trajectory in case people request it before a new shift of 00126 // attention or other event occurrs: 00127 itsHasNewInput = true; 00128 itsCurrTime = q.now(); 00129 itsScaleSurpriseControl.SSCinputRawImage(itsInput); 00130 } 00131 00132 // ###################################################################### 00133 void SimulationViewerSurpCont:: 00134 onSimEventClockTick(SimEventQueue& q, rutz::shared_ptr<SimEventClockTick>& e) 00135 { 00136 // this seems bogus... is it really necessary? 00137 itsCurrTime = q.now(); 00138 } 00139 00140 // ###################################################################### 00141 void SimulationViewerSurpCont::saveResults(const nub::ref<FrameOstream>& ofs) 00142 { 00143 // update our internal time: 00144 double msecs = itsCurrTime.msecs(); 00145 00146 LINFO("Running Surprise Control on Sample Input time %f ms",msecs); 00147 00148 00149 LFATAL("FIXME"); 00150 //// itsScaleSurpriseControl.SSCprocessFrame(itsBrain); 00151 00152 LINFO("Saving Surprise Control Output"); 00153 Image<PixRGB<byte> > bimage; 00154 00155 Image<PixRGB<float> > outImage = itsScaleSurpriseControl.SSCgetFrame(); 00156 bimage = outImage; 00157 00158 00159 ofs->writeRGB(bimage, "SSC", FrameInfo("ScaleSurpriseControl final image", 00160 SRC_POS)); 00161 00162 Image<PixRGB<float> > diffImage = 00163 itsScaleSurpriseControl.SSCgetDiffImage(false); 00164 bimage = diffImage; 00165 00166 ofs->writeRGB(bimage, "SSC-diff", 00167 FrameInfo("ScaleSurpriseControl diff image",SRC_POS)); 00168 00169 diffImage = itsScaleSurpriseControl.SSCgetDiffImage(true); 00170 bimage = diffImage; 00171 00172 ofs->writeRGB(bimage, "SSC-diff-norm", 00173 FrameInfo("ScaleSurpriseControl diff image normalized",SRC_POS)); 00174 00175 if(itsDrawDiffParts.getVal()) 00176 { 00177 std::vector<Image<PixRGB<float> > > diffParts = 00178 itsScaleSurpriseControl.SSCgetDiffParts(); 00179 std::vector<Image<PixRGB<float> > >::const_iterator diffPartsItr = 00180 diffParts.begin(); 00181 ushort type = 0; 00182 while(diffPartsItr != diffParts.end()) 00183 { 00184 bimage = *diffPartsItr; 00185 char name[100]; 00186 if(type == 0) 00187 sprintf(name,"SSC-diffParts-H1-"); 00188 else if(type == 1) 00189 sprintf(name,"SSC-diffParts-H2-"); 00190 else if(type == 2) 00191 sprintf(name,"SSC-diffParts-S-"); 00192 else if(type == 3) 00193 sprintf(name,"SSC-diffParts-V-"); 00194 else 00195 sprintf(name,"SSC-diffParts-%d-",type); 00196 std::string prefix = name; 00197 std::string frameInfo = "ScaleSurpriseControl difference "; 00198 frameInfo = frameInfo + prefix; 00199 ofs->writeRGB(bimage, prefix, FrameInfo(frameInfo,SRC_POS)); 00200 ++diffPartsItr; type++; 00201 } 00202 } 00203 00204 if(itsDrawBetaParts.getVal()) 00205 { 00206 std::vector<Image<float> > betaParts = 00207 itsScaleSurpriseControl.SSCgetBetaParts(false); 00208 std::vector<Image<float> >::const_iterator betaPartsItr = 00209 betaParts.begin(); 00210 ushort type = 0; 00211 while(betaPartsItr != betaParts.end()) 00212 { 00213 bimage = *betaPartsItr; 00214 char name[100]; 00215 sprintf(name,"SSC-betaParts-%s-",sc_channel_name_abv[type].c_str()); 00216 std::string prefix = name; 00217 std::string frameInfo = "ScaleSurpriseControl beta "; 00218 frameInfo = frameInfo + prefix; 00219 ofs->writeRGB(bimage, prefix, FrameInfo(frameInfo,SRC_POS)); 00220 ++betaPartsItr; type++; 00221 } 00222 00223 betaParts = itsScaleSurpriseControl.SSCgetBetaParts(true); 00224 betaPartsItr = betaParts.begin(); 00225 type = 0; 00226 while(betaPartsItr != betaParts.end()) 00227 { 00228 bimage = *betaPartsItr; 00229 char name[100]; 00230 sprintf(name,"SSC-betaParts-norm-%s-",sc_channel_name_abv[type].c_str()); 00231 std::string prefix = name; 00232 std::string frameInfo = "ScaleSurpriseControl beta norm"; 00233 frameInfo = frameInfo + prefix; 00234 ofs->writeRGB(bimage, prefix, FrameInfo(frameInfo,SRC_POS)); 00235 ++betaPartsItr; type++; 00236 } 00237 } 00238 00239 if(itsDrawBiasParts.getVal()) 00240 { 00241 std::vector<Image<PixRGB<float> > > biasH1; 00242 std::vector<Image<PixRGB<float> > > biasH2; 00243 std::vector<Image<PixRGB<float> > > biasS; 00244 std::vector<Image<PixRGB<float> > > biasV; 00245 00246 itsScaleSurpriseControl.SSCgetBiasParts(biasH1,biasH2,biasS,biasV); 00247 00248 std::vector<Image<PixRGB<float> > >::const_iterator biasH1Itr = 00249 biasH1.begin(); 00250 std::vector<Image<PixRGB<float> > >::const_iterator biasH2Itr = 00251 biasH2.begin(); 00252 std::vector<Image<PixRGB<float> > >::const_iterator biasSItr = 00253 biasS.begin(); 00254 std::vector<Image<PixRGB<float> > >::const_iterator biasVItr = 00255 biasV.begin(); 00256 00257 ushort scale = 0; 00258 00259 while(biasH1Itr != biasH1.end()) 00260 { 00261 char name[100]; 00262 00263 bimage = *biasH1Itr; 00264 sprintf(name,"SSC-biasParts-H1-%d-",scale); 00265 std::string prefix = name; 00266 std::string frameInfo = "ScaleSurpriseControl biasH1 "; 00267 frameInfo = frameInfo + prefix; 00268 ofs->writeRGB(bimage, prefix, FrameInfo(frameInfo,SRC_POS)); 00269 00270 bimage = *biasH2Itr; 00271 sprintf(name,"SSC-biasParts-H2-%d-",scale); 00272 prefix = name; 00273 frameInfo = "ScaleSurpriseControl biasH2 "; 00274 frameInfo = frameInfo + prefix; 00275 ofs->writeRGB(bimage, prefix, FrameInfo(frameInfo,SRC_POS)); 00276 00277 bimage = *biasSItr; 00278 sprintf(name,"SSC-biasParts-S-%d-",scale); 00279 prefix = name; 00280 frameInfo = "ScaleSurpriseControl biasS "; 00281 frameInfo = frameInfo + prefix; 00282 ofs->writeRGB(bimage, prefix, FrameInfo(frameInfo,SRC_POS)); 00283 00284 bimage = *biasVItr; 00285 sprintf(name,"SSC-biasParts-V-%d-",scale); 00286 prefix = name; 00287 frameInfo = "ScaleSurpriseControl biasV "; 00288 frameInfo = frameInfo + prefix; 00289 ofs->writeRGB(bimage, prefix, FrameInfo(frameInfo,SRC_POS)); 00290 ++biasH1Itr; ++biasH2Itr; ++biasSItr; ++biasVItr; scale++; 00291 } 00292 } 00293 00294 if(itsDrawSeperableParts.getVal()) 00295 { 00296 std::vector<Image<PixRGB<float> > > Zimgs; 00297 std::vector<Image<PixRGB<float> > > Yimgs; 00298 00299 itsScaleSurpriseControl.SSCgetSeperableParts(Zimgs,Yimgs,false); 00300 00301 std::vector<Image<PixRGB<float> > >::const_iterator Zitr = Zimgs.begin(); 00302 std::vector<Image<PixRGB<float> > >::const_iterator Yitr = Yimgs.begin(); 00303 00304 ushort scale = 0; 00305 00306 while(Zitr != Zimgs.end()) 00307 { 00308 00309 char name[100]; 00310 bimage = *Zitr; 00311 sprintf(name,"SSC-seperable-parts-Z-%d-",scale); 00312 std::string prefix = name; 00313 std::string frameInfo = "Seperable Parts Z"; 00314 frameInfo = frameInfo + prefix; 00315 ofs->writeRGB(bimage, prefix, FrameInfo(frameInfo,SRC_POS)); 00316 00317 bimage = *Yitr; 00318 sprintf(name,"SSC-seperable-parts-Y-%d-",scale); 00319 prefix = name; 00320 frameInfo = "Seperable Parts Y"; 00321 frameInfo = frameInfo + prefix; 00322 ofs->writeRGB(bimage, prefix, FrameInfo(frameInfo,SRC_POS)); 00323 00324 ++Zitr; ++Yitr; scale++; 00325 } 00326 } 00327 } 00328 00329 // ###################################################################### 00330 void SimulationViewerSurpCont:: 00331 onSimEventSaveOutput(SimEventQueue& q, rutz::shared_ptr<SimEventSaveOutput>& e) 00332 { 00333 this->save1(e->sinfo()); 00334 } 00335 00336 // ###################################################################### 00337 void SimulationViewerSurpCont::save1(const ModelComponentSaveInfo& sinfo) 00338 { 00339 // get the OFS to save to, assuming sinfo is of type 00340 // SimModuleSaveInfo (will throw a fatal exception otherwise): 00341 nub::ref<FrameOstream> ofs = 00342 dynamic_cast<const SimModuleSaveInfo&>(sinfo).ofs; 00343 00344 saveResults(ofs); 00345 } 00346 00347 // ##################################################################### 00348 Image<PixRGB<float> > SimulationViewerSurpCont::getResult() 00349 { 00350 return itsScaleSurpriseControl.SSCgetFrame(); 00351 } 00352 00353 // ##################################################################### 00354 std::vector<Image<PixRGB<float> > > SimulationViewerSurpCont::getDiffImages() 00355 { 00356 return itsScaleSurpriseControl.SSCgetDiffParts(); 00357 } 00358 00359 // ##################################################################### 00360 std::vector<Image<float> > SimulationViewerSurpCont::getBetaImages() 00361 { 00362 return itsScaleSurpriseControl.SSCgetBetaParts(); 00363 } 00364 00365 // ##################################################################### 00366 void SimulationViewerSurpCont::drawTime(Image<PixRGB<byte> >& image) 00367 { 00368 char txt[20]; sprintf(txt, " %dms ", int(itsCurrTime.msecs() + 0.4999)); 00369 writeText(image, Point2D<int>(0, 0), txt); 00370 } 00371 00372 // ###################################################################### 00373 /* So things look consistent in everyone's emacs... */ 00374 /* Local Variables: */ 00375 /* indent-tabs-mode: nil */ 00376 /* End: */