SimulationViewerEyeHand.C

00001 /*!@file Neuro/SimulationViewerEyeMvt.C comparison between saliency and
00002   human eye movements */
00003 
00004 // //////////////////////////////////////////////////////////////////// //
00005 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2000-2003   //
00006 // by the University of Southern California (USC) and the iLab at USC.  //
00007 // See http://iLab.usc.edu for information about this project.          //
00008 // //////////////////////////////////////////////////////////////////// //
00009 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00010 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00011 // in Visual Environments, and Applications'' by Christof Koch and      //
00012 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00013 // pending; application number 09/912,225 filed July 23, 2001; see      //
00014 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00015 // //////////////////////////////////////////////////////////////////// //
00016 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00017 //                                                                      //
00018 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00019 // redistribute it and/or modify it under the terms of the GNU General  //
00020 // Public License as published by the Free Software Foundation; either  //
00021 // version 2 of the License, or (at your option) any later version.     //
00022 //                                                                      //
00023 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00024 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00025 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00026 // PURPOSE.  See the GNU General Public License for more details.       //
00027 //                                                                      //
00028 // You should have received a copy of the GNU General Public License    //
00029 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00030 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00031 // Boston, MA 02111-1307 USA.                                           //
00032 // //////////////////////////////////////////////////////////////////// //
00033 //
00034 // Primary maintainer for this file: Dicky Nauli Sihite <sihite@usc.edu>
00035 // $HeadURL:
00036 // $Id:
00037 //
00038 
00039 #include "Neuro/SimulationViewerEyeHand.H"
00040 
00041 #include "Channels/ChannelBase.H"
00042 #include "Channels/ChannelOpts.H" // for LevelSpec option
00043 #include "Component/OptionManager.H"
00044 #include "Component/ModelOptionDef.H"
00045 #include "Image/ColorOps.H"    // for contrastModulate()
00046 #include "Image/CutPaste.H"    // for concatX()
00047 #include "Image/DrawOps.H"     // for colGreyCombo()
00048 #include "Image/FilterOps.H"   // for lowPass3()
00049 #include "Image/MathOps.H"     // for scramble()
00050 #include "Image/Transforms.H"  // for segmentObjectClean(), contour2D()
00051 #include "Image/ShapeOps.H"    // for rescale()
00052 #include "Neuro/NeuroOpts.H"
00053 #include "Neuro/NeuroSimEvents.H"
00054 #include "Neuro/SpatialMetrics.H"
00055 #include "Psycho/EyeData.H"
00056 #include "Psycho/HandData.H"
00057 #include "Simulation/SimEventQueue.H"
00058 #include "Transport/FrameInfo.H"
00059 #include "Transport/FrameOstream.H"
00060 #include "Util/sformat.H"
00061 #include "rutz/trace.h"
00062 
00063 #include <fstream>
00064 #include <iostream>
00065 
00066 // for some reason this is not recognized when it appears in NeuroOpts.C
00067 const ModelOptionDef OPT_SMhistoryQlen =
00068   { MODOPT_ARG(uint), "SMhistoryQ", &MOC_DISPLAY, OPTEXP_CORE,
00069     "Keep a historical queue of salieny maps (one per new input frame), "
00070     "and report the history of saliency values over the entire queue "
00071     "and at a saccade target location, for each saccade",
00072     "sm-history-qlen", '\0', "<uint>", "0" };
00073 
00074 
00075 // ######################################################################
00076 SimulationViewerEyeHand::
00077 SimulationViewerEyeHand(OptionManager& mgr, const std::string& descrName,
00078                        const std::string& tagName) :
00079   SimulationViewer(mgr, descrName, tagName),
00080   SIMCALLBACK_INIT(SimEventClockTick),
00081   SIMCALLBACK_INIT(SimEventSaveOutput),
00082   itsMetrics(new SpatialMetrics(mgr)),
00083   itsSaveTraj(&OPT_SVsaveTraj, this),
00084   itsDelayCacheSize(&OPT_SVEMdelayCacheSize, this),
00085   itsMaxCacheSize(&OPT_SVEMmaxCacheSize, this),
00086   itsSampleAtStart(&OPT_SVEMsampleAtStart, this),
00087   itsDisplaySacNum(&OPT_SVEMdisplaySacNum, this),
00088   itsDisplayPatch(&OPT_SVdisplayPatch, this),
00089   itsPatchSize(&OPT_SVpatchSize, this),
00090   itsEraseMarker(&OPT_SVeraseMarker, this),
00091   itsDisplayFOA(&OPT_SVdisplayFOA, this),
00092   itsLevelSpec(&OPT_LevelSpec, this),
00093   itsOutFname(&OPT_SVEMoutFname, this),
00094   itsPriorRandomDistro(&OPT_SVEMpriorRandomDistro, this),
00095   itsUseSaccadeInBlink(&OPT_SVEMuseSaccadeInBlink, this),
00096   itsUseDiagColor(&OPT_SVEMuseDiagColors, this),
00097   itsLabelEyePatch(&OPT_SVEMlabelEyePatch, this),
00098   itsNumRandomSamples(&OPT_SVEMnumRandomSamples, this),
00099   itsMaxComboWidth(&OPT_SVEMmaxComboWidth, this),
00100   itsSMhistoryQlen(&OPT_SMhistoryQlen, this),
00101   itsDisplayHand(&OPT_SVHandDisplay, this),
00102   itsSaveCombo(&OPT_SVEMsaveMegaCombo, this),
00103   itsDelayCache(), itsMaxCache(), itsHeadSM(), 
00104   itsDrawings(), itsCurrTime(),  itsFrameNumber(-1), 
00105   itsHeaderCrafted(false), itsOutFields(), itsEyeStyles(), 
00106   itsEyeTargets(), itsHandTargets(), itsSMhistory(), itsOutFile(0), 
00107   itsRandFile(0),itsRawToRetOffset()
00108 {
00109 GVX_TRACE(__PRETTY_FUNCTION__);
00110 
00111   this->addSubComponent(itsMetrics);
00112 
00113   // disable IOR:
00114   LINFO("NOTE: disabling IOR and SE, selecting FixedSaccadeController");
00115   getManager().setOptionValString(&OPT_IORtype, "None");
00116   getManager().setOptionValString(&OPT_ShapeEstimatorMode, "None");
00117 
00118   // select an eyetrack EyeHeadController:
00119   getManager().setOptionValString(&OPT_EyeHeadControllerType, "EyeTrack");
00120 
00121   // Select HandTrack for the HandControllerType
00122   getManager().setOptionValString(&OPT_HandControllerType, "HandTrack");
00123 
00124   // change default to --display-additive=false; user can still
00125   // override this on the command-line
00126   getManager().setOptionValString(&OPT_SVdisplayAdditive, "false");
00127 }
00128 
00129 // ######################################################################
00130 SimulationViewerEyeHand::~SimulationViewerEyeHand()
00131 {
00132 GVX_TRACE(__PRETTY_FUNCTION__);
00133 }
00134 
00135 // ######################################################################
00136 void SimulationViewerEyeHand::start1()
00137 {
00138 GVX_TRACE(__PRETTY_FUNCTION__);
00139 
00140   if (itsDelayCacheSize.getVal())
00141     itsDelayCache.setMaxSize(itsDelayCacheSize.getVal());
00142 
00143   if (itsMaxCacheSize.getVal())
00144     itsMaxCache.setMaxSize(itsMaxCacheSize.getVal());
00145 
00146   // open output file:
00147   if (itsOutFname.getVal().empty() == false) {
00148     if (itsOutFile) delete itsOutFile;
00149     itsOutFile = new std::ofstream(itsOutFname.getVal().c_str());
00150     if (itsOutFile->is_open() == false)
00151       LFATAL("Cannot open '%s' for writing", itsOutFname.getVal().c_str());
00152   }
00153 
00154   // open random file if one
00155   if (itsPriorRandomDistro.getVal().empty() == false) {
00156     if (itsRandFile) delete itsRandFile;
00157     itsRandFile = new std::ifstream(itsPriorRandomDistro.getVal().c_str());
00158     if (itsRandFile->is_open() == false)
00159       LFATAL("Cannot open '%s' for reading",
00160              itsPriorRandomDistro.getVal().c_str());
00161     else {
00162       while (! itsRandFile->eof()) {
00163         Point2D<int> pr;
00164         (*itsRandFile) >> pr.i >> pr.j;
00165         randPoints.push_back(pr);
00166       }
00167       itsRandFile->close();
00168     }
00169   }
00170   
00171   // set our SM history queue length:
00172   if (itsSMhistoryQlen.getVal())
00173     itsSMhistory.setMaxSize(itsSMhistoryQlen.getVal());
00174 
00175   SimulationViewer::start1();
00176 }
00177 
00178 // ######################################################################
00179 void SimulationViewerEyeHand::stop1()
00180 {
00181 GVX_TRACE(__PRETTY_FUNCTION__);
00182 
00183  if (itsOutFile)  {delete itsOutFile;  itsOutFile  = 0;}
00184  if (itsRandFile) {delete itsRandFile; itsRandFile = 0;}
00185 }
00186 
00187 // ######################################################################
00188 void SimulationViewerEyeHand::
00189 onSimEventClockTick(SimEventQueue& q, rutz::shared_ptr<SimEventClockTick>& ect)
00190 {
00191   // get the current time (mostly for SVEyeRegion)
00192   itsCurrTime = q.now();
00193   // any new input frame from the retina? If so, let's initialize or
00194   // clear all our drawings:
00195   bool gotnewinput = false;
00196   if (SeC<SimEventRetinaImage> e = q.check<SimEventRetinaImage>(this)) {
00197     itsDrawings.resize(e->frame().getDims(), true);
00198     gotnewinput = true;
00199     itsFrameNumber++; // used for derived class SVEyeRegion
00200     // this member is never used in SVEyeHand
00201   }
00202 
00203   // Let's update our sliding caches with the current SM/AGM/etc:
00204   const Image<float> currsm = getMap(q, false);
00205 
00206   // if we have a delay cache, we push the SM into it and push
00207   // what comes out of it into our max-computation cache;
00208   // otherwise, we directly push the SM into the max-computation
00209   // cache:
00210   if (itsDelayCacheSize.getVal())
00211     {
00212       // let's push the SM into our delay sliding cache:
00213       itsDelayCache.push_back(currsm);
00214 
00215       // transfer oldest delay image to our max computation cache:
00216       if (int(itsDelayCache.size()) >= itsDelayCacheSize.getVal())
00217           itsHeadSM = itsDelayCache.front();        
00218       else
00219         itsHeadSM.resize(currsm.getDims(), true);  // blank map
00220     }
00221   else 
00222     itsHeadSM = currsm;  // no delay cache
00223 
00224   // if we have a saliency history queue, and a new input, push the
00225   // current sm into it:
00226   if (itsSMhistoryQlen.getVal() && gotnewinput)
00227     itsSMhistory.push_back(itsHeadSM);
00228 
00229   // if we have a max cache, let's push the (possibly delayed) sm into it:
00230   if (itsMaxCacheSize.getVal()) itsMaxCache.push_back(itsHeadSM);
00231 
00232   // all right, get the latest retina image (whether new or not) as we
00233   // will need it to do coordinate transforms, coordinate clippings, etc:
00234   SeC<SimEventRetinaImage> retev = q.check<SimEventRetinaImage>(this, SEQ_ANY);
00235   if (retev.is_valid() == false)
00236     LFATAL("I need some SimEventRetinaImage in the queue to function.");
00237   // set the offset
00238   itsRawToRetOffset = retev->offset();
00239 
00240   // ## Hand-related stuffs
00241   //{@
00242   
00243   // any hand-tracker action?
00244   SeC<SimEventHandTrackerData> ee = q.check<SimEventHandTrackerData>(this);
00245   
00246   while(ee.is_valid()) {
00247     // parse that event:
00248     const rutz::shared_ptr<HandData> trackCurrPos = ee->data();
00249     const uint tnum = ee->trackerNum();
00250     const std::string tfn = ee->trackerFilename();
00251     // add entry to style vector when we get a new trackernum
00252     if (itsHandStyles.size() <= tnum) {
00253       itsHandStyles.resize(tnum+1);
00254       
00255       itsHandStyles[tnum].pSize = itsPatchSize.getVal();
00256       itsHandStyles[tnum].col = ee->trackerColor();
00257       
00258       // give distinct label, TODO: maybe write a switch for this
00259       //      itsHandStyles[tnum].label = convertToString(tnum);
00260       //huge hack right now, relies on directory tree data/<subject>/ pieces
00261       itsHandStyles[tnum].label = tfn.substr(tfn.find("data")+5,2);
00262     }
00263     
00264     // draw the conditions of current hand conditions:
00265     if (itsDisplayHand.getVal()) {
00266 
00267       // erase previous hand markers first
00268       if (itsEraseMarker.getVal()) itsDrawings.resize(retev->frame().getDims(), true);
00269       
00270       // draw marker at hand position
00271       drawHand(trackCurrPos, tnum);
00272     }
00273     
00274     // do we want to do any other sample processing? 
00275     // (for SVHandRegion or other derived classes)
00276     extraSampleProcessing(trackCurrPos);
00277     
00278     // any more events like that in the queue?
00279     ee = q.check<SimEventHandTrackerData>(this);
00280   }
00281   //@}
00282 
00283   // ## Eye-related Stuffs
00284 
00285   // any eye-tracker action?
00286   SeC<SimEventEyeTrackerData> e = q.check<SimEventEyeTrackerData>(this);
00287 
00288   while(e.is_valid()) {
00289     // parse that event:
00290     const rutz::shared_ptr<EyeData> trackCurrPos = e->data();
00291     const uint tnum = e->trackerNum();
00292     const std::string tfn = e->trackerFilename();
00293     // add entry to style vector when we get a new trackernum
00294     if (itsEyeStyles.size() <= tnum) {
00295       itsEyeStyles.resize(tnum+1);
00296          
00297       itsEyeStyles[tnum].pSize = itsPatchSize.getVal();
00298       itsEyeStyles[tnum].col = e->trackerColor();
00299 
00300       // give distinct label, TODO: maybe write a switch for this
00301       //      itsEyeStyles[tnum].label = convertToString(tnum);
00302       //huge hack right now, relies on directory tree data/<subject>/ pieces
00303       itsEyeStyles[tnum].label = tfn.substr(tfn.find("data")+5,2);
00304     }
00305 
00306     // draw small patch at current eye position:
00307     if (itsDisplayPatch.getVal()) 
00308       {
00309         // erase previous eye markers first
00310         if (itsEraseMarker.getVal()) itsDrawings.resize(retev->frame().getDims(), true);
00311 
00312         // draw marker at eye position
00313         drawEye(trackCurrPos, tnum);
00314       }
00315 
00316     // do we want to do any other sample processing? 
00317     // (for SVEyeRegion or other derived classes)
00318     extraSampleProcessing(trackCurrPos);
00319 
00320     bool isUsableTrace = true;
00321     // if we can't use saccades in blink and it's in blink, don't process the event
00322     if (itsUseSaccadeInBlink.getVal() == false && trackCurrPos->isInBlink() == true) isUsableTrace = false;
00323     // NB: what about combined saccades?
00324 
00325     // are we starting a saccade/special region with extra data?
00326     if (trackCurrPos->hasSaccadeTargetData() && isUsableTrace)
00327       {
00328         // extract the saccade destination for display:
00329         Point2D<int> dp = rawToRet(trackCurrPos->saccadeTarget());
00330 
00331         CLINFO("**** Tracker %03d: Saccade to (%d, %d) at %.1fms ****",
00332                tnum, dp.i, dp.j, q.now().msecs());
00333         // check that saccade target is okay:
00334         if (itsDrawings.coordsOk(dp) == false)
00335           CLINFO("Tracker %03d: Saccade target (%d, %d) out of bounds "
00336                  "-- IGNORING", tnum, dp.i, dp.j);
00337         else
00338           // do some saliency sampling at saccade target, but may
00339           // have to be delayed to end of a saccade... So for now
00340           // let's just push the eye data into a temporary storage:
00341           itsEyeTargets[tnum] = trackCurrPos;
00342       }
00343 
00344     // is it time to take a bunch of samples at the saccade target of
00345     // this tracker? Either we want to sample at the start of a
00346     // saccade and just pushed some saccade data, or we want to sample
00347     // at end and we are ending the saccade during which we pushed
00348     // some data:
00349     std::map<int, rutz::shared_ptr<EyeData> >::iterator
00350       itr = itsEyeTargets.find(tnum);
00351 
00352     if (itr != itsEyeTargets.end() && isUsableTrace &&
00353         (itsSampleAtStart.getVal() || trackCurrPos->isInSaccade() == false))
00354     {
00355       // extract the saccade target data:
00356       const rutz::shared_ptr<EyeData> trackEventSample = itr->second;
00357       itsEyeTargets.erase(itr); // we are now taking care of this one
00358 
00359       // changed from trackCurrPos->...();
00360       if(!(trackEventSample->hasSaccadeTargetData()) )
00361          LFATAL("Tracker %03d: Marked event has incomplete targeting data.", tnum);
00362  
00363       Point2D<int> trackerTarget = rawToRet(trackEventSample->saccadeTarget());
00364 
00365       LINFO("Tracker %03d [%s]: Taking saccade samples at (%d, %d) at time %.1fms",
00366             tnum, tfn.c_str(), trackerTarget.i, trackerTarget.j, q.now().msecs());
00367 
00368       // draw circle around target:
00369       if (itsDisplayFOA.getVal()) drawFOA(trackerTarget, tnum);
00370 
00371       // indicate saccade number?
00372       if (itsDisplaySacNum.getVal())
00373         {
00374           if (tnum == 0) {
00375             std::string fld("typenum"); //subject to change
00376             if (trackCurrPos->hasMetaData(fld.c_str()))
00377               writeText(itsDrawings, Point2D<int>(0, 0),
00378                         sformat(" %1g ", 
00379                                 trackEventSample->getMetaDataField("typenum")).c_str(),
00380                         PixRGB<byte>(1, 1, 1));
00381             else
00382               LFATAL("--svem-display-sacnum does not have numbered saccades available");
00383           }
00384           else
00385             LFATAL("--svem-display-sacnum only supported with 1 tracker");
00386         } 
00387 
00388       if (itsOutFile) {
00389         LINFO("writing to file %s........", itsOutFname.getVal().c_str());
00390         (*itsOutFile) << craftSVEMOutput(tfn, trackEventSample) << std::endl;
00391       }
00392     }//end sampling at saccade target
00393 
00394     // any more events like that in the queue?
00395     e = q.check<SimEventEyeTrackerData>(this);
00396   }
00397 
00398 
00399 }
00400 
00401 // ######################################################################
00402 void SimulationViewerEyeHand::drawEye(const rutz::shared_ptr<EyeData> rawPos, const uint tN)
00403 {
00404   // convert to retinal coordinates (accounting for any shifting,
00405   // embedding, etc):
00406   Point2D<int> currPos = rawToRet(rawPos->position());
00407 
00408   // note: these functions are tolerant to 
00409   // coordinates outside the actual image area
00410   PixRGB<byte> col;
00411   // select a drawing color:
00412   // note: we use 'almost black' (1,1,1) instead of black so that we can later
00413   // composite our drawings with various images, with pure black
00414   // then being treated as transparent):
00415   if (!itsUseDiagColor.getVal())
00416     {
00417       col = itsEyeStyles[tN].col;
00418       // dec/inc will clamp/saturate properly
00419       if (rawPos->isInSaccade()) col += byte(64);
00420       if (rawPos->isInBlink()) col -= byte(128);  
00421     }
00422   else //diagnostic colors
00423       {
00424         byte r = 1, g = 1, b = 1;
00425         if (rawPos->isInSaccade()) g = 255;  //green for saccade
00426         else if (rawPos->isInSmoothPursuit()) 
00427           {r = 255;b = 255;} // magenta
00428         else if (rawPos->isInFixation()) b = 255;                // blue
00429         if (rawPos->isInCombinedSaccade()) {b = 255;g = 255;}    //cyan
00430 
00431         col = PixRGB<byte>(r,g,b);    //default black
00432         if (rawPos->isInBlink()) col-=byte(128); // let's just make the cursor darker
00433       }
00434   drawPatchBB(itsDrawings, currPos, itsEyeStyles[tN].pSize, 
00435               col, PixRGB<byte>(1));
00436 
00437   //output the label we got next to the patch
00438   if(itsLabelEyePatch.getVal()) { 
00439     writeText(itsDrawings, currPos+itsEyeStyles[tN].pSize/2,
00440               itsEyeStyles[tN].label.c_str(),
00441               PixRGB<byte>(255,255,255), PixRGB<byte>(0,0,0),
00442               SimpleFont::FIXED(10), true);
00443   }
00444 }
00445 
00446 // ######################################################################
00447 void SimulationViewerEyeHand::drawHand(const rutz::shared_ptr<HandData> rawPos, const uint tN)
00448 {
00449   /* The output that we expect on screen:
00450    * .--------------------.
00451    * | |                  |
00452    * | |                  |
00453    * | Y                  |
00454    * | | B B B B ...      |
00455    * |  ------X--------   |
00456    * '--------------------'
00457    * X & Y = x & y (left-right & accl-brake) direction respectively
00458    *         It will show a line for range of movement and a box
00459    *         indicating the current position
00460    * B's   = buttons, there will be circles with numbers on it indicating
00461    *         whether the button is pressed(green) or not(red)
00462    *         Hopefully the B's on screen is wrapped around incase too many
00463    *         buttons are avail
00464    */
00465 
00466   // Our screen dimensions
00467   const Dims dims = itsDrawings.getDims();
00468   
00469   // Default one grid cell based on the screen dimensions
00470   // Assuming we always have 4:3 tv ratio to get a square grid
00471   // TODO: make this in the option
00472   const Dims numgrid (80,60);//Under 640x480, its expected to have 8x8 per grid
00473   const Dims grid (dims.w()/numgrid.w(),dims.h()/numgrid.h()); 
00474 
00475   PixRGB<byte> colTransp = PixRGB<byte> (  0, 0, 0);
00476   PixRGB<byte> colRed    = PixRGB<byte> (255, 0, 0);
00477   PixRGB<byte> colB;
00478 
00479 
00480   if (rawPos->isValid()) {
00481     // Get raw X and Y position
00482     int rawX = rawPos->getX();
00483     int rawY = rawPos->getY();
00484 
00485     // Get the dims of X and Y position
00486     /*! Assuming grid [1,1]'s coord is (8,8) --> [ 0, 0] is (  0,  0)
00487      *                                       --> [80,60] is (640,480)
00488      *  We only want the X to be in between grid [2,59] to [79,59]
00489      *  and Y in [1,1] to [1,58]. We set those instead of [1,59] to prevent
00490      *  collision incase both meet eachother (rawX=0/255, rawY=0/255)
00491      *  |----------------|-------[]------|
00492      *  X1               XC      X_      X2
00493      */
00494     Point2D<int> X1 (getGridCoord(grid,             2, numgrid.h()-1));
00495     Point2D<int> X2 (getGridCoord(grid, numgrid.w()-1, numgrid.h()-1));
00496     Point2D<int> XC ((127*(X2.i-X1.i)/255)+X1.i,X1.j);
00497     Point2D<int> XD (0,grid.h()/2);
00498     Point2D<int> Y1 (getGridCoord(grid,             1,             1));
00499     Point2D<int> Y2 (getGridCoord(grid,             1, numgrid.h()-2));
00500     Point2D<int> YC (grid.w(),(127*(Y2.j-Y1.j)/255)+Y1.j);
00501     Point2D<int> YD (grid.w()/2,0);
00502     // The patch coords
00503     Point2D<int> X_ ((rawX*(X2.i-X1.i)/255)+X1.i, dims.h()-grid.h());
00504     Point2D<int> Y_ (grid.w(), (rawY*(Y2.j-Y1.j)/255)+Y1.j);
00505 
00506     // First draw the white line for the X-Y coords
00507     const PixRGB<byte> colWhite (255,255,255);
00508     // The X-line
00509     drawLine(itsDrawings, X1, X2, colWhite);
00510     drawLine(itsDrawings, X1-XD, X1+XD, colWhite);
00511     drawLine(itsDrawings, XC-XD, XC+XD, colWhite);
00512     drawLine(itsDrawings, X2-XD, X2+XD, colWhite);
00513     // The Y-line
00514     drawLine(itsDrawings, Y1, Y2, colWhite);
00515     drawLine(itsDrawings, Y1-YD, Y1+YD, colWhite);
00516     drawLine(itsDrawings, YC-YD, YC+YD, colWhite);
00517     drawLine(itsDrawings, Y2-YD, Y2+YD, colWhite);
00518 
00519 
00520     // Draw the box for the X-Y
00521     drawPatchBB(itsDrawings, X_, itsHandStyles[tN].pSize,   
00522                 colWhite, PixRGB<byte>(1));
00523     drawPatchBB(itsDrawings, Y_, itsHandStyles[tN].pSize,   
00524                 colWhite, PixRGB<byte>(1));
00525   
00526     // For the buttons
00527     if (rawPos->isButtonEmpty() == false) {  
00528       for (uint i = 0; i < rawPos->numButton(); i++) {
00529         // Blue if pressed, red if not
00530         if (rawPos->isPressed(i))
00531           colB = colRed;
00532         else
00533           colB = colTransp;
00534 
00535         // For buttons position, we want them to be @ bottom of the screen
00536         // TODO: wrapped around in case the buttons grid over [80,_]
00537 
00538         // Write text of numbers into this thing
00539         writeText(itsDrawings,
00540                   Point2D<int>(getGridCoord(grid, 3*i+10, numgrid.h()-5)),
00541                   sformat("%2d",i+1).c_str(),
00542                   PixRGB<byte>(255), // Text color
00543                   colB, // Background color
00544                   SimpleFont::FIXED(10), // Font
00545                   false); // BG transparent?
00546   
00547         // Draw circle
00548         //drawCircle(itsDrawings, Point2D<int>(getGridCoord(grid, 3*i+3, 1)),6,colB,1);
00549       }
00550     }
00551   }//if rawPos->isValid()
00552 
00553   // For Mouse
00554   writeText(itsDrawings,
00555             Point2D<int>(getGridCoord(grid, 7, numgrid.h()-8)),
00556             sformat("Mouse: ").c_str(),
00557             PixRGB<byte>(255), // Text color
00558             PixRGB<byte>(0), // Background color
00559             SimpleFont::FIXED(10), // Font
00560             false); // BG transparent?
00561   if (rawPos->isMouseValid()) {
00562     // raw mouse position
00563     Point2D<int> mousePos;
00564 
00565     // computed mouse position
00566     if (rawPos->getNativeX() > 0 && rawPos->getNativeY() > 0) // valid dim
00567       // Absolute relation to the movie
00568       //mousePos=Point2D<int>(rawPos->getMouseX()*dims.w()/rawPos->getNativeX(),
00569       //                    rawPos->getMouseY()*dims.h()/rawPos->getNativeY());
00570 
00571       // Relative to the inside, fixed value
00572       if (rawPos->getNativeX() == 1024)
00573         mousePos=Point2D<int>((rawPos->getMouseX()*583/rawPos->getNativeX())+37,
00574                               (rawPos->getMouseY()*438/rawPos->getNativeY())+15);
00575       else if (rawPos->getNativeX() == 800)
00576         mousePos=Point2D<int>((rawPos->getMouseX()*602/rawPos->getNativeX())+11,
00577                               (rawPos->getMouseY()*439/rawPos->getNativeY())+11);
00578       else
00579         mousePos=Point2D<int>(rawPos->getMouseX()*dims.w()/rawPos->getNativeX(),
00580                               rawPos->getMouseY()*dims.h()/rawPos->getNativeY());
00581     else
00582       mousePos=Point2D<int>(rawPos->getMouseX(), rawPos->getMouseY());
00583 
00584     // draw the mouse position
00585     drawPatchBB(itsDrawings, mousePos, itsHandStyles[tN].pSize,
00586                 PixRGB<byte>(127), PixRGB<byte>(1));
00587     //}
00588     // drawing the letters: Mouse: L M R
00589     // the letter LMR will go red if pressed
00590     colB = rawPos->getMouseBL()?colRed:colTransp;
00591     writeText(itsDrawings,
00592               Point2D<int>(getGridCoord(grid, 15, numgrid.h()-8)),
00593               sformat("L").c_str(),
00594               PixRGB<byte>(255), // Text color
00595               colB,
00596               SimpleFont::FIXED(10), // Font
00597               false); // BG transparent?
00598     colB = rawPos->getMouseBM()?colRed:colTransp;
00599     writeText(itsDrawings,
00600               Point2D<int>(getGridCoord(grid, 18, numgrid.h()-8)),
00601               sformat("M").c_str(),
00602               PixRGB<byte>(255), // Text color
00603               colB,
00604               SimpleFont::FIXED(10), // Font
00605               false); // BG transparent?
00606     colB = rawPos->getMouseBR()?colRed:colTransp;
00607     writeText(itsDrawings,
00608               Point2D<int>(getGridCoord(grid, 21, numgrid.h()-8)),
00609               sformat("R").c_str(),
00610               PixRGB<byte>(255), // Text color
00611               colB,
00612               SimpleFont::FIXED(10), // Font
00613               false); // BG transparent?
00614   }
00615   else {
00616     writeText(itsDrawings,
00617               Point2D<int>(getGridCoord(grid, 15, numgrid.h()-8)),
00618               sformat("n/a").c_str(),
00619               PixRGB<byte>(255), // Text color
00620               PixRGB<byte>(0),
00621               SimpleFont::FIXED(10), // Font
00622               false); // BG transparent?
00623   }
00624 
00625   // For Keyboard
00626   writeText(itsDrawings,
00627             Point2D<int>(getGridCoord(grid,25,numgrid.h()-8)),
00628             sformat("Key:").c_str(),
00629             PixRGB<byte>(255),
00630             PixRGB<byte>(0),
00631             SimpleFont::FIXED(10),
00632             false);
00633   if (!rawPos->isKeyboardEmpty()) {
00634     //LINFO("%s", rawPos->getKeyboard());
00635     writeText(itsDrawings,
00636               Point2D<int>(getGridCoord(grid,31,numgrid.h()-8)),
00637               sformat(" %s ",rawPos->getKeyboard()).c_str(),
00638               //rawPos->getKeyboard(),
00639               PixRGB<byte>(255),
00640               PixRGB<byte>(0),
00641               SimpleFont::FIXED(10),
00642               false);
00643   }
00644   else {
00645     writeText(itsDrawings,
00646               Point2D<int>(getGridCoord(grid,31,numgrid.h()-8)),
00647               sformat("                         ").c_str(),
00648               PixRGB<byte>(255),
00649               PixRGB<byte>(0),
00650               SimpleFont::FIXED(10),
00651               false);
00652   }
00653 }
00654 
00655 
00656 // ######################################################################
00657 void SimulationViewerEyeHand::drawFOA(const Point2D<int> target, const uint tN)
00658 {
00659   const PixRGB<byte> col = itsEyeStyles[tN].col;
00660   const int radius = itsMetrics->getFoveaRadius();
00661   drawCircle(itsDrawings, target, radius, col, 2);
00662 }
00663 
00664 // ######################################################################
00665 std::string SimulationViewerEyeHand::craftSVEMOutput(const std::string tfn, 
00666                         const rutz::shared_ptr<EyeData> data)
00667 {
00668   // Write some measurements to a string. Here, we just put all the stats
00669   // into a text string, and we will send that off to be
00670   // written to disk.
00671   // The first time we run this, we build a header for the file 
00672   // to know what stats to read. This is done in the craft submodules.  
00673 
00674   std::string output = 
00675     sformat("%s ", tfn.c_str());
00676   if(!itsHeaderCrafted) itsOutFields.push_back("filename");
00677  
00678   // add model free output from the original eyetrack-data file 
00679   output += craftModelFreeOutput(data);
00680 
00681   // construct the saliency map, 
00682   // if requested from the max of several previous maps 
00683   Image<float> smap;
00684   if (itsMaxCacheSize.getVal()) 
00685     smap = itsMaxCache.getMax();
00686   else smap = itsHeadSM;
00687 
00688   // add samples of saliency map at the target / in the frame
00689   output += craftSMSamples(data, smap);
00690 
00691   // add history of saliency map across past frames for key locations
00692   if (itsSMhistoryQlen.getVal()) output += craftSMHistory(data, smap);
00693 
00694   // write the header line now - only once for each output 
00695   if(!itsHeaderCrafted)
00696     {
00697       writeHeader();
00698       itsHeaderCrafted = true;
00699     }
00700   return output;
00701 }
00702 
00703 // ######################################################################
00704 std::string SimulationViewerEyeHand::
00705 craftModelFreeOutput(const rutz::shared_ptr<EyeData> data)
00706 {
00707   // legacy behavior - this was the old eyesal output for the model-free numbers
00708   // retained for reference
00709   /*
00710     std::string output =
00711     sformat("%s %d %d  %d %d  %g %g %g %g", tfn.c_str(), rawdp.i, rawdp.j,
00712     itsCurrTarget.i, itsCurrTarget.j, d->pupilDiameter(), d->saccadeAmplitude(),
00713     d->saccadeDuration(), d->saccadeTime());
00714   */
00715 
00716   Point2D<int> target = rawToRet(data->saccadeTarget());
00717   std::string output = sformat("%d %d ", target.i, target.j);
00718   
00719   // start writing fields for the header
00720   if(!itsHeaderCrafted) 
00721     { 
00722       itsOutFields.push_back("ret_targetx"); 
00723       itsOutFields.push_back("ret_targety");
00724     }
00725   
00726   // output all the extra (now precomputed) parameters, 
00727   // default from Eye-markup is saccade target first, 
00728   // then amplitude and peak velocity, then time onset and offset of saccade.
00729   // then the interval between this saccade and the next, 
00730   // and lastly the number of the saccade (discriminates between saccades 
00731   // in blink, etc.) 
00732   std::string foo;
00733   for (ParamMap::key_iterator iter = data->getMetaData()->keys_begin();
00734        iter != data->getMetaData()->keys_end();
00735              ++iter)
00736     {
00737       // get rid of any leading asterisk
00738       foo = *iter;
00739       
00740       // add this to the queue of arguments
00741       if(!itsHeaderCrafted) itsOutFields.push_back(foo);
00742       output += sformat (" %g", data->getMetaDataField(foo));
00743     }
00744 
00745   return output;
00746 }
00747 
00748 // ######################################################################
00749 std::string SimulationViewerEyeHand::craftSMSamples(const rutz::shared_ptr<EyeData> data, 
00750                                                    const Image<float> smap)
00751 {
00752   std::string output;
00753   // get the map level to scale things down:
00754   int sml = itsLevelSpec.getVal().mapLevel();
00755 
00756   // get target
00757   Point2D<int> target = rawToRet(data->saccadeTarget());
00758   
00759   // get ready to take samples in the saliency map and in all
00760   // conspicuity maps, at the scale of the saliency map:
00761   const int radius = itsMetrics->getFoveaRadius();
00762   const int rad = radius >> sml;
00763   Point2D<int> p(target.i >> sml, target.j >> sml);
00764   
00765   // just in case the image has weird dims, make sure the sampling
00766   // location is in it:
00767 
00768   p.clampToDims(smap.getDims());
00769 
00770   // get the min/max/avg and saliency samples:
00771   float mi, ma, av; getMinMaxAvg(smap, mi, ma, av);
00772 
00773   // get a sample around the saccade target location:
00774   float val = getLocalMax(smap, p, rad);
00775 
00776   // add necessary headers
00777   if(!itsHeaderCrafted) 
00778     {
00779       itsOutFields.push_back("sal_val");
00780       itsOutFields.push_back("sal_min");
00781       itsOutFields.push_back("sal_max");
00782       itsOutFields.push_back("sal_av");
00783     }
00784   
00785   output = sformat("  %g %g %g %g ", val, mi, ma, av);
00786 
00787   // now get a bunch of random samples from the saliency map:
00788   if (itsNumRandomSamples.getVal() < smap.getSize())
00789     {
00790       // ok, we want fewer samples than are actually in the
00791       // saliency map, so choose them randomly:
00792       for (int k = 0; k < itsNumRandomSamples.getVal(); ++k) 
00793         {
00794           if (itsPriorRandomDistro.getVal().empty()) // uniform random
00795             {
00796               Point2D<int> randp(randomUpToNotIncluding(smap.getWidth()),
00797                                  randomUpToNotIncluding(smap.getHeight()));
00798               output += sformat(" %g %d %d", getLocalMax(smap, randp, rad),
00799                                 randp.i, randp.j);
00800             }
00801           else // take random points from file
00802             {
00803               const int randn = randomUpToNotIncluding(randPoints.size());
00804               Point2D<int> randp(randPoints[randn].i >> sml,
00805                                  randPoints[randn].j >> sml);
00806               output += sformat(" %g %d %d", getLocalMax(smap, randp, rad),
00807                                 randPoints[randn].i, randPoints[randn].j);
00808             }
00809           if(!itsHeaderCrafted) 
00810             {
00811               itsOutFields.push_back("rand_sal");
00812               itsOutFields.push_back("rand_x");
00813               itsOutFields.push_back("rand_y");
00814             }
00815         }
00816     }
00817   else
00818     {
00819       // ok, we want more or as many samples as are in the
00820       // saliency map, so just dump each map value once:
00821       Point2D<int> rp;
00822       for (rp.j = 0; rp.j < smap.getHeight(); ++rp.j)
00823         for (rp.i = 0; rp.i < smap.getWidth(); ++rp.i)
00824           {
00825             output += sformat(" %g %d %d", getLocalMax(smap, rp, rad),
00826                               rp.i, rp.j);
00827             if(!itsHeaderCrafted) 
00828               {
00829                 itsOutFields.push_back("rand_sal");
00830                 itsOutFields.push_back("rand_x");
00831                 itsOutFields.push_back("rand_y");
00832               }
00833           }
00834     }
00835   return output;
00836 }
00837 
00838 // ######################################################################
00839 std::string SimulationViewerEyeHand::craftSMHistory(const rutz::shared_ptr<EyeData> data,
00840                                                    Image<float> smap)
00841 {
00842   std::string output;
00843   // get the map level to scale things down:
00844   int sml = itsLevelSpec.getVal().mapLevel();
00845   
00846   // get target
00847   Point2D<int> target = rawToRet(data->saccadeTarget());
00848 
00849   // get ready to take samples in the saliency map and in all
00850   // conspicuity maps, at the scale of the saliency map:
00851   const int radius = itsMetrics->getFoveaRadius();
00852   const int rad = radius >> sml;
00853   Point2D<int> p(target.i >> sml, target.j >> sml);
00854   
00855   // save history of saliency values at saccade target?
00856   output += std::string("   ");
00857       
00858   // also get saccade start location, scaled down to SM size:
00859   Point2D<int> pp(data->position());
00860   pp.i >>= sml; pp.j >>= sml; pp.clampToDims(smap.getDims());
00861   
00862   // note: even though the queue may not be full yet, we
00863   // will always write itsSMhistoryQlen samples here, to
00864   // facilitate reading the data into matlab using
00865   // dlmread(). Saliency values for non-existent maps will
00866   // be set to 0:
00867   size_t ql = itsSMhistory.size();
00868   for (uint i = 0; i < itsSMhistoryQlen.getVal(); ++ i) 
00869     {
00870       if (i < ql)
00871         {
00872           // get min, max and avg saliency values over history:
00873           float mmi, mma, mmav;
00874           getMinMaxAvg(itsSMhistory[ql - 1 - i], mmi, mma, mmav);
00875           
00876           // and also value at one random location:
00877           Point2D<int> randp;
00878           if (itsPriorRandomDistro.getVal().empty()) // uniform random
00879             {
00880               randp.i = randomUpToNotIncluding(smap.getWidth());
00881               randp.j = randomUpToNotIncluding(smap.getHeight());
00882             }
00883           else // prior from file
00884             {
00885               const int randn =
00886                 randomUpToNotIncluding(randPoints.size());
00887               randp.i = randPoints[randn].i >> sml;
00888               randp.j = randPoints[randn].j >> sml;
00889             }
00890           
00891           output +=
00892             sformat(" %g %g %g %g %g",
00893                     getLocalMax(itsSMhistory[ql - 1 - i], p, rad),
00894                     getLocalMax(itsSMhistory[ql - 1 - i], pp, rad),
00895                     mma, mmav,
00896                     getLocalMax(itsSMhistory[ql - 1 - i], randp, rad));
00897         }
00898       else output += std::string(" 0.0 0.0 0.0 0.0 0.0");
00899       if(!itsHeaderCrafted) 
00900         {
00901           itsOutFields.push_back("hist_sacendsal");
00902           itsOutFields.push_back("hist_sacstartsal");
00903           itsOutFields.push_back("hist_avgsal");
00904           itsOutFields.push_back("hist_maxsal");
00905           itsOutFields.push_back("hist_randsal");      
00906         }
00907     }
00908   return output;
00909 }
00910 
00911 // ######################################################################
00912 void SimulationViewerEyeHand::writeHeader()
00913 {
00914   LINFO("writing header for file %s", itsOutFname.getVal().c_str());
00915   (*itsOutFile) << "#" << itsOutFields[0];
00916   for(uint i = 1; i < itsOutFields.size(); i++)
00917     (*itsOutFile) << " " << itsOutFields[i];
00918   (*itsOutFile) << std::endl;
00919 }
00920 
00921 // ######################################################################
00922 void SimulationViewerEyeHand::extraSampleProcessing(const rutz::shared_ptr<EyeData>) {}
00923 // empty here, written so that EyeRegion can do extra output on each eye position
00924 
00925 // ######################################################################
00926 void SimulationViewerEyeHand::extraSampleProcessing(const rutz::shared_ptr<HandData>) {}
00927 // empty here, written so that EyeRegion can do extra output on each eye position
00928 
00929 // ######################################################################
00930 Image< PixRGB<byte> > SimulationViewerEyeHand::getTraj(SimEventQueue& q)
00931 {
00932 GVX_TRACE(__PRETTY_FUNCTION__);
00933 
00934   // get the latest retina image:
00935   Image< PixRGB<byte> > input;
00936   if (SeC<SimEventRetinaImage> e = q.check<SimEventRetinaImage>(this, SEQ_ANY))
00937     input = e->frame().colorByte();
00938   else
00939     LFATAL("Could not find required SimEventRetinaImage");
00940 
00941   // make a composite of the input + the drawings:
00942   Image< PixRGB<byte> > comp = composite(itsDrawings, input);
00943 
00944   // return a plain traj?
00945   if (itsSaveTraj.getVal()) return comp;
00946 
00947   // let's get the current saliency map (normalized):
00948   const Dims dims = input.getDims();
00949   Image<float> sm = getMap(q, false);
00950   if (sm.initialized()) sm = rescaleOpt(sm, dims, itsDisplayInterp.getVal());
00951   else sm.resize(dims, true); // blank
00952   Image< PixRGB<byte> > smc = toRGB(Image<byte>(sm));
00953 
00954   // make a composite of the instantaneous SM + the drawings:
00955   Image< PixRGB<byte> > smcomp = composite(itsDrawings, smc);
00956 
00957   // otherwise, return mega combo; we have two formats: if we have a
00958   // max-cache, it will be a 4-image format, otherwise a 2-image
00959   // format:
00960   Image< PixRGB<byte> > ret;
00961   if (itsMaxCacheSize.getVal())
00962     {
00963       // 4-image format
00964       Image<float> maxsm =
00965         rescaleOpt(itsMaxCache.getMax(), dims, itsDisplayInterp.getVal());
00966       Image< PixRGB<byte> > maxsmc = toRGB(Image<byte>(maxsm));
00967 
00968       ret = concatX(concatY(input, smcomp),
00969                     concatY(comp, composite(itsDrawings, maxsmc)));
00970 
00971       drawGrid(ret, dims.w()-1, dims.h()-1, 3, 3, PixRGB<byte>(128, 128, 128));
00972     }
00973   else
00974     {
00975       // 2-image format:
00976       ret = concatX(comp, smcomp);
00977       drawLine(ret, Point2D<int>(dims.w()-1, 0), Point2D<int>(dims.w()-1, dims.h()-1),
00978                PixRGB<byte>(255, 255, 0), 1);
00979       drawLine(ret, Point2D<int>(dims.w(), 0), Point2D<int>(dims.w(), dims.h()-1),
00980                PixRGB<byte>(255, 255, 0), 1);
00981     }
00982 
00983   // make sure image is not unreasonably large:
00984   while (ret.getWidth() > itsMaxComboWidth.getVal())
00985     ret = decY(lowPass3y(decX(lowPass3x(ret))));
00986 
00987   return ret;
00988 }
00989 
00990 // ######################################################################
00991 void SimulationViewerEyeHand::
00992 onSimEventSaveOutput(SimEventQueue& q, rutz::shared_ptr<SimEventSaveOutput>& e)
00993 {
00994   this->save1(e->sinfo());
00995 }
00996 
00997 // ######################################################################
00998 void SimulationViewerEyeHand::save1(const ModelComponentSaveInfo& sinfo)
00999 {
01000 GVX_TRACE(__PRETTY_FUNCTION__);
01001   SimEventQueue *q = dynamic_cast<const SimModuleSaveInfo&>(sinfo).q;
01002 
01003   // update the trajectory:
01004   Image< PixRGB<byte> > res = getTraj(*q);
01005 
01006   // save results?
01007   if (itsSaveCombo.getVal() || itsSaveTraj.getVal())
01008     {
01009       // get the OFS to save to, assuming sinfo is of type
01010       // SimModuleSaveInfo (will throw a fatal exception otherwise):
01011       nub::ref<FrameOstream> ofs =
01012         dynamic_cast<const SimModuleSaveInfo&>(sinfo).ofs;
01013 
01014       ofs->writeRGB(res, "T",
01015                     FrameInfo("SimulationViewerEyeHand trajectory", SRC_POS));
01016 
01017     }
01018 }
01019 
01020 // ######################################################################
01021 /* So things look consistent in everyone's emacs... */
01022 /* Local Variables: */
01023 /* indent-tabs-mode: nil */
01024 /* End: */
Generated on Sun May 8 08:41:04 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3