psycho-oculomotorcapture.C

00001 /*!@file AppPsycho/psycho-movie.C Psychophysics display of movies */
00002 
00003 // //////////////////////////////////////////////////////////////////// //
00004 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2001 by the //
00005 // University of Southern California (USC) and the iLab at USC.         //
00006 // See http://iLab.usc.edu for information about this project.          //
00007 // //////////////////////////////////////////////////////////////////// //
00008 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00009 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00010 // in Visual Environments, and Applications'' by Christof Koch and      //
00011 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00012 // pending; application number 09/912,225 filed July 23, 2001; see      //
00013 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00014 // //////////////////////////////////////////////////////////////////// //
00015 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00016 //                                                                      //
00017 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00018 // redistribute it and/or modify it under the terms of the GNU General  //
00019 // Public License as published by the Free Software Foundation; either  //
00020 // version 2 of the License, or (at your option) any later version.     //
00021 //                                                                      //
00022 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00023 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00024 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00025 // PURPOSE.  See the GNU General Public License for more details.       //
00026 //                                                                      //
00027 // You should have received a copy of the GNU General Public License    //
00028 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00029 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00030 // Boston, MA 02111-1307 USA.                                           //
00031 // //////////////////////////////////////////////////////////////////// //
00032 //
00033 // Primary maintainer for this file: Po-He Tseng <ptseng@usc.edu>
00034 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/AppPsycho/psycho-movie-fixicon.C $
00035 // $Id: psycho-movie-fixicon.C 13712 2010-07-28 21:00:40Z itti $
00036 
00037 #include "Component/ModelManager.H"
00038 #include "Image/Image.H"
00039 #include "Raster/Raster.H"
00040 #include "Media/MPEGStream.H"
00041 #include "Media/MediaOpts.H"
00042 #include "Psycho/PsychoDisplay.H"
00043 #include "Psycho/EyeTrackerConfigurator.H"
00044 #include "Psycho/EyeTracker.H"
00045 #include "Psycho/PsychoOpts.H"
00046 #include "Component/EventLog.H"
00047 #include "Component/ComponentOpts.H"
00048 #include "Util/MathFunctions.H"
00049 #include "Util/Types.H"
00050 #include "Video/VideoFrame.H"
00051 #include "rutz/time.h"
00052 
00053 #include "SDL/SDL_rotozoom.h"
00054 
00055 #include <deque>
00056 #include <sstream>
00057 
00058 /*
00059 // ######################################################################
00060 static int waitForFixation(const Point2D<int> fparr [], 
00061                                                                            const int arraylen,
00062                                                                            const int radius,         //in pixel
00063                                                                  double fixlen,            // in msec
00064                                                                                                          nub::soft_ref<EyeTracker> et, 
00065                                                                                                          nub::soft_ref<PsychoDisplay> d,
00066                                                                                                          const bool do_drift_correction = false)
00067 {
00068         Point2D<int> ip;                                
00069         double dist [arraylen], sdist;
00070         double tt=0, adur=0, ax=0, ay=0; 
00071         Timer timer, timer2;
00072         fixlen = fixlen/1000;
00073         int status = 1;
00074 
00075         // clean up queue for key response
00076         while(d->checkForKey() != -1);
00077 
00078         // wait for fixation
00079         timer2.reset();
00080         while(timer.getSecs() < fixlen) {
00081                 ip = et->getFixationPos();
00082 
00083                 // check distance between fixation point to all possible targets
00084                 for (int i=0; i<arraylen; i++)
00085                         dist[i] = sqrt(pow(ip.i-fparr[i].i, 2) + pow(ip.j-fparr[i].j, 2));
00086 
00087                 // get the closest target
00088                 if (arraylen == 1)
00089                         sdist = dist[0];
00090                 else
00091                         sdist = std::min(dist[0], dist[1]);
00092 
00093                 // inside of tolerance?
00094                 if (sdist > radius && (ip.i!=-1 && ip.j!=-1)){
00095                         timer.reset();
00096                 }else{
00097                         tt = timer.getSecs()-tt;                        
00098                         adur += tt;
00099                         ax += tt * ip.i;
00100                         ay += tt * ip.j;        
00101                 }
00102 
00103                 //LINFO("(%i, %i) - (%i, %i) dist: %f, %f", fparr[0].i, fparr[0].j, ip.i, ip.j, sdist, timer.getSecs());
00104 
00105                 // time out
00106                 if (timer2.getSecs() > 5 || d->checkForKey() > 0){
00107                         d->pushEvent("bad trial - time out / no response.");
00108                         status = -1;
00109                         break;
00110                 }
00111         }                               
00112         
00113         // do drift correction
00114         if (status == 1 && do_drift_correction == true) {
00115                 // when there's only 1 target on screen                 
00116                 et->manualDriftCorrection(Point2D<double>(ax/adur, ay/adur), 
00117                                                                                           Point2D<double>(fparr[0].i, fparr[0].j));             
00118                 LINFO("drift correction: (%i, %i) (%i, %i)", (int)(ax/adur), (int)(ay/adur), fparr[0].i, fparr[0].j);
00119         }
00120 
00121         return status;
00122 }
00123 */
00124 
00125 // ######################################################################
00126 static int submain(const int argc, char** argv)
00127 {
00128   MYLOGVERB = LOG_INFO;  // suppress debug messages
00129 
00130   // Instantiate a ModelManager:
00131   ModelManager manager("Psycho Oculomotor Capture");
00132 
00133   // Instantiate our various ModelComponents:
00134   nub::soft_ref<InputMPEGStream> mp
00135     (new InputMPEGStream(manager, "Input MPEG Stream", "InputMPEGStream"));
00136   manager.addSubComponent(mp);
00137 
00138   nub::soft_ref<EventLog> el(new EventLog(manager));
00139   manager.addSubComponent(el);
00140 
00141   nub::soft_ref<EyeTrackerConfigurator>
00142     etc(new EyeTrackerConfigurator(manager));
00143   manager.addSubComponent(etc);
00144 
00145   nub::soft_ref<PsychoDisplay> d(new PsychoDisplay(manager));
00146   manager.addSubComponent(d);
00147 
00148   manager.setOptionValString(&OPT_InputMPEGStreamPreload, "true");
00149   manager.setOptionValString(&OPT_EventLogFileName, "psychodata.psy");
00150   manager.setOptionValString(&OPT_EyeTrackerType, "EL");
00151 
00152   // Parse command-line:
00153   if (manager.parseCommandLine(argc, argv, "<# of trials>", 1, 1) == false)
00154     return(1);
00155 
00156         // construct an array to indicate target location
00157   int ntrial = fromStr<int>(manager.getExtraArg(0).c_str());
00158         LINFO("Total Trials: %i", ntrial);
00159         int trials[ntrial];
00160         for (int i=0; i< ntrial; i++)
00161                 trials[i] = i % 6;
00162 
00163   // randomize stimulus presentation order:
00164         int trialindex[ntrial];
00165   for (int i = 0; i < ntrial; i ++) 
00166                 trialindex[i] = i;
00167   randShuffle(trialindex, ntrial);
00168 
00169   // hook our various babies up and do post-command-line configs:
00170   nub::soft_ref<EyeTracker> et = etc->getET();
00171   d->setEyeTracker(et);
00172   d->setEventLog(el);
00173   et->setEventLog(el);
00174 
00175   // EyeLink opens the screen for us, so make sure SDLdisplay is slave:
00176   if (etc->getModelParamString("EyeTrackerType").compare("EL") == 0)
00177     d->setModelParamVal("SDLslaveMode", true);
00178 
00179   // let's get all our ModelComponent instances started:
00180   manager.start();
00181 
00182   // vision test (generate random letters again if pressing 'r')
00183         LINFO("*****************************************************************************");
00184         LINFO("visual acuity test: [r] to regenerate another string; other keys to continue.");
00185         int key;
00186         do {
00187                 d->displayRandomText(6, 8);
00188         key = d->waitForKey(true);
00189         } while(key == 114 || key == 101 || key == 116 || key == 102);
00190 
00191         // let's do an eye tracker calibration:
00192         d->pushEventBegin("Calibration");
00193         et->setBackgroundColor(d);
00194         et->calibrate(d);
00195         d->pushEventEnd("Calibration");
00196 
00197         LINFO("Press any key to start......");
00198   d->displayText("Press any key to start......", true, 0, 10);
00199   d->waitForKey(true);
00200 
00201         // preparation for the main loop
00202         const int x = d->getWidth()/2;
00203         const int y = d->getHeight()/2;
00204         const int bradius = x/2.5; // radius for displaying 6 possible target locations
00205         const int sradius = 66;  // radius for target/distractor circle
00206         const int isradius = sradius - 8;
00207 
00208         //const PixRGB<byte> tcolor = PixRGB<byte>(255,0,0); // target color (red)
00209         const PixRGB<byte> tcolor = PixRGB<byte>(0,255,0); // target color (green)
00210         const PixRGB<byte> dcolor = PixRGB<byte>(255,0,0); // distractor color (red)
00211         const PixRGB<byte> white = PixRGB<byte>(255,255,255); // distractor color (red)
00212         const PixRGB<byte> bcolor = d->getGrey();          // background color
00213 
00214         // calulate circle locations
00215         // clockwise: 0, right; 1, lower-right; 2, lower-left; 3, left; 4, upper-left; 5, upper-right
00216         int tcirpos[6][2];
00217         int dcirpos[6][2];
00218         const double PI = 3.141592;
00219         for (int i=0; i<6; i++) {
00220                 // potential target locations
00221                 tcirpos[i][0] = x + bradius * cos(i*PI/3);      
00222                 tcirpos[i][1] = y + bradius * sin(i*PI/3);
00223 
00224                 // potential distractor locations
00225                 dcirpos[i][0] = x + bradius * cos(i*PI/3 + PI/6);       
00226                 dcirpos[i][1] = y + bradius * sin(i*PI/3 + PI/6);       
00227         }
00228 
00229         // create arrow for pointing direction
00230         const int arrowsize = 16;
00231         SDL_Surface *arrowsurf = SDL_CreateRGBSurface(SDL_SWSURFACE, arrowsize, arrowsize, 24, 0, 0, 0, 0);
00232         SDL_Rect arect;
00233         arect.x = 0; arect.y = 0; arect.w = arrowsize; arect.h = arrowsize;
00234         SDL_FillRect(arrowsurf, &arect, d->getUint32color(bcolor));
00235         arect.x = 1; arect.y = 1; arect.w = arrowsize-1; arect.h = (int)arrowsize/5;
00236         SDL_FillRect(arrowsurf, &arect, d->getUint32color(tcolor));
00237         arect.w = (int)arrowsize/5; arect.h = arrowsize-1;
00238         SDL_FillRect(arrowsurf, &arect, d->getUint32color(tcolor));
00239 
00240         // create some T and non-T for discrimination
00241         const int sizeT = 10;
00242         SDL_Surface *isT = SDL_CreateRGBSurface(SDL_SWSURFACE, sizeT, sizeT, 24, 0, 0, 0, 0);
00243         SDL_Surface *noT = SDL_CreateRGBSurface(SDL_SWSURFACE, sizeT, sizeT, 24, 0, 0, 0, 0);
00244         arect.x = 0; arect.y = 0; arect.w = sizeT; arect.h = sizeT; // background
00245         SDL_FillRect(isT, &arect, d->getUint32color(bcolor));
00246         SDL_FillRect(noT, &arect, d->getUint32color(bcolor));
00247         arect.x = sizeT/2; arect.y = 0; arect.w = 1; arect.h = sizeT; //vertical bar
00248         SDL_FillRect(isT, &arect, d->getUint32color(tcolor));
00249         SDL_FillRect(noT, &arect, d->getUint32color(tcolor));   
00250         arect.x = 0; arect.y = 0; arect.w = sizeT; arect.h = 1; //horizontal bar, T
00251         SDL_FillRect(isT, &arect, d->getUint32color(tcolor));
00252         arect.x = 0; arect.y = 2; arect.w = sizeT; arect.h = 1; //horizontal bar, non-T
00253         SDL_FillRect(noT, &arect, d->getUint32color(tcolor));
00254 
00255         // initialze T and Not-T index that will be randomized later
00256         int tntidx[6];
00257         for(int i=0; i< 6; i++)
00258                 tntidx[i] = i;
00259 
00260         // main loop
00261         double delay;
00262         int j, iscontrol, ang, thisTarget, thisDistractor, thisSOA;
00263         int tnt[6], correctArray[ntrial];
00264         SDL_Surface *surf = SDL_CreateRGBSurface(SDL_SWSURFACE, d->getWidth(), d->getHeight(), 24, 0, 0, 0, 0);
00265         SDL_Rect rect, rr; rect.x = 0; rect.y = 0; rect.w = d->getWidth(); rect.h = d->getHeight();
00266         Timer timer;
00267         double rt, avg;
00268         avg = 0; 
00269 
00270         for (int i=0; i < ntrial; i++) {
00271                 thisTarget = trials[trialindex[i]];
00272                 thisSOA = rand()%150;
00273                 thisDistractor = rand()%6;
00274                 if (rand()%10 < 1){
00275                         iscontrol = 1;
00276                 } else {
00277                         iscontrol = 0;
00278                 }
00279 
00280                 delay = 1000000; // + randomUpToIncluding(1000000);
00281                 SDL_FillRect(surf, &rect, d->getUint32color(d->getGrey())); 
00282 
00283                 // clean screen and do drift correction
00284                 et->recalibrate(d,13);
00285                 d->clearScreen();
00286         
00287                 // randomize T and Not-T
00288                 randShuffle(tntidx, 6);
00289                 for (j=0; j<6; j++)
00290                         tnt[tntidx[j]] = j % 2; //0, non-T; 1, T
00291 
00292                 // track eye
00293                 d->displayFilledCircle(x, y, 3, white, true);
00294                 usleep(900000);
00295                 et->track(true);
00296                 usleep(100000);
00297 
00298                 // display all possible target locations
00299         filledCircleRGBA(surf, x, y, 3, white.red(), white.green(), white.blue(), 0XFF);
00300                 for (j=0; j<6; j++){
00301                 filledCircleRGBA(surf, tcirpos[j][0], tcirpos[j][1], sradius, tcolor.red(), tcolor.green(), tcolor.blue(), 0XFF);
00302                 filledCircleRGBA(surf, tcirpos[j][0], tcirpos[j][1], isradius, bcolor.red(), bcolor.green(), bcolor.blue(), 0XFF);
00303                 
00304                         rr.x = tcirpos[j][0]-isT->w/2; rr.y = tcirpos[j][1]-isT->h/2;
00305                         if (tnt[j] == 0) {
00306                                 SDL_BlitSurface(noT, NULL, surf, &rr);
00307                         } else {
00308                                 SDL_BlitSurface(isT, NULL, surf, &rr);
00309                         }
00310                 }
00311 
00312                 // is it an control trial?
00313                 if (iscontrol == 1) {
00314                 filledCircleRGBA(surf, dcirpos[thisDistractor][0], dcirpos[thisDistractor][1], sradius, dcolor.red(), dcolor.green(), dcolor.blue(), 0XFF);
00315                 filledCircleRGBA(surf, dcirpos[thisDistractor][0], dcirpos[thisDistractor][1], isradius, bcolor.red(), bcolor.green(), bcolor.blue(), 0XFF);
00316                 }
00317 
00318                 // display array
00319                 d->displaySurface(surf, -2, true);
00320                 d->pushEvent("Array UP");
00321 
00322                 // waiting before target comes up
00323                 usleep(delay);
00324 
00325                 // show direction
00326                 ang = 45 - 60 * thisTarget - 180; 
00327                 SDL_Surface *tmpsurf = rotozoomSurface(arrowsurf, ang, 1, 0.5);
00328                 rr.x = d->getWidth()/2-tmpsurf->w/2; rr.y = d->getHeight()/2-tmpsurf->h/2;
00329                 SDL_BlitSurface(tmpsurf, NULL, surf, &rr); 
00330                 LINFO("thisTarget: %i, isT: %i", thisTarget, tnt[thisTarget]);
00331                 
00332                 // show 
00333                 timer.reset();
00334                 if (iscontrol == 1) {
00335 
00336                         d->displaySurface(surf, -2, true);
00337                         d->pushEvent(sformat("Target direction shown: %d", thisTarget));
00338                         d->pushEvent("Distractor shown: control condition");
00339 
00340                 }else if (thisSOA < 34){
00341 
00342                 filledCircleRGBA(surf, dcirpos[thisDistractor][0], dcirpos[thisDistractor][1], sradius, dcolor.red(), dcolor.green(), dcolor.blue(), 0XFF);
00343                 filledCircleRGBA(surf, dcirpos[thisDistractor][0], dcirpos[thisDistractor][1], isradius, bcolor.red(), bcolor.green(), bcolor.blue(), 0XFF);
00344                         d->displaySurface(surf, -2, true);
00345                         d->pushEvent(sformat("Target direction shown: %d", thisTarget));
00346                         d->pushEvent(sformat("Distractor shown: %f (SOA = %d ms)", thisDistractor+0.5, thisSOA));
00347                 
00348                 } else {
00349 
00350                         // display instruction direction first
00351                         d->displaySurface(surf, -2, true);
00352                         d->pushEvent(sformat("Target direction shown: %d", thisTarget));
00353 
00354                         // wait
00355                         usleep(thisSOA*1000);
00356 
00357                         // display distractor
00358                 filledCircleRGBA(surf, dcirpos[thisDistractor][0], dcirpos[thisDistractor][1], sradius, dcolor.red(), dcolor.green(), dcolor.blue(), 0XFF);
00359                 filledCircleRGBA(surf, dcirpos[thisDistractor][0], dcirpos[thisDistractor][1], isradius, bcolor.red(), bcolor.green(), bcolor.blue(), 0XFF);
00360                         d->displaySurface(surf, -2, true);
00361                         d->pushEvent(sformat("Distractor shown: %f (SOA = %d ms)", thisDistractor+0.5, thisSOA));
00362                 
00363                 }       
00364                 SDL_FreeSurface(tmpsurf);
00365 
00366                 // wait for response    
00367                 //LINFO("Press 4 for T, 6 for Not-T, or Space Bar to skip.");
00368                 do {
00369                         key = d->waitForKey();
00370                 } while (key != 52 && key != 54 && key != 32);
00371                 rt = timer.getMilliSecs();
00372                 int kk = 0;
00373                 if (key==52)
00374                         kk = 1;
00375                 else if (key==32)
00376                         kk = -1; // space bar
00377 
00378                 d->pushEvent(sformat("Responded %d (%f ms), correct %d", kk, rt, kk==tnt[thisTarget]));
00379                 LINFO("Responded %d (%f ms), correct %d", kk, rt, kk==tnt[thisTarget]);
00380                 avg = onlineMean(avg, rt, i+1);
00381 
00382                 d->clearScreen();
00383 
00384                 usleep(50000);
00385                 et->track(false);
00386         
00387                 correctArray[i] = (kk==tnt[thisTarget]);
00388 
00389                 // display percentage completed
00390                 double completion = 100*((double)i+1)/double(ntrial);
00391                 if ((int)completion % 5 == 0 && completion - (int)completion == 0){
00392                 
00393                         d->clearScreen();
00394                         d->displayText(sformat("%i %% completed", (int)completion), true, 0, 10);
00395                         usleep(1000000);
00396                 }
00397 
00398                 // take a break for a quarter of trials
00399                 if ((int)completion%50 == 0 && i<ntrial-1 && completion-(int)completion==0) {
00400                 d->clearScreen();
00401                 d->displayText("Please Take a Break", true, 0, 10);
00402 
00403                         // performance information
00404                         int cc = 0;
00405                         for (j=0; j<=i; j++)
00406                                 if (correctArray[j]==1)
00407                                         cc++;
00408                         LINFO("Average RT: %f ms, Correctness: %f%%", avg, 100*(double)cc/((double)i+1));
00409 
00410                         LINFO("Break time.  Press [Space] to continue, or [ESC] to terminate the experiment.");
00411                 d->waitForKey(true);
00412                 d->displayText("Calibration", true, 0, 10);
00413       d->pushEventBegin("Calibration");
00414       et->calibrate(d);
00415       d->pushEventEnd("Calibration");
00416                 }
00417         }
00418 
00419         SDL_FreeSurface(arrowsurf);
00420         SDL_FreeSurface(surf);
00421 
00422         // performance information
00423         int cc = 0;
00424         for (j=0; j<ntrial; j++)
00425                 if (correctArray[j]==1)
00426                         cc++;
00427         LINFO("Average RT: %f ms, Correctness: %f%%", avg, 100*(double)cc/((double)ntrial));
00428 
00429   d->clearScreen();
00430   d->displayText("Experiment complete. Thank you!", true, 0, 10);
00431   d->waitForKey(true);
00432 
00433   // stop all our ModelComponents
00434   manager.stop();
00435 
00436   // all done!
00437   return 0;
00438 }
00439 
00440 // ######################################################################
00441 extern "C" int main(const int argc, char** argv)
00442 {
00443   // simple wrapper around submain() to catch exceptions (because we
00444   // want to allow PsychoDisplay to shut down cleanly; otherwise if we
00445   // abort while SDL is in fullscreen mode, the X server won't return
00446   // to its original resolution)
00447   try
00448     {
00449       return submain(argc, argv);
00450     }
00451   catch (...)
00452     {
00453       REPORT_CURRENT_EXCEPTION;
00454     }
00455 
00456   return 1;
00457 }
00458 
00459 // ######################################################################
00460 /* So things look consistent in everyone's emacs... */
00461 /* Local Variables: */
00462 /* indent-tabs-mode: nil */
00463 /* End: */
Generated on Sun May 8 08:40:09 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3