psycho-oculomotorcapture2.C

00001 /*!@file AppPsycho/psycho-movie.C Psychophysics display of movies */
00002 
00003 // //////////////////////////////////////////////////////////////////// //
00004 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2001 by the //
00005 // University of Southern California (USC) and the iLab at USC.         //
00006 // See http://iLab.usc.edu for information about this project.          //
00007 // //////////////////////////////////////////////////////////////////// //
00008 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00009 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00010 // in Visual Environments, and Applications'' by Christof Koch and      //
00011 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00012 // pending; application number 09/912,225 filed July 23, 2001; see      //
00013 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00014 // //////////////////////////////////////////////////////////////////// //
00015 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00016 //                                                                      //
00017 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00018 // redistribute it and/or modify it under the terms of the GNU General  //
00019 // Public License as published by the Free Software Foundation; either  //
00020 // version 2 of the License, or (at your option) any later version.     //
00021 //                                                                      //
00022 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00023 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00024 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00025 // PURPOSE.  See the GNU General Public License for more details.       //
00026 //                                                                      //
00027 // You should have received a copy of the GNU General Public License    //
00028 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00029 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00030 // Boston, MA 02111-1307 USA.                                           //
00031 // //////////////////////////////////////////////////////////////////// //
00032 //
00033 // Primary maintainer for this file: Po-He Tseng <ptseng@usc.edu>
00034 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/AppPsycho/psycho-movie-fixicon.C $
00035 // $Id: psycho-movie-fixicon.C 13712 2010-07-28 21:00:40Z itti $
00036 
00037 #include "Component/ModelManager.H"
00038 #include "Image/Image.H"
00039 #include "Raster/Raster.H"
00040 #include "Media/MPEGStream.H"
00041 #include "Media/MediaOpts.H"
00042 #include "Psycho/PsychoDisplay.H"
00043 #include "Psycho/EyeTrackerConfigurator.H"
00044 #include "Psycho/EyeTracker.H"
00045 #include "Psycho/PsychoOpts.H"
00046 #include "Component/EventLog.H"
00047 #include "Component/ComponentOpts.H"
00048 #include "Util/MathFunctions.H"
00049 #include "Util/Types.H"
00050 #include "Video/VideoFrame.H"
00051 #include "rutz/time.h"
00052 
00053 #include "SDL/SDL_rotozoom.h"
00054 
00055 #include <deque>
00056 #include <sstream>
00057 
00058 /*
00059 // ######################################################################
00060 static int waitForFixation(const Point2D<int> fparr [], 
00061                                                                            const int arraylen,
00062                                                                            const int radius,         //in pixel
00063                                                                  double fixlen,            // in msec
00064                                                                                                          nub::soft_ref<EyeTracker> et, 
00065                                                                                                          nub::soft_ref<PsychoDisplay> d,
00066                                                                                                          const bool do_drift_correction = false)
00067 {
00068         Point2D<int> ip;                                
00069         double dist [arraylen], sdist;
00070         double tt=0, adur=0, ax=0, ay=0; 
00071         Timer timer, timer2;
00072         fixlen = fixlen/1000;
00073         int status = 1;
00074 
00075         // clean up queue for key response
00076         while(d->checkForKey() != -1);
00077 
00078         // wait for fixation
00079         timer2.reset();
00080         while(timer.getSecs() < fixlen) {
00081                 ip = et->getFixationPos();
00082 
00083                 // check distance between fixation point to all possible targets
00084                 for (int i=0; i<arraylen; i++)
00085                         dist[i] = sqrt(pow(ip.i-fparr[i].i, 2) + pow(ip.j-fparr[i].j, 2));
00086 
00087                 // get the closest target
00088                 if (arraylen == 1)
00089                         sdist = dist[0];
00090                 else
00091                         sdist = std::min(dist[0], dist[1]);
00092 
00093                 // inside of tolerance?
00094                 if (sdist > radius && (ip.i!=-1 && ip.j!=-1)){
00095                         timer.reset();
00096                 }else{
00097                         tt = timer.getSecs()-tt;                        
00098                         adur += tt;
00099                         ax += tt * ip.i;
00100                         ay += tt * ip.j;        
00101                 }
00102 
00103                 //LINFO("(%i, %i) - (%i, %i) dist: %f, %f", fparr[0].i, fparr[0].j, ip.i, ip.j, sdist, timer.getSecs());
00104 
00105                 // time out
00106                 if (timer2.getSecs() > 5 || d->checkForKey() > 0){
00107                         d->pushEvent("bad trial - time out / no response.");
00108                         status = -1;
00109                         break;
00110                 }
00111         }                               
00112         
00113         // do drift correction
00114         if (status == 1 && do_drift_correction == true) {
00115                 // when there's only 1 target on screen                 
00116                 et->manualDriftCorrection(Point2D<double>(ax/adur, ay/adur), 
00117                                                                                           Point2D<double>(fparr[0].i, fparr[0].j));             
00118                 LINFO("drift correction: (%i, %i) (%i, %i)", (int)(ax/adur), (int)(ay/adur), fparr[0].i, fparr[0].j);
00119         }
00120 
00121         return status;
00122 }
00123 */
00124 
00125 // ######################################################################
00126 static int submain(const int argc, char** argv)
00127 {
00128   MYLOGVERB = LOG_INFO;  // suppress debug messages
00129 
00130   // Instantiate a ModelManager:
00131   ModelManager manager("Psycho Oculomotor Capture");
00132 
00133   // Instantiate our various ModelComponents:
00134   nub::soft_ref<InputMPEGStream> mp
00135     (new InputMPEGStream(manager, "Input MPEG Stream", "InputMPEGStream"));
00136   manager.addSubComponent(mp);
00137 
00138   nub::soft_ref<EventLog> el(new EventLog(manager));
00139   manager.addSubComponent(el);
00140 
00141   nub::soft_ref<EyeTrackerConfigurator>
00142     etc(new EyeTrackerConfigurator(manager));
00143   manager.addSubComponent(etc);
00144 
00145   nub::soft_ref<PsychoDisplay> d(new PsychoDisplay(manager));
00146   manager.addSubComponent(d);
00147 
00148   manager.setOptionValString(&OPT_InputMPEGStreamPreload, "true");
00149   manager.setOptionValString(&OPT_EventLogFileName, "psychodata.psy");
00150   manager.setOptionValString(&OPT_EyeTrackerType, "EL");
00151 
00152   // Parse command-line:
00153   if (manager.parseCommandLine(argc, argv, "<# of trials>", 1, 1) == false)
00154     return(1);
00155 
00156         // construct an array to indicate target location
00157   int ntrial = fromStr<int>(manager.getExtraArg(0).c_str());
00158         LINFO("Total Trials: %i", ntrial);
00159         int trials[ntrial];
00160         for (int i=0; i< ntrial; i++)
00161                 trials[i] = i % 6;
00162 
00163   // randomize stimulus presentation order:
00164         int trialindex[ntrial];
00165   for (int i = 0; i < ntrial; i ++) 
00166                 trialindex[i] = i;
00167   randShuffle(trialindex, ntrial);
00168 
00169   // hook our various babies up and do post-command-line configs:
00170   nub::soft_ref<EyeTracker> et = etc->getET();
00171   d->setEyeTracker(et);
00172   d->setEventLog(el);
00173   et->setEventLog(el);
00174 
00175   // EyeLink opens the screen for us, so make sure SDLdisplay is slave:
00176   if (etc->getModelParamString("EyeTrackerType").compare("EL") == 0)
00177     d->setModelParamVal("SDLslaveMode", true);
00178 
00179   // let's get all our ModelComponent instances started:
00180   manager.start();
00181 
00182   // vision test (generate random letters again if pressing 'r')
00183         LINFO("*****************************************************************************");
00184         LINFO("visual acuity test: [r] to regenerate another string; other keys to continue.");
00185         int key;
00186         do {
00187                 d->displayRandomText(6, 8);
00188         key = d->waitForKey(true);
00189         } while(key == 114 || key == 101 || key == 116 || key == 102);
00190 
00191         // let's do an eye tracker calibration:
00192         d->pushEventBegin("Calibration");
00193         et->setBackgroundColor(d);
00194         et->calibrate(d);
00195         d->pushEventEnd("Calibration");
00196 
00197         LINFO("Press any key to start......");
00198   d->displayText("Press any key to start......", true, 0, 10);
00199   d->waitForKey(true);
00200 
00201         // preparation for the main loop
00202         const int x = d->getWidth()/2;
00203         const int y = d->getHeight()/2;
00204         const int bradius = x/2.5; // radius for displaying 6 possible target locations
00205         const int sradius = 66;  // radius for target/distractor circle
00206         const int isradius = sradius - 8;
00207 
00208         //const PixRGB<byte> tcolor = PixRGB<byte>(255,0,0); // target color (red)
00209         const PixRGB<byte> tcolor = PixRGB<byte>(0,255,0); // target color (green)
00210         const PixRGB<byte> dcolor = PixRGB<byte>(255,0,0); // distractor color (red)
00211         const PixRGB<byte> black  = PixRGB<byte>(255,255,255);   // black color (red)
00212         const PixRGB<byte> bcolor = d->getGrey();          // background color
00213 
00214         // calulate circle locations
00215         // clockwise: 0, right; 1, lower-right; 2, lower-left; 3, left; 4, upper-left; 5, upper-right
00216         int tcirpos[6][2];
00217         int dcirpos[6][2];
00218         const double PI = 3.141592;
00219         for (int i=0; i<6; i++) {
00220                 // potential target locations
00221                 tcirpos[i][0] = x + bradius * cos(i*PI/3);      
00222                 tcirpos[i][1] = y + bradius * sin(i*PI/3);
00223 
00224                 // potential distractor locations
00225                 dcirpos[i][0] = x + bradius * cos(i*PI/3 + PI/6);       
00226                 dcirpos[i][1] = y + bradius * sin(i*PI/3 + PI/6);       
00227         }
00228 
00229         // create arrow for pointing direction
00230         const int arrowsize = 16;
00231         SDL_Surface *arrowsurf = SDL_CreateRGBSurface(SDL_SWSURFACE, arrowsize, arrowsize, 24, 0, 0, 0, 0);
00232         SDL_Rect arect;
00233         arect.x = 0; arect.y = 0; arect.w = arrowsize; arect.h = arrowsize;
00234         SDL_FillRect(arrowsurf, &arect, d->getUint32color(bcolor));
00235         arect.x = 1; arect.y = 1; arect.w = arrowsize-1; arect.h = (int)arrowsize/5;
00236         SDL_FillRect(arrowsurf, &arect, d->getUint32color(tcolor));
00237         arect.w = (int)arrowsize/5; arect.h = arrowsize-1;
00238         SDL_FillRect(arrowsurf, &arect, d->getUint32color(tcolor));
00239 
00240         // create some T and non-T for discrimination
00241         const int sizeT = 10;
00242         SDL_Surface *isT = SDL_CreateRGBSurface(SDL_SWSURFACE, sizeT, sizeT, 24, 0, 0, 0, 0);
00243         SDL_Surface *noT = SDL_CreateRGBSurface(SDL_SWSURFACE, sizeT, sizeT, 24, 0, 0, 0, 0);
00244         arect.x = 0; arect.y = 0; arect.w = sizeT; arect.h = sizeT; // background
00245         SDL_FillRect(isT, &arect, d->getUint32color(bcolor));
00246         SDL_FillRect(noT, &arect, d->getUint32color(bcolor));
00247         arect.x = sizeT/2; arect.y = 0; arect.w = 1; arect.h = sizeT; //vertical bar
00248         SDL_FillRect(isT, &arect, d->getUint32color(tcolor));
00249         SDL_FillRect(noT, &arect, d->getUint32color(tcolor));   
00250         arect.x = 0; arect.y = 0; arect.w = sizeT; arect.h = 1; //horizontal bar, T
00251         SDL_FillRect(isT, &arect, d->getUint32color(tcolor));
00252         arect.x = 0; arect.y = 2; arect.w = sizeT; arect.h = 1; //horizontal bar, non-T
00253         SDL_FillRect(noT, &arect, d->getUint32color(tcolor));
00254 
00255         SDL_Surface *isTd = SDL_CreateRGBSurface(SDL_SWSURFACE, sizeT, sizeT, 24, 0, 0, 0, 0);
00256         SDL_Surface *noTd = SDL_CreateRGBSurface(SDL_SWSURFACE, sizeT, sizeT, 24, 0, 0, 0, 0);
00257         arect.x = 0; arect.y = 0; arect.w = sizeT; arect.h = sizeT; // background
00258         SDL_FillRect(isTd, &arect, d->getUint32color(bcolor));
00259         SDL_FillRect(noTd, &arect, d->getUint32color(bcolor));
00260         arect.x = sizeT/2; arect.y = 0; arect.w = 1; arect.h = sizeT; //vertical bar
00261         SDL_FillRect(isTd, &arect, d->getUint32color(dcolor));
00262         SDL_FillRect(noTd, &arect, d->getUint32color(dcolor));  
00263         arect.x = 0; arect.y = 0; arect.w = sizeT; arect.h = 1; //horizontal bar, T
00264         SDL_FillRect(isTd, &arect, d->getUint32color(dcolor));
00265         arect.x = 0; arect.y = 2; arect.w = sizeT; arect.h = 1; //horizontal bar, non-T
00266         SDL_FillRect(noTd, &arect, d->getUint32color(dcolor));
00267 
00268   // initialze T and Not-T index that will be randomized later
00269   int tntidx[6];
00270   for(int i=0; i< 6; i++)
00271     tntidx[i] = i;
00272 
00273         // main loop
00274         double delay;
00275         int j, iscontrol, thisTarget, thisDistractor, thisSOA;
00276         int tnt[6], correctArray[ntrial];
00277         SDL_Surface *surf = SDL_CreateRGBSurface(SDL_SWSURFACE, d->getWidth(), d->getHeight(), 24, 0, 0, 0, 0);
00278         SDL_Rect rect, rr; rect.x = 0; rect.y = 0; rect.w = d->getWidth(); rect.h = d->getHeight();
00279         Timer timer;
00280         double rt, avg;
00281         avg = 0; 
00282 
00283         for (int i=0; i < ntrial; i++) {
00284                 thisTarget = trials[trialindex[i]];
00285                 thisSOA = rand()%150;
00286                 thisDistractor = rand()%6;
00287                 if (rand()%10 < 1){
00288                         iscontrol = 1;
00289                 } else {
00290                         iscontrol = 0;
00291                 }
00292 
00293                 delay = 1000000; // + randomUpToIncluding(1000000);
00294                 SDL_FillRect(surf, &rect, d->getUint32color(d->getGrey())); 
00295 
00296                 // clean screen and do drift correction
00297                 et->recalibrate(d,13);
00298                 d->clearScreen();
00299         
00300     // randomize T and Not-T
00301     randShuffle(tntidx, 6);
00302     for (j=0; j<6; j++)
00303       tnt[tntidx[j]] = j % 2; //0, non-T; 1, T
00304 
00305                 // track eye
00306                 d->displayFilledCircle(x, y, 3, black, true);
00307                 usleep(900000);
00308                 et->track(true);
00309                 usleep(100000);
00310 
00311                 // display all possible target locations
00312                 for (j=0; j<6; j++){
00313                 filledCircleRGBA(surf, tcirpos[j][0], tcirpos[j][1], sradius, tcolor.red(), tcolor.green(), tcolor.blue(), 0XFF);
00314                 filledCircleRGBA(surf, tcirpos[j][0], tcirpos[j][1], isradius, bcolor.red(), bcolor.green(), bcolor.blue(), 0XFF);
00315                 
00316                         rr.x = tcirpos[j][0]-isT->w/2; rr.y = tcirpos[j][1]-isT->h/2;
00317                         if (tnt[j] == 0) {
00318                                 SDL_BlitSurface(noT, NULL, surf, &rr);
00319                         } else {
00320                                 SDL_BlitSurface(isT, NULL, surf, &rr);
00321                         }
00322                 }
00323 
00324                 // is it an control trial?
00325                 if (iscontrol == 1) {
00326                 filledCircleRGBA(surf, dcirpos[thisDistractor][0], dcirpos[thisDistractor][1], sradius, tcolor.red(), tcolor.green(), tcolor.blue(), 0XFF);
00327                 filledCircleRGBA(surf, dcirpos[thisDistractor][0], dcirpos[thisDistractor][1], isradius, bcolor.red(), bcolor.green(), bcolor.blue(), 0XFF);
00328                 }
00329 
00330                 // display array
00331         filledCircleRGBA(surf, x, y, 3, black.red(), black.green(), black.blue(), 0XFF);
00332                 d->displaySurface(surf, -2, true);
00333                 d->pushEvent("Array UP");
00334 
00335                 // waiting before target comes up
00336                 usleep(delay);
00337 
00338                 // show direction
00339                 LINFO("thisTarget: %i, isT: %i", thisTarget, tnt[thisTarget]);
00340                 
00341                 // show 
00342                 for (j=0; j<6; j++)
00343                         if (j != thisTarget){
00344                         filledCircleRGBA(surf, tcirpos[j][0], tcirpos[j][1], sradius, dcolor.red(), dcolor.green(), dcolor.blue(), 0XFF);
00345                         filledCircleRGBA(surf, tcirpos[j][0], tcirpos[j][1], isradius, bcolor.red(), bcolor.green(), bcolor.blue(), 0XFF);
00346 
00347                                 rr.x = tcirpos[j][0]-isT->w/2; rr.y = tcirpos[j][1]-isT->h/2;
00348                                 if (tnt[j] == 0) {
00349                                         SDL_BlitSurface(noTd, NULL, surf, &rr);
00350                                 } else {
00351                                         SDL_BlitSurface(isTd, NULL, surf, &rr);
00352                                 }
00353                         }
00354 
00355                 timer.reset();
00356                 if (thisSOA < 34){
00357                 filledCircleRGBA(surf, dcirpos[thisDistractor][0], dcirpos[thisDistractor][1], sradius, dcolor.red(), dcolor.green(), dcolor.blue(), 0XFF);
00358                 filledCircleRGBA(surf, dcirpos[thisDistractor][0], dcirpos[thisDistractor][1], isradius, bcolor.red(), bcolor.green(), bcolor.blue(), 0XFF);
00359                         d->displaySurface(surf, -2, true);
00360                         d->pushEvent(sformat("Target direction shown: %d", thisTarget));
00361                         if (iscontrol == 1)
00362                                 d->pushEvent("Distractor shown: control condition");
00363                         else
00364                                 d->pushEvent(sformat("Distractor shown: %f (SOA = %d ms)", thisDistractor+0.5, thisSOA));
00365                 
00366                 } else {
00367 
00368                         // display instruction direction first
00369                         d->displaySurface(surf, -2, true);
00370                         d->pushEvent(sformat("Target direction shown: %d", thisTarget));
00371 
00372                         // wait
00373                         usleep(thisSOA*1000);
00374 
00375                         // display distractor
00376                 filledCircleRGBA(surf, dcirpos[thisDistractor][0], dcirpos[thisDistractor][1], sradius, dcolor.red(), dcolor.green(), dcolor.blue(), 0XFF);
00377                 filledCircleRGBA(surf, dcirpos[thisDistractor][0], dcirpos[thisDistractor][1], isradius, bcolor.red(), bcolor.green(), bcolor.blue(), 0XFF);
00378                         d->displaySurface(surf, -2, true);
00379                         d->pushEvent(sformat("Distractor shown: %f (SOA = %d ms)", thisDistractor+0.5, thisSOA));
00380                 
00381                 }       
00382 
00383                 // wait for response    
00384                 //LINFO("Press 4 for T, 6 for Not-T, or Space Bar to skip.");
00385                 do {
00386                         key = d->waitForKey();
00387                 } while (key != 52 && key != 54 && key != 32);
00388                 rt = timer.getMilliSecs();
00389                 int kk = 0;
00390                 if (key==52)
00391                         kk = 1;
00392                 else if (key==32)
00393                         kk = -1; // space bar
00394 
00395                 d->pushEvent(sformat("Responded %d (%f ms), correct %d", kk, rt, kk==tnt[thisTarget]));
00396                 LINFO("Responded %d (%f ms), correct %d", kk, rt, kk==tnt[thisTarget]);
00397                 avg = onlineMean(avg, rt, i+1);
00398 
00399                 d->clearScreen();
00400 
00401                 usleep(50000);
00402                 et->track(false);
00403         
00404                 correctArray[i] = (kk==tnt[thisTarget]);
00405 
00406                 // display percentage completed
00407                 double completion = 100*((double)i+1)/double(ntrial);
00408                 if ((int)completion % 5 == 0 && completion - (int)completion == 0){
00409                 
00410                         d->clearScreen();
00411                         d->displayText(sformat("%i %% completed", (int)completion), true, 0, 10);
00412                         usleep(1000000);
00413                 }
00414 
00415                 // take a break for a quarter of trials
00416                 if ((int)completion%50 == 0 && i<ntrial-1 && completion-(int)completion==0) {
00417                 d->clearScreen();
00418                 d->displayText("Please Take a Break", true, 0, 10);
00419 
00420                         // performance information
00421                         int cc = 0;
00422                         for (j=0; j<=i; j++)
00423                                 if (correctArray[j]==1)
00424                                         cc++;
00425                         LINFO("Average RT: %f ms, Correctness: %f%%", avg, 100*(double)cc/((double)i+1));
00426 
00427                         LINFO("Break time.  Press [Space] to continue, or [ESC] to terminate the experiment.");
00428                 d->waitForKey(true);
00429                 d->displayText("Calibration", true, 0, 10);
00430       d->pushEventBegin("Calibration");
00431       et->calibrate(d);
00432       d->pushEventEnd("Calibration");
00433                 }
00434         }
00435 
00436         SDL_FreeSurface(arrowsurf);
00437         SDL_FreeSurface(surf);
00438 
00439         // performance information
00440         int cc = 0;
00441         for (j=0; j<ntrial; j++)
00442                 if (correctArray[j]==1)
00443                         cc++;
00444         LINFO("Average RT: %f ms, Correctness: %f%%", avg, 100*(double)cc/((double)ntrial));
00445 
00446   d->clearScreen();
00447   d->displayText("Experiment complete. Thank you!", true, 0, 10);
00448   d->waitForKey(true);
00449 
00450   // stop all our ModelComponents
00451   manager.stop();
00452 
00453   // all done!
00454   return 0;
00455 }
00456 
00457 // ######################################################################
00458 extern "C" int main(const int argc, char** argv)
00459 {
00460   // simple wrapper around submain() to catch exceptions (because we
00461   // want to allow PsychoDisplay to shut down cleanly; otherwise if we
00462   // abort while SDL is in fullscreen mode, the X server won't return
00463   // to its original resolution)
00464   try
00465     {
00466       return submain(argc, argv);
00467     }
00468   catch (...)
00469     {
00470       REPORT_CURRENT_EXCEPTION;
00471     }
00472 
00473   return 1;
00474 }
00475 
00476 // ######################################################################
00477 /* So things look consistent in everyone's emacs... */
00478 /* Local Variables: */
00479 /* indent-tabs-mode: nil */
00480 /* End: */
Generated on Sun May 8 08:40:09 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3