psycho-foe.C

00001 /*!@file AppPsycho/psycho-search.C Psychophysics display for a search for a
00002   target that is presented to the observer prior to the search */
00003 // //////////////////////////////////////////////////////////////////// //
00004 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2001 by the //
00005 // University of Southern California (USC) and the iLab at USC.         //
00006 // See http://iLab.usc.edu for information about this project.          //
00007 // //////////////////////////////////////////////////////////////////// //
00008 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00009 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00010 // in Visual Environments, and Applications'' by Christof Koch and      //
00011 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00012 // pending; application number 09/912,225 filed July 23, 2001; see      //
00013 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00014 // //////////////////////////////////////////////////////////////////// //
00015 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00016 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00017 // redistribute it and/or modify it under the terms of the GNU General  //
00018 // Public License as published by the Free Software Foundation; either  //
00019 // version 2 of the License, or (at your option) any later version.     //
00020 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00021 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00022 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00023 // PURPOSE.  See the GNU General Public License for more details.       //
00024 //                                                                      //
00025 // You should have received a copy of the GNU General Public License    //
00026 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00027 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00028 // Boston, MA 02111-1307 USA.                                           //
00029 // //////////////////////////////////////////////////////////////////// //
00030 // Primary maintainer for this file: Laurent Itti <itti@usc.edu>
00031 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/AppPsycho/psycho-foe.C $
00032 // $Id: psycho-foe.C 12962 2010-03-06 02:13:53Z irock $
00033 
00034 #include "Component/ModelManager.H"
00035 #include "Image/ColorOps.H" // for makeRGB()
00036 #include "Image/CutPaste.H" // for inplacePaste()
00037 #include "Image/Image.H"
00038 #include "Image/MathOps.H"  // for inplaceSpeckleNoise()
00039 #include "Psycho/PsychoDisplay.H"
00040 #include "Psycho/EyeTrackerConfigurator.H"
00041 #include "Psycho/EyeTracker.H"
00042 #include "Psycho/PsychoOpts.H"
00043 #include "Component/EventLog.H"
00044 #include "Component/ComponentOpts.H"
00045 #include "Raster/Raster.H"
00046 #include "Util/MathFunctions.H"
00047 #include "Util/StringUtil.H"
00048 #include "Util/Types.H"
00049 #include "Video/VideoFrame.H"
00050 #include "rutz/time.h"
00051 #include "Raster/Raster.H"
00052 #include <deque>
00053 #include "Media/MPEGStream.H"
00054 #include "Media/MediaOpts.H"
00055 
00056 #include <ctype.h>
00057 #include <vector>
00058 #include <string>
00059 #include <fstream>
00060 #include <sstream>
00061 #include <iostream>
00062 
00063 #define CACHELEN 150
00064 #define NMASK 10
00065 
00066 using namespace std;
00067 //! number of frames in the mask
00068 
00069 // ######################################################################
00070 static bool cacheFrame(nub::soft_ref<InputMPEGStream>& mp,
00071                        std::deque<VideoFrame>& cache)
00072 {
00073   const VideoFrame frame = mp->readVideoFrame();
00074   if (!frame.initialized()) return false; // end of stream
00075   cache.push_front(frame);
00076   return true;
00077 }
00078 
00079 static int submain(const int argc, char** argv)
00080 {
00081   MYLOGVERB = LOG_INFO;  // suppress debug messages
00082   ModelManager manager("Psycho Movie");
00083   nub::soft_ref<PsychoDisplay> d(new PsychoDisplay(manager));
00084   manager.addSubComponent(d);
00085   nub::soft_ref<EyeTrackerConfigurator> etc(new EyeTrackerConfigurator(manager));
00086   manager.addSubComponent(etc);
00087   nub::soft_ref<EventLog> el(new EventLog(manager));
00088   manager.addSubComponent(el);
00089   nub::soft_ref<InputMPEGStream> mp (new InputMPEGStream(manager, "Input MPEG Stream", "InputMPEGStream"));
00090   manager.addSubComponent(mp);
00091   manager.setOptionValString(&OPT_InputMPEGStreamPreload, "true");
00092 
00093   // Parse command-line:
00094   // Subject - MaleOrFemale -  pilot - Experiment Number - Run Number - Movie Type
00095   if (manager.parseCommandLine(argc, argv," We have 6 args:  Subject - MaleOrFemale -  pilot - Experiment Number - Run Number - Movie Type ", 6, 6)==false){
00096         cout<<"please give the following arguments:\n 1)Subject-Initials 2)M/F 3)Is-Pilot(1 or 0) 4)ExpSetNumber 5)Run-Number 6)movie type(1,2,3)"<< endl;
00097              return(1);
00098         }
00099         string Sub=manager.getExtraArg(0);
00100         string Gender=manager.getExtraArg(1);
00101         string pilot=manager.getExtraArg(2);
00102         int isPilot=(int)pilot[0]-48;//
00103                 cout<<isPilot<<"is a pilot experiment"<<endl;
00104         string experiment_set_number=manager.getExtraArg(3);
00105         int experiment_set_no=(int)experiment_set_number[0]-48;//atoi(PT[0].c_str());
00106                 cout<<"experiment set number is:"<<experiment_set_no<<endl;
00107         string Run=manager.getExtraArg(4);
00108         string DataDir="/home2/tmp/u/elno/research/exp1/data/";
00109         string psyFileName=DataDir + Sub + "_" + Gender + "__" + pilot +"__Set_Number--"+ experiment_set_number+"__RUN--"+Run;
00110         string movie_type=manager.getExtraArg(5);
00111         //int bg_type=(int)movie_type[0]-48;//atoi(PT[0].c_str());
00112 
00113         manager.setOptionValString(&OPT_EventLogFileName, psyFileName);
00114         manager.setOptionValString(&OPT_EyeTrackerType, "ISCAN");
00115 
00116         // Push Events for this one :-)
00117         d->pushEvent(std::string("Run=") + Run + "\nSubject=" + Sub +"\nGender="+ Gender +"\n" );
00118         d->pushEvent(std::string("Set=") + experiment_set_number + "\n");
00119         d->pushEvent(std::string("MovieType=") + movie_type + "\n");
00120         if (isPilot) {d->pushEvent("Piloting");}
00121         else {d->pushEvent("Control");}
00122 
00123         nub::soft_ref<EyeTracker> et = etc->getET();
00124         d->setEyeTracker(et);
00125         d->setEventLog(el);
00126         et->setEventLog(el);
00127 
00128         if (etc->getModelParamString("EyeTrackerType").compare("EL") == 0)
00129         d->setModelParamVal("SDLslaveMode", true);
00130         manager.start();
00131 
00132 
00133 // Getting the foreground images from the set we needed
00134 
00135    string imagelisthomeDir="/home2/tmp/u/elno/research/exp1/stim/fgImages";
00136    string imagelistsetFile=imagelisthomeDir+"/set"+experiment_set_number.c_str()+"/imagelist";
00137 
00138           cout<<"Trying to open imagelist : "<<imagelistsetFile<< endl;
00139 
00140   FILE *f = fopen(imagelistsetFile.c_str(), "r");
00141   if (f == NULL) LFATAL("Cannot read stimulus file");
00142   char line[1024];
00143   std::vector<std::string> ilist, tlist, rlist, slist, vlist;
00144 
00145 // Get the FOE movie directory
00146         string foeDir="/home2/tmp/u/elno/research/exp1/stim/bgMovies/foe/";
00147         for (int foe=1; foe<6; foe++){
00148                 std::string s;
00149                 std::stringstream out;
00150                 s="";
00151                 out << foe;
00152                 s = out.str();
00153                 string foeFileName=foeDir + s + "/";
00154                 vlist.push_back(foeFileName); // FOE movies back ground
00155         }
00156 
00157       LINFO("\n video : %s,",vlist[0].c_str());
00158       LINFO("\n video : %s,",vlist[1].c_str());
00159       LINFO("\n video : %s,",vlist[2].c_str());
00160       LINFO("\n video : %s,",vlist[3].c_str());
00161       LINFO("\n video : %s,",vlist[4].c_str());
00162 
00163   int correct = 0, accuracy = 100, total = 0;
00164   while(fgets(line, 1024, f)){
00165         std::vector<std::string> tokens;
00166               LINFO("line reads %s",line);
00167               split(line," ", std::back_inserter(tokens));
00168       // now line is at the imagelet, line2 at the image
00169       tlist.push_back(std::string(tokens[0])); // ONLY THE TARGET
00170       ilist.push_back(std::string(tokens[1])); // IMAGE WITH TARGET
00171       rlist.push_back(std::string(tokens[2])); // Image to report Position of the target
00172       slist.push_back(std::string(tokens[3])); // spec file for the search array
00173 
00174       LINFO("\nNew pair \nline1 reads: %s, \nline2 reads:%s, \nline 3 reads %s,\nline 4 reads %s",tokens[0].c_str(), tokens[1].c_str(),tokens[2].c_str(), tokens[3].c_str());
00175     }
00176   fclose(f);
00177   // randomize stimulus presentation order:
00178   int nimg = ilist.size(); int imindex[nimg];
00179   for (int i = 0; i < nimg; i ++) imindex[i] = i;
00180        randShuffle(imindex, nimg);
00181 
00182   int n_vids = ilist.size();
00183   int vid_index[n_vids];
00184   for (int i = 0; i < n_vids; i ++) vid_index[i] = i;
00185        randShuffle(vid_index, n_vids);
00186 
00187   for (int i = 0; i < n_vids; i ++)
00188         vid_index[i] = vid_index[i]%5;
00189   // let's display an ISCAN calibration grid:
00190   d->clearScreen();
00191   d->displayISCANcalib();
00192   d->waitForKey();
00193   // let's do an eye tracker calibration:
00194   d->displayText("<SPACE> to calibrate; other key to skip");
00195 
00196   int c = d->waitForKey();
00197   if (c == ' ') d->displayEyeTrackerCalibration(3, 3);
00198 
00199   //  if (c == ' ') d->displayEyeTrackerCalibration(3, 3);
00200 /*
00201   if (c == ' ')
00202     {
00203       et->track(true);
00204       et->calibrateOnline(d);
00205       et->track(false);
00206     }
00207 */
00208 
00209   // we are ready to start:
00210   d->clearScreen();
00211   d->displayText("<SPACE> to start experiment");
00212   d->waitForKey();
00213   //******************* main loop:***********************//
00214   for (int trial = 0; trial < nimg; trial++) {
00215         // Trial information push down
00216                 std::string stt;
00217                 std::stringstream outt;
00218                 outt << trial;
00219                 stt = outt.str();
00220                 d->pushEvent(std::string("===== set number / trial number target image:  ") + experiment_set_number + std::string(" === ")+ stt + std::string(" ====="));
00221 
00222         int imnum = imindex[trial];
00223              int v_num = vid_index[trial];
00224 
00225                 LINFO("Loading array image: '%s' ... /target: '%s' .../report: '%s'... /video: '%s' ... /spec: '%s'   ", ilist[imnum].c_str(), tlist[imnum].c_str(), rlist[imnum].c_str(), vlist[v_num].c_str(), slist[imnum].c_str());
00226 
00227         // load up the images and show a fixation cross on a blank screen:
00228         // get the imagelet and place it at a random position:
00229                 if(!Raster::fileExists(tlist[imnum]))
00230                 {
00231                         manager.stop();
00232                         LFATAL("i couldnt find target image file %s", tlist[imnum].c_str());
00233                 }
00234         //Get target image
00235         Image< PixRGB<byte> > img = Raster::ReadRGB(tlist[imnum]);
00236         Image< PixRGB<byte> > rndimg(d->getDims(), NO_INIT);
00237         rndimg.clear(d->getGrey());
00238         int rndx = 0,rndy = 0;
00239         SDL_Surface *surf1 = d->makeBlittableSurface(img, true);
00240         char buf[256];
00241                 if(!Raster::fileExists(ilist[imnum]))
00242                 {
00243                         manager.stop();
00244                         LFATAL("i couldnt find array image file %s", ilist[imnum].c_str());
00245                 }
00246         img = Raster::ReadRGB(ilist[imnum]);
00247         SDL_Surface *surf2 = d->makeBlittableSurface(img, true);
00248         //randShuffle(mindex, NMASK);
00249         // load up the reporting number image:
00250                 if(!Raster::fileExists(rlist[imnum]))
00251                 {
00252                         manager.stop();
00253                         LFATAL(" i couldnt find report image file %s", rlist[imnum].c_str());
00254                 }
00255 
00256         img = Raster::ReadRGB(rlist[imnum]);
00257         SDL_Surface *surf3 = d->makeBlittableSurface(img, true);
00258 
00259         // give a chance to other processes if single-CPU:
00260     d->clearScreen();
00261     usleep(200000);
00262     // ready to go whenever the user is ready:
00263     d->displayFixationBlink();
00264     //d->waitForKey();
00265     d->waitNextRequestedVsync(false, true);
00266 
00267     //************************ Display target************************//
00268     sprintf(buf, "===== Showing imagelet: %s at (%d, %d) =====", tlist[imnum].c_str(), rndx, rndy);
00269     d->pushEvent(std::string("===== Showing target image: ") + tlist[imnum] + std::string(" ====="));
00270     d->pushEvent(buf);
00271  if (trial<5){
00272             d->displayText("Find this target among others:");
00273             usleep(2000000);
00274 }
00275     d->displaySurface(surf1, 0, true);
00276     usleep(3000000);
00277     d->clearScreen();
00278     //************************ Display array************************//
00279       LINFO("Buffering foe video '%s'...",(vlist[v_num]).c_str());
00280      // LINFO("Loading %s", manager.getExtraArg(ilist[imnum]).c_str());
00281       Image<PixRGB<byte> > cimage = Raster::ReadRGB(ilist[imnum]);
00282       //transparent pixel should be in upper left hand corner
00283       PixRGB<byte> currTransPixel = cimage[0];
00284       // cache initial movie frames:
00285       bool streaming = true;
00286       LINFO("Buffering '%s'...",(vlist[v_num]).c_str());
00287       int video_index= int(9.0 * randomDouble())+1;
00288         std::string sv="";
00289         std::stringstream outv;
00290         outv << video_index;
00291         sv = outv.str();
00292         sv=vlist[v_num].c_str()+movie_type+"-"+sv+".mpg";
00293       mp->setFileName(sv.c_str());
00294       std::deque<VideoFrame> cache;
00295       for (uint j = 0; j < CACHELEN; j ++)
00296         {
00297           streaming = cacheFrame(mp, cache);
00298           if (streaming == false) break;  // all movie frames got cached!
00299         }
00300       // give a chance to other processes (useful on single-CPU machines):
00301       sleep(1); system("/bin/sync");
00302       // display fixation to indicate that we are ready:
00303       d->displayFixation();
00304       // ready to go whenever the user is ready:
00305       d->waitForKey(true); int frame = 0;
00306       d->waitNextRequestedVsync(false, true);
00307       d->pushEvent(std::string("===== Playing movie: ") + sv.c_str() + " =====");
00308       // start the eye tracker:
00309       et->track(true);
00310       // blink the fixation:
00311       d->displayFixationBlink();
00312       // show the image:
00313       d->pushEvent(std::string("===== Showing search image: ") + ilist[imnum] + std::string(" ====="));
00314       // create an overlay:
00315       d->createVideoOverlay(VIDFMT_YUV420P,mp->getWidth(),mp->getHeight()); // mpeg stream returns YUV420P
00316      // play the movie:
00317 
00318       rutz::time start = rutz::time::wall_clock_now();
00319 
00320     Timer timer(100);
00321     timer.reset();
00322     double startTime=timer.getSecs();
00323     double EndTime=timer.getSecs();
00324 
00325     // wait for key; it will record reaction time in the logs:
00326 
00327     bool timeUp=false;
00328 
00329       while(cache.size() && d->checkForKey()<0 &&  !timeUp ) {
00330         if(timer.getSecs() > startTime +10)
00331         {
00332           timeUp=true;
00333           d->pushEvent(std::string("===== Time Up ====="));
00334         }
00335           if (streaming) streaming = cacheFrame(mp, cache);
00336           VideoFrame vidframe = cache.back();
00337           d->displayVideoOverlay_image(vidframe, frame,
00338                                        SDLdisplay::NEXT_VSYNC,
00339                                        cimage, currTransPixel,4);
00340           cache.pop_back();
00341           ++frame;
00342 
00343         EndTime=timer.getSecs();
00344       }
00345 
00346       rutz::time stop = rutz::time::wall_clock_now();
00347       const double secs = (stop-start).sec();
00348       const double responsetime = EndTime - startTime;
00349 
00350         std::string stdob;
00351         std::stringstream outdob;
00352         outdob << secs;
00353         stdob = outdob.str();
00354 
00355         std::string stdob2;
00356         std::stringstream outdob2;
00357         outdob2 << responsetime;
00358         stdob2 = outdob2.str();
00359 
00360 
00361       d->pushEvent(std::string("==== Total Time:  ") + stdob + std::string(" =====") );
00362       d->pushEvent(std::string("==== Response Time:  ") + stdob2 + std::string(" =====") );
00363 
00364       LINFO("%d frames in %.02f sec (~%.02f fps)", frame, secs, frame/secs);
00365       d->destroyYUVoverlay();
00366 
00367       et->track(false); //we just want to record eye movements while subjects view the array   // stop the eye tracker:
00368       d->clearScreen();  // sometimes 2 clearScreen() are necessary
00369       d->clearScreen();
00370       usleep(2000000);
00371       d->displayText("Use this image to report target's position: ");
00372       usleep(2000000);
00373     //d->waitForKey();
00374     //*********************** Display reporting image**************//
00375     // show the reporting image:
00376     d->pushEvent(std::string("===== Showing reporting image: ") + rlist[imnum] + std::string(" ====="));
00377     d->displaySurface(surf3, 0, true);
00378     usleep(3000000);
00379     usleep(5000);
00380     d->displayText("Input the target number: \n\n\n [press enter]");
00381     string inputString = d->getString('\n');
00382     //check user response
00383     char tmp[40];
00384     //lets open the spec file and extract the target number
00385 
00386 
00387         string buffer=slist[imnum].c_str();
00388         int s = buffer.size();
00389         string::iterator it;
00390         for(int x = 0; x < s ; x++)
00391         {
00392                 if(buffer.at(x) == '\n')
00393                 {
00394                         it = buffer.begin() + x;
00395                         buffer.erase(it);
00396                         s--;                                                        // Correction of string length
00397                 }
00398         }
00399 
00400 
00401     ifstream specFile(buffer.c_str(), ifstream::in);
00402     bool found =false;
00403     string testLine;
00404     std::vector<std::string> specTokens;
00405 
00406    if(specFile.is_open())
00407       {
00408         while(!specFile.eof() && !found)
00409           {
00410             getline(specFile, testLine);
00411             string::size_type loc = testLine.find("target", 0);
00412 
00413             if(loc != string::npos)
00414                found = true;
00415           }
00416         if(!found)
00417           {
00418             manager.stop();
00419             LFATAL("couldnt find the target number from spec file");
00420           }
00421 
00422         split(testLine," ", std::back_inserter(specTokens));
00423 
00424         std::string responseString;
00425         responseString = inputString;
00426         int intResponse = atoi(responseString.c_str()), intActual = atoi(specTokens[1].c_str());
00427 
00428         total++;
00429 
00430         if (intResponse == intActual)
00431             {
00432               correct ++;
00433               accuracy = correct * 100 / total;
00434               sprintf(tmp, "Correct! Accuracy is %d%%", accuracy);
00435               d->displayText(tmp);
00436               d->pushEvent(std::string("===== Correct ====="));
00437               usleep(500000);
00438             }
00439           else
00440             {
00441               accuracy = correct * 100 / total;
00442               sprintf(tmp, "Wrong! Accuracy is %d%%", accuracy);
00443               d->displayText(tmp);
00444               d->pushEvent(std::string("===== Wrong ====="));
00445               usleep(500000);
00446             }
00447 
00448         specFile.close();
00449       }
00450     else
00451       {
00452         d->displayText("no target file found!");
00453         LFATAL("couldnt open the file -%s-",slist[imnum].c_str());
00454       }
00455 
00456     // free the imagelet and image:
00457     SDL_FreeSurface(surf1); SDL_FreeSurface(surf2); SDL_FreeSurface(surf3);
00458 
00459     // let's do a quiinckie eye tracker calibration once in a while:
00460     /*if (im > 0 && im % 20 == 0) {
00461       d->displayText("Ready for quick recalibration");
00462       d->waitForKey();
00463       d->displayEyeTrackerCalibration(3, 3);
00464       d->clearScreen();
00465       d->displayText("Ready to continue with the images");
00466       d->waitForKey();
00467       }*/
00468 
00469     //allow for a break after 50 trials then recalibrate
00470 
00471     if (trial==50)
00472       {
00473         d->displayText("You may take a break press space to continue when ready");
00474         d->waitForKey();
00475         // let's display an ISCAN calibration grid:
00476         d->clearScreen();
00477         d->displayISCANcalib();
00478         d->waitForKey();
00479         // let's do an eye tracker calibration:
00480         d ->displayText("<SPACE> to calibrate; other key to skip");
00481         int c = d->waitForKey();
00482         if (c == ' ') d->displayEyeTrackerCalibration(3, 3);
00483      }
00484     et->recalibrate(d,15);
00485   }
00486 
00487   d->clearScreen();
00488   d->displayText("Experiment complete. Thank you!");
00489   d->waitForKey();
00490 
00491   // stop all our ModelComponents
00492   manager.stop();
00493 
00494   // all done!
00495   return 0;
00496 }
00497 
00498 // ######################################################################
00499 
00500 extern "C" int main(const int argc, char** argv)
00501 {
00502   // simple wrapper around submain() to catch exceptions (because we
00503   // want to allow PsychoDisplay to shut down cleanly; otherwise if we
00504   // abort while SDL is in fullscreen mode, the X server won't return
00505   // to its original resolution)
00506   try
00507     {
00508       return submain(argc, argv);
00509     }
00510   catch (...)
00511     {
00512       REPORT_CURRENT_EXCEPTION;
00513     }
00514   return 1;
00515 }
00516 // ######################################################################
00517 /* So things look consistent in everyone's emacs... */
00518 /* Local Variables: */
00519 /* indent-tabs-mode: nil */
00520 /* End: */
Generated on Sun May 8 08:40:08 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3