psycho-mplayer2.C

00001 /*!@file AppPsycho/psycho-mplayertextsearch.C Movie to text search sequence.
00002    A set of movie clips and a text file with a set of questions and answers are given at the cmdline.  Each trial consists of a movie clip (played in mplayer with audio), followed by a question and a regular grid of answers.
00003 */
00004 
00005 // //////////////////////////////////////////////////////////////////// //
00006 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2001 by the //
00007 // University of Southern California (USC) and the iLab at USC.         //
00008 // See http://iLab.usc.edu for information about this project.          //
00009 // //////////////////////////////////////////////////////////////////// //
00010 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00011 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00012 // in Visual Environments, and Applications'' by Christof Koch and      //
00013 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00014 // pending; application number 09/912,225 filed July 23, 2001; see      //
00015 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00016 // //////////////////////////////////////////////////////////////////// //
00017 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00018 //                                                                      //
00019 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00020 // redistribute it and/or modify it under the terms of the GNU General  //
00021 // Public License as published by the Free Software Foundation; either  //
00022 // version 2 of the License, or (at your option) any later version.     //
00023 //                                                                      //
00024 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00025 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00026 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00027 // PURPOSE.  See the GNU General Public License for more details.       //
00028 //                                                                      //
00029 // You should have received a copy of the GNU General Public License    //
00030 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00031 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00032 // Boston, MA 02111-1307 USA.                                           //
00033 // //////////////////////////////////////////////////////////////////// //
00034 //
00035 // Primary maintainer for this file: John Shen <shenjohn@usc.edu>
00036 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/AppPsycho/psycho-mplayer2.C $
00037 
00038 #include "Component/ModelManager.H"
00039 #include "Image/Image.H"
00040 #include "Image/DrawOps.H"
00041 #include "Image/SimpleFont.H"
00042 #include "Psycho/PsychoDisplay.H"
00043 #include "Psycho/EyeTrackerConfigurator.H"
00044 #include "Psycho/EyeTracker.H"
00045 #include "Psycho/PsychoOpts.H"
00046 #include "Component/EventLog.H"
00047 #include "Component/ComponentOpts.H"
00048 #include "Util/Types.H"
00049 #include "Util/StringConversions.H"
00050 #include "Util/StringUtil.H"
00051 #include "Psycho/MPlayerWrapper.H"
00052 #include "Devices/SimpleLED.H"
00053 #include <fstream>
00054 
00055 #define HDEG 54.9
00056 
00057 typedef struct trial
00058 {
00059   std::string itsClip;
00060   std::string itsQuestion;
00061   std::vector<std::string> itsChoices;
00062   Image<PixRGB<byte> > itsQimage;
00063   Image<PixRGB<byte> > itsSimage;
00064   int itsFamily;
00065 } SearchTrial;
00066 
00067 // ######################################################################
00068 int submain(const int argc, char** argv)
00069 {
00070 
00071   // ********************************************************************
00072   // *** This portion initializes all the components ********************
00073   // ********************************************************************
00074 
00075   MYLOGVERB = LOG_INFO;  // suppress debug messages
00076 
00077   // Instantiate a ModelManager:
00078   ModelManager manager("Psycho Text");
00079 
00080   // Instantiate our various ModelComponents:
00081   nub::soft_ref<PsychoDisplay> d(new PsychoDisplay(manager));
00082   manager.addSubComponent(d);
00083 
00084   nub::soft_ref<EyeTrackerConfigurator>
00085     etc(new EyeTrackerConfigurator(manager));
00086   manager.addSubComponent(etc);
00087 
00088   nub::soft_ref<EventLog> el(new EventLog(manager));
00089   manager.addSubComponent(el);
00090 
00091   nub::soft_ref<MPlayerWrapper> player(new MPlayerWrapper(manager));
00092   manager.addSubComponent(player);
00093 
00094   nub::soft_ref<SimpleLED> recordLight(new SimpleLED(manager));
00095   manager.addSubComponent(recordLight);
00096 
00097   manager.setOptionValString(&OPT_EventLogFileName, "psychodata.psy");
00098   manager.setOptionValString(&OPT_EyeTrackerType, "ISCAN");
00099 
00100   // Parse command-line:
00101   if (manager.parseCommandLine(argc, argv,
00102                                "<textfile> visual-angle-of-single-character grid-rows grid-columns",
00103                                4, 4)==false)
00104     return(1);
00105 
00106   // create an image frame for each sentence in our text file and store
00107   // it in a vector before we start the experiment, then we can just
00108   // present each frame like in psycho still
00109 
00110   //First read the text file and all the sentences
00111   //load our file
00112   std::ifstream *itsFile;
00113   itsFile = new std::ifstream(manager.getExtraArg(0).c_str());
00114 
00115   //error if no file
00116   if (itsFile->is_open() == false)
00117     LFATAL("Cannot open '%s' for reading",manager.getExtraArg(0).c_str());
00118 
00119   //some storage variables
00120   std::string line;
00121   std::string clipstem = "";
00122   std::vector<SearchTrial> expt(100);
00123   uint num_trials = 0, num_stems = 0;
00124   std::vector<uint> curr_stem_index;
00125 
00126   //loop through lines of file
00127   while (!itsFile->eof())
00128     {
00129       getline(*itsFile, line, '\n');
00130 
00131       //store the sentence and type (question or statement)
00132         if (line[0] == '>')//video
00133         {
00134           line.erase(0,1);
00135           expt[num_trials].itsClip = line;
00136 
00137           //clip filename has format <stem>[a-z].avi
00138           if(line.compare(0,line.size()-5,clipstem) != 0) //new stem
00139             {
00140               num_stems++;
00141               clipstem = line.substr(0,line.size()-5);
00142               curr_stem_index.push_back(num_trials);
00143             }
00144           expt[num_trials].itsFamily = num_stems;
00145           num_trials++;
00146         }
00147         else if (line[0] == '#') //question, always one line
00148         {
00149           line.erase(0,1);
00150           expt[num_trials-1].itsQuestion = line;
00151         }
00152         else if (line[0] == '!') //choice, first choice
00153         {
00154           line.erase(0,1);
00155           expt[num_trials-1].itsChoices.push_back(line);
00156         }
00157         else if (line[0] == '&')//sub for a carriage return
00158         {
00159           //not handled yet
00160         }
00161         else //choice, subsequent choices
00162         {
00163           expt[num_trials-1].itsChoices.push_back(line);
00164         }
00165     }
00166   itsFile->close();
00167 
00168   //now we have stored all of our sentences, lets create our search images
00169   int w = d->getWidth();//width and height of SDL surface
00170   int h = d->getHeight();
00171 
00172   double fontsize = fromStr<double>(manager.getExtraArg(1));
00173   uint fontwidth = uint(fontsize * w / HDEG);
00174   SimpleFont fnt = SimpleFont::fixedMaxWidth(fontwidth); //font
00175 
00176   //store a grid of equally spaced coordinates in a gridrows x gridcols grid;
00177   const uint gridrows = fromStr<uint>(manager.getExtraArg(2));
00178   const uint gridcols = fromStr<uint>(manager.getExtraArg(3));
00179   const uint gridslots = gridrows*gridcols;
00180   std::vector<int> x_coords(gridslots);
00181   std::vector<int> y_coords(gridslots);
00182   for (uint i = 0; i < gridrows; i++)
00183     {
00184     for(uint j = 0; j < gridcols; j++)
00185       {
00186         x_coords[gridcols*i+j] = (int( double(w*(j+1)) / (gridcols+1)));
00187         y_coords[gridcols*i+j] = (int( double(h*(i+1)) / (gridrows+1)));
00188      }
00189     }
00190 
00191   Point2D<int> tanchor;
00192   for (uint i = 0; i < num_trials; i++)
00193   {
00194     int space = 0;
00195     int hanchor = int(h/2) - int(fnt.h()/2); //center character half a height behind
00196     expt[i].itsQimage.resize(w,h);
00197     expt[i].itsQimage.clear(d->getGrey());
00198 
00199     space = int( double(w - fnt.w() * expt[i].itsQuestion.size()) / 2.0 );
00200     tanchor = Point2D<int>(space, hanchor);
00201 
00202     writeText(expt[i].itsQimage,tanchor,expt[i].itsQuestion.c_str(),
00203                     PixRGB<byte>(0,0,0),
00204                     d->getGrey(),
00205                     fnt);
00206 
00207     expt[i].itsSimage.resize(w,h);
00208     expt[i].itsSimage.clear(d->getGrey());
00209     for (uint j = 0; j < expt[i].itsChoices.size(); j++)
00210     {
00211       //place each choice in its place
00212       if(j >= gridslots) //if there are too many choices
00213       {
00214         LDEBUG("Trial %d, clip %s: Too many answer choices for the grid", i, expt[i].itsClip.c_str());
00215         break;
00216       }
00217       space = x_coords[j] - int( double(fnt.w() * expt[i].itsChoices[j].length()) / 2.0);
00218       hanchor = y_coords[j] - int(fnt.h()/2);
00219       tanchor = Point2D<int>(space, hanchor);
00220 
00221       writeText(expt[i].itsSimage,tanchor,expt[i].itsChoices[j].c_str(),
00222                     PixRGB<byte>(0,0,0),
00223                     d->getGrey(),
00224                     fnt);
00225     }
00226   }
00227 
00228   // hook our various babies up and do post-command-line configs:
00229   nub::soft_ref<EyeTracker> et = etc->getET();
00230   d->setEyeTracker(et);
00231   d->setEventLog(el);
00232   et->setEventLog(el);
00233   player->setEventLog(el);
00234 
00235   // let's get all our ModelComponent instances started:
00236   manager.start();
00237 
00238   //turn the LED on
00239   recordLight->turnOn();
00240   el->pushEvent("===== pilot light on =====");
00241 
00242   // let's do an eye tracker calibration:
00243   et->calibrate(d);
00244 
00245   d->clearScreen();
00246   d->displayText("<space> for random order, other key for ordered play.");
00247   int c = d->waitForKey();
00248   d->clearScreen();
00249 
00250   int plan[num_trials];
00251   for (uint i = 0; i < num_trials; i++) plan[i] = i;
00252   if (c == ' ') //randomize while preserving order of dialogues
00253     {
00254       LINFO("Randomizing trials...");
00255 
00256       for(uint i = 0; i < num_trials; i++)
00257         plan[i] = expt[i].itsFamily-1;
00258       randShuffle(plan, num_trials);
00259 
00260       for(uint i = 0; i < num_trials; i++)
00261         plan[i]=curr_stem_index[plan[i]]++;
00262 
00263     }
00264 
00265   //turn the LED off
00266   recordLight->turnOff();
00267   el->pushEvent("===== pilot light off =====");
00268 
00269   std::string currvideo;
00270   // main loop:
00271   for (uint ii = 0; ii < num_trials; ii ++)
00272     {
00273                         currvideo = expt[plan[ii]].itsClip;
00274       //announce the video playing
00275       //next step - load clips & timings from text file instead of a prompt
00276       LDEBUG("Playing '%s'...",currvideo.c_str());
00277       player->setSourceVideo(currvideo);
00278 
00279       //give a chance to other processes (useful on single-CPU machines):
00280       sleep(1); if (system("/bin/sync")) LERROR("error in sunc()");
00281 
00282       //turn the LED off
00283       recordLight->turnOff();
00284       el->pushEvent("pilot light off");
00285 
00286       // display fixation to indicate that we are ready:
00287       d->displayFixation();
00288 
00289       // ready to go whenever the user is ready:
00290       d->waitForKey(true);
00291       d->displayFixationBlink();
00292 
00293       // start the eye tracker:
00294       et->track(true);
00295 
00296       //play the movie
00297       d->waitNextRequestedVsync(false, true);
00298       el->pushEvent(std::string("===== Playing movie: ") +
00299                                                                                 currvideo + " =====");
00300       player->runfromSDL(d);
00301 
00302       //turn the LED on
00303       recordLight->turnOn();
00304       el->pushEvent("pilot light on");
00305 
00306       et->track(false);
00307                         // load up the frame and show a fixation cross on a blank screen:
00308       d->clearScreen();
00309       d->displayFixation();
00310       Image<PixRGB<byte> > image;
00311 
00312       //start with the question
00313       image = expt[plan[ii]].itsQimage;
00314       SDL_Surface* surf = d->makeBlittableSurface(image, true);
00315       LINFO("question '%d' ready.", ii);
00316 
00317       // ready to go whenever the user is ready:
00318       d->waitForKey();
00319       d->waitNextRequestedVsync(false, true);
00320 
00321       d->pushEvent(std::string("===== Showing Question: question#") +
00322                      toStr<int>(plan[ii])  + " =====");
00323 
00324       et->track(true);
00325       // blink the fixation:
00326       d->displayFixationBlink();
00327 
00328       // show the image:
00329       d->displaySurface(surf, -2);
00330 
00331       // wait for key:
00332       c = d->waitForKey();
00333 
00334       // free the image:
00335       SDL_FreeSurface(surf);
00336 
00337       // make sure display is off before we stop the tracker:
00338       d->clearScreen();
00339 
00340       //next with the search answers
00341       image = expt[plan[ii]].itsSimage;
00342       surf = d->makeBlittableSurface(image, true);
00343 
00344       LINFO("sentence '%d' ready.", plan[ii]);
00345 
00346       d->displayFixation();
00347 
00348       // ready to go whenever the user is ready:
00349       d->waitForKey();
00350       d->waitNextRequestedVsync(false, true);
00351 
00352       d->pushEvent(std::string("===== Showing Answer Grid: answer#") +
00353                      toStr<int>(plan[ii])  + " =====");
00354 
00355       // blink the fixation:
00356       d->displayFixationBlink();
00357 
00358       // show the image:
00359       d->displaySurface(surf, -2);
00360 
00361       // wait for key:
00362       c = d->waitForKey();
00363 
00364       // free the image:
00365       SDL_FreeSurface(surf);
00366 
00367       // make sure display if off before we stop the tracker:
00368       d->clearScreen();
00369 
00370       // stop the eye tracker:
00371       usleep(50000);
00372       et->track(false);
00373 
00374       // every 15 trials let's do a quickie eye tracker recalibration:
00375                         et->recalibrate(d,15);
00376                         d->clearScreen();
00377     }
00378 
00379   d->clearScreen();
00380   d->displayText("Experiment complete. Thank you!");
00381   d->waitForKey();
00382 
00383 
00384   //turn the LED off (should be in LED->stop as well)
00385   recordLight->turnOff();
00386   el->pushEvent("===== pilot light off =====");
00387 
00388         // stop all our ModelComponents
00389   manager.stop();
00390 
00391   // all done!
00392   return 0;
00393 }
00394 
00395 extern "C" int main(const int argc, char** argv)
00396 {
00397   // simple wrapper around submain() to catch exceptions (because we
00398   // want to allow PsychoDisplay to shut down cleanly; otherwise if we
00399   // abort while SDL is in fullscreen mode, the X server won't return
00400   // to its original resolution)
00401   try
00402     {
00403       return submain(argc, argv);
00404     }
00405   catch (...)
00406     {
00407       REPORT_CURRENT_EXCEPTION;
00408     }
00409 
00410   return 1;
00411 }
00412 
00413 // ######################################################################
00414 /* So things look consistent in everyone's emacs... */
00415 /* Local Variables: */
00416 /* indent-tabs-mode: nil */
00417 /* End: */
Generated on Sun May 8 08:40:09 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3