app-chimpanzee.C

Go to the documentation of this file.
00001 /*!@file Demo/app-chimpanzee.C Saliency-driven chimpanzee head */
00002 
00003 // //////////////////////////////////////////////////////////////////// //
00004 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2001 by the //
00005 // University of Southern California (USC) and the iLab at USC.         //
00006 // See http://iLab.usc.edu for information about this project.          //
00007 // //////////////////////////////////////////////////////////////////// //
00008 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00009 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00010 // in Visual Environments, and Applications'' by Christof Koch and      //
00011 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00012 // pending; application number 09/912,225 filed July 23, 2001; see      //
00013 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00014 // //////////////////////////////////////////////////////////////////// //
00015 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00016 //                                                                      //
00017 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00018 // redistribute it and/or modify it under the terms of the GNU General  //
00019 // Public License as published by the Free Software Foundation; either  //
00020 // version 2 of the License, or (at your option) any later version.     //
00021 //                                                                      //
00022 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00023 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00024 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00025 // PURPOSE.  See the GNU General Public License for more details.       //
00026 //                                                                      //
00027 // You should have received a copy of the GNU General Public License    //
00028 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00029 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00030 // Boston, MA 02111-1307 USA.                                           //
00031 // //////////////////////////////////////////////////////////////////// //
00032 //
00033 // Primary maintainer for this file: Zack Gossman <gossman@usc.edu>
00034 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/Demo/app-chimpanzee.C $
00035 // $Id: app-chimpanzee.C 11630 2009-08-28 01:35:32Z dberg $
00036 //
00037 
00038 #include "Component/ModelManager.H"
00039 #include "Demo/SaliencyMT.H"
00040 #include "Devices/DeviceOpts.H"
00041 #include "Devices/FrameGrabberConfigurator.H"
00042 #include "GUI/XWindow.H"
00043 #include "Image/CutPaste.H"     // for inplacePaste()
00044 #include "Image/Image.H"
00045 #include "Image/Pixels.H"
00046 #include "Neuro/NeuroOpts.H"
00047 #include "Neuro/NeuroSimEvents.H"
00048 #include "Neuro/EyeHeadControllerConfigurator.H"
00049 #include "Neuro/SaccadeController.H"
00050 #include "Raster/Raster.H"
00051 #include "Simulation/SimEventQueue.H"
00052 #include "Simulation/SimEventQueueConfigurator.H"
00053 #include "Transport/FrameIstream.H"
00054 #include "Util/Timer.H"
00055 #include "Devices/BeoMonkey.H"
00056 
00057 #include <arpa/inet.h>
00058 #include <fcntl.h>
00059 #include <netdb.h>
00060 #include <signal.h>
00061 #include <stdlib.h>
00062 #include <unistd.h>
00063 
00064 #include "Image/Kernels.H"
00065 #include "Image/Convolver.H"
00066 
00067 //! Number of frames over which average framerate is computed
00068 #define NAVG 20
00069 
00070 #define EVEL 10  //should depend on distance
00071 #define HVEL 10   //should depend on distance
00072 //! Factor to display the sm values as greyscale:
00073 #define SMFAC 0.05F
00074 
00075 static bool goforever = true;  //!< Will turn false on interrupt signal
00076 
00077 //! Signal handler (e.g., for control-C)
00078 void terminate(int s)
00079 { LERROR("*** INTERRUPT ***"); goforever = false; }
00080 
00081 // ######################################################################
00082 int main(const int argc, const char **argv)
00083 {
00084   MYLOGVERB = LOG_INFO;
00085 
00086   // instantiate a model manager (for camera input):
00087   ModelManager manager("SaliencyMT Tester");
00088 
00089   // Instantiate our various ModelComponents:
00090   nub::soft_ref<SimEventQueueConfigurator>
00091     seqc(new SimEventQueueConfigurator(manager));
00092   manager.addSubComponent(seqc);
00093 
00094   nub::ref<FrameGrabberConfigurator>
00095     gbc(new FrameGrabberConfigurator(manager));
00096   manager.addSubComponent(gbc);
00097 
00098   nub::ref<SaliencyMT> smt(new SaliencyMT(manager));
00099   manager.addSubComponent(smt);
00100 
00101   nub::ref<EyeHeadControllerConfigurator>
00102     ehcc(new EyeHeadControllerConfigurator(manager));
00103   manager.addSubComponent(ehcc);
00104 
00105   nub::ref<BeoMonkey> bc(new BeoMonkey(manager));
00106   manager.addSubComponent(bc);
00107 
00108   // Set the appropriate defaults for our machine that is connected to
00109   // the chimpanzee robot head:
00110   manager.exportOptions(MC_RECURSE);
00111   manager.setOptionValString(&OPT_FrameGrabberType, "V4L");
00112   manager.setOptionValString(&OPT_FrameGrabberChannel, "1");
00113   manager.setOptionValString(&OPT_FrameGrabberHue, "0");
00114   manager.setOptionValString(&OPT_FrameGrabberContrast, "16384");
00115   manager.setOptionValString(&OPT_FrameGrabberDims, "320x240");
00116   //  manager.setOptionValString(&OPT_SaccadeControllerType, "Threshfric");
00117   manager.setOptionValString(&OPT_EyeHeadControllerType, "Monkey");
00118   manager.setOptionValString(&OPT_SaccadeControllerEyeType, "Monkey");
00119   manager.setOptionValString(&OPT_SaccadeControllerHeadType, "Monkey");
00120   manager.setOptionValString(&OPT_SCeyeMaxIdleSecs, "500.0");
00121   manager.setOptionValString(&OPT_SCeyeThreshMinOvert, "1.0");
00122   manager.setOptionValString(&OPT_SCeyeThreshMaxCovert, "3.0");
00123   manager.setOptionValString(&OPT_SCeyeThreshMinNum, "3");
00124   //  manager.setOptionValString(&OPT_SCeyeSpringK, "1000000.0");
00125 
00126   // Parse command-line:
00127   if (manager.parseCommandLine(argc, argv, "", 0, 0) == false) return(1);
00128 
00129   // do post-command-line configs:
00130   nub::soft_ref<SimEventQueue> seq = seqc->getQ();
00131 
00132   nub::soft_ref<FrameIstream> gb = gbc->getFrameGrabber();
00133   if (gb.isInvalid())
00134     LFATAL("You need to select a frame grabber type via the "
00135            "--fg-type=XX command-line option for this program "
00136            "to be useful");
00137   const int w = gb->getWidth(), h = gb->getHeight();
00138 
00139   nub::ref<EyeHeadController> ehc = ehcc->getEHC();
00140 
00141   const int foa_size = std::min(w, h) / 12;
00142   manager.setModelParamVal("InputFrameDims", Dims(w, h),
00143                            MC_RECURSE | MC_IGNORE_MISSING);
00144   manager.setModelParamVal("SCeyeStartAtIP", true,
00145                            MC_RECURSE | MC_IGNORE_MISSING);
00146   manager.setModelParamVal("SCeyeInitialPosition", Point2D<int>(w/2,h/2),
00147                            MC_RECURSE | MC_IGNORE_MISSING);
00148   manager.setModelParamVal("FOAradius", foa_size,
00149                            MC_RECURSE | MC_IGNORE_MISSING);
00150   manager.setModelParamVal("FoveaRadius", foa_size,
00151                            MC_RECURSE | MC_IGNORE_MISSING);
00152 
00153   // catch signals and redirect them to terminate for clean exit:
00154   signal(SIGHUP, terminate); signal(SIGINT, terminate);
00155   signal(SIGQUIT, terminate); signal(SIGTERM, terminate);
00156   signal(SIGALRM, terminate);
00157 
00158   // get prepared to grab, move, display, etc:
00159   uint frame = 0U;                  // count the frames
00160   uint lastframe = 0U;              // last frame sent for processing
00161   Point2D<int> lastpoint(w/2, h/2);      // last point of fixation
00162 
00163   uint64 avgtime = 0; int avgn = 0; // for average framerate
00164   float fps = 0.0F;                 // to display framerate
00165   Timer tim;                        // for computation of framerate
00166   Timer masterclock;                // master clock for simulations
00167 
00168   int sml = 3;                      // pyramid level of saliency map
00169   Image<float> sm(w >> sml, h >> sml, ZEROS); // saliency map
00170   Point2D<int> fixation(-1, -1);         // coordinates of eye fixation
00171 
00172   // image buffer for display:
00173   Image<PixRGB<byte> > disp(w * 2, h + 20, ZEROS);
00174   disp += PixRGB<byte>(128);
00175   XWindow xwin(disp.getDims(), -1, -1, "USC Chimpanzee Demo");
00176 
00177   char info[1000];  // general text buffer for various info messages
00178 
00179 
00180   // ######################################################################
00181   try {
00182     // let's do it!
00183     manager.start();
00184 
00185     // get the frame grabber to start streaming:
00186     gb->startStream();
00187 
00188     // initialize the timers:
00189     tim.reset(); masterclock.reset();
00190 
00191     //set up a convoltion mask
00192     //  Image<float> gauss = gaussian<float>(0.0F,25.0F,0);
00193     //  Convolver cg(gauss,gauss.getDims());
00194 
00195     while(goforever)
00196       {
00197         // grab image:
00198         Image< PixRGB<byte> > ima = gb->readRGB();
00199 
00200         // display image:
00201         inplacePaste(disp, ima, Point2D<int>(0, 0));
00202         Image<float> dispsm = sm * SMFAC;
00203         inplacePaste(disp, Image<PixRGB<byte> >
00204                      (toRGB(quickInterpolate(dispsm, 1 << sml))),
00205                      Point2D<int>(w, 0));
00206 
00207         //        ehc->evolve(*seq);
00208 
00209         Point2D<int> ceye(-1, -1), chead(-1, -1);
00210 
00211         if (SeC<SimEventSaccadeStatusEye> e =
00212             seq->check<SimEventSaccadeStatusEye>(0)) ceye = e->position();
00213         if (SeC<SimEventSaccadeStatusHead> e =
00214             seq->check<SimEventSaccadeStatusHead>(0)) chead = e->position();
00215 
00216         if (ceye.isValid()) fixation = ceye;
00217         Point2D<int> fix2(fixation); fix2.i += w;
00218         if (fixation.i >= 0)
00219           {
00220             drawDisk(disp, fixation, foa_size/6+2, PixRGB<byte>(20, 50, 255));
00221             drawDisk(disp, fixation, foa_size/6, PixRGB<byte>(255, 255, 20));
00222             drawDisk(disp, fix2, foa_size/6+2, PixRGB<byte>(20, 50, 255));
00223             drawDisk(disp, fix2, foa_size/6, PixRGB<byte>(255, 255, 20));
00224           }
00225 
00226         xwin.drawImage(disp);
00227 
00228         // are we ready to process a new frame? if so, send our new one:
00229         if (smt->outputReady())
00230           {
00231             // let's get the previous results, if any:
00232             Image<float> out = smt->getOutput();
00233             if (out.initialized()) sm = out;
00234 
00235             //Lets blur the saliency map a little
00236             // sm = cg.fftConvolve(sm);
00237 
00238             // find most salient location and feed saccade controller:
00239             float maxval; Point2D<int> currwin; findMax(sm, currwin, maxval);
00240             WTAwinner newwin =
00241               WTAwinner::buildFromSMcoords(currwin, sml, true,
00242                                            masterclock.getSimTime(),
00243                                            maxval, false);
00244             if (newwin.isValid())
00245               {
00246                 //      rutz::shared_ptr<SimEventWTAwinner>
00247                 //   e(new SimEventWTAwinner(0, newwin));
00248                 // seq->post(e);
00249               }
00250 
00251             // feed our current image as next one to process:
00252             smt->newInput(decXY(ima));
00253             lastframe = frame;
00254             lastpoint.i = newwin.p.i * 2; lastpoint.j = newwin.p.j * 2;
00255             //LINFO("Processing frame %u", frame);
00256           }
00257 
00258         // compute and show framerate and stats over the last NAVG frames:
00259         avgtime += tim.getReset(); avgn ++;
00260         if (avgn == NAVG)
00261           {
00262             fps = 1000.0F / float(avgtime) * float(avgn);
00263             avgtime = 0; avgn = 0;
00264           }
00265 
00266 
00267        // float x = ((float)lastpoint.i/640)*2 -1;
00268        // float y = ((float)lastpoint.j/480)*2 -1;
00269        float xeye = (float)ceye.i;
00270        float yeye = (float)ceye.j;
00271        float xhead = (float)chead.i;
00272        float yhead = (float)chead.j;
00273 
00274         // create an info string:
00275         sprintf(info, "USC Chimpanzee - %06u / %06u - [%03f %03f] - %.1ffps ",
00276                 frame, lastframe, xeye, yeye, fps);
00277 
00278         writeText(disp, Point2D<int>(0, h), info,
00279                   PixRGB<byte>(255), PixRGB<byte>(127));
00280 
00281         //make a movement
00282         if (bc->isQueEmpty())
00283           {
00284             LINFO("%f %f/n",xhead,yhead);
00285             if ((xeye != -1) & (yeye != -1))
00286               {
00287                 xeye = -1*((xeye/320)*2-1);
00288                 yeye = -1*((yeye/240)*2-1);
00289                 std::deque<Position> p = bc->blend(bc->getPathServo(BeoMonkey::H_EYE,xeye,EVEL),bc->getPathServo(BeoMonkey::V_EYE,yeye,EVEL));
00290                 bc-> addSequence(p);
00291               }
00292 
00293             if ((xhead != -1) & (yhead != -1))
00294               {
00295                 xhead = -1*((xhead/320)*2-1);
00296                 yhead = -1*((yhead/240)*2-1);
00297                 std::deque<Position> p = bc->blend(bc->getPathServo(BeoMonkey::H_HEAD,xhead,HVEL),bc->getPathServo(BeoMonkey::V_HEAD,yhead,HVEL));
00298                 bc-> addSequence(p);
00299               }
00300 
00301           }
00302            bc->nextTimeStep();
00303 
00304         // ready for next frame:
00305         ++ frame;
00306         while(seq->now() < masterclock.getSimTime()) seq->evolve();
00307       }
00308 
00309     // get ready to terminate:
00310     manager.stop();
00311 
00312   } catch ( ... ) { };
00313 
00314   return 0;
00315 }
00316 
00317 // ######################################################################
00318 /* So things look consistent in everyone's emacs... */
00319 /* Local Variables: */
00320 /* indent-tabs-mode: nil */
00321 /* End: */
Generated on Sun May 8 08:40:37 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3