beobot-Gist-Sal-Nav-master.C

Go to the documentation of this file.
00001 /*!@file Beobot/beobot-Gist-Sal-Nav-master.C Robot navigation using a
00002   combination saliency and gist.
00003   Run beobot-Gist-Sal-Nav-master at A to do vision
00004   Run beobot-Gist-Sal-Nav        at B to move                           */
00005 
00006 // //////////////////////////////////////////////////////////////////// //
00007 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2001 by the //
00008 // University of Southern California (USC) and the iLab at USC.         //
00009 // See http://iLab.usc.edu for information about this project.          //
00010 // //////////////////////////////////////////////////////////////////// //
00011 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00012 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00013 // in Visual Environments, and Applications'' by Christof Koch and      //
00014 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00015 // pending; application number 09/912,225 filed July 23, 2001; see      //
00016 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00017 // //////////////////////////////////////////////////////////////////// //
00018 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00019 //                                                                      //
00020 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00021 // redistribute it and/or modify it under the terms of the GNU General  //
00022 // Public License as published by the Free Software Foundation; either  //
00023 // version 2 of the License, or (at your option) any later version.     //
00024 //                                                                      //
00025 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00026 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00027 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00028 // PURPOSE.  See the GNU General Public License for more details.       //
00029 //                                                                      //
00030 // You should have received a copy of the GNU General Public License    //
00031 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00032 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00033 // Boston, MA 02111-1307 USA.                                           //
00034 // //////////////////////////////////////////////////////////////////// //
00035 //
00036 // Primary maintainer for this file: Christian Siagian <siagian@usc.edu>
00037 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/Beobot/beobot-Gist-Sal-Nav-master.C $
00038 // $Id: beobot-Gist-Sal-Nav-master.C 10982 2009-03-05 05:11:22Z itti $
00039 //
00040 ////////////////////////////////////////////////////////
00041 // beobot-Gist-Sal-Nav-Master.C <input_train.txt>
00042 //
00043 // a version of Gist/test-Gist-Sal-Nav.C to be run on the Beobot
00044 //
00045 
00046 #include "Channels/ChannelOpts.H"
00047 #include "Component/GlobalOpts.H"
00048 #include "Component/ModelManager.H"
00049 #include "Component/ModelOptionDef.H"
00050 #include "Devices/FrameGrabberConfigurator.H"
00051 #include "GUI/XWinManaged.H"
00052 #include "Gist/FFN.H"
00053 #include "Gist/trainUtils.H"
00054 #include "Image/ColorOps.H"
00055 #include "Image/CutPaste.H"
00056 #include "Image/DrawOps.H"
00057 #include "Image/ImageCache.H"
00058 #include "Image/MathOps.H"
00059 #include "Image/MatrixOps.H"
00060 #include "Image/Pixels.H"
00061 #include "Image/Transforms.H"
00062 #include "Media/MPEGStream.H"
00063 #include "Media/MediaOpts.H"
00064 #include "Media/MediaSimEvents.H"
00065 #include "Neuro/GistEstimatorStd.H"
00066 #include "Neuro/NeuroOpts.H"
00067 #include "Neuro/NeuroSimEvents.H"
00068 #include "Neuro/ShapeEstimator.H"
00069 #include "Neuro/ShapeEstimatorModes.H"
00070 #include "Neuro/SpatialMetrics.H"
00071 #include "Neuro/StdBrain.H"
00072 #include "Neuro/gistParams.H"
00073 #include "Raster/Raster.H"
00074 #include "SIFT/Histogram.H"
00075 #include "Transport/FrameIstream.H"
00076 #include "Util/Timer.H"
00077 #include "Beobot/GridMap.H"
00078 //#include "Beobot/TopologicalMap.H"
00079 #include "Simulation/SimEventQueueConfigurator.H"
00080 
00081 
00082 #define W_ASPECT_RATIO  320 // ideal minimum width for display
00083 #define H_ASPECT_RATIO  240 // ideal minimum height for display
00084 
00085 CloseButtonListener wList;
00086 XWinManaged *inputWin;
00087 XWinManaged *salWin;
00088 
00089 XWinManaged *dispWin;
00090 int wDisp, hDisp, sDisp, scaleDisp;
00091 int wDispWin,  hDispWin;
00092 
00093 // gist display
00094 int pcaW = 16, pcaH = 5;
00095 int winBarW = 5, winBarH = 25;
00096 
00097 // ######################################################################
00098 void setupDispWin(int w, int h);
00099 
00100 Image< PixRGB<byte> > getDispImg
00101 (Image< PixRGB<byte> > img, Image<float> gistImg,
00102  Image<float> gistPcaImg, Image<float> outHistImg);
00103 
00104 // ######################################################################
00105 // Main function
00106 int main(const int argc, const char **argv)
00107 {
00108   MYLOGVERB = LOG_INFO;  // suppress debug messages
00109 
00110   // Instantiate a ModelManager:
00111   ModelManager manager("Beobot: Navigation Model");
00112 
00113   // we cannot use saveResults() on our various ModelComponent objects
00114   // here, so let's not export the related command-line options.
00115   manager.allowOptions(OPTEXP_ALL & (~OPTEXP_SAVE));
00116 
00117   // Instantiate our various ModelComponents:
00118   nub::soft_ref<SimEventQueueConfigurator>
00119     seqc(new SimEventQueueConfigurator(manager));
00120   manager.addSubComponent(seqc);
00121 
00122   nub::soft_ref<FrameGrabberConfigurator>
00123     gbc(new FrameGrabberConfigurator(manager));
00124   manager.addSubComponent(gbc);
00125 
00126   nub::soft_ref<StdBrain> brain(new StdBrain(manager));
00127   manager.addSubComponent(brain);
00128 
00129   nub::ref<SpatialMetrics> metrics(new SpatialMetrics(manager));
00130   manager.addSubComponent(metrics);
00131 
00132   manager.exportOptions(MC_RECURSE);
00133   metrics->setFOAradius(30); // FIXME
00134   metrics->setFoveaRadius(30); // FIXME
00135   manager.setOptionValString(&OPT_MaxNormType, "FancyOne");
00136   manager.setOptionValString(&OPT_UseRandom, "false");
00137   //  manager.setOptionValString("ShapeEstimatorMode","SaliencyMap");
00138   //  manager.setOptionValString(&OPT_ShapeEstimatorMode,"ConspicuityMap");
00139   manager.setOptionValString(&OPT_ShapeEstimatorMode, "FeatureMap");
00140   manager.setOptionValString(&OPT_ShapeEstimatorSmoothMethod, "Chamfer");
00141   //manager.setOptionValString(&OPT_ShapeEstimatorSmoothMethod, "Gaussian");
00142   manager.setOptionValString(&OPT_RawVisualCortexChans,"OIC");
00143   //manager.setOptionValString(&OPT_IORtype, "ShapeEstFM");
00144   manager.setOptionValString(&OPT_IORtype, "Disc");
00145 
00146   // setting up the GIST ESTIMATOR
00147   manager.setOptionValString(&OPT_GistEstimatorType,"Std");
00148 
00149   // Request a bunch of option aliases (shortcuts to lists of options):
00150   REQUEST_OPTIONALIAS_NEURO(manager);
00151 
00152   // Parse command-line:
00153   if (manager.parseCommandLine(argc, argv, "<input_train.txt>",
00154                                1, 1) == false)
00155     return(1);
00156 
00157   // do post-command-line configs
00158   nub::soft_ref<FrameIstream> gb = gbc->getFrameGrabber();
00159   if (gb.isInvalid())
00160     LFATAL("You need to select a frame grabber type via the "
00161            "--fg-type=XX command-line option for this program "
00162            "to be useful -- ABORT");
00163   int w = gb->getWidth(), h = gb->getHeight();
00164   std::string dims = convertToString(Dims(w, h));
00165   manager.setOptionValString(&OPT_InputFrameDims, dims);
00166   nub::soft_ref<SimEventQueue> seq = seqc->getQ();
00167 
00168   // frame delay in seconds
00169   double fdelay = 33.3667/1000.0; // real time
00170 
00171   // let's get all our ModelComponent instances started:
00172   manager.start();
00173 
00174   // main loop:
00175   SimTime prevstime = SimTime::ZERO();
00176   int fNum = 0;
00177   Image< PixRGB<byte> > inputImg;
00178   Image< PixRGB<byte> > dispImg;
00179   Image< PixRGB<byte> > tImg;
00180 
00181   // instantiate a 3-layer feed-forward network
00182   // initialize with the provided parameters
00183   rutz::shared_ptr<FeedForwardNetwork> ffn_place(new FeedForwardNetwork());
00184   FFNtrainInfo pcInfo(manager.getExtraArg(0).c_str());
00185   ffn_place->init3L(pcInfo.h1Name, pcInfo.h2Name, pcInfo.oName,
00186                     pcInfo.redFeatSize, pcInfo.h1size, pcInfo.h2size,
00187                     pcInfo.nOutput, 0.0, 0.0);
00188 
00189   // setup the PCA eigenvector
00190   Image<double> pcaVec =
00191     setupPcaIcaMatrix(pcInfo.trainFolder+pcInfo.evecFname,
00192                       pcInfo.oriFeatSize, pcInfo.redFeatSize);
00193 
00194   // get the frame grabber to start streaming:
00195   gb->startStream();
00196 
00197   // MAIN LOOP
00198   while(1)
00199   {
00200     // has the time come for a new frame?
00201     // LATER ON GIST WILL DECIDE IF WE WANT TO SLOW THINGS DOWN
00202     if (fNum == 0 ||
00203         ((seq->now() - 0.5 * (prevstime - seq->now())).secs() - fNum * fdelay
00204          > fdelay))
00205       {
00206         // NEED CONDITION TO END THE LOOP
00207         //if (??) break;
00208 
00209         // grab a frame
00210         inputImg = gb->readRGB();
00211         tImg = inputImg;
00212 
00213         // setup  display  at the start of stream
00214         // NOTE: wDisp, hDisp, and sDisp are modified here
00215         if (fNum == 0) setupDispWin(w, h);
00216         //inputWin->drawImage(inputImg,0,0);
00217 
00218         // pass input to brain:
00219         seq->post(rutz::make_shared(new SimEventInputFrame(brain.get(), GenericFrame(inputImg), 0)));
00220         LINFO("\nnew frame :%d",fNum);
00221 
00222         // get the gist feature vector
00223         Image<double> cgist;
00224         if (SeC<SimEventGistOutput> ee = seq->check<SimEventGistOutput>(brain.get())) cgist = ee->gv();
00225         else LFATAL("No gist output in the queue");
00226 
00227         // reduce feature dimension (if available)
00228         Image<double> in;
00229         if(pcInfo.isPCA) in = matrixMult(pcaVec, cgist);
00230         else in = cgist;
00231 
00232         // recognize the place
00233         ffn_place->run3L(in);
00234         rutz::shared_ptr<Histogram> resHist(new Histogram(pcInfo.nOutput));
00235 
00236         for(uint i = 0; i < pcInfo.nOutput; i++)
00237           {
00238             LINFO("pl[%3d]: %.4f",i,ffn_place->getOutput().getVal(i));
00239             resHist->addValue(i,ffn_place->getOutput().getVal(i));
00240           }
00241 
00242         // display or save the visuals
00243         LFATAL("FIXME SimEventGistOutput does not contain an image and should be updated");
00244         /*
00245         dispImg = getDispImg
00246           (tImg,
00247            ge->getGistImage(sDisp),
00248            getPcaIcaFeatImage(in, pcaW, pcaH, sDisp*2),
00249            resHist->getHistogramImage(wDisp,sDisp*2 *pcaH, 0.0, 1.0));
00250         */
00251 
00252         // have to press a key to continue to the next frame
00253         dispWin->drawImage(dispImg,0,0);
00254         Raster::waitForKey();
00255 
00256         // if we got an image, save it:
00257 //         if (inputImg.initialized())
00258 //           {
00259 //             std::string base = "../data/CameraCalib/beobotCF_";
00260 //             Raster::WriteRGB
00261 //               (inputImg, PNM, sformat("%s%03d",base.c_str(),fNum));
00262 //             LINFO("saving %s%03d", base.c_str(), fNum);
00263 //           }
00264 
00265        // increment frame count
00266         fNum++;
00267       }
00268 
00269     // evolve brain:
00270     prevstime = seq->now(); // time before current step
00271     const SimStatus status = seq->evolve();
00272 
00273     // process if salient location is selected
00274     if (SeC<SimEventWTAwinner> e = seq->check<SimEventWTAwinner>(0))
00275     {
00276       const Point2D<int> winner = e->winner().p;
00277 
00278       // use Shape estimator to focus on the attended region
00279       Image<float> fmask; std::string label;
00280       if (SeC<SimEventShapeEstimatorOutput>
00281           e = seq->check<SimEventShapeEstimatorOutput>(0))
00282         { fmask = e->smoothMask(); label = e->winningLabel(); }
00283 
00284       Image<float> roiImg;
00285       if (fmask.initialized())
00286         roiImg = fmask * luminance(inputImg);
00287       else
00288         roiImg = luminance(inputImg);
00289 
00290       drawCircle(roiImg, winner, 10, 0.0f, 1);
00291       LINFO("\nFrame: %d, winner: (%d,%d) in %s\n\n",
00292             fNum, winner.i, winner.j, label.c_str());
00293       //salWin->drawImage(roiImg,0,0);
00294       //Raster::waitForKey();
00295       //doSomeSalStuff(img, brain, winner);
00296     }
00297 
00298     if (SIM_BREAK == status) // Brain decided it's time to quit
00299       break;
00300   }
00301 
00302   // stop all our ModelComponents
00303   manager.stop();
00304 
00305   // all done!
00306   return 0;
00307 }
00308 
00309 // ######################################################################
00310 // setup display window for visualization purposes
00311 void setupDispWin(int w, int h)
00312 {
00313   // figure out the best display w, h, and scale
00314 
00315   // check if both dimensions of the image
00316   // are much smaller than the desired resolution
00317   scaleDisp = 1;
00318   while (w*scaleDisp < W_ASPECT_RATIO*.75 && h*scaleDisp < H_ASPECT_RATIO*.75)
00319     scaleDisp++;
00320 
00321   // check if the height is longer aspect-ratio-wise
00322   // this is because the whole display is setup wrt/ to it
00323   wDisp = w*scaleDisp; hDisp = h*scaleDisp;
00324   if(wDisp/(0.0 + W_ASPECT_RATIO) > hDisp/(0.0 + H_ASPECT_RATIO))
00325     hDisp = (int)(wDisp / (0.0 + W_ASPECT_RATIO) * H_ASPECT_RATIO)+1;
00326   else
00327     wDisp = (int)(hDisp / (0.0 + H_ASPECT_RATIO) * W_ASPECT_RATIO)+1;
00328 
00329   // add slack so that the gist feature entry is square
00330   sDisp = (hDisp/NUM_GIST_FEAT + 1);
00331   hDisp =  sDisp * NUM_GIST_FEAT;
00332 
00333   // add space for all the visuals
00334   wDispWin = wDisp + sDisp * NUM_GIST_COL;
00335   hDispWin = hDisp + sDisp * pcaH * 2;
00336 
00337   dispWin  = new XWinManaged(Dims(wDispWin, hDispWin), 0, 0, "dispImg");
00338   wList.add(dispWin);
00339 
00340   inputWin = new XWinManaged(Dims(w, h), w, 0, "input" );
00341   wList.add(inputWin);
00342 
00343   salWin   = new XWinManaged(Dims(w, h), 2*w, 0, "Sal" );
00344   wList.add(salWin);
00345 }
00346 
00347 // ######################################################################
00348 // get display image for visualization purposes
00349 Image< PixRGB<byte> > getDispImg (Image< PixRGB<byte> > img,
00350                                   Image<float> gistImg,
00351                                   Image<float> gistPcaImg,
00352                                   Image<float> outHistImg)
00353 {
00354   Image< PixRGB<byte> > dispImg(wDispWin, hDispWin, ZEROS);
00355   int w = img.getWidth(); int h = img.getHeight();
00356 
00357   // grid the displayed input image
00358   drawGrid(img, w/4,h/4,1,1,PixRGB<byte>(255,255,255));
00359   inplacePaste(dispImg, img,        Point2D<int>(0, 0));
00360 
00361   // display the gist features
00362   inplaceNormalize(gistImg, 0.0f, 255.0f);
00363   inplacePaste(dispImg, Image<PixRGB<byte> >(gistImg),    Point2D<int>(wDisp, 0));
00364 
00365   // display the PCA gist features
00366   inplaceNormalize(gistPcaImg, 0.0f, 255.0f);
00367   inplacePaste(dispImg, Image<PixRGB<byte> >(gistPcaImg), Point2D<int>(wDisp, hDisp));
00368 
00369   // display the classifier output histogram
00370   inplaceNormalize(outHistImg, 0.0f, 255.0f);
00371   inplacePaste(dispImg, Image<PixRGB<byte> >(outHistImg), Point2D<int>(0, hDisp));
00372 
00373   // draw lines delineating the information
00374   drawLine(dispImg, Point2D<int>(0,hDisp),
00375            Point2D<int>(wDispWin,hDisp),
00376            PixRGB<byte>(255,255,255),1);
00377   drawLine(dispImg, Point2D<int>(wDisp-1,0),
00378            Point2D<int>(wDisp-1,hDispWin-1),
00379            PixRGB<byte>(255,255,255),1);
00380   return dispImg;
00381 }
00382 
00383 // ######################################################################
00384 /* So things look consistent in everyone's emacs... */
00385 /* Local Variables: */
00386 /* indent-tabs-mode: nil */
00387 /* End: */
Generated on Sun May 8 08:40:11 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3