test-ObjSearch.C

Go to the documentation of this file.
00001 /*! @file ObjRec/test-ObjSearch.C test various visual search alg */
00002 
00003 // //////////////////////////////////////////////////////////////////// //
00004 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2000-2005   //
00005 // by the University of Southern California (USC) and the iLab at USC.  //
00006 // See http://iLab.usc.edu for information about this project.          //
00007 // //////////////////////////////////////////////////////////////////// //
00008 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00009 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00010 // in Visual Environments, and Applications'' by Christof Koch and      //
00011 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00012 // pending; application number 09/912,225 filed July 23, 2001; see      //
00013 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00014 // //////////////////////////////////////////////////////////////////// //
00015 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00016 //                                                                      //
00017 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00018 // redistribute it and/or modify it under the terms of the GNU General  //
00019 // Public License as published by the Free Software Foundation; either  //
00020 // version 2 of the License, or (at your option) any later version.     //
00021 //                                                                      //
00022 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00023 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00024 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00025 // PURPOSE.  See the GNU General Public License for more details.       //
00026 //                                                                      //
00027 // You should have received a copy of the GNU General Public License    //
00028 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00029 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00030 // Boston, MA 02111-1307 USA.                                           //
00031 // //////////////////////////////////////////////////////////////////// //
00032 //
00033 // Primary maintainer for this file: Lior Elazary <elazary@usc.edu>
00034 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/ObjRec/test-ObjSearch.C $
00035 // $Id: test-ObjSearch.C 10982 2009-03-05 05:11:22Z itti $
00036 //
00037 
00038 
00039 #include "Component/ModelManager.H"
00040 #include "Image/Image.H"
00041 #include "Image/ImageSet.H"
00042 #include "Image/ShapeOps.H"
00043 #include "Image/DrawOps.H"
00044 #include "Image/FilterOps.H"
00045 #include "Image/ColorOps.H"
00046 #include "Image/Transforms.H"
00047 #include "Image/MathOps.H"
00048 #include "Neuro/StdBrain.H"
00049 #include "Neuro/VisualCortex.H"
00050 #include "Neuro/VisualCortexConfigurator.H"
00051 #include "Neuro/NeuroOpts.H"
00052 #include "Media/TestImages.H"
00053 #include "Media/SceneGenerator.H"
00054 #include "Media/MediaSimEvents.H"
00055 #include "Channels/DescriptorVec.H"
00056 #include "Channels/ComplexChannel.H"
00057 #include "Channels/SubmapAlgorithmBiased.H"
00058 #include "Simulation/SimEventQueue.H"
00059 #include "Simulation/SimulationOpts.H"
00060 #include "Simulation/SimEventQueueConfigurator.H"
00061 #include "Neuro/NeuroSimEvents.H"
00062 #include "Learn/Bayes.H"
00063 #include "GUI/DebugWin.H"
00064 #include "ObjRec/BayesianBiaser.H"
00065 
00066 
00067 int classifyImage(Image<PixRGB<byte> > & img, DescriptorVec &descVec, Bayes &bayesNet);
00068 int classifyLocation(Point2D<int> &loc, DescriptorVec &descVec, Bayes &bayesNet,
00069     double *prob, double *statSig, std::vector<Bayes::ClassInfo> &classesInfo);
00070 void biasVC(ComplexChannel &vc, Bayes &bayesNet, int objId);
00071 Point2D<int> evolveBrain(Image<PixRGB<byte> > &img, DescriptorVec& descVec, int ii=-1);
00072 int checkWinnerLoc(TestImages &testImages, uint scene, Point2D<int> &winner, std::string biasedObj);
00073 std::string getWinnerLoc(TestImages &testImages, uint scene, Point2D<int> &winner);
00074 
00075 ModelManager *mgr;
00076 
00077 const char* Labels[14]={"house", "residential", "commercial", "agriculture",  "road",  "airport",
00078   "bridge",  "submerged house", "submerged residential", "submerged commercial",
00079   "submerged agriculture", "submerged road", "submerged airport",  "submerged bridge"};
00080 int foveaRadius = 0;
00081 
00082 
00083 int main(const int argc, const char **argv)
00084 {
00085 
00086   MYLOGVERB = LOG_INFO;
00087   mgr = new ModelManager("Test ObjRec");
00088 
00089   nub::soft_ref<SimEventQueueConfigurator>
00090     seqc(new SimEventQueueConfigurator(*mgr));
00091   mgr->addSubComponent(seqc);
00092 
00093   //our brain
00094   nub::ref<StdBrain>  brain(new StdBrain(*mgr));
00095   mgr->addSubComponent(brain);
00096 
00097   mgr->exportOptions(MC_RECURSE);
00098   mgr->setOptionValString(&OPT_RawVisualCortexChans, "IOC");
00099   //mgr.setOptionValString(&OPT_RawVisualCortexChans, "I");
00100   //mgr->setOptionValString(&OPT_RawVisualCortexChans, "GNO");
00101   //mgr.setOptionValString(&OPT_RawVisualCortexChans, "N");
00102   //manager.setOptionValString(&OPT_UseOlderVersion, "false");
00103   // set the FOA and fovea radii
00104   mgr->setOptionValString(&OPT_SaliencyMapType, "Fast");
00105   mgr->setOptionValString(&OPT_SMfastInputCoeff, "1");
00106 
00107   mgr->setOptionValString(&OPT_WinnerTakeAllType, "Fast");
00108   mgr->setOptionValString(&OPT_SimulationTimeStep, "0.2");
00109 
00110   mgr->setModelParamVal("FOAradius", 50, MC_RECURSE);
00111   mgr->setModelParamVal("FoveaRadius", 50, MC_RECURSE);
00112 
00113 
00114   mgr->setOptionValString(&OPT_IORtype, "Disc");
00115 
00116 
00117   if (mgr->parseCommandLine(
00118         (const int)argc, (const char**)argv, "<Network file> <scenes set xml file> <obj to bias>", 3, 3) == false)
00119     return 1;
00120 
00121   mgr->start();
00122 
00123   bool debug = 1;
00124   ComplexChannel *cc =
00125     &*dynCastWeak<ComplexChannel>(brain->getVC());
00126 
00127   //Get a new descriptor vector
00128   DescriptorVec descVec(*mgr, "Descriptor Vector", "DecscriptorVec", cc);
00129   //Get  new classifier
00130   Bayes bayesNet(descVec.getFVSize(), 1000);
00131 
00132   const char *bayesNetFile = mgr->getExtraArg(0).c_str();
00133   const char *imageSetFile = mgr->getExtraArg(1).c_str();
00134   bayesNet.load(bayesNetFile);
00135 
00136   //get command line options
00137   int objToBias = bayesNet.getClassId(mgr->getExtraArg(2).c_str());
00138 
00139   foveaRadius = mgr->getModelParamVal<int>("FoveaRadius", MC_RECURSE);
00140 
00141   LINFO("******* Biasing for %i\n", objToBias);
00142   printf("Biasing for %i:%s\n", objToBias, mgr->getExtraArg(2).c_str());
00143 
00144   //load the images
00145   TestImages testImages(imageSetFile, TestImages::XMLFILE);
00146 
00147   descVec.setFoveaSize(foveaRadius);
00148 
00149 
00150   //bias for the object
00151   if (objToBias != -1)
00152   {
00153     //objToBias++;
00154     biasVC(*cc, bayesNet, objToBias);
00155   }
00156 
00157 
00158   int totalScenes = 0; //the number of scenes presented to the network
00159 
00160   for (uint scene=0; scene<testImages.getNumScenes(); scene++) //look at all the scenes
00161   {
00162     TestImages::SceneData sceneData = testImages.getSceneData(scene);
00163 
00164     totalScenes++;
00165     Image<PixRGB<byte> > sceneImg = testImages.getScene(scene);
00166 
00167     LINFO("Display scene %i", scene);
00168     Point2D<int> winner = evolveBrain(sceneImg, descVec); //evolve the biased brain
00169 
00170     int sacc = 1;
00171     printf("scene:%i:%s b:%i \n", scene, sceneData.filename.c_str(), objToBias);
00172     Image<PixRGB<byte> > tmp = sceneImg;
00173     Point2D<int> lastWinner = winner;
00174 
00175     char logfile[255];
00176     sprintf(logfile, "%s.dat", sceneData.filename.c_str());
00177     FILE *fp = fopen(logfile, "w");
00178 
00179     for(sacc=0; sacc<30; sacc++)
00180     {
00181       //classify the object under the fovea
00182       double prob = 0, statsSig = 0;
00183       std::vector<Bayes::ClassInfo> classesInfo;
00184       printf("Classify: %i\n", sacc);
00185       int cls = classifyLocation(winner, descVec, bayesNet,
00186           &prob, &statsSig, classesInfo);
00187       //int foundClsId = bayesNet.getClassId(objData.description.c_str());
00188 
00189       //printf("(%ix%i):", winner.i, winner.j);
00190       std::string trueClsName = getWinnerLoc(testImages, scene, winner);
00191       printf("%i:%i:%f:%f\n",
00192           bayesNet.getClassId(trueClsName.c_str()), cls, prob, statsSig);
00193 
00194       fprintf(fp, "%i %i %i %i ",
00195           winner.i, winner.j,
00196           bayesNet.getClassId(trueClsName.c_str()), cls);
00197       for (uint i=0; i<classesInfo.size(); i++)
00198         fprintf(fp, "%i %f %f ",
00199             classesInfo[i].classID, classesInfo[i].prob,
00200             classesInfo[i].statSig);
00201       fprintf(fp, "\n");
00202 
00203       // printf("lum %i Obj %i is class %i BiasedObj %i\n", lum, objId, cls, biasedObj);
00204      // LINFO("Object %i %s: Class %i:%s BiasingFor: %i",
00205       //    foundClsId, objData.description.c_str(),
00206       //    cls, bayesNet.getClassName(cls), objToBias);
00207 
00208      // if (debug)
00209      // {
00210         drawLine(tmp, lastWinner, winner, PixRGB<byte>(255, 0, 0), 3);
00211         if (bayesNet.getClassId(trueClsName.c_str()) ==  cls)
00212           drawCircle(tmp, winner, foveaRadius, PixRGB<byte>(0, 255, 0), 3);
00213         else
00214           drawCircle(tmp, winner, foveaRadius, PixRGB<byte>(255, 0, 0), 3);
00215 
00216         lastWinner = winner;
00217         //testImages.labelScene(scene, tmp);
00218 
00219      // }
00220 
00221      //if (objFound)
00222      //   break;
00223 
00224      Image<PixRGB<byte> > nullImg;
00225      winner = evolveBrain(nullImg, descVec); //evolve the biased brain to get a new winner
00226 
00227     }
00228     fclose(fp);
00229 
00230 
00231     if (debug)
00232     {
00233       //draw the labeled outline
00234       char info[255];
00235       if (objToBias != -1)
00236       {
00237         sprintf(info, "Biasing for %s\n", bayesNet.getClassName(objToBias));
00238         writeText(tmp, Point2D<int>(0,0), info, PixRGB<byte>(255), PixRGB<byte>(0));
00239       }
00240 
00241       testImages.labelScene(scene, tmp);
00242     }
00243 
00244 
00245     printf("] found in %i saccades\n", sacc);
00246     char filename[255];
00247     sprintf(filename, "%s_sacc.ppm", sceneData.filename.c_str());
00248     LINFO("Write image %s\n", filename);
00249     testImages.labelScene(scene, tmp);
00250     Raster::WriteRGB(tmp, filename);
00251     //SHOWIMG(tmp);
00252 
00253   }
00254   printf("Total scenes %i\n", totalScenes);
00255 
00256 // stop all our ModelComponents
00257   mgr->stop();
00258 
00259   return 0;
00260 
00261 }
00262 
00263 void biasVC(ComplexChannel &vc, Bayes &bayesNet, int objId)
00264 {
00265   //Set mean and sigma to bias submap
00266   BayesianBiaser bb(bayesNet, objId, -1, true);
00267   vc.accept(bb);
00268 
00269   setSubmapAlgorithmBiased(vc);
00270 }
00271 
00272 Point2D<int> evolveBrain(Image<PixRGB<byte> > &img, DescriptorVec& descVec, int ii)
00273 {
00274 
00275   nub::ref<StdBrain>  brain = dynCastWeak<StdBrain>(mgr->subComponent("Brain"));
00276   nub::ref<SimEventQueueConfigurator> seqc =
00277     dynCastWeak<SimEventQueueConfigurator>(mgr->subComponent("SimEventQueueConfigurator"));
00278   nub::soft_ref<SimEventQueue> seq  = seqc->getQ();
00279 
00280   LINFO("Evolve Brain");
00281 
00282   if (mgr->started()){    //give the image to the brain
00283 
00284     if (img.initialized())
00285       {
00286         //place the image in the inputFrame queue
00287         rutz::shared_ptr<SimEventInputFrame>
00288           e(new SimEventInputFrame(brain.get(), GenericFrame(img), 0));
00289         seq->post(e);
00290        // brain->input(img, seq);
00291         descVec.setInputImg(img);
00292       }
00293 
00294     SimTime end_time = seq->now() + SimTime::MSECS(3.0);
00295 
00296     while (seq->now() < end_time)
00297     {
00298       brain->evolve(*seq); //evolve the brain
00299 
00300       // Any new WTA winner?
00301       if (SeC<SimEventWTAwinner> e = seq->check<SimEventWTAwinner>(brain.get()))
00302       {
00303        const Point2D<int> winner = e->winner().p;
00304 
00305         //get the saliency map output
00306        // if (SeC<SimEventSaliencyMapOutput> smo =
00307        //     seq->check<SimEventSaliencyMapOutput>(brain.get(), SEQ_ANY))
00308        // {
00309        //   Image<float> img = smo->sm();
00310        //   //SHOWIMG(rescale(img, img.getWidth()*16, img.getHeight()*16));
00311        // }
00312         seq->evolve();
00313         return winner;
00314       }
00315 
00316 
00317       seq->evolve();
00318       LINFO("Evolve 1\n");
00319 
00320     }
00321   }
00322 
00323   return Point2D<int>();
00324 
00325 }
00326 
00327 
00328 int classifyImage(Image<PixRGB<byte> > & img, DescriptorVec &descVec, Bayes &bayesNet)
00329 {
00330   Point2D<int> winner = evolveBrain(img, descVec); //evolve the brain
00331 
00332   //get the descriptor
00333   descVec.setFovea(winner);
00334   descVec.buildRawDV(); //build the descriptor vector
00335 
00336   //get the resulting feature vector
00337   std::vector<double> FV = descVec.getFV();
00338 
00339   // printf("%i %i ", winner.i, winner.j);
00340   //  for(uint i=0; i<FV.size(); i++)
00341   //     printf("%f ", FV[i]);
00342 
00343   //classify
00344 
00345   int cls = bayesNet.classify(FV);
00346 
00347 
00348   if (cls == -1) //check for errors
00349     return -1;
00350   else
00351     return cls;
00352 
00353 }
00354 
00355 
00356 int classifyLocation(Point2D<int> &loc, DescriptorVec &descVec, Bayes &bayesNet,
00357     double *prob, double *statSig, std::vector<Bayes::ClassInfo> &classesInfo)
00358 {
00359 
00360   //get the descriptor
00361   descVec.setFovea(loc);
00362   descVec.buildRawDV(); //build the descriptor vector
00363 
00364   //get the resulting feature vector
00365   std::vector<double> FV = descVec.getFV();
00366 
00367   //classify
00368   //  printf("FV: ");
00369   //  for(uint i=0; i<FV.size(); i++)
00370   //    printf("%f ", FV[i]);
00371   //  printf("\n");
00372 
00373   int cls = -1;
00374   if (prob != NULL)
00375     classesInfo = bayesNet.classifyRange(FV, cls);
00376     //cls = bayesNet.classify(FV, prob);
00377   else
00378     cls = bayesNet.classify(FV);
00379 
00380   if (cls == -1) //check for errors
00381     return -1;
00382   else
00383     return cls;
00384 
00385 }
00386 
00387 int checkWinnerLoc(TestImages &testImages, uint scene, Point2D<int> &winner, std::string biasedObj)
00388 {
00389 
00390   LINFO("Checkign for %s at %ix%i", biasedObj.c_str(), winner.i, winner.j);
00391   for(uint obj=0; obj<testImages.getNumObj(scene); obj++)
00392   {
00393     TestImages::ObjData objData = testImages.getObjectData(scene, obj, false);
00394 
00395     //find the object dimention from the polygon
00396     if (objData.polygon.size() > 0)
00397     {
00398       Point2D<int> upperLeft = objData.polygon[0];
00399       Point2D<int> lowerRight = objData.polygon[0];
00400 
00401       for(uint i=0; i<objData.polygon.size(); i++)
00402       {
00403         //find the bounds for the crop
00404         if (objData.polygon[i].i < upperLeft.i) upperLeft.i = objData.polygon[i].i;
00405         if (objData.polygon[i].j < upperLeft.j) upperLeft.j = objData.polygon[i].j;
00406 
00407         if (objData.polygon[i].i > lowerRight.i) lowerRight.i = objData.polygon[i].i;
00408         if (objData.polygon[i].j > lowerRight.j) lowerRight.j = objData.polygon[i].j;
00409       }
00410 
00411       //check if point is within the polygon
00412       for(int y=upperLeft.j; y<lowerRight.j; y++)
00413         for(int x=upperLeft.i; x<lowerRight.i; x++)
00414         {
00415           if(testImages.pnpoly(objData.polygon, winner)) //if the point is outsize the image
00416           {
00417             if (objData.description == biasedObj)
00418             {
00419               printf("Match %s with %s\n", objData.description.c_str(), biasedObj.c_str());
00420               return 1;
00421             }
00422           }
00423         }
00424     }
00425 
00426   }
00427 
00428   return 0;
00429 }
00430 
00431 std::string getWinnerLoc(TestImages &testImages, uint scene, Point2D<int> &winner)
00432 {
00433 
00434   for(uint obj=0; obj<testImages.getNumObj(scene); obj++)
00435   {
00436     TestImages::ObjData objData = testImages.getObjectData(scene, obj, false);
00437 
00438     //find the object dimention from the polygon
00439     if (objData.polygon.size() > 0)
00440     {
00441       Point2D<int> upperLeft = objData.polygon[0];
00442       Point2D<int> lowerRight = objData.polygon[0];
00443 
00444       for(uint i=0; i<objData.polygon.size(); i++)
00445       {
00446         //find the bounds for the crop
00447         if (objData.polygon[i].i < upperLeft.i) upperLeft.i = objData.polygon[i].i;
00448         if (objData.polygon[i].j < upperLeft.j) upperLeft.j = objData.polygon[i].j;
00449 
00450         if (objData.polygon[i].i > lowerRight.i) lowerRight.i = objData.polygon[i].i;
00451         if (objData.polygon[i].j > lowerRight.j) lowerRight.j = objData.polygon[i].j;
00452       }
00453 
00454       //check if point is within the polygon
00455       for(int y=upperLeft.j; y<lowerRight.j; y++)
00456         for(int x=upperLeft.i; x<lowerRight.i; x++)
00457         {
00458           if(testImages.pnpoly(objData.polygon, winner)) //if the point is outsize the image
00459           {
00460             return objData.description.c_str();
00461             /*if (objData.description == biasedObj)
00462             {
00463               printf("Match %s with %s\n", objData.description.c_str(), biasedObj.c_str());
00464               return 1;
00465             }*/
00466           }
00467         }
00468     }
00469 
00470   }
00471 
00472   return std::string("NULL");
00473 }
Generated on Sun May 8 08:41:08 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3