SceneUnderstanding.C

Go to the documentation of this file.
00001 /*!@file SceneUnderstanding/SceneUnderstanding.C  */
00002 
00003 // //////////////////////////////////////////////////////////////////// //
00004 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2000-2005   //
00005 // by the University of Southern California (USC) and the iLab at USC.  //
00006 // See http://iLab.usc.edu for information about this project.          //
00007 // //////////////////////////////////////////////////////////////////// //
00008 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00009 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00010 // in Visual Environments, and Applications'' by Christof Koch and      //
00011 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00012 // pending; application number 09/912,225 filed July 23, 2001; see      //
00013 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00014 // //////////////////////////////////////////////////////////////////// //
00015 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00016 //                                                                      //
00017 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00018 // redistribute it and/or modify it under the terms of the GNU General  //
00019 // Public License as published by the Free Software Foundation; either  //
00020 // version 2 of the License, or (at your option) any later version.     //
00021 //                                                                      //
00022 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00023 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00024 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00025 // PURPOSE.  See the GNU General Public License for more details.       //
00026 //                                                                      //
00027 // You should have received a copy of the GNU General Public License    //
00028 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00029 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00030 // Boston, MA 02111-1307 USA.                                           //
00031 // //////////////////////////////////////////////////////////////////// //
00032 //
00033 // Primary maintainer for this file: Lior Elazary <elazary@usc.edu>
00034 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/plugins/SceneUnderstanding/SceneUnderstanding.C $
00035 // $Id: SceneUnderstanding.C 13413 2010-05-15 21:00:11Z itti $
00036 //
00037 
00038 #ifndef SCENEUNDERSTANDING_SCENEUNDERSTANDING_C_DEFINED
00039 #define SCENEUNDERSTANDING_SCENEUNDERSTANDING_C_DEFINED
00040 
00041 #include "plugins/SceneUnderstanding/SceneUnderstanding.H"
00042 
00043 #include "Channels/SubmapAlgorithmBiased.H"
00044 #include "ObjRec/BayesianBiaser.H"
00045 
00046 #include <math.h>
00047 #include <fcntl.h>
00048 #include <limits>
00049 #include <string>
00050 
00051 // ######################################################################
00052 SceneUnderstanding::SceneUnderstanding(ModelManager *mgr, nub::ref<StdBrain> &brain) :
00053   itsMgr(mgr), itsBrain(brain)
00054 {
00055 
00056   //generate a complexChannel from Visual cortex to pass to DescriptorVec
00057   ComplexChannel *cc = &*dynCastWeak<ComplexChannel>(itsBrain->getVC());
00058 
00059   OptionManager *mm = (OptionManager *)itsMgr;
00060 
00061   //get the seq component
00062   nub::ref<SimEventQueueConfigurator> seqc =
00063     dynCastWeak<SimEventQueueConfigurator>(itsMgr->subComponent("SimEventQueueConfigurator"));
00064   itsSEQ = seqc->getQ();
00065 
00066   LINFO("Start 1222");
00067   //Build the descriptor vector
00068   itsDescriptorVec = new DescriptorVec(*mm,
00069       "Descriptor Vector", "DecscriptorVec", cc);
00070 
00071 
00072   //set up featuers and 2 classes
00073   itsBayes = new Bayes(itsDescriptorVec->getFVSize(), 0); //start with no classes
00074 
00075 
00076   //get a new working memory
00077   itsWorkingMemory = new WorkingMemory;
00078 
00079   //set the feature names
00080   for (uint i = 0; i < cc->numSubmaps(); i++)
00081   {
00082     const std::string name = cc->getSubmapName(i);
00083     itsBayes->setFeatureName(i, name.c_str());
00084   }
00085 
00086   //The prolog engine
00087   itsProlog = new SWIProlog(0, NULL);
00088 
00089 
00090 }
00091 
00092 // ######################################################################
00093 SceneUnderstanding::~SceneUnderstanding()
00094 {
00095 
00096 }
00097 
00098 // ######################################################################
00099 nub::ref<StdBrain> SceneUnderstanding::getBrainPtr()
00100 {
00101   return itsBrain;
00102 }
00103 
00104 // ######################################################################
00105 Bayes* SceneUnderstanding::getBayesPtr()
00106 {
00107   return itsBayes;
00108 }
00109 
00110 // ######################################################################
00111 SWIProlog* SceneUnderstanding::getPrologPtr()
00112 {
00113   return itsProlog;
00114 }
00115 
00116 // ######################################################################
00117 DescriptorVec* SceneUnderstanding::getDescriptorVecPtr()
00118 {
00119   return itsDescriptorVec;
00120 }
00121 
00122 // ######################################################################
00123 void SceneUnderstanding::setImage(Image<PixRGB<byte> > &img)
00124 {
00125   itsImg = img;
00126   itsBrain->input(img, itsSEQ);
00127   itsWorkingMemory->setImage(img);
00128   itsDescriptorVec->setInputImg(img);
00129 }
00130 
00131 // ######################################################################
00132 float SceneUnderstanding::evolveBrain()
00133 {
00134   float interestLevel = 0.0F;
00135 
00136   //if (itsMgr->started() && img.initialized()){         //give the image to the brain
00137   if (itsImg.initialized()){         //give the image to the brain
00138     //itsBrain->time();
00139 
00140     LINFO("No new input");
00141     bool keep_going = true;
00142     while (keep_going){
00143       itsBrain->evolve(itsSEQ);
00144       const SimStatus status = itsSEQ->evolve();
00145       if (status == SIM_BREAK) {
00146         LINFO("V %d\n", (int)(itsSEQ->now().msecs()) );
00147         keep_going = false;
00148       }
00149       if (itsBrain->gotCovertShift()) // new attended location
00150       {
00151 
00152         const Point2D<int> winner = itsBrain->getLastCovertPos();
00153         const float winV = itsBrain->getLastCovertAgmV();
00154 
00155         interestLevel = (winV * 1000.0f) * 1; //the gain shold be based on context
00156         LINFO("##### Winner (%d,%d) at %fms : interestLevel=%f#####",
00157             winner.i, winner.j, itsSEQ->now().msecs(),interestLevel);
00158 
00159 
00160         keep_going = false;
00161 
00162         itsCurrentFoveaLoc = winner;
00163       }
00164       if (itsSEQ->now().secs() > 3.0) {
00165         LINFO("##### Time limit reached #####");
00166         keep_going = false;
00167       }
00168       LINFO("Evolve brain");
00169     }
00170 
00171   }
00172 
00173   return interestLevel;
00174 }
00175 
00176 // ######################################################################
00177 Point2D<int> SceneUnderstanding::getFoveaLoc()
00178 {
00179   return itsCurrentFoveaLoc;
00180 }
00181 
00182 // ######################################################################
00183 std::vector<double> SceneUnderstanding::getFV(Point2D<int> foveaLoc)
00184 {
00185 
00186   //build descriptive vector
00187   itsDescriptorVec->setFovea(foveaLoc);
00188   Dims foveaSize = itsDescriptorVec->getFoveaSize();
00189 
00190   itsDescriptorVec->buildRawDV();
00191 
00192   //get the resulting feature vector
00193   std::vector<double> FV = itsDescriptorVec->getFV();
00194 
00195   return FV;
00196 
00197 }
00198 
00199 // ######################################################################
00200 int SceneUnderstanding::classifyFovea(Point2D<int> foveaLoc, double *prob)
00201 {
00202 
00203   std::vector<double> FV = getFV(foveaLoc);
00204 
00205   int cls = itsBayes->classify(FV, prob);
00206 
00207   return cls;
00208 
00209 }
00210 
00211 // ######################################################################
00212 void SceneUnderstanding::learn(Point2D<int> foveaLoc, const char *name)
00213 {
00214 
00215   std::vector<double> FV = getFV(foveaLoc);
00216   itsBayes->learn(FV, name);
00217 
00218 }
00219 
00220 // ######################################################################
00221 bool SceneUnderstanding::biasFor(const char *name)
00222 {
00223   int cls = itsBayes->getClassId(name);
00224   if (cls != -1 ) //we know about this object
00225   {
00226 
00227     ComplexChannel *cc = &*dynCastWeak<ComplexChannel>(itsBrain->getVC());
00228 
00229     //Set mean and sigma to bias submap
00230     BayesianBiaser bb(*itsBayes, cls, -1, true);
00231     cc->accept(bb);
00232 
00233     //Tell the channels we want to bias
00234     setSubmapAlgorithmBiased(*cc);
00235 
00236     itsBrain->input(itsImg, itsSEQ); //Set a new input to the brain
00237     itsDescriptorVec->setInputImg(itsImg);
00238     return true;
00239   } else {
00240     LINFO("Object %s is unknown", name);
00241     return false;
00242   }
00243 
00244 }
00245 
00246 // ######################################################################
00247 std::string SceneUnderstanding::highOrderRec()
00248 {
00249 
00250   static bool moreQuestions = true;
00251   std::vector<std::string> args;
00252   std::string question;
00253   std::string sceneType;
00254 
00255   if(moreQuestions)
00256   {
00257 
00258     //get the scene type
00259     args.clear();
00260     args.push_back(std::string());
00261     if( itsProlog->query("go", args) )
00262     {
00263       LINFO("Scene is %s", args[0].c_str());
00264       sceneType = args[0];
00265     } else {
00266       LINFO("Something is worng");
00267     }
00268 
00269     //check if we have any questions
00270     args.clear();
00271     args.push_back(std::string());
00272     if( itsProlog->query("getQuestion", args) )
00273     {
00274       LINFO("Question is %s", args[0].c_str());
00275       question = args[0];
00276     } else {
00277       LINFO("No more Question" );
00278       moreQuestions = false;
00279     }
00280 
00281     if (moreQuestions)
00282     {
00283       //get the command and object to look for
00284       std::string cmd = question.substr(0,2);
00285       std::string objLookingFor = question.substr(2,question.size());
00286 
00287       sceneType += std::string(".   Looking for: ") + question;
00288 
00289       switch(cmd[0])
00290       {
00291         case 'e':   //check if object exists
00292           //Attempt to answer the questions by finding the features
00293           if (biasFor(objLookingFor.c_str()) )
00294           {
00295             //for now only once time and give up
00296             evolveBrain(); //TODO: get intrest level and decide when to stop searching
00297 
00298             //recognize the object under the fovea
00299             int cls = classifyFovea(itsCurrentFoveaLoc);
00300             if (cls != -1)
00301             {
00302               std::string objName = std::string(itsBayes->getClassName(cls));
00303               std::string predicate;
00304 
00305               LINFO("Found object %s", objName.c_str());
00306               if (objName ==  objLookingFor)
00307               {
00308                 predicate = "setYes"; //we found it
00309                 sceneType += std::string(".   object found. ");
00310 
00311                 PixRGB<byte> col(0,255,0);
00312                 itsBrain->getSV()->setColorNormal(col);
00313               }
00314               else
00315               {
00316                 predicate = "setNo";  //we did not find it
00317                 sceneType += std::string(".   object not found. ");
00318                 PixRGB<byte> col(255,255,0);
00319                 itsBrain->getSV()->setColorNormal(col);
00320               }
00321 
00322               //Set what we know about the scene
00323               LINFO("Asserting %s %s", predicate.c_str(), objLookingFor.c_str());
00324               args.clear();
00325               std::string obj = "e_" + objLookingFor;
00326               args.push_back(obj);
00327               if( itsProlog->query((char *)predicate.c_str(), args) )
00328               {
00329                 LINFO("Assert OK");
00330               } else {
00331                 LINFO("Fatal Assertion");
00332               }
00333             }
00334 
00335           } else {
00336             LINFO("I am sorry, but I dont know anything about ");
00337           }
00338           break;
00339 
00340         case 'c': //count the number of times the object exists
00341           //Attempt to answer the questions by finding the features
00342           if (biasFor(objLookingFor.c_str()) )
00343           {
00344             //Set the obj count to 0 (if obj is not set) to indicate that we already looked for the object
00345             args.clear();
00346             std::string obj = "c_" + objLookingFor;
00347             args.push_back(obj);
00348             if( itsProlog->query("resetCount", args) )
00349             {
00350               LINFO("incCount OK");
00351             } else {
00352               LINFO("incCount");
00353             }
00354 
00355             float interestLevel = 10;
00356             int count = 0;
00357             for(int i=0; i<20 && interestLevel > 5.0F; i++)
00358             {
00359               interestLevel = evolveBrain();
00360 
00361               double prob=0;
00362               //recognize the object under the fovea
00363               int cls = classifyFovea(itsCurrentFoveaLoc, &prob);
00364               if (cls != -1)
00365               {
00366                 std::string objName = std::string(itsBayes->getClassName(cls));
00367                 std::string predicate;
00368 
00369                 LINFO("%i: Found object %s(%f). Interest %f", i, objName.c_str(), prob, interestLevel);
00370 
00371                 //remeber the object under the fovea
00372                 itsWorkingMemory->setMemory(itsCurrentFoveaLoc, cls, itsDescriptorVec->getFoveaSize().h()/2);
00373 
00374                 //Set what we know about the scene
00375                 //TODO. need to optemize, since during our search we incounter objects
00376                 //but we dont count for them. What happens to open that have we don't know about?
00377                 //that is have very low probability
00378               //  if (objName ==  objLookingFor)
00379                // {
00380                   count++;
00381                   args.clear();
00382                   std::string obj = "c_" + objName; //objLookingFor;
00383                   args.push_back(obj);
00384                   if( itsProlog->query("incCount", args) )
00385                   {
00386                     LINFO("incCount OK");
00387                   } else {
00388                     LINFO("incCount");
00389                   }
00390 
00391                   //Set the FOA to a color indicating that we found the object
00392                   int confCol = int(-1.0*prob);
00393 
00394                   if (confCol > 255) confCol = 255;
00395                   confCol = 255-confCol;
00396 
00397                   PixRGB<byte> col(0,confCol,0);
00398                   itsBrain->getSV()->setColorNormal(col);
00399               //  }
00400               //  else
00401               //  {
00402               //    //Set the FOA to a color indicating that we did not find the object
00403               //    PixRGB<byte> col(255,255,0);
00404               //    itsBrain->getSV()->setColorNormal(col);
00405 
00406               //  }
00407 
00408               }
00409             }
00410 
00411             char num[100];
00412             sprintf(num, "%i", count);
00413             sceneType += std::string(".   found: ") + std::string(num);
00414           } else {
00415             LINFO("I am sorry, but I dont know anything about that");
00416           }
00417           break;
00418       }
00419 
00420       //show debug
00421       args.clear();
00422       if(itsProlog->query("debug", args) )
00423       {
00424         LINFO("Debug Ok");
00425       } else {
00426         LINFO("Fatal Question");
00427       }
00428 
00429       //empty the questions list
00430       args.clear();
00431       if(!itsProlog->query("undoQuestions", args) )
00432       {
00433         LINFO("Question Deleted");
00434       } else {
00435         LINFO("Fatal Question");
00436       }
00437 
00438     } else {
00439       //get the scene type
00440       args.clear();
00441       args.push_back(std::string());
00442       if( itsProlog->query("hypothesize", args) )
00443       {
00444         LINFO("Scene is %s", args[0].c_str());
00445         sceneType = args[0];
00446       } else {
00447         LINFO("Something is worng");
00448       }
00449 
00450       //No more questions, retract answers
00451       args.clear();
00452       if( itsProlog->query("undo", args) )
00453       {
00454         LINFO("Removed all Assertion");
00455       } else {
00456         LINFO("Can not remove all Assertion");
00457       }
00458       moreQuestions = true;
00459 
00460     }
00461 
00462 
00463   }
00464 
00465  // //restore FOA color back to normal
00466  // PixRGB<byte> col(255,255,0);
00467  // itsBrain->getSV()->setColorNormal(col);
00468 
00469   return sceneType;
00470 }
00471 
00472 
00473 // ######################################################################
00474 /* So things look consistent in everyone's emacs... */
00475 /* Local Variables: */
00476 /* indent-tabs-mode: nil */
00477 /* End: */
00478 
00479 #endif //SCENEUNDERSTANDING_SCENEUNDERSTANDING_C_DEFINED
00480 
Generated on Sun May 8 08:41:09 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3