Contours.C

Go to the documentation of this file.
00001 /*!@file SceneUnderstanding/Contours.C  */
00002 
00003 // //////////////////////////////////////////////////////////////////// //
00004 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2000-2005   //
00005 // by the University of Southern California (USC) and the iLab at USC.  //
00006 // See http://iLab.usc.edu for information about this project.          //
00007 // //////////////////////////////////////////////////////////////////// //
00008 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00009 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00010 // in Visual Environments, and Applications'' by Christof Koch and      //
00011 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00012 // pending; application number 09/912,225 filed July 23, 2001; see      //
00013 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00014 // //////////////////////////////////////////////////////////////////// //
00015 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00016 //                                                                      //
00017 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00018 // redistribute it and/or modify it under the terms of the GNU General  //
00019 // Public License as published by the Free Software Foundation; either  //
00020 // version 2 of the License, or (at your option) any later version.     //
00021 //                                                                      //
00022 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00023 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00024 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00025 // PURPOSE.  See the GNU General Public License for more details.       //
00026 //                                                                      //
00027 // You should have received a copy of the GNU General Public License    //
00028 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00029 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00030 // Boston, MA 02111-1307 USA.                                           //
00031 // //////////////////////////////////////////////////////////////////// //
00032 //
00033 // Primary maintainer for this file: Lior Elazary <elazary@usc.edu>
00034 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/plugins/SceneUnderstanding/Contours.C $
00035 // $Id: Contours.C 13906 2010-09-10 01:08:31Z lior $
00036 //
00037 
00038 #ifndef Contours_C_DEFINED
00039 #define Contours_C_DEFINED
00040 
00041 
00042 #include "Image/OpenCVUtil.H"
00043 #include "plugins/SceneUnderstanding/LFLineFitter/LFLineFitter.h"
00044 #include "plugins/SceneUnderstanding/Contours.H"
00045 #include "plugins/SceneUnderstanding/V1.H"
00046 
00047 #include "Image/DrawOps.H"
00048 #include "Image/MathOps.H"
00049 #include "Image/Kernels.H"
00050 #include "Image/ColorMap.H"
00051 #include "Image/FilterOps.H"
00052 #include "Image/Transforms.H"
00053 #include "Image/fancynorm.H"
00054 #include "Image/Convolutions.H"
00055 #include "Simulation/SimEventQueue.H"
00056 #include "GUI/DebugWin.H"
00057 #include <math.h>
00058 #include <fcntl.h>
00059 #include <limits>
00060 #include <string>
00061 
00062 const ModelOptionCateg MOC_Contours = {
00063   MOC_SORTPRI_3,   "Contours-Related Options" };
00064 
00065 // Used by: SimulationViewerEyeMvt
00066 const ModelOptionDef OPT_ContoursShowDebug =
00067   { MODOPT_ARG(bool), "ContoursShowDebug", &MOC_Contours, OPTEXP_CORE,
00068     "Show debug img",
00069     "contours-debug", '\0', "<true|false>", "false" };
00070 
00071 
00072 //Define the inst function name
00073 SIMMODULEINSTFUNC(Contours);
00074 
00075 // ######################################################################
00076 Contours::Contours(OptionManager& mgr, const std::string& descrName,
00077     const std::string& tagName) :
00078   SimModule(mgr, descrName, tagName),
00079   SIMCALLBACK_INIT(SimEventV2Output),
00080   SIMCALLBACK_INIT(SimEventContoursBias),
00081   SIMCALLBACK_INIT(SimEventSaveOutput),
00082   SIMCALLBACK_INIT(SimEventUserInput),
00083   itsShowDebug(&OPT_ContoursShowDebug, this)
00084 {
00085 
00086   itsCurrentContour = -1; //for debug view
00087 }
00088 
00089 // ######################################################################
00090 Contours::~Contours()
00091 {
00092 }
00093 
00094 // ######################################################################
00095 void Contours::onSimEventV2Output(SimEventQueue& q,
00096                                   rutz::shared_ptr<SimEventV2Output>& e)
00097 {
00098   itsTensorFields = e->getTensorFields();
00099   evolve(q);
00100 
00101 
00102 
00103   //Layout<PixRGB<byte> > layout = getDebugImage();
00104   //Image<PixRGB<byte> > tmp = layout.render();
00105   //SHOWIMG(tmp);
00106   
00107 }
00108 
00109 // ######################################################################
00110 void Contours::onSimEventContoursBias(SimEventQueue& q,
00111                                   rutz::shared_ptr<SimEventContoursBias>& e)
00112 {
00113   itsBiasContours.clear();
00114   itsBiasContours = e->getContours();
00115 
00116 
00117   LINFO("Size %i", (uint)itsBiasContours.size());
00118   //Bias V1
00119 
00120   if (itsBiasContours.size() > 0)
00121   {
00122     std::vector<V1::SpatialBias> v1Bias;
00123     for(uint i=0; i<itsBiasContours.size(); i++)
00124     {
00125       Point2D<int> center(0,0);
00126       for(uint j=0; j<itsBiasContours[i].points.size(); j++)
00127       {
00128        // tmp.setVal(itsBiasContours[i].points[j], 255.0F);
00129 
00130         center += itsBiasContours[i].points[j];
00131       }
00132       center /= itsBiasContours[i].size();
00133 
00134       //Image<float> tmp(320,240,ZEROS);
00135       //drawCircle(tmp, center, 11, 255.0F);
00136       //SHOWIMG(tmp);
00137 
00138       //Bias V1
00139       V1::SpatialBias sb;
00140       sb.loc = center;
00141       sb.threshold = 0.01;
00142       sb.dims = Dims(50,50);
00143       v1Bias.push_back(sb);
00144 
00145     }
00146     q.post(rutz::make_shared(new SimEventV1Bias(this, v1Bias)));
00147     
00148   }
00149 
00150 }
00151 
00152 
00153 // ######################################################################
00154 void Contours::onSimEventSaveOutput(SimEventQueue& q, rutz::shared_ptr<SimEventSaveOutput>& e)
00155 {
00156   if (itsShowDebug.getVal())
00157     {
00158       // get the OFS to save to, assuming sinfo is of type
00159       // SimModuleSaveInfo (will throw a fatal exception otherwise):
00160       nub::ref<FrameOstream> ofs =
00161         dynamic_cast<const SimModuleSaveInfo&>(e->sinfo()).ofs;
00162       Layout<PixRGB<byte> > disp = getDebugImage();
00163       if (disp.initialized())
00164         ofs->writeRgbLayout(disp, "Contours", FrameInfo("Contours", SRC_POS));
00165     }
00166 }
00167 
00168 // ######################################################################
00169 void Contours::onSimEventUserInput(SimEventQueue& q, rutz::shared_ptr<SimEventUserInput>& e)
00170 {
00171 
00172   LINFO("Got event %s %ix%i key=%i",
00173       e->getWinName(),
00174       e->getMouseClick().i,
00175       e->getMouseClick().j,
00176       e->getKey());
00177 
00178   if (strcmp(e->getWinName(), "Contours"))
00179     return;
00180  
00181   switch(e->getKey())
00182   {
00183     case 10: //1
00184       itsCurrentContour++;
00185       if (itsCurrentContour > (int)itsContours.size()) itsCurrentContour = itsContours.size()-1;
00186       break;
00187     case 24: //q
00188       itsCurrentContour--;
00189       if (itsCurrentContour < 0) itsCurrentContour = 0;
00190       break;
00191     case 38: //a show all contours
00192       itsCurrentContour=-1;
00193       break;
00194   }
00195   
00196 
00197   evolve(q);
00198 
00199 }
00200 
00201 // ######################################################################
00202 void Contours::evolve(SimEventQueue& q)
00203 {
00204 
00205 
00206 
00207   //Process any biasing
00208 
00209   int cid=0; 
00210   itsContours.clear();
00211   //if (itsBiasContours.size() > 0)
00212   //{
00213   //  for(uint i=0; i<itsBiasContours.size(); i++)
00214   //  {
00215   //    for(uint j=0; j<itsBiasContours[i].points.size(); j++)
00216   //    {
00217   //      contoursImg.setVal(itsBiasContours[i].points[j], cid);
00218   //      itsBiasContours[i].ori.push_back(edgesOri.getVal(itsBiasContours[i].points[j]));
00219   //    }
00220   //    itsContours.push_back(itsBiasContours[i]);
00221   //    cid++;
00222   //  }
00223   //}
00224 
00225 
00226 
00227   Image<float> edgesMag; //the edges mag map
00228   Image<float> edgesOri; //the edges ori map
00229   for(uint cidx=0; cidx<3; cidx++)
00230   {
00231     if (!edgesMag.initialized())
00232     {
00233       edgesMag = Image<float>(itsTensorFields[cidx].getDims(), ZEROS);
00234       edgesOri = Image<float>(itsTensorFields[cidx].getDims(), ZEROS);
00235     }
00236 
00237     EigenSpace eigen = getTensorEigen(itsTensorFields[cidx].getTensorField());
00238     Image<float> features = eigen.l1-eigen.l2;
00239     for(uint i=0; i<features.size(); i++)
00240       if (features.getVal(i) > edgesMag.getVal(i))
00241       {
00242         edgesMag.setVal(i, features.getVal(i));
00243         float u = eigen.e1[1].getVal(i);
00244         float v = eigen.e1[0].getVal(i);
00245         edgesOri.setVal(i,atan(-u/v));
00246       }
00247   }
00248   //SHOWIMG(edgesMag);
00249 
00250   Image<float> contoursImg; //mask for which contours exsists
00251   std::vector<Point2D<int> > pointList = getPointList(edgesMag, edgesOri, contoursImg);
00252 
00253   //Go through the points and follow the contour
00254   for(uint i=0; i<pointList.size(); i++)
00255   {
00256     if (contoursImg.getVal(pointList[i]) == UNKNOWN) //since we dont know were this pixel belongs too lets follow it
00257     {
00258       Contour contour = followContour(cid, pointList[i], contoursImg, edgesMag, edgesOri);
00259       if (contour.points.size() > 5)
00260       {
00261         itsContours.push_back(contour);
00262         cid++;
00263       }
00264     }
00265   }
00266 
00267 
00268   //sort the contours
00269   std::sort(itsContours.begin(), itsContours.end(), ContourCmp());
00270 
00271   //////**** Debug ****////
00272   Image<PixRGB<byte> > byEdges = toRGB(itsTensorFields[2].getTokensMag(true));
00273   Image<PixRGB<byte> > tmp(byEdges.getDims(), ZEROS);
00274 
00275   for(uint i=0; i<itsContours.size(); i++)
00276   {
00277     Contour& contour = itsContours[i];
00278     for(uint j=0; j<contour.points.size(); j++)
00279       tmp.setVal(contour.points[j], PixRGB<byte>(0,255,0));
00280   }
00281   //SHOWIMG(tmp);
00282 
00283 
00284 
00285 
00286   q.post(rutz::make_shared(new SimEventContoursOutput(this, itsContours, edgesMag)));
00287 }
00288 
00289 Contours::Contour Contours::followContour(int idx, Point2D<int> startLoc, Image<float>& contoursImg,
00290     const Image<float>& edgesMag, const Image<float>& edgesOri)
00291 {
00292   float origEdgeMag = edgesMag.getVal(startLoc);
00293 
00294 
00295   Image<float> t = edgesMag;
00296   inplaceNormalize(t, 0.0F, 255.0F);
00297   Image<PixRGB<byte> > tmp = t;
00298 
00299 
00300   Contour fcontour;
00301   fcontour.points.push_back(startLoc);
00302   fcontour.ori.push_back(edgesOri.getVal(startLoc));
00303   contoursImg.setVal(startLoc, idx);
00304 
00305   Contour contour;
00306 
00307   //follow the contour one way
00308   Point2D<int> pEdge(-1,-1);
00309   for(uint i=0; i<fcontour.points.size(); i++)
00310   {
00311     Point2D<int> newEdge = findEdgeToFollow(fcontour.points[i], origEdgeMag,
00312         edgesMag, edgesOri, contoursImg, pEdge);
00313 
00314     if (newEdge.isValid())
00315     {
00316       pEdge = newEdge;
00317       tmp.setVal(newEdge, PixRGB<byte>(0,255,0));
00318       contoursImg.setVal(newEdge,idx);
00319       fcontour.points.push_back(newEdge);
00320       fcontour.ori.push_back(edgesOri.getVal(newEdge));
00321     }
00322   }
00323 
00324   //Try to follow the other side of the contour
00325   Point2D<int> newEdge = findEdgeToFollow(startLoc, origEdgeMag,
00326       edgesMag, edgesOri, contoursImg, pEdge);
00327 
00328   if (newEdge.isValid()) //If we found this, then follow
00329   {
00330     tmp.setVal(newEdge, PixRGB<byte>(0,255,0));
00331 
00332     Contour bcontour;
00333     bcontour.points.push_back(newEdge);
00334     bcontour.ori.push_back(edgesOri.getVal(newEdge));
00335     contoursImg.setVal(newEdge,idx);
00336 
00337     for(uint i=0; i<bcontour.points.size(); i++)
00338     {
00339       Point2D<int> newEdge = findEdgeToFollow(bcontour.points[i], origEdgeMag,
00340           edgesMag, edgesOri, contoursImg, pEdge);
00341 
00342       if (newEdge.isValid())
00343       {
00344         tmp.setVal(newEdge, PixRGB<byte>(0,255,0));
00345         contoursImg.setVal(newEdge,idx);
00346         bcontour.points.push_back(newEdge);
00347         bcontour.ori.push_back(edgesOri.getVal(newEdge));
00348       }
00349       //Image<PixRGB<byte> > tt = rescale(tmp, tmp.getDims()*3); 
00350       //SHOWIMG(tt);
00351     }
00352 
00353     //combine the contours in order so the contour will be continues
00354     for(int i=bcontour.size()-1; i>=0; i--)
00355     {
00356       contour.points.push_back(bcontour.points[i]);
00357       contour.ori.push_back(bcontour.ori[i]);
00358     }
00359 
00360   }
00361 
00362   //combine the contours in order so the contour will be continues
00363   for(uint i=0; i<fcontour.size(); i++)
00364   {
00365     contour.points.push_back(fcontour.points[i]);
00366     contour.ori.push_back(fcontour.ori[i]);
00367   }
00368 
00369   return contour;
00370 }
00371 
00372 Point2D<int> Contours::findEdgeToFollow(const Point2D<int>& edgeLoc,
00373     const float origEdgeMag,
00374     const Image<float>& edgesMag, const Image<float>& edgesOri,
00375     const Image<float>& contoursImg,
00376     const Point2D<int>& pEdge)
00377 {
00378   const int radius = 1; //search within this radius
00379   const float magDiffThreshold = 0.2;
00380 
00381   //float edgeMag = edgesMag.getVal(edgeLoc);
00382   //LINFO("Find edge Loc %ix%i", edgeLoc.i, edgeLoc.j);
00383 
00384   Point2D<int> newEdge(-1,-1);
00385 
00386   //Weights for biasing
00387   float w1 = 1;
00388   float w2 = 1;
00389   float w3 = 1.5;
00390 
00391   float maxDiff = 0;
00392   //Search for a new edge to add
00393   //LINFO("Getting edge to follow\n");
00394   for(int x=edgeLoc.i-radius; x<=edgeLoc.i+radius; x++)
00395     for(int y=edgeLoc.j-radius; y<=edgeLoc.j+radius; y++)
00396     {
00397       Point2D<int> loc(x,y);
00398       if (edgeLoc != loc &&
00399           contoursImg.coordsOk(loc) &&
00400           contoursImg.getVal(loc) == UNKNOWN)
00401       {
00402         float mag = 1.0F - (fabs(origEdgeMag - edgesMag.getVal(loc))/origEdgeMag);
00403 
00404         if (mag > magDiffThreshold) 
00405         {
00406           float edgeOri = edgesOri.getVal(edgeLoc);
00407 
00408           float dist = 1.0F - (edgeLoc.squdist(loc)/(2*radius*2));
00409 
00410           //Limit the search to only 90 degrees from us
00411           float oriDiff = 1.0F - (angDiff(edgeOri, edgesOri.getVal(loc))/(M_PI));
00412 
00413           //Point2D<int> t1 = pEdge - edgeLoc;
00414           //Point2D<int> t2 = loc - edgeLoc;
00415           //double edgeAng = angDiff(atan2(double(t1.j), double(t1.i)), atan2(double(t2.j), double(t2.i)));
00416 
00417           float diff = w1*mag + w2*dist + w3*oriDiff;
00418           //LINFO("m %f d %f o(%f,%f) %f eo: %f total: %f",
00419           //    mag, dist, edgeOri*180/M_PI, edgesOri.getVal(loc)*180/M_PI, oriDiff, edgeAng*180/M_PI, diff);
00420           if (diff > maxDiff)
00421           {
00422             newEdge = loc;
00423             maxDiff = diff;
00424           } 
00425         }
00426       }
00427     }
00428   //LINFO("Choose %f", maxDiff);
00429 
00430   return newEdge;
00431 }
00432 
00433 
00434 
00435 std::vector<Point2D<int> > Contours::getPointList(TensorVoting& tv, Image<float>& contoursImg,
00436     Image<float>& edgesMag, Image<float>& edgesOri)
00437 {
00438   float maxMag = 1000;
00439   int nBins = 100;
00440   //float threshold = 0.1;
00441 
00442   //initialize the contoursImg; 
00443   contoursImg = Image<float>(tv.getDims(), NO_INIT);
00444   
00445   EigenSpace eigen = getTensorEigen(tv.getTensorField());
00446   Image<float> features = eigen.l1-eigen.l2;
00447   inplaceNormalize(features, 0.0F, maxMag);
00448 
00449   edgesMag = features;
00450   //SHOWIMG(edgesMag);
00451   edgesOri = Image<float>(features.getDims(), ZEROS);
00452 
00453   //Seperate the location above threshold into bins so we can sort them fast 
00454   //Mark the locations that we will be used in the contours img
00455   //
00456   std::vector<std::vector<Point2D<int> > > pointBins(nBins);
00457   for(int j=0; j<features.getHeight(); j++)
00458     for(int i=0; i<features.getWidth(); i++)
00459     {
00460       float val = features.getVal(i,j);
00461       if (val > 0) //threshold*maxMag)
00462       {
00463         contoursImg.setVal(i,j, UNKNOWN);
00464         //Find the bin this val belongs to
00465         int idx = (int)(val * nBins / maxMag);
00466         if (idx >= nBins) idx = nBins - 1; //if val == maxMag;
00467         pointBins[idx].push_back(Point2D<int>(i,j));
00468 
00469         //Get the direction of the vote from e1, while the weight is l1-l2
00470         float u = eigen.e1[1].getVal(i,j);
00471         float v = eigen.e1[0].getVal(i,j);
00472         edgesOri.setVal(i,j, atan(-u/v));
00473       } else {
00474         contoursImg.setVal(i,j, NOTDEFINED);
00475       }
00476     }
00477 
00478   //return a sorted list;
00479   std::vector<Point2D<int> > points;
00480   for(int i=pointBins.size()-1; i > -1; i--)
00481   {
00482     for(uint j=0; j<pointBins[i].size(); j++)
00483       points.push_back(pointBins[i][j]);
00484   }
00485 
00486   return points;
00487 
00488 }
00489 
00490 std::vector<Point2D<int> > Contours::getPointList(Image<float>& inMag, Image<float>& inOri, Image<float>& contoursImg)
00491 {
00492   float maxMag = 1000;
00493   int nBins = 100;
00494   float threshold = 0.1;
00495 
00496   //initialize the contoursImg; 
00497   contoursImg = Image<float>(inMag.getDims(), NO_INIT);
00498   inplaceNormalize(inMag, 0.0F, maxMag);
00499 
00500   //Seperate the location above threshold into bins so we can sort them fast 
00501   //Mark the locations that we will be used in the contours img
00502   //
00503   std::vector<std::vector<Point2D<int> > > pointBins(nBins);
00504   for(int j=0; j<inMag.getHeight(); j++)
00505     for(int i=0; i<inMag.getWidth(); i++)
00506     {
00507       float val = inMag.getVal(i,j);
00508       if (val > threshold*maxMag)
00509       {
00510         contoursImg.setVal(i,j, UNKNOWN);
00511         //Find the bin this val belongs to
00512         int idx = (int)(val * nBins / maxMag);
00513         if (idx >= nBins) idx = nBins - 1; //if val == maxMag;
00514         pointBins[idx].push_back(Point2D<int>(i,j));
00515       } else {
00516         contoursImg.setVal(i,j, NOTDEFINED);
00517       }
00518     }
00519 
00520   //return a sorted list;
00521   std::vector<Point2D<int> > points;
00522   for(int i=pointBins.size()-1; i > -1; i--)
00523   {
00524     for(uint j=0; j<pointBins[i].size(); j++)
00525       points.push_back(pointBins[i][j]);
00526   }
00527 
00528   return points;
00529 
00530 }
00531 
00532 
00533 Layout<PixRGB<byte> > Contours::getDebugImage()
00534 {
00535   Layout<PixRGB<byte> > outDisp;
00536 
00537 
00538   Layout<PixRGB<byte> > tensorDisp;
00539   Image<PixRGB<byte> > lumEdges = toRGB(itsTensorFields[0].getTokensMag(true));
00540   Image<PixRGB<byte> > rgEdges = toRGB(itsTensorFields[1].getTokensMag(true));
00541   Image<PixRGB<byte> > byEdges = toRGB(itsTensorFields[2].getTokensMag(true));
00542 
00543   tensorDisp = hcat(lumEdges, rgEdges);
00544   tensorDisp = hcat(tensorDisp, byEdges);
00545 
00546 
00547   //ShowContour;
00548   Image<PixRGB<byte> > contourImg(lumEdges.getDims(), ZEROS);
00549   ColorMap cm = ColorMap::LINES(100);
00550   if (itsCurrentContour != -1)
00551   {
00552       Contour& contour = itsContours[itsCurrentContour];
00553       for(uint j=0; j<contour.points.size(); j++)
00554         contourImg.setVal(contour.points[j], cm[itsCurrentContour%cm.size()]);
00555 
00556       char msg[255];
00557       sprintf(msg, "C: %i", itsCurrentContour);
00558       writeText(contourImg, Point2D<int>(0,0), msg,
00559           PixRGB<byte>(255,255,255),
00560           PixRGB<byte>(0,0,0));
00561       
00562   } else {
00563     for(uint i=0; i<itsContours.size(); i++)
00564     {
00565       Contour& contour = itsContours[i];
00566       for(uint j=0; j<contour.points.size(); j++)
00567         contourImg.setVal(contour.points[j], cm[i%cm.size()]);
00568     }
00569   }
00570   //for(uint i=0; i<itsBiasContours.size(); i++)
00571   //{
00572   //  for(uint j=0; j<itsBiasContours[i].points.size(); j++)
00573   //    contourImg.setVal(itsBiasContours[i].points[j], PixRGB<byte>(0,255,0));
00574   //}
00575 
00576   outDisp = contourImg; //vcat(tensorDisp, contourImg);
00577   
00578   return outDisp;
00579 
00580 }
00581 
00582 
00583 // ######################################################################
00584 /* So things look consistent in everyone's emacs... */
00585 /* Local Variables: */
00586 /* indent-tabs-mode: nil */
00587 /* End: */
00588 
00589 #endif
00590 
Generated on Sun May 8 08:41:09 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3