test-ObjRec.C

Go to the documentation of this file.
00001 /*!@file TestSuite/test-ObjRec.C Test Varius object rec code */
00002 
00003 // //////////////////////////////////////////////////////////////////// //
00004 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2001 by the //
00005 // University of Southern California (USC) and the iLab at USC.         //
00006 // See http://iLab.usc.edu for information about this project.          //
00007 // //////////////////////////////////////////////////////////////////// //
00008 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00009 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00010 // in Visual Environments, and Applications'' by Christof Koch and      //
00011 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00012 // pending; application number 09/912,225 filed July 23, 2001; see      //
00013 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00014 // //////////////////////////////////////////////////////////////////// //
00015 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00016 //                                                                      //
00017 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00018 // redistribute it and/or modify it under the terms of the GNU General  //
00019 // Public License as published by the Free Software Foundation; either  //
00020 // version 2 of the License, or (at your option) any later version.     //
00021 //                                                                      //
00022 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00023 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00024 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00025 // PURPOSE.  See the GNU General Public License for more details.       //
00026 //                                                                      //
00027 // You should have received a copy of the GNU General Public License    //
00028 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00029 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00030 // Boston, MA 02111-1307 USA.                                           //
00031 // //////////////////////////////////////////////////////////////////// //
00032 //
00033 // Primary maintainer for this file: Lior Elazary
00034 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/TestSuite/test-ObjRec.C $
00035 // $Id: test-ObjRec.C 12962 2010-03-06 02:13:53Z irock $
00036 //
00037 
00038 #include "Component/GlobalOpts.H"
00039 #include "Component/ModelManager.H"
00040 #include "Component/ModelOptionDef.H"
00041 #include "Component/ModelParam.H"
00042 #include "Component/ModelParamBatch.H"
00043 #include "GUI/XWindow.H"
00044 #include "Image/Image.H"
00045 #include "Image/ColorOps.H"
00046 #include "Image/CutPaste.H"
00047 #include "Image/ShapeOps.H"
00048 #include "Image/Rectangle.H"
00049 #include "Image/MathOps.H"
00050 #include "Image/MatrixOps.H"
00051 #include "Image/Transforms.H"
00052 #include "Image/Convolutions.H"
00053 #include "Media/FrameSeries.H"
00054 #include "Media/TestImages.H"
00055 #include "nub/ref.h"
00056 #include "Raster/GenericFrame.H"
00057 #include "Transport/FrameInfo.H"
00058 #include "Raster/Raster.H"
00059 #include "Util/Types.H"
00060 #include "Util/log.H"
00061 #include "Util/Timer.H"
00062 #include "TestSuite/ObjRecBrain.h"
00063 
00064 #include <fstream>
00065 #include <iostream>
00066 #include <iomanip>
00067 #include <string>
00068 #include <unistd.h>
00069 #include <cstdlib>
00070 #include <cstdlib>
00071 #include <dlfcn.h>
00072 
00073 static const ModelOptionDef OPT_ObjRecTrainingMode =
00074   { MODOPT_FLAG, "ObjRecTrainingMode", &MOC_GENERAL, OPTEXP_CORE,
00075     "Whether to traing the classifier or recognize ",
00076     "training-mode", '\0', "", "false" };
00077 
00078 static const ModelOptionDef OPT_ObjRecFilterObject =
00079   { MODOPT_ARG_STRING, "ObjRecFilterObject", &MOC_GENERAL, OPTEXP_CORE,
00080     "Binary recognition. Is this object there or not. ",
00081     "filter-object", '\0', "<string>", "" };
00082 
00083 static const ModelOptionDef OPT_ObjRecOutputROCFile =
00084   { MODOPT_ARG_STRING, "ObjRecOutputROCFile", &MOC_GENERAL, OPTEXP_CORE,
00085     "The file name to output the ROC data to. ",
00086     "roc-file", '\0', "<string>", "" };
00087 
00088 static const ModelOptionDef OPT_ObjRecOutputTimingFile =
00089   { MODOPT_ARG_STRING, "ObjRecOutputTimingFile", &MOC_GENERAL, OPTEXP_CORE,
00090     "The file name to output timing information. ",
00091     "timing-file", '\0', "<string>", "" };
00092 
00093 static const ModelOptionDef OPT_ObjRecOutputResultsFile =
00094   { MODOPT_ARG_STRING, "ObjRecOutputResultsFile", &MOC_GENERAL, OPTEXP_CORE,
00095     "The file name to output full results information to. "
00096     "This will include the frame number, the scene filename, which object we "
00097     "had, what we labeled it and the confidence. Only for recognition mode.",
00098     "results-file", '\0', "<string>", "" };
00099 
00100 static const ModelOptionDef OPT_ObjRecObjectsDBFile =
00101   { MODOPT_ARG_STRING, "ObjRecObjectsDBFile", &MOC_GENERAL, OPTEXP_CORE,
00102     "The file name to use as the object database ",
00103     "objects-db-file", '\0', "<string>", "objects.dat" };
00104 
00105 
00106 struct ResultData
00107 {
00108   int frame;
00109   std::string objName;
00110   std::string labelName;
00111   float confidence;
00112 
00113   ResultData(int f, std::string& obj, std::string& label, float c) :
00114     frame(f),
00115     objName(obj),
00116     labelName(label),
00117     confidence(c)
00118   {}
00119 };
00120 
00121 bool ResultDataCmp(const ResultData& r1, const ResultData& r2)
00122 {
00123   return r1.confidence > r2.confidence;
00124 }
00125 
00126 
00127 
00128 int main(const int argc, const char **argv)
00129 {
00130 
00131   MYLOGVERB = LOG_INFO;
00132   ModelManager manager("Test Object Rec");
00133 
00134   OModelParam<bool> optTrainingMode(&OPT_ObjRecTrainingMode, &manager);
00135   OModelParam<std::string> optFilterObject(&OPT_ObjRecFilterObject, &manager);
00136   OModelParam<std::string> optOutputROCFile(&OPT_ObjRecOutputROCFile, &manager);
00137   OModelParam<std::string> optOutputTimingFile(&OPT_ObjRecOutputTimingFile, &manager);
00138   OModelParam<std::string> optOutputResultsFile(&OPT_ObjRecOutputResultsFile, &manager);
00139   OModelParam<std::string> optObjectsDBFile(&OPT_ObjRecObjectsDBFile, &manager);
00140 
00141   nub::ref<InputFrameSeries> ifs(new InputFrameSeries(manager));
00142   manager.addSubComponent(ifs);
00143 
00144   nub::ref<OutputFrameSeries> ofs(new OutputFrameSeries(manager));
00145   manager.addSubComponent(ofs);
00146 
00147   manager.exportOptions(MC_RECURSE);
00148 
00149   if (manager.parseCommandLine(
00150         (const int)argc, (const char**)argv, "<ObjRecBrainLib>", 1, 1) == false)
00151     return 1;
00152 
00153   std::string libFile = manager.getExtraArg(0);
00154   LDEBUG("Loading %s", libFile.c_str());
00155   void* brainLib = dlopen(libFile.c_str(), RTLD_LAZY);
00156   if (!brainLib)
00157     LFATAL("Can load library: %s (%s)", libFile.c_str(), dlerror());
00158 
00159   //Load the symbols
00160   dlerror(); //reset any errors
00161   CreateObjRecBrain* createBrain = (CreateObjRecBrain*) dlsym(brainLib, "createObjRecBrain");
00162   DestoryObjRecBrain* destoryBrain = (DestoryObjRecBrain*) dlsym(brainLib, "destoryObjRecBrain");
00163 
00164   if (!createBrain  || !destoryBrain)
00165     LFATAL("Can not find the create and destory symbols: %s", dlerror());
00166 
00167   ObjRecBrain* brain = createBrain(optObjectsDBFile.getVal());
00168 
00169   manager.start();
00170 
00171   ifs->startStream();
00172 
00173   Timer timer;
00174   FILE* timingFP = NULL;
00175   if (optOutputTimingFile.getVal().size() > 0)
00176   {
00177     timingFP = fopen(optOutputTimingFile.getVal().c_str(), "w");
00178     if (timingFP == NULL)
00179       LFATAL("Can not open timing file: %s",
00180           optOutputTimingFile.getVal().c_str());
00181   }
00182 
00183 
00184   timer.reset();
00185   if (optTrainingMode.getVal())
00186     brain->preTraining();
00187   else
00188     brain->preRecognition();
00189   float preTime = timer.getSecs();
00190 
00191   if (timingFP)
00192     fprintf(timingFP, "%s %f\n",
00193         optTrainingMode.getVal() ? "PreTraining" : "PreRecognition",
00194         preTime);
00195 
00196   FILE* resultsFP = NULL;
00197   if (optOutputResultsFile.getVal().size() > 0 && !optTrainingMode.getVal())
00198   {
00199     resultsFP = fopen(optOutputResultsFile.getVal().c_str(), "w");
00200     if (resultsFP == NULL)
00201       LFATAL("Can not open results file: %s",
00202           optOutputResultsFile.getVal().c_str());
00203   }
00204   std::vector<ResultData> results;
00205 
00206   double totalTime = 0;
00207   unsigned long totalNumFrames = 0;
00208   while(1)
00209   {
00210     Image< PixRGB<byte> > inputImg;
00211     const FrameState is = ifs->updateNext();
00212     if (is == FRAME_COMPLETE)
00213       break;
00214 
00215     //grab the images
00216     GenericFrame input = ifs->readFrame();
00217     if (!input.initialized())
00218       break;
00219     inputImg = input.asRgb();
00220 
00221     //Get the metadata and find if we have the object name in the scene
00222     rutz::shared_ptr<GenericFrame::MetaData>
00223       metaData = input.getMetaData(std::string("SceneData"));
00224     if (metaData.get() != 0) {
00225       rutz::shared_ptr<TestImages::SceneData> sceneData;
00226       sceneData.dyn_cast_from(metaData);
00227 
00228       ObjectData labeledObj;
00229 
00230       if (optFilterObject.getVal().size() > 0)
00231       {
00232         labeledObj.name = "no_" + optFilterObject.getVal();
00233         labeledObj.confidence = -1;
00234 
00235         //Sech and see if we have this object in the scene
00236         for (uint i = 0; i < sceneData->objects.size(); i++) {
00237           TestImages::ObjData objData = sceneData->objects[i];
00238           if (optFilterObject.getVal() == objData.name)
00239             labeledObj.name = objData.name;
00240         }
00241       } else {
00242         //Take the first object
00243         for (uint i = 0; i < sceneData->objects.size() && i<1; i++) {
00244           TestImages::ObjData objData = sceneData->objects[i];
00245           labeledObj.name = objData.name;
00246         }
00247       }
00248 
00249       double frameTime = -1;
00250       if (optTrainingMode.getVal())
00251       {
00252         timer.reset();
00253         brain->onTraining(inputImg, labeledObj);
00254         frameTime = timer.getSecs();
00255       } else {
00256         timer.reset();
00257         ObjectData obj = brain->onRecognition(inputImg);
00258         frameTime = timer.getSecs();
00259 
00260         float confidence = obj.confidence;
00261 
00262         //Invert the confidence since we are more confident that this is not the object
00263         //so we care less confident that this is the object
00264         if ( optFilterObject.getVal() != obj.name )
00265           confidence = 1/confidence;
00266 
00267         results.push_back(ResultData(ifs->frame(),
00268               labeledObj.name,
00269               obj.name,
00270               confidence));
00271 
00272         if (resultsFP)
00273           fprintf(resultsFP, "%i %s %s %s %f\n",
00274               ifs->frame(), sceneData->filename.c_str(),
00275               labeledObj.name.c_str(),
00276               obj.name.c_str(), confidence);
00277 
00278       }
00279 
00280       if (timingFP)
00281         fprintf(timingFP, "%i %f\n",
00282             ifs->frame(), frameTime);
00283       totalNumFrames++;
00284       totalTime += frameTime;
00285 
00286     }
00287     ofs->writeRGB(inputImg, "input", FrameInfo("input", SRC_POS));
00288     usleep(10000);
00289   }
00290 
00291   if (resultsFP)
00292     fclose(resultsFP);
00293 
00294   timer.reset();
00295   if (optTrainingMode.getVal())
00296     brain->postTraining();
00297   else
00298     brain->postRecognition();
00299   float postTime = timer.getSecs();
00300   if (timingFP)
00301     fprintf(timingFP, "%s %f\n",
00302         optTrainingMode.getVal() ? "PostTraining" : "PostRecognition",
00303         postTime);
00304 
00305   if (timingFP)
00306     fclose(timingFP);
00307 
00308   //Calculate ROC curve and AP
00309   if (!optTrainingMode.getVal())
00310   {
00311     std::sort(results.begin(), results.end(), ResultDataCmp);
00312     std::vector<float> tp;
00313     int numPosExamples = 0;
00314     for(uint i=0; i<results.size(); i++)
00315     {
00316       //Calculate true positive
00317       if (optFilterObject.getVal() == results[i].objName)
00318       {
00319         numPosExamples++;
00320         if (tp.size() > 0)
00321           tp.push_back(tp.at(i-1)+1);
00322         else
00323           tp.push_back(1);
00324       } else {
00325         if (tp.size() > 0)
00326           tp.push_back(tp.at(i-1));
00327         else
00328           tp.push_back(0);
00329       }
00330     }
00331 
00332     std::vector<float> rec;
00333     std::vector<float> prec;
00334     for(uint i=0; i<tp.size(); i++)
00335     {
00336       rec.push_back(tp[i]/numPosExamples);
00337       prec.push_back(tp[i]/(i+1));
00338     }
00339 
00340     ////Output the precision recall curve
00341     FILE* rocFP = NULL;
00342     if (optOutputROCFile.getVal().size() > 0)
00343     {
00344       rocFP = fopen(optOutputROCFile.getVal().c_str(), "w");
00345       if (rocFP == NULL)
00346         LFATAL("Can not open roc file: %s",
00347             optOutputROCFile.getVal().c_str());
00348     }
00349 
00350     if (rocFP)
00351     {
00352       for(uint i=0; i<rec.size(); i++)
00353         fprintf(rocFP, "%f %f\n", rec[i], prec[i]);
00354       fclose(rocFP);
00355     }
00356 
00357     //Calculate the average precision
00358     double ap=0;
00359     double step = 0.1;
00360 
00361     for(double t=0; t<=1; t+=step)
00362     {
00363       double maxPrec = 0;
00364       for(uint i=0; i<rec.size(); i++)
00365         if (rec[i] >= t)
00366           if (prec[i] > maxPrec)
00367             maxPrec = prec[i];
00368 
00369       ap += (maxPrec / ((1/step)+1) ); //take the average
00370     }
00371     printf("Stats: Frames:%lu FPS:%f AP:%f\n",
00372         totalNumFrames, (double)totalNumFrames/totalTime, ap);
00373   } else {
00374     printf("Stats: Frames:%lu FPS:%f \n",
00375         totalNumFrames, (double)totalNumFrames/totalTime);
00376   }
00377 
00378 
00379 
00380   destoryBrain(brain);
00381 
00382   //unload the library
00383   dlclose(brainLib);
00384 
00385 
00386   return 0;
00387 }
00388 
00389 
00390 // ######################################################################
00391 /* So things look consistent in everyone's emacs... */
00392 /* Local Variables: */
00393 /* indent-tabs-mode: nil */
00394 /* End: */
Generated on Sun May 8 08:41:08 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3