CudaSaliency.C

00001 #include "CudaSaliency.H"
00002 #include "Util/Timer.H"
00003 #include "CUDA/CudaPyramidOps.H"
00004 #include "CUDA/CudaShapeOps.H"
00005 #include "CUDA/CudaTransforms.H"
00006 #include "CUDA/CudaPyrBuilder.H"
00007 #include "CUDA/CudaDrawOps.H"
00008 #include "CUDA/CudaNorm.H"
00009 #include "CUDA/CudaRandom.H"
00010 #include "Component/ModelOptionDef.H"
00011 
00012 // relative feature weights:
00013 #define IWEIGHT 0.7
00014 #define CWEIGHT 1.0
00015 #define OWEIGHT 1.0
00016 #define FWEIGHT 1.0
00017 #define SWEIGHT 0.7
00018 #define COLOR_THRESH 0.1F
00019 
00020 #define delta_min  3
00021 #define delta_max  4
00022 #define level_min  0
00023 #define level_max  2
00024 #define maxdepth   (level_max + delta_max + 1)
00025 #define sml        4
00026 #define normtyp    VCXNORM_FANCY
00027 #define MAXNORMITERS 1
00028 
00029 static const ModelOptionCateg MOC_CSM = {
00030   MOC_SORTPRI_3, "CudaSaliencyMap-related Options" };
00031 
00032 // Used by: CudaSaliency
00033 const ModelOptionDef OPT_CsmCudaDevice =
00034   { MODOPT_ARG(int), "CsmCudaDevice", &MOC_CSM, OPTEXP_CORE,
00035     "Number of frames in which the IOR map decays by half",
00036     "csm-cuda-device", '\0', "<int>", "0" };
00037 
00038 // Used by: EnvSaliencyMap
00039 const ModelOptionDef OPT_CsmNMostSalientLoc =
00040   { MODOPT_ARG(int), "CsmNMostSalientLoc", &MOC_CSM, OPTEXP_CORE,
00041     "Return a vector with the stop n most salient locations",
00042     "csm-nmost-salient-loc", '\0', "int", "1" };
00043 
00044 // Used by: CudaSaliency
00045 const ModelOptionDef OPT_CsmIorHalfLife =
00046   { MODOPT_ARG(double), "CsmIorHalfLife", &MOC_CSM, OPTEXP_CORE,
00047     "Number of frames in which the IOR map decays by half",
00048     "csm-ior-halflife", '\0', "<double>", "6.5" };
00049 
00050 // Used by: CudaSaliency
00051 const ModelOptionDef OPT_CsmIorStrength =
00052   { MODOPT_ARG(double), "CsmIorStrength", &MOC_CSM, OPTEXP_CORE,
00053     "Magnitude of IOR (useful range 0..255)",
00054     "csm-ior-strength", '\0', "<double>", "16.0" };
00055 
00056 // Used by: CudaSaliency
00057 const ModelOptionDef OPT_CsmIorRadius =
00058   { MODOPT_ARG(double), "CsmIorRadius", &MOC_CSM, OPTEXP_CORE,
00059     "Radius of IOR",
00060     "csm-ior-radius", '\0', "<double>", "32.0" };
00061 
00062 // Used by: CudaSaliency
00063 const ModelOptionDef OPT_CsmPatchSize =
00064   { MODOPT_ARG(double), "CsmPatchSize", &MOC_CSM, OPTEXP_CORE,
00065     "Patch size",
00066     "csm-patch-size", '\0', "<double>", "72.0" };
00067 
00068 // Used by: CudaSaliency
00069 const ModelOptionDef OPT_CsmInertiaRadius =
00070   { MODOPT_ARG(double), "CsmInertiaRadius", &MOC_CSM, OPTEXP_CORE,
00071     "Radius of inertia blob",
00072     "csm-inertia-radius", '\0', "<double>", "32.0" };
00073 
00074 // Used by: CudaSaliency
00075 const ModelOptionDef OPT_CsmInertiaStrength =
00076   { MODOPT_ARG(double), "CsmInertiaStrength", &MOC_CSM, OPTEXP_CORE,
00077     "Initial strength of inertia blob",
00078     "csm-inertia-strength", '\0', "<double>", "100.0" };
00079 
00080 // Used by: CudaSaliency
00081 const ModelOptionDef OPT_CsmInertiaHalfLife =
00082   { MODOPT_ARG(double), "CsmInertiaHalfLife", &MOC_CSM, OPTEXP_CORE,
00083     "Number of frames in which the inertia blob decays by half",
00084     "csm-inertia-halflife", '\0', "<double>", "6.5" };
00085 
00086 // Used by: CudaSaliency
00087 const ModelOptionDef OPT_CsmInertiaShiftThresh =
00088   { MODOPT_ARG(double), "CsmInertiaShiftThresh", &MOC_CSM, OPTEXP_CORE,
00089     "Distance threshold for inertia shift",
00090     "csm-inertia-thresh", '\0', "<double>", "5.0" };
00091 
00092 
00093 // ######################################################################
00094 CudaSaliency::CudaSaliency(OptionManager& mgr,
00095            const std::string& descrName,
00096                            const std::string& tagName):
00097   ModelComponent(mgr, descrName, tagName),
00098   itsCudaDevice(&OPT_CsmCudaDevice, this),
00099   itsInertiaLoc(-1,-1),
00100   itsCurrentInertiaFactor(1.0),
00101   itsInertiaMap(),
00102   itsInhibitionMap(),
00103   itsNMostSalientLoc(&OPT_CsmNMostSalientLoc, this, ALLOW_ONLINE_CHANGES),
00104   itsInertiaHalfLife(&OPT_CsmInertiaHalfLife, this, ALLOW_ONLINE_CHANGES),
00105   itsInertiaStrength(&OPT_CsmInertiaStrength, this, ALLOW_ONLINE_CHANGES),
00106   itsInertiaRadius(&OPT_CsmInertiaRadius, this, ALLOW_ONLINE_CHANGES),
00107   itsInertiaShiftThresh(&OPT_CsmInertiaShiftThresh, this, ALLOW_ONLINE_CHANGES),
00108   itsIorHalfLife(&OPT_CsmIorHalfLife, this, ALLOW_ONLINE_CHANGES),
00109   itsIorStrength(&OPT_CsmIorStrength, this, ALLOW_ONLINE_CHANGES),
00110   itsIorRadius(&OPT_CsmIorRadius, this, ALLOW_ONLINE_CHANGES),
00111   itsPatchSize(&OPT_CsmPatchSize, this, ALLOW_ONLINE_CHANGES),
00112   itsDynamicFactor(1.0)
00113 
00114 {
00115   gotLum = false; gotRGBY = false; gotSaliency = false;
00116   mp = GLOBAL_DEVICE_MEMORY; dev = -1;
00117   numMotionDirs = 4;
00118   numOrientationDirs = 4;
00119 }
00120 
00121 void CudaSaliency::setDevice(MemoryPolicy mp_in, int dev_in)
00122 {
00123   mp = mp_in;
00124   itsCudaDevice.setVal(dev_in);
00125   itsSalMaxLoc.resize(itsNMostSalientLoc.getVal());
00126   itsSalMax.resize(itsNMostSalientLoc.getVal());
00127 }
00128 
00129 // ######################################################################
00130 void CudaSaliency::paramChanged(ModelParamBase* const param,
00131                                   const bool valueChanged,
00132                                   ParamClient::ChangeStatus* status)
00133 {
00134   if (param == &itsCudaDevice)
00135     {
00136       dev = itsCudaDevice.getVal();
00137     }
00138   if (param == &itsNMostSalientLoc)
00139     {
00140       itsSalMaxLoc.resize(itsNMostSalientLoc.getVal());
00141       itsSalMax.resize(itsNMostSalientLoc.getVal());
00142     }
00143 }
00144 
00145 
00146 // ######################################################################
00147 void CudaSaliency::start1()
00148 {
00149   cudaSetSeed(dev);
00150   cudaSizeRandomBuffer(randBuf,mp,dev);
00151   reichardtPyr = new CudaReichardtPyrBuilder<float>[numMotionDirs];
00152   for(int i=0;i<numMotionDirs;i++)
00153     {
00154       double direction = 360.0*double(i)/double(numMotionDirs);
00155       reichardtPyr[i] =
00156         CudaReichardtPyrBuilder<float>(cos(direction*M_PI/180.0),-sin(direction*M_PI/180.0),
00157                                        Gaussian5,direction+90.0);
00158     }
00159 }
00160 
00161 // ######################################################################
00162 void CudaSaliency::stop2()
00163 {
00164   delete [] reichardtPyr;
00165 }
00166 
00167 // ######################################################################
00168 CudaSaliency::~CudaSaliency()
00169 { }
00170 
00171 // ######################################################################
00172 void CudaSaliency::doInput(const Image< PixRGB<byte> > img)
00173 {
00174   LINFO("new input.....");
00175   Image<PixRGB<float> > fimg = img;
00176   CudaImage<PixRGB<float> > tmp = CudaImage<PixRGB<float> >(fimg,mp,dev);
00177   doCudaInput(tmp);
00178 }
00179 
00180 void CudaSaliency::clearMaps()
00181 {
00182   if(outmap.initialized())
00183     outmap.clear();
00184   if(intensityMap.initialized())
00185     intensityMap.clear();
00186   if(colorMap.initialized())
00187     colorMap.clear();
00188   if(orientationMap.initialized())
00189     orientationMap.clear();
00190   if(flickerMap.initialized())
00191     flickerMap.clear();
00192   if(motionMap.initialized())
00193     motionMap.clear();
00194 }
00195 
00196 void CudaSaliency::doCudaInput(const CudaImage< PixRGB<float> > img)
00197 {
00198   int inDev = img.getMemoryDevice();
00199   ASSERT(inDev==dev);
00200   colima = img;
00201   clearMaps();
00202   // Reset the flags
00203   gotLum = false; gotRGBY = false;
00204   // Clear the outmap
00205   // also kill any old output and internals:
00206   Timer tim;
00207   //printf("Timing saliency\n");
00208   tim.reset();
00209   runSaliency();
00210   LINFO("Done! %fms", tim.getSecs() * 1000.0F);
00211 }
00212 
00213 // ######################################################################
00214 bool CudaSaliency::outputReady()
00215 {
00216   return gotSaliency;
00217 }
00218 
00219 // ######################################################################
00220 Image<float> CudaSaliency::getOutput()
00221 {
00222   return convmap.exportToImage();
00223 }
00224 
00225 CudaImage<float> CudaSaliency::getCudaOutput()
00226 {
00227   return convmap;
00228 }
00229 
00230 CudaImage<float> CudaSaliency::getIMap()
00231 {
00232   return intensityMap;
00233 }
00234 
00235 CudaImage<float> CudaSaliency::getCMap()
00236 {
00237   return colorMap;
00238 }
00239 
00240 CudaImage<float> CudaSaliency::getOMap()
00241 {
00242   return orientationMap;
00243 }
00244 
00245 CudaImage<float> CudaSaliency::getFMap()
00246 {
00247   return flickerMap;
00248 }
00249 
00250 CudaImage<float> CudaSaliency::getMMap()
00251 {
00252   return motionMap;
00253 }
00254 
00255 CudaImage<float> CudaSaliency::getInertiaMap()
00256 {
00257   return itsInertiaMap;
00258 }
00259 
00260 CudaImage<float> CudaSaliency::getInhibitionMap()
00261 {
00262   return itsInhibitionMap;
00263 }
00264 
00265 std::vector<Point2D<int> > CudaSaliency::getSalMaxLoc()
00266 {
00267   return itsSalMaxLoc;
00268 }
00269 
00270 std::vector<float> CudaSaliency::getSalMax()
00271 {
00272   return itsSalMax;
00273 }
00274 
00275 double CudaSaliency::getPatchSize()
00276 {
00277   return itsPatchSize.getVal();
00278 }
00279 
00280 CudaImage<float> CudaSaliency::cudaPostChannel(CudaImage<float> curImage, PyramidType ptyp, float orientation, float weight, CudaImage<float>& outmap)
00281 {
00282   // compute pyramid:
00283   CudaImageSet<float> pyr =
00284     cudaBuildPyrGeneric(curImage, 0, maxdepth, ptyp, orientation);
00285   CudaImage<float> cmap = processPyramid(pyr);
00286 
00287   // multiply by conspicuity coefficient:
00288   if (weight != 1.0F) cmap *= weight;
00289   // Add to saliency map:
00290   if (outmap.initialized()) outmap += cmap;
00291   else outmap = cmap;
00292   return cmap;
00293 }
00294 
00295 CudaImage<float> CudaSaliency::processPyramid(CudaImageSet<float> pyr)
00296 {
00297   // alloc conspicuity map and clear it:
00298   CudaImage<float> cmap;
00299 
00300   // intensities is the max-normalized weighted sum of IntensCS:
00301   for (int delta = delta_min; delta <= delta_max; delta ++)
00302     for (int lev = level_min; lev <= level_max; lev ++)
00303       {
00304         CudaImage<float> tmp = cudaCenterSurround(pyr, lev, lev + delta, true);
00305         tmp = cudaDownSize(tmp, pyr[sml].getWidth(), pyr[sml].getHeight());
00306         tmp = cudaMaxNormalize(tmp, MAXNORMMIN, MAXNORMMAX, normtyp,MAXNORMITERS);
00307         if(cmap.initialized())
00308           cmap += tmp;
00309         else
00310           cmap = tmp;
00311       }
00312 
00313   cudaInplaceAddBGnoise(cmap, 25.0F,randBuf);
00314 
00315 
00316   if (normtyp == VCXNORM_MAXNORM)
00317     cmap = cudaMaxNormalize(cmap, MAXNORMMIN, MAXNORMMAX, normtyp,MAXNORMITERS);
00318   else
00319     cmap = cudaMaxNormalize(cmap, 0.0f, 0.0f, normtyp,MAXNORMITERS);
00320 
00321 
00322   return cmap;
00323 }
00324 
00325 void CudaSaliency::calcInertia(const CudaImage<float> & salMap)
00326 {
00327   const MemoryPolicy mp = salMap.getMemoryPolicy();
00328   int dev = salMap.getMemoryDevice();
00329   ASSERT(mp != HOST_MEMORY);
00330 
00331   if (itsInertiaLoc.i < 0 || itsInertiaLoc.j < 0
00332       || (itsSalMaxLoc[0].squdist(itsInertiaLoc)
00333           > (itsInertiaShiftThresh.getVal()
00334              * itsInertiaShiftThresh.getVal())))
00335     {
00336       itsCurrentInertiaFactor = 1.0;
00337       LDEBUG("inertia shift to (%d,%d)",
00338              itsSalMaxLoc[0].i, itsSalMaxLoc[0].j);
00339     }
00340   else
00341     {
00342       const float factor =
00343         (itsDynamicFactor * itsInertiaHalfLife.getVal()) > 0
00344         ? pow(0.5, 1.0/(itsDynamicFactor * itsInertiaHalfLife.getVal()))
00345         : 0.0f;
00346       itsCurrentInertiaFactor *= factor;
00347     }
00348 
00349   {
00350     if (itsInertiaMap.getDims() != salMap.getDims() ||
00351           itsInertiaMap.getMemoryDevice() != dev)
00352       itsInertiaMap = CudaImage<float>(salMap.getDims(), ZEROS, mp, dev);
00353 
00354     itsInertiaLoc = itsSalMaxLoc[0];
00355 
00356     const double s =  itsInertiaStrength.getVal() * itsDynamicFactor * itsCurrentInertiaFactor;
00357     const double r_inv = itsInertiaRadius.getVal() > 0.0
00358       ? (1.0 / itsInertiaRadius.getVal()) : 0.0;
00359     Dims tile = CudaDevices::getDeviceTileSize1D(dev);
00360     int w = itsInertiaMap.getWidth();
00361     int h = itsInertiaMap.getHeight();
00362     cuda_c_inertiaMap(itsInertiaMap.getCudaArrayPtr(),s,r_inv,itsInertiaLoc.i,itsInertiaLoc.j,tile.w(),tile.h(),w,h);
00363   }
00364 }
00365 
00366 void CudaSaliency::calcInhibition(const CudaImage<float> & salMap)
00367 {
00368   const MemoryPolicy mp = salMap.getMemoryPolicy();
00369   int dev = salMap.getMemoryDevice();
00370   ASSERT(mp != HOST_MEMORY);
00371   if (itsDynamicFactor * itsIorStrength.getVal() > 0.0)
00372     {
00373       if (itsInhibitionMap.getDims() != salMap.getDims() ||
00374           itsInhibitionMap.getMemoryDevice() != dev)
00375         itsInhibitionMap = CudaImage<float>(salMap.getDims(), ZEROS, mp, dev);
00376 
00377       const float factor =
00378         (itsDynamicFactor * itsIorHalfLife.getVal()) > 0
00379         ? pow(0.5, 1.0/(itsDynamicFactor * itsIorHalfLife.getVal()))
00380         : 0.0f;
00381 
00382       Dims tile = CudaDevices::getDeviceTileSize1D(dev);
00383       int w = itsInhibitionMap.getWidth();
00384       int h = itsInhibitionMap.getHeight();
00385 
00386       cuda_c_inhibitionMap(itsInhibitionMap.getCudaArrayPtr(),factor,itsDynamicFactor*itsIorStrength.getVal(),itsIorRadius.getVal(),
00387                            itsSalMaxLoc[0].i,itsSalMaxLoc[0].j,tile.w(),tile.h(),w,h);
00388     }
00389   else
00390     itsInhibitionMap.clear(0);
00391 
00392 }
00393 
00394 
00395 void CudaSaliency::calcIntensity(const CudaImage<PixRGB<float> > & colImage, CudaImage<float>& outmap)
00396 {
00397   if(gotLum == false)
00398   {
00399     curLum = cudaLuminance(colImage);
00400     // compute pyramid:
00401     curLumPyr = cudaBuildPyrGeneric(curLum, 0, maxdepth, Gaussian5, 0.0F);
00402     CudaImage<float> cmap = processPyramid(curLumPyr);
00403     // multiply by conspicuity coefficient:
00404     if (IWEIGHT != 1.0F) cmap *= IWEIGHT;
00405     // Add to saliency map:
00406     if (outmap.initialized()) outmap += cmap;
00407     else outmap = cmap;
00408     intensityMap=cmap;
00409     gotLum = true;
00410   }
00411 }
00412 
00413 
00414 void CudaSaliency::calcColor(const CudaImage<PixRGB<float> > & colImage, CudaImage<float>& outmap)
00415 {
00416   if(gotRGBY == false)
00417   {
00418     cudaGetRGBY(colImage,rg,by,COLOR_THRESH);
00419     gotRGBY = true;
00420   }
00421   CudaImage<float> col=(rg+by)/2.0F;
00422   colorMap = cudaPostChannel(col,Gaussian5,0.0F,CWEIGHT,outmap);
00423 }
00424 
00425 void CudaSaliency::calcOrientation(const CudaImage<PixRGB<float> > & colImage, float orientation, CudaImage<float>& outmap)
00426 {
00427   if(gotLum == false)
00428   {
00429     curLum = cudaLuminance(colImage);
00430     gotLum = true;
00431   }
00432   CudaImage<float> o = cudaPostChannel(curLum,Oriented5,orientation,OWEIGHT,outmap);
00433   if(orientationMap.initialized())
00434     orientationMap += o;
00435   else
00436     orientationMap = o;
00437 }
00438 
00439 void CudaSaliency::calcFlicker(const CudaImage<PixRGB<float> >& colImage, CudaImage<float>& outmap)
00440 {
00441   CudaImage<float> curImage;
00442   if(gotLum == false)
00443   {
00444     curLum = cudaLuminance(colima);
00445     gotLum = true;
00446   }
00447   if (prevLum.initialized() == false)
00448     {
00449       prevLum = curLum;
00450       curImage = CudaImage<float>(curLum.getDims(), ZEROS, mp, dev);
00451     }
00452   else
00453     {
00454       curImage = curLum - prevLum;
00455       prevLum = curLum;
00456     }
00457   flickerMap = cudaPostChannel(curImage,Gaussian5,0.0F,FWEIGHT,outmap);
00458 }
00459 
00460 void CudaSaliency::calcMotion(const CudaImage<PixRGB<float> > & colImage, int motionIndex)
00461 {
00462   if(gotLum == false)
00463   {
00464     curLum = cudaLuminance(colImage);
00465     gotLum = true;
00466   }
00467 
00468   CudaImageSet<float> motPyr = reichardtPyr[motionIndex].build(curLum,0,maxdepth);
00469   CudaImage<float> m = processPyramid(motPyr);
00470   if(motionMap.initialized())
00471     motionMap += m;
00472   else
00473     motionMap = m;
00474 }
00475 
00476 void CudaSaliency::runSaliency()
00477 {
00478   Timer tim;
00479   gotSaliency = false;
00480   tim.reset();
00481   calcIntensity(colima,outmap);
00482   //LINFO("Intensity %fms", tim.getSecs() * 1000.0F);
00483 
00484   tim.reset();
00485   calcColor(colima,outmap);
00486   //LINFO("Color %fms", tim.getSecs() * 1000.0F);
00487   tim.reset();
00488 
00489   //LINFO("Orientation %fms", tim.getSecs() * 1000.0F);
00490   for(int i=0;i<numOrientationDirs;i++)
00491     {
00492       calcOrientation(colima,180.0*double(i)/double(numOrientationDirs),outmap);
00493     }
00494 
00495   tim.reset();
00496   calcFlicker(colima,outmap);
00497   //LINFO("Flicker %fms", tim.getSecs() * 1000.0F);
00498 
00499   for(int i=0;i<numMotionDirs;i++)
00500     {
00501       calcMotion(colima,i);
00502     }
00503 
00504   // Max norm the combined motion maps
00505   motionMap = cudaMaxNormalize(motionMap, MAXNORMMIN, MAXNORMMAX, normtyp,MAXNORMITERS);
00506   motionMap *=10;
00507   // Add to saliency map:
00508   if (outmap.initialized()) outmap += motionMap;
00509   else outmap = motionMap;
00510   // Subtract the inhibition map
00511   if(itsInhibitionMap.initialized())
00512     outmap -= itsInhibitionMap;
00513   cudaInplaceClamp(outmap,0,255);
00514   convmap = outmap;
00515   double w_ratio = outmap.getWidth()/double(colima.getWidth());
00516   double h_ratio = outmap.getHeight()/double(colima.getHeight());
00517   int smallPatchWidth = itsPatchSize.getVal()*w_ratio;
00518   int smallPatchHeight = itsPatchSize.getVal()*h_ratio;
00519   Dims rectDims = Dims(smallPatchHeight,smallPatchHeight);
00520   CudaImage<float> tmpmap = outmap;
00521 
00522   // Get the N most salient locations
00523   for(int i=0;i<itsNMostSalientLoc.getVal();i++)
00524     {
00525       cudaFindMax(tmpmap,itsSalMaxLoc[i],itsSalMax[i]);
00526       int modI = std::max(0,std::min(itsSalMaxLoc[i].i-smallPatchWidth/2,int(outmap.getWidth()-smallPatchWidth/2)));
00527       int modJ = std::max(0,std::min(itsSalMaxLoc[i].j-smallPatchHeight/2,int(outmap.getHeight()-smallPatchHeight/2)));  
00528 
00529       Rectangle tmprect = Rectangle(Point2D<int>(modI,modJ),rectDims);
00530       if(i+1<itsNMostSalientLoc.getVal())
00531         cudaDrawFilledRect(tmpmap,tmprect,0);
00532     }
00533   // Calculate inertia and inhibition
00534   calcInhibition(outmap);
00535   if(!itsInhibitionMap.initialized())
00536     itsInhibitionMap = CudaImage<float>(outmap.getDims(), ZEROS, mp, dev);
00537   //calcInertia(outmap);
00538   gotSaliency = true;
00539 }
Generated on Sun May 8 08:40:36 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3