00001 /*!@file Surprise/RemoveSurprise.H attempt to remove surprise from image */ 00002 00003 // //////////////////////////////////////////////////////////////////// // 00004 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2001 by the // 00005 // University of Southern California (USC) and the iLab at USC. // 00006 // See http://iLab.usc.edu for information about this project. // 00007 // //////////////////////////////////////////////////////////////////// // 00008 // Major portions of the iLab Neuromorphic Vision Toolkit are protected // 00009 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency // 00010 // in Visual Environments, and Applications'' by Christof Koch and // 00011 // Laurent Itti, California Institute of Technology, 2001 (patent // 00012 // pending; application number 09/912,225 filed July 23, 2001; see // 00013 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status). // 00014 // //////////////////////////////////////////////////////////////////// // 00015 // This file is part of the iLab Neuromorphic Vision C++ Toolkit. // 00016 // // 00017 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can // 00018 // redistribute it and/or modify it under the terms of the GNU General // 00019 // Public License as published by the Free Software Foundation; either // 00020 // version 2 of the License, or (at your option) any later version. // 00021 // // 00022 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope // 00023 // that it will be useful, but WITHOUT ANY WARRANTY; without even the // 00024 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR // 00025 // PURPOSE. See the GNU General Public License for more details. // 00026 // // 00027 // You should have received a copy of the GNU General Public License // 00028 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write // 00029 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, // 00030 // Boston, MA 02111-1307 USA. // 00031 // //////////////////////////////////////////////////////////////////// // 00032 // 00033 // Primary maintainer for this file: T. Nathan Mundhenk <mundhenk@usc.edu> 00034 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/Surprise/RemoveSurprise.H $ 00035 // $Id: RemoveSurprise.H 6795 2006-06-29 20:45:32Z rjpeters $ 00036 // 00037 00038 #ifndef REMOVE_SURPRISE_H_DEFINED 00039 #define REMOVE_SURPRISE_H_DEFINED 00040 00041 #include <fstream> 00042 #include <iostream> 00043 #include <math.h> 00044 #include <string> 00045 #include <vector> 00046 #include <deque> 00047 00048 #include "Raster/Raster.H" 00049 #include "Util/Assert.H" 00050 #include "Util/Timer.H" 00051 #include "Image/Image.H" 00052 #include "Image/Pixels.H" 00053 00054 #define SMALL_SALIENCY 0.005F 00055 #define NEAR_BLACK 0.005F 00056 #define SCALE_CHANNELS 6 00057 00058 using namespace std; 00059 00060 //! smooth surprise in a movie 00061 template <class PIXTYPE, class BETATYPE, class FLOAT> class RemoveSurprise 00062 { 00063 public: 00064 RemoveSurprise(const ushort sizeX, const ushort sizeY); 00065 RemoveSurprise(); 00066 ~RemoveSurprise(); 00067 //! Init the basic variables 00068 void RSinit(const ushort sizeX, const ushort sizeY); 00069 //! set true to use a more true kalman filter 00070 void RSuseTrueKalman(const bool useTK); 00071 //! use max level from conspicuity map rather than sum the levels 00072 void RSuseMaxLevel(const bool useML); 00073 //! set the bias to something other than 1.0F 00074 void RSsetConspicBias(const FLOAT CO, const FLOAT MO, 00075 const FLOAT OR, const FLOAT IN); 00076 //! set the bias to something other than 1.0F 00077 void RSsetConspicBias(const FLOAT IN, const FLOAT DR, 00078 const FLOAT FL, const FLOAT GA, 00079 const FLOAT RG, const FLOAT BY); 00080 //! set the bias over a single axis 00081 void RSsetAxisBias(const FLOAT X, const FLOAT Y, const FLOAT Z); 00082 //! set the bias per H2SV channel 00083 void RSsetH2SVBias(const FLOAT H1, const FLOAT H2, 00084 const FLOAT S, const FLOAT V); 00085 //! set the decay term over the beta map 00086 void RSsetLambda(const FLOAT lambda); 00087 //! set the bias to desaturate color at surprising locations 00088 void RSsetDesatBias(const FLOAT desat); 00089 //! input the raw image frame for processing 00090 void RSinputRawImage(const Image<PixRGB<FLOAT> >& rawImage, 00091 const uint frame); 00092 //! input the salmap for this frame 00093 void RSinputSalMap(const Image<FLOAT>& salMap); 00094 //! input the color conspicuity map for this frame 00095 void RSinputConspicCO(const Image<FLOAT>& conspicCO); 00096 //! input the motion conspicuity map for this frame 00097 void RSinputConspicMO(const Image<FLOAT>& conspicMO); 00098 //! input the orientation conspicuity map for this frame 00099 void RSinputConspicOR(const Image<FLOAT>& conspicOR); 00100 //! input the intensity conspicuity map for this frame 00101 void RSinputConspicIN(const Image<FLOAT>& conspicIN); 00102 //! input the orientation conspicuity map for this frame 00103 void RSinputConspicDR(const Image<FLOAT>& conspicDR); 00104 //! input the orientation conspicuity map for this frame 00105 void RSinputConspicFL(const Image<FLOAT>& conspicFL); 00106 //! input the orientation conspicuity map for this frame 00107 void RSinputConspicGA(const Image<FLOAT>& conspicGA); 00108 //! input the orientation conspicuity map for this frame 00109 void RSinputConspicRG(const Image<FLOAT>& conspicRG); 00110 //! input the orientation conspicuity map for this frame 00111 void RSinputConspicBY(const Image<FLOAT>& conspicBY); 00112 //! Input a bayes weight image to bias surprise reduction 00113 void RSinputBayesWeightImage(const Image<FLOAT> &bayesImage); 00114 //! Input correlationMatrices for weight augmentation 00115 void RSsetCorrWeightMat(const std::vector<std::vector<FLOAT> > corAnti, 00116 const std::vector<std::vector<FLOAT> > corBase); 00117 //! create the seperable filters per image 00118 void RScreateSepFilters(const FLOAT spatSigma, 00119 const FLOAT tempSigma, 00120 const FLOAT stdDevSize); 00121 //! find parameters over convolution 00122 void RSfindConvolutionEndPoints(); 00123 //! process a single frame of video 00124 void RSprocessFrame(); 00125 //! process a single frame of video using seperable filters 00126 void RSprocessFrameSeperable(); 00127 //! process each axis by itself 00128 void RSseperateConv(const char axis); 00129 //! compute a new weight based on feature correlation matrix, used in f bias 00130 FLOAT RScomputeCor(const FLOAT IN, const FLOAT DR, const FLOAT FL, 00131 const FLOAT GA, const FLOAT RG, const FLOAT BY, 00132 const short idx) const; 00133 00134 //! process and return the final image 00135 Image<PixRGB<FLOAT> > RSgetFrame(); 00136 //! process and return the filtered image 00137 Image<PixRGB<FLOAT> > RSgetOutImage() const; 00138 //! get the raw PIXTYPE output image 00139 Image<PIXTYPE> RSgetRawOutImage() const; 00140 //! get the input image converted as PIXTYPE 00141 Image<PIXTYPE> RSgetRawInImage() const; 00142 //! get the beta image of smoothed salMaps used in filter 00143 Image<BETATYPE> RSgetBetaImage() const; 00144 private: 00145 //! The basic frame buffer, holds either smoothed or non-smoothed 00146 std::deque<Image<PIXTYPE> > itsFrameBuffer; 00147 //! intermediate processed image 00148 std::vector<Image<PIXTYPE> > itsInterImage; 00149 //! intermediate processed image 00150 std::vector<Image<PIXTYPE> > itsInterImageNorm; 00151 //! Anti image elementary correlation matrix 00152 std::vector<std::vector<FLOAT> > itsAntiCorrelationMat; 00153 //! Base image elementary correlation matrix 00154 std::vector<std::vector<FLOAT> > itsBaseCorrelationMat; 00155 //! The input for this frame 00156 Image<PIXTYPE> itsInImage; 00157 //! The output for this frame 00158 Image<PIXTYPE> itsOutImage; 00159 //! The output for this frame 00160 Image<BETATYPE> itsBetaImage; 00161 //! The final image after we process by the surprise map 00162 Image<PixRGB<FLOAT> > itsFinalImage; 00163 //! the current saliency map 00164 Image<FLOAT> itsSalMap; 00165 //! color conspicuity map 00166 Image<FLOAT> itsConspicCO; 00167 //! motion conspicuity map 00168 Image<FLOAT> itsConspicMO; 00169 //! orientation conspicuty map 00170 Image<FLOAT> itsConspicOR; 00171 //! intensity conspicuity map 00172 Image<FLOAT> itsConspicIN; 00173 //! input maps of the post surprise processed image Direction 00174 Image<FLOAT> itsConspicDR; 00175 //! input maps of the post surprise processed image Flicker 00176 Image<FLOAT> itsConspicFL; 00177 //! input maps of the post surprise processed image Gaussian 00178 Image<FLOAT> itsConspicGA; 00179 //! input maps of the post surprise processed image Red/Green 00180 Image<FLOAT> itsConspicRG; 00181 //! input maps of the post surprise processed image Blue/Yellow 00182 Image<FLOAT> itsConspicBY; 00183 //! input a bayesian bias image 00184 Image<FLOAT> itsBayesWeightImage; 00185 //! store the start point for this convolution in X 00186 Image<ushort> itsXStart; 00187 //! store the start point for this convolution in Y 00188 Image<ushort> itsYStart; 00189 //! store the start point for this convolution in Z 00190 Image<ushort> itsZStart; 00191 //! store the stop point for this convolution in X 00192 Image<ushort> itsXStop; 00193 //! store the stop point for this convolution in Y 00194 Image<ushort> itsYStop; 00195 //! store the stop point for this convolution in Z 00196 Image<ushort> itsZStop; 00197 //! store the start point for this convolution in X 00198 Image<ushort> itsKXStart; 00199 //! store the start point for this convolution in Y 00200 Image<ushort> itsKYStart; 00201 //! store the start point for this convolution in Z 00202 Image<ushort> itsKZStart; 00203 //! store messages that tell us it we did not init something 00204 std::vector<std::string> itsInitMessage; 00205 //! store the X part of the kernel 00206 std::vector<FLOAT> itsKalmanKernelX; 00207 //! store the Y part of the kernel 00208 std::vector<FLOAT> itsKalmanKernelY; 00209 //! store the Z part of the kernel 00210 std::vector<FLOAT> itsKalmanKernelZ; 00211 //! store messages that tell us it we did not init something 00212 std::vector<bool> itsInit; 00213 //! blank pixel used in run time typing 00214 PixHyper<FLOAT,6> itsHyper6; 00215 //! blank pixel used in run time typing 00216 PixHyper<FLOAT,4> itsHyper4; 00217 //! The color bias 00218 FLOAT itsCObias; 00219 //! The motion bias 00220 FLOAT itsMObias; 00221 //! The orientation bias 00222 FLOAT itsORbias; 00223 //! The intensity bias 00224 FLOAT itsINbias; 00225 //! The directional bias 00226 FLOAT itsDRbias; 00227 //! The flicker bias 00228 FLOAT itsFLbias; 00229 //! The guassian bias 00230 FLOAT itsGAbias; 00231 //! The red/green bias 00232 FLOAT itsRGbias; 00233 //! The blue/yellow bias 00234 FLOAT itsBYbias; 00235 //! The decay term over beta 00236 FLOAT itsLambda; 00237 //! Bias over the X axis 00238 FLOAT itsXbias; 00239 //! Bias over the Y axis 00240 FLOAT itsYbias; 00241 //! Bias over the Z axis 00242 FLOAT itsZbias; 00243 //! Bias to H1 00244 FLOAT itsH1bias; 00245 //! Bias to H2 00246 FLOAT itsH2bias; 00247 //! Bias to S 00248 FLOAT itsSbias; 00249 //! Bias to V 00250 FLOAT itsVbias; 00251 //! Bias to desatuate surprsing color 00252 FLOAT itsDesatbias; 00253 //! The current frame number 00254 uint itsFrameNumber; 00255 //! the size of the X kernel 00256 ushort itsKernelSizeX; 00257 //! the size of the Y kernel 00258 ushort itsKernelSizeY; 00259 //! the size of the Z kernel 00260 ushort itsKernelSizeZ; 00261 //! the size of the image in X 00262 ushort itsImageSizeX; 00263 //! the size of the image in Y 00264 ushort itsImageSizeY; 00265 bool itsUseTrueKalman; 00266 bool itsInitBuffer; 00267 //! Use max or combined surprise values 00268 bool itsUseMaxLevel; 00269 //! Use correlation matrix for biased surprise removal 00270 bool itsUseCorrMatrixSet; 00271 //! turn on using bayesian weighting of surprise reduction 00272 bool itsUseBayesWeightImage; 00273 }; 00274 #endif 00275
1.6.3