Retina.H

Go to the documentation of this file.
00001 /*!@file Neuro/Retina.H simulation of a human retina */
00002 
00003 // //////////////////////////////////////////////////////////////////// //
00004 // The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2000-2003   //
00005 // by the University of Southern California (USC) and the iLab at USC.  //
00006 // See http://iLab.usc.edu for information about this project.          //
00007 // //////////////////////////////////////////////////////////////////// //
00008 // Major portions of the iLab Neuromorphic Vision Toolkit are protected //
00009 // under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
00010 // in Visual Environments, and Applications'' by Christof Koch and      //
00011 // Laurent Itti, California Institute of Technology, 2001 (patent       //
00012 // pending; application number 09/912,225 filed July 23, 2001; see      //
00013 // http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
00014 // //////////////////////////////////////////////////////////////////// //
00015 // This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
00016 //                                                                      //
00017 // The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
00018 // redistribute it and/or modify it under the terms of the GNU General  //
00019 // Public License as published by the Free Software Foundation; either  //
00020 // version 2 of the License, or (at your option) any later version.     //
00021 //                                                                      //
00022 // The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
00023 // that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
00024 // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
00025 // PURPOSE.  See the GNU General Public License for more details.       //
00026 //                                                                      //
00027 // You should have received a copy of the GNU General Public License    //
00028 // along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
00029 // to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
00030 // Boston, MA 02111-1307 USA.                                           //
00031 // //////////////////////////////////////////////////////////////////// //
00032 //
00033 // Primary maintainer for this file: Laurent Itti <itti@usc.edu>
00034 // $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/Neuro/Retina.H $
00035 // $Id: Retina.H 14634 2011-03-24 00:33:16Z dberg $
00036 //
00037 
00038 #ifndef RETINA_H_DEFINED
00039 #define RETINA_H_DEFINED
00040 
00041 #include "Channels/InputFrame.H"
00042 #include "Component/ModelComponent.H"
00043 #include "Component/ModelParam.H"
00044 #include "Image/Image.H"
00045 #include "Image/ImageSet.H"
00046 #include "Image/Pixels.H"
00047 #include "Image/Point2D.H"
00048 #include "Image/LevelSpec.H"
00049 #include "Media/MediaSimEvents.H"
00050 #include "Neuro/NeuroSimEvents.H"
00051 #include "SpaceVariant/SpaceVariantModule.H"
00052 #include "SpaceVariant/SVChanLevels.H"
00053 #include "Simulation/SimEvents.H"
00054 #include "Simulation/SimModule.H"
00055 #include "Util/Timer.H"
00056 #include "Util/Types.H"
00057 
00058 class ModelManager;
00059 class SpatialMetrics;
00060 
00061 // ######################################################################
00062 //! The retina base class
00063 /*! The Retina is the entry point to the Brain and VisualCortex, and this is a base class that mostly specifies the
00064   interface. Every input image to be processed first passes through the Retina (which is a subcomponent of Brain). The
00065   Retina may transform the input in various ways, e.g., apply a foveation filter, embed the input into a larger
00066   background framing image, or shift the image to current eye position. The base class does nothing in order to not
00067   constrain the interface. Derived classes implement different strategies. */
00068 class Retina : public SimModule
00069 {
00070 public:
00071   //! Constructor
00072   /*! See ModelComponent.H for details */
00073   Retina(OptionManager& mgr, const std::string& descrName = "Retina",
00074          const std::string& tagName = "Retina");
00075 
00076   //! Destructor
00077   virtual ~Retina();
00078 };
00079 
00080 // ######################################################################
00081 class RetinaAdapter : public Retina {
00082 public:
00083   //! Constructor
00084   /*! See ModelComponent.H for details */
00085   RetinaAdapter(OptionManager& mgr, const std::string& descrName = "RetinaAdapter",
00086          const std::string& tagName = "RetinaAdapter");
00087 
00088   //! Destructor
00089   virtual ~RetinaAdapter();
00090 
00091 protected:
00092   //! Callback for when a new input frame is available
00093   SIMCALLBACK_DECLARE(RetinaAdapter, SimEventInputFrame);
00094 
00095   //! Callback for when a new eye movement is available
00096   SIMCALLBACK_DECLARE(RetinaAdapter, SimEventSaccadeStatusEye);
00097 
00098   //! Callback for every time we should save our outputs
00099   SIMCALLBACK_DECLARE(RetinaAdapter, SimEventSaveOutput);
00100 
00101   //! filename of a clipmask image to load; or empty string for no clipmask
00102   OModelParam<std::string> itsClipMaskFname;
00103 
00104   //! Border for our raw input rectangle, in pixels
00105   OModelParam<int> itsRawInpRectBorder;
00106 
00107   //! Initial eye position (to foveate first frame)
00108   OModelParam<Point2D<int> > itsInitialEyePosition;
00109 
00110   //! Whether to allow pyramid caches in the InputFrame objects that we create
00111   OModelParam<bool> itsEnablePyramidCaches;
00112 
00113   //! Overridden so we can load our clip mask file, if any
00114   virtual void start1();
00115 
00116   //! Radius of our fovea
00117   OModelParam<int> itsFoveaRadius;
00118 
00119   //! Save our raw input?
00120   OModelParam<bool> itsSaveInput;
00121 
00122   //! Save our output image?
00123   OModelParam<bool> itsSaveOutput;
00124 
00125   //! Image file to use for input framing
00126   OModelParam<std::string> itsFramingImageName;
00127 
00128   //! Offset to apply for input framing
00129   OModelParam<Point2D<int> > itsFramingImagePos;
00130 
00131   //! Pyramid depth to use when foveating the input, or 0 for no foveation
00132   OModelParam<uint> itsFoveateInputDepth;
00133 
00134   //! shift input to eye position?
00135   OModelParam<bool> itsShiftInput;
00136 
00137   //! background color to use when shifting input to eye position
00138   OModelParam< PixRGB<byte> > itsShiftInputBGcol;
00139 
00140   //! crop input to a given FOV?
00141   OModelParam<Dims> itsInputFOV;
00142 
00143   //! Save our internal pyramid?
00144   OModelParam<bool> itsSavePyr;
00145 
00146   //! Blank out visual inputs during blinks?
00147   OModelParam<bool> itsBlankBlink;
00148 
00149   //! filename of a retinal mask image to load; or empty string for no mask
00150   OModelParam<std::string> itsRetMaskFname;
00151 
00152   //! flip input image horizontally?
00153   OModelParam<bool> itsFlipHoriz;
00154 
00155   //! flip input image vertically?
00156   OModelParam<bool> itsFlipVertic;
00157 
00158   //! Compute a new output
00159   /*! Do not overload this in derived classes, instead overload transform() below. */
00160   Image<PixRGB<byte> > getOutput(const Image<PixRGB<byte> >& inp, const Point2D<int>& eyepos, const bool inBlink);
00161 
00162   //! post our input frame with a SimEventRetinaImage message.
00163   //! Derived classes may want to post messages derived from
00164   //! SimEventRetinaImage
00165   virtual void postInputFrame(SimEventQueue& q, InputFrame& ifr);
00166 
00167 //! Transform the image after it has already been shifted, cropped, foveated, etc
00168   virtual Image<PixRGB<byte> > transform(const Image<PixRGB<byte> >& image) = 0;
00169 
00170   //! Get a rectangle delineating the raw input image area
00171   /*! As the raw input may be embedded into a background, shifted,
00172     cropped by a field of view, etc., this function allows users to
00173     recover the boundaries of the original frames. Note that the
00174     rectangle will be possibly cropped to fit within the dims of the
00175     Retina's output. A border may be specified that will be used to
00176     shrink (if positive) or enlarge (if negative) the returned
00177     rectangle (and will be applied before the rectangle is cropped to
00178     dims). The default implementation in the base class is to just
00179     return a rectangle at (0,0) with dims of the raw input. */
00180   virtual Rectangle getRawInputRectangle(const Dims& indims, const Dims& outdims) const;
00181   
00182   //! Get raw to retinal offset
00183   Point2D<int> getRawToRetinalOffset() const;
00184   
00185   //! Save our internals
00186   /*! Depending on our ModelParam settings, we can save our raw input
00187       frames (with name prefix "RETIN-"), and/or our foveated output
00188       images (prefix "RETOUT-"), and/or our pyramid frames (with name
00189       prefix "RET<level>-"). */
00190   virtual void save1(const ModelComponentSaveInfo& sinfo);
00191 
00192   Image<byte> itsClipMask;  //!< mask to eliminate parts of scene
00193   Point2D<int> itsEyePos;   //!< current eye position
00194   bool itsEyeBlinkStatus;   //!< are we in blink?
00195 
00196   Image<PixRGB<byte> > itsRawInput; //!< copy of input in case we want to save it
00197   Image<PixRGB<byte> > itsOutput; //!< copy of output in case we want to save it
00198 
00199   Image< PixRGB<byte> > itsFramingImage; //!< image used for framing
00200   Point2D<int> itsRetinalShift;
00201   Image<byte> itsRetMask;
00202   ImageSet< PixRGB<byte> > itsMultiRetina;
00203 };
00204 
00205 
00206 // ######################################################################
00207 //! Retina configurator
00208 // ######################################################################
00209 /*! This will export the --retina-type=XX command-line option and will
00210   instantiate a Retina of the desired type as the option gets assigned a
00211   value. As this happens, new options may become available in the
00212   command-line. To see them, use --help AFTER you have chosen the type
00213   to use. The current Retina may be retrieved using getRET(). */
00214 class RetinaConfigurator : public ModelComponent
00215 {
00216 public:
00217   //! Constructor
00218   RetinaConfigurator(OptionManager& mgr,
00219                      const std::string& descrName = "Retina Configurator",
00220                      const std::string& tagName = "RetinaConfigurator");
00221 
00222   //! destructor
00223   virtual ~RetinaConfigurator();
00224 
00225   //! Get the chosen Retina
00226   /*! You should call this during start() of the ModelComponent that
00227       needs the Retina. This is guaranteed to be a non-null object
00228       (though it may be "stub" type, e.g. RetinaStub). */
00229   nub::ref<Retina> getRET() const;
00230 
00231 protected:
00232   OModelParam<std::string> itsType; //!< type of retina
00233 
00234   //! Intercept people changing our ModelParam
00235   /*! See ModelComponent.H; as parsing the command-line or reading a
00236     config file sets our name, we'll also here instantiate a
00237     retina of the proper type (and export its options) */
00238   virtual void paramChanged(ModelParamBase* const param,
00239                             const bool valueChanged,
00240                             ParamClient::ChangeStatus* status);
00241 
00242 private:
00243   nub::ref<Retina> itsRET; // the retina
00244 };
00245 
00246 // ######################################################################
00247 //! "Stub" Retina implementation
00248 // ######################################################################
00249 /*! Does nothing but copy its input to its output. */
00250 class RetinaStub : public Retina
00251 {
00252 public:
00253   //! Constructor
00254   RetinaStub(OptionManager& mgr,
00255              const std::string& descrName = "RetinaStub",
00256              const std::string& tagName = "RetinaStub");
00257 
00258   //! Destructor
00259   virtual ~RetinaStub();
00260 
00261 protected:
00262   //! Callback for when a new input frame is available
00263   SIMCALLBACK_DECLARE(RetinaStub, SimEventInputFrame);
00264 };
00265 
00266 // ######################################################################
00267 //! Standard Retina implementation
00268 // ######################################################################
00269 /*! Can do foveation, shifting, framing, etc all inherited from RetinaAdapter. */
00270 class RetinaStd : public RetinaAdapter
00271 {
00272 public:
00273   //! Constructor
00274   /*! See ModelComponent.H for details */
00275   RetinaStd(OptionManager& mgr, const std::string& descrName = "RetinaStd",
00276             const std::string& tagName = "RetinaStd");
00277 
00278   //! Destructor
00279   virtual ~RetinaStd();
00280 
00281 protected:
00282   //! Transform the image after it has already been shifted, cropped, foveated, etc
00283   /*! In RetinaStd, this is a no-op */
00284   virtual Image<PixRGB<byte> > transform(const Image<PixRGB<byte> >& image);
00285 };
00286 
00287 // ######################################################################
00288 /*! Space variant transform using a foveated model. Could be used as a model
00289     of spatial input of retinal ganglion cell's, thalamus, colliculus or 
00290     early visual cortex */
00291 // ######################################################################
00292 /*! Can do foveation, shifting, flipping, framing in log-polar coords. */
00293 class RetinaCT : public RetinaAdapter
00294 {
00295 public:
00296   //! Constructor
00297   /*! See ModelComponent.H for details */
00298   RetinaCT(OptionManager& mgr,
00299            const std::string& descrName = "Cortical Transform",
00300            const std::string& tagName = "RetinaCT");
00301   
00302   //this implementation just returns a rectangle at 0,0 with the size of the input for now
00303   Rectangle getRawInputRectangle(const Dims& indims, const Dims& outdims) const;
00304   
00305   //! Destructor
00306   virtual ~RetinaCT();
00307   
00308 protected:
00309   virtual void start1();
00310   
00311   //! post a SimEventRetinaSpaceVariantmage
00312   virtual void postInputFrame(SimEventQueue& q, InputFrame& ifr);
00313   
00314   //! Transform the image after it has already been shifted, cropped, foveated, etc
00315   /*! In RetinaStd, this is a no-op */
00316   virtual Image<PixRGB<byte> > transform(const Image<PixRGB<byte> >& image);
00317 
00318   OModelParam<float> itsSurrFac;
00319   OModelParam<SVChanLevels> itsLevels;
00320   
00321 private:
00322   nub::ref<SpaceVariantModule> itsTransform;
00323   rutz::shared_ptr<PyramidCache<PixRGB<float> > > itsRgbCache;
00324   rutz::shared_ptr<PyramidCache<float> > itsFloatCache;
00325 };
00326 
00327 #endif
00328 
00329 // ######################################################################
00330 /* So things look consistent in everyone's emacs... */
00331 /* Local Variables: */
00332 /* indent-tabs-mode: nil */
00333 /* End: */
Generated on Sun May 8 08:41:03 2011 for iLab Neuromorphic Vision Toolkit by  doxygen 1.6.3