001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.feature.local.detector.dog.extractor;
031
032
033import org.openimaj.feature.OrientedFeatureVector;
034import org.openimaj.image.FImage;
035import org.openimaj.image.feature.local.descriptor.gradient.GradientFeatureProvider;
036import org.openimaj.image.feature.local.descriptor.gradient.GradientFeatureProviderFactory;
037import org.openimaj.image.feature.local.descriptor.gradient.SIFTFeatureProvider;
038import org.openimaj.image.feature.local.extraction.GradientScaleSpaceImageExtractorProperties;
039import org.openimaj.image.feature.local.extraction.ScaleSpaceImageExtractorProperties;
040import org.openimaj.image.processing.convolution.FImageGradients;
041
042
043/**
044 * <p>
045 * Class capable of extracting local descriptors from a circular region
046 * in an image defined by its scale and centre. The actual feature 
047 * extracted is determined by the {@link GradientFeatureProvider} that
048 * is provided by the {@link GradientFeatureProviderFactory} set during
049 * construction.
050 * </p>
051 * <p>
052 * The GradientFeatureExtractor first calculates the dominant orientation
053 * of the image patch described by the {@link ScaleSpaceImageExtractorProperties}
054 * and then iterates over the pixels in an oriented square, centered on the
055 * interest point, passing the gradient and magnitude values of the respective
056 * pixel to the {@link GradientFeatureProvider}.
057 * </p>
058 * <p>
059 * The size of the sampling square, relative to scale is set by a single parameter,
060 * magnification. For some types of feature provider, this number
061 * might need to be set based on the internal settings of the provider. For example,
062 * with a {@link SIFTFeatureProvider} this will probably be set to a constant multiplied
063 * by the number of spatial bins of the feature. For SIFT, this constant is typically 
064 * around 3, so with a standard 4-spatial binned SIFT provider, the magnification
065 * factor of the extractor should be about 12.
066 * </p>
067 * 
068 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
069 *
070 */
071public class GradientFeatureExtractor implements ScaleSpaceFeatureExtractor<OrientedFeatureVector, FImage> {
072        AbstractDominantOrientationExtractor dominantOrientationExtractor;
073        
074        GradientFeatureProviderFactory factory;
075        
076        private GradientScaleSpaceImageExtractorProperties<FImage> currentGradientProperties = new GradientScaleSpaceImageExtractorProperties<FImage>();
077        
078        /**
079         * The magnification factor determining the size of the sampling
080         * region relative to the scale of the interest point.
081         */
082        protected float magnification = 12;
083                
084        /**
085         * Construct with the given orientation extractor and gradient feature provider.
086         * The default magnification factor of 12 is used.
087         * 
088         * @param dominantOrientationExtractor the orientation extractor
089         * @param factory the gradient feature provider
090         */
091        public GradientFeatureExtractor(AbstractDominantOrientationExtractor dominantOrientationExtractor, GradientFeatureProviderFactory factory) {
092                this.dominantOrientationExtractor = dominantOrientationExtractor;
093                this.factory = factory;
094        }
095        
096        /**
097         * Construct with the given orientation extractor, gradient feature provider
098         * and magnification factor determining the size of the sampling
099         * region relative to the scale of the interest point.
100         * 
101         * @param dominantOrientationExtractor the orientation extractor
102         * @param factory the gradient feature provider
103         * @param magnification the magnification factor.
104         */
105        public GradientFeatureExtractor(AbstractDominantOrientationExtractor dominantOrientationExtractor, GradientFeatureProviderFactory factory, float magnification) {
106                this(dominantOrientationExtractor, factory);
107                this.magnification = magnification;
108        }
109
110        @Override
111        public OrientedFeatureVector[] extractFeature(ScaleSpaceImageExtractorProperties<FImage> properties) {
112                GradientScaleSpaceImageExtractorProperties<FImage> gprops = getCurrentGradientProps(properties);
113
114                float [] dominantOrientations = dominantOrientationExtractor.extractFeatureRaw(gprops);
115
116                OrientedFeatureVector[] ret = new OrientedFeatureVector[dominantOrientations.length];
117
118                for (int i=0; i<dominantOrientations.length; i++) {
119                        ret[i] = createFeature(dominantOrientations[i]);
120                }
121
122                return ret;
123        }
124
125        /**
126         * Get the GradientScaleSpaceImageExtractorProperties for the given properties.
127         * The returned properties are the same as the input properties, but with the
128         * gradient images added. 
129         * 
130         * For efficiency, this method always returns the same cached GradientScaleSpaceImageExtractorProperties,
131         * and internally updates this as necessary. The gradient images are only recalculated
132         * when the input image from the input properties is different to the cached one.
133         * 
134         * @param properties input properties
135         * @return cached GradientScaleSpaceImageExtractorProperties 
136         */
137        public GradientScaleSpaceImageExtractorProperties<FImage> getCurrentGradientProps(ScaleSpaceImageExtractorProperties<FImage> properties) {
138                if (properties.image != currentGradientProperties.image) {
139                        currentGradientProperties.image = properties.image;
140
141                        //only if the size of the image has changed do we need to reset the gradient and orientation images. 
142                        if (currentGradientProperties.orientation == null || 
143                                        currentGradientProperties.orientation.height != currentGradientProperties.image.height || 
144                                        currentGradientProperties.orientation.width != currentGradientProperties.image.width) { 
145                                currentGradientProperties.orientation = new FImage(currentGradientProperties.image.width, currentGradientProperties.image.height);
146                                currentGradientProperties.magnitude = new FImage(currentGradientProperties.image.width, currentGradientProperties.image.height);                                
147                        }
148
149                        FImageGradients.gradientMagnitudesAndOrientations(currentGradientProperties.image, currentGradientProperties.magnitude, currentGradientProperties.orientation);
150                }
151                
152                currentGradientProperties.x = properties.x;
153                currentGradientProperties.y = properties.y;
154                currentGradientProperties.scale = properties.scale;
155                
156                return currentGradientProperties;
157        }
158
159        /*
160         * Iterate over the pixels in a sampling patch around the given feature coordinates
161         * and pass the information to a feature provider that will extract the relevant
162         * feature vector.
163         */
164        protected OrientedFeatureVector createFeature(final float orientation) {
165                final float fx = currentGradientProperties.x;
166                final float fy = currentGradientProperties.y;
167                final float scale = currentGradientProperties.scale; 
168                
169                //create a new feature provider and initialise it with the dominant orientation
170                GradientFeatureProvider sfe = factory.newProvider();
171                sfe.setPatchOrientation(orientation);
172                
173                //the integer coordinates of the patch
174                final int ix = Math.round(fx);
175                final int iy = Math.round(fy);
176
177                final float sin = (float) Math.sin(orientation);
178                final float cos = (float) Math.cos(orientation);
179
180                //get the amount of extra sampling outside the unit square requested by the feature
181                final float oversampling = sfe.getOversamplingAmount();
182                
183                //this is the size of the unit bounding box of the patch in the image in pixels
184                final float boundingBoxSize = magnification * scale;
185                
186                //the amount of extra sampling per side in pixels
187                final float extraSampling = oversampling * boundingBoxSize;
188                
189                //the actual sampling area is bigger than the boundingBoxSize by an extraSampling on each side
190                final float samplingBoxSize = extraSampling + boundingBoxSize + extraSampling;
191                
192                //In the image, the box (with sides parallel to the image frame) that contains the
193                //sampling box is:
194                final float orientedSamplingBoxSize = Math.abs(sin * samplingBoxSize) + Math.abs(cos * samplingBoxSize);
195                
196                //now half the size and round to an int so we can iterate
197                final int orientedSamplingBoxHalfSize = Math.round(orientedSamplingBoxSize / 2.0f);
198
199                //get the images and their size
200                final FImage mag = currentGradientProperties.magnitude;
201                final FImage ori = currentGradientProperties.orientation;
202                final int width = mag.width;
203                final int height = mag.height;
204                
205                //now pass over all the pixels in the image that *might* contribute to the sampling area
206                for (int y = -orientedSamplingBoxHalfSize; y <= orientedSamplingBoxHalfSize; y++) {
207                        for (int x = -orientedSamplingBoxHalfSize; x <= orientedSamplingBoxHalfSize; x++) {
208                                int px = x + ix;
209                                int py = y + iy;
210                                
211                                //check if the pixel is in the image bounds; if not ignore it
212                                if (px >= 0 && px < width && py >= 0 && py < height) {
213                                        //calculate the actual position of the sample in the patch coordinate system
214                                        float sx = 0.5f + ((-sin * y + cos * x) - (fx - ix)) / boundingBoxSize;
215                                        float sy = 0.5f + ((cos * y + sin * x) - (fy - iy)) / boundingBoxSize;
216                                        
217                                        //if the pixel is in the bounds of the sampling area then add it
218                                        if (sx > -oversampling && sx < 1 + oversampling && sy > -oversampling && sy < 1 + oversampling) {
219                                                sfe.addSample(sx, sy, mag.pixels[py][px], ori.pixels[py][px]);
220                                        }
221                                }
222                        }
223                }
224
225                return sfe.getFeatureVector();
226        }
227}