001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.image.feature.local.detector.dog.extractor; 031 032 033import org.openimaj.feature.OrientedFeatureVector; 034import org.openimaj.image.FImage; 035import org.openimaj.image.feature.local.descriptor.gradient.GradientFeatureProvider; 036import org.openimaj.image.feature.local.descriptor.gradient.GradientFeatureProviderFactory; 037import org.openimaj.image.feature.local.descriptor.gradient.SIFTFeatureProvider; 038import org.openimaj.image.feature.local.extraction.GradientScaleSpaceImageExtractorProperties; 039import org.openimaj.image.feature.local.extraction.ScaleSpaceImageExtractorProperties; 040import org.openimaj.image.processing.convolution.FImageGradients; 041 042 043/** 044 * <p> 045 * Class capable of extracting local descriptors from a circular region 046 * in an image defined by its scale and centre. The actual feature 047 * extracted is determined by the {@link GradientFeatureProvider} that 048 * is provided by the {@link GradientFeatureProviderFactory} set during 049 * construction. 050 * </p> 051 * <p> 052 * The GradientFeatureExtractor first calculates the dominant orientation 053 * of the image patch described by the {@link ScaleSpaceImageExtractorProperties} 054 * and then iterates over the pixels in an oriented square, centered on the 055 * interest point, passing the gradient and magnitude values of the respective 056 * pixel to the {@link GradientFeatureProvider}. 057 * </p> 058 * <p> 059 * The size of the sampling square, relative to scale is set by a single parameter, 060 * magnification. For some types of feature provider, this number 061 * might need to be set based on the internal settings of the provider. For example, 062 * with a {@link SIFTFeatureProvider} this will probably be set to a constant multiplied 063 * by the number of spatial bins of the feature. For SIFT, this constant is typically 064 * around 3, so with a standard 4-spatial binned SIFT provider, the magnification 065 * factor of the extractor should be about 12. 066 * </p> 067 * 068 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 069 * 070 */ 071public class GradientFeatureExtractor implements ScaleSpaceFeatureExtractor<OrientedFeatureVector, FImage> { 072 AbstractDominantOrientationExtractor dominantOrientationExtractor; 073 074 GradientFeatureProviderFactory factory; 075 076 private GradientScaleSpaceImageExtractorProperties<FImage> currentGradientProperties = new GradientScaleSpaceImageExtractorProperties<FImage>(); 077 078 /** 079 * The magnification factor determining the size of the sampling 080 * region relative to the scale of the interest point. 081 */ 082 protected float magnification = 12; 083 084 /** 085 * Construct with the given orientation extractor and gradient feature provider. 086 * The default magnification factor of 12 is used. 087 * 088 * @param dominantOrientationExtractor the orientation extractor 089 * @param factory the gradient feature provider 090 */ 091 public GradientFeatureExtractor(AbstractDominantOrientationExtractor dominantOrientationExtractor, GradientFeatureProviderFactory factory) { 092 this.dominantOrientationExtractor = dominantOrientationExtractor; 093 this.factory = factory; 094 } 095 096 /** 097 * Construct with the given orientation extractor, gradient feature provider 098 * and magnification factor determining the size of the sampling 099 * region relative to the scale of the interest point. 100 * 101 * @param dominantOrientationExtractor the orientation extractor 102 * @param factory the gradient feature provider 103 * @param magnification the magnification factor. 104 */ 105 public GradientFeatureExtractor(AbstractDominantOrientationExtractor dominantOrientationExtractor, GradientFeatureProviderFactory factory, float magnification) { 106 this(dominantOrientationExtractor, factory); 107 this.magnification = magnification; 108 } 109 110 @Override 111 public OrientedFeatureVector[] extractFeature(ScaleSpaceImageExtractorProperties<FImage> properties) { 112 GradientScaleSpaceImageExtractorProperties<FImage> gprops = getCurrentGradientProps(properties); 113 114 float [] dominantOrientations = dominantOrientationExtractor.extractFeatureRaw(gprops); 115 116 OrientedFeatureVector[] ret = new OrientedFeatureVector[dominantOrientations.length]; 117 118 for (int i=0; i<dominantOrientations.length; i++) { 119 ret[i] = createFeature(dominantOrientations[i]); 120 } 121 122 return ret; 123 } 124 125 /** 126 * Get the GradientScaleSpaceImageExtractorProperties for the given properties. 127 * The returned properties are the same as the input properties, but with the 128 * gradient images added. 129 * 130 * For efficiency, this method always returns the same cached GradientScaleSpaceImageExtractorProperties, 131 * and internally updates this as necessary. The gradient images are only recalculated 132 * when the input image from the input properties is different to the cached one. 133 * 134 * @param properties input properties 135 * @return cached GradientScaleSpaceImageExtractorProperties 136 */ 137 public GradientScaleSpaceImageExtractorProperties<FImage> getCurrentGradientProps(ScaleSpaceImageExtractorProperties<FImage> properties) { 138 if (properties.image != currentGradientProperties.image) { 139 currentGradientProperties.image = properties.image; 140 141 //only if the size of the image has changed do we need to reset the gradient and orientation images. 142 if (currentGradientProperties.orientation == null || 143 currentGradientProperties.orientation.height != currentGradientProperties.image.height || 144 currentGradientProperties.orientation.width != currentGradientProperties.image.width) { 145 currentGradientProperties.orientation = new FImage(currentGradientProperties.image.width, currentGradientProperties.image.height); 146 currentGradientProperties.magnitude = new FImage(currentGradientProperties.image.width, currentGradientProperties.image.height); 147 } 148 149 FImageGradients.gradientMagnitudesAndOrientations(currentGradientProperties.image, currentGradientProperties.magnitude, currentGradientProperties.orientation); 150 } 151 152 currentGradientProperties.x = properties.x; 153 currentGradientProperties.y = properties.y; 154 currentGradientProperties.scale = properties.scale; 155 156 return currentGradientProperties; 157 } 158 159 /* 160 * Iterate over the pixels in a sampling patch around the given feature coordinates 161 * and pass the information to a feature provider that will extract the relevant 162 * feature vector. 163 */ 164 protected OrientedFeatureVector createFeature(final float orientation) { 165 final float fx = currentGradientProperties.x; 166 final float fy = currentGradientProperties.y; 167 final float scale = currentGradientProperties.scale; 168 169 //create a new feature provider and initialise it with the dominant orientation 170 GradientFeatureProvider sfe = factory.newProvider(); 171 sfe.setPatchOrientation(orientation); 172 173 //the integer coordinates of the patch 174 final int ix = Math.round(fx); 175 final int iy = Math.round(fy); 176 177 final float sin = (float) Math.sin(orientation); 178 final float cos = (float) Math.cos(orientation); 179 180 //get the amount of extra sampling outside the unit square requested by the feature 181 final float oversampling = sfe.getOversamplingAmount(); 182 183 //this is the size of the unit bounding box of the patch in the image in pixels 184 final float boundingBoxSize = magnification * scale; 185 186 //the amount of extra sampling per side in pixels 187 final float extraSampling = oversampling * boundingBoxSize; 188 189 //the actual sampling area is bigger than the boundingBoxSize by an extraSampling on each side 190 final float samplingBoxSize = extraSampling + boundingBoxSize + extraSampling; 191 192 //In the image, the box (with sides parallel to the image frame) that contains the 193 //sampling box is: 194 final float orientedSamplingBoxSize = Math.abs(sin * samplingBoxSize) + Math.abs(cos * samplingBoxSize); 195 196 //now half the size and round to an int so we can iterate 197 final int orientedSamplingBoxHalfSize = Math.round(orientedSamplingBoxSize / 2.0f); 198 199 //get the images and their size 200 final FImage mag = currentGradientProperties.magnitude; 201 final FImage ori = currentGradientProperties.orientation; 202 final int width = mag.width; 203 final int height = mag.height; 204 205 //now pass over all the pixels in the image that *might* contribute to the sampling area 206 for (int y = -orientedSamplingBoxHalfSize; y <= orientedSamplingBoxHalfSize; y++) { 207 for (int x = -orientedSamplingBoxHalfSize; x <= orientedSamplingBoxHalfSize; x++) { 208 int px = x + ix; 209 int py = y + iy; 210 211 //check if the pixel is in the image bounds; if not ignore it 212 if (px >= 0 && px < width && py >= 0 && py < height) { 213 //calculate the actual position of the sample in the patch coordinate system 214 float sx = 0.5f + ((-sin * y + cos * x) - (fx - ix)) / boundingBoxSize; 215 float sy = 0.5f + ((cos * y + sin * x) - (fy - iy)) / boundingBoxSize; 216 217 //if the pixel is in the bounds of the sampling area then add it 218 if (sx > -oversampling && sx < 1 + oversampling && sy > -oversampling && sy < 1 + oversampling) { 219 sfe.addSample(sx, sy, mag.pixels[py][px], ori.pixels[py][px]); 220 } 221 } 222 } 223 } 224 225 return sfe.getFeatureVector(); 226 } 227}