diff --git a/+benchmarks/RepeatabilityBenchmark.m b/+benchmarks/RepeatabilityBenchmark.m index ce86cdd..6b758f8 100644 --- a/+benchmarks/RepeatabilityBenchmark.m +++ b/+benchmarks/RepeatabilityBenchmark.m @@ -139,13 +139,9 @@ properties Opts = struct(... 'overlapError', 0.4,... - 'normaliseFrames', true,... 'cropFrames', true,... - 'magnification', 3,... - 'warpMethod', 'linearise',... 'mode', 'repeatability',... - 'descriptorsDistanceMetric', 'L2',... - 'normalisedScale', 30); + 'descriptorsDistanceMetric', 'L2'); end properties(Constant, Hidden) @@ -174,8 +170,8 @@ obj.checkInstall(varargin); end - function [score numMatches bestMatches reprojFrames] = ... - testFeatureExtractor(obj, featExtractor, tf, imageAPath, imageBPath) + function [score numMatches bestMatches] = ... + testFeatureExtractor(obj, featExtractor, dataset, imgAId, imgBId) % testFeatureExtractor Image feature extractor repeatability % REPEATABILITY = obj.testFeatureExtractor(FEAT_EXTRACTOR, TF, % IMAGEAPATH, IMAGEBPATH) computes the repeatability REP of a image @@ -211,6 +207,8 @@ import benchmarks.*; import helpers.*; + imageAPath = dataset.getImagePath(imgAId); + imageBPath = dataset.getImagePath(imgBId); obj.info('Comparing frames from det. %s and images %s and %s.',... featExtractor.Name,getFileName(imageAPath),... getFileName(imageBPath)); @@ -228,29 +226,29 @@ if obj.ModesOpts(obj.Opts.mode).matchDescs [framesA descriptorsA] = featExtractor.extractFeatures(imageAPath); [framesB descriptorsB] = featExtractor.extractFeatures(imageBPath); - [score numMatches bestMatches reprojFrames] = obj.testFeatures(... - tf, imageASize, imageBSize, framesA, framesB,... + [score numMatches bestMatches] = obj.testFeatures(... + dataset, imgAId, imgBId, imageASize, imageBSize, framesA, framesB,... descriptorsA, descriptorsB); else [framesA] = featExtractor.extractFeatures(imageAPath); [framesB] = featExtractor.extractFeatures(imageBPath); - [score numMatches bestMatches reprojFrames] = ... - obj.testFeatures(tf,imageASize, imageBSize,framesA, framesB); + [score numMatches bestMatches] = ... + obj.testFeatures(dataset, imgAId, imgBId, imageASize, imageBSize,framesA, framesB); end if featExtractor.UseCache - results = {score numMatches bestMatches reprojFrames}; + results = {score numMatches bestMatches}; obj.storeResults(results, resultsKey); end else - [score numMatches bestMatches reprojFrames] = cachedResults{:}; + [score numMatches bestMatches] = cachedResults{:}; obj.debug('Results loaded from cache'); end end - function [score numMatches matches reprojFrames] = ... - testFeatures(obj, tf, imageASize, imageBSize, framesA, framesB, ... - descriptorsA, descriptorsB) + function [score numMatches matches] = ... + testFeatures(obj, dataset, imgAId, imgBId, imageASize, ... + imageBSize, framesA, framesB, descriptorsA, descriptorsB) % testFeatures Compute repeatability of given image features % [SCORE NUM_MATCHES] = obj.testFeatures(TF, IMAGE_A_SIZE, % IMAGE_B_SIZE, FRAMES_A, FRAMES_B, DESCS_A, DESCS_B) Compute @@ -284,8 +282,11 @@ matchDescriptors = obj.ModesOpts(obj.Opts.mode).matchDescs; if isempty(framesA) || isempty(framesB) - matches = zeros(size(framesA,2)); reprojFrames = {}; + matches = zeros(size(framesA,2)); obj.info('Nothing to compute.'); + score = 0; + numMatches = 0; + matches = []; return; end if exist('descriptorsA','var') && exist('descriptorsB','var') @@ -299,7 +300,7 @@ score = 0; numMatches = 0; startTime = tic; - normFrames = obj.Opts.normaliseFrames; + overlapError = obj.Opts.overlapError; overlapThresh = 1 - overlapError; @@ -308,57 +309,29 @@ framesA = localFeatures.helpers.frameToEllipse(framesA) ; framesB = localFeatures.helpers.frameToEllipse(framesB) ; - % map frames from image A to image B and viceversa - reprojFramesA = warpEllipse(tf, framesA,... - 'Method',obj.Opts.warpMethod) ; - reprojFramesB = warpEllipse(inv(tf), framesB,... - 'Method',obj.Opts.warpMethod) ; - % optionally remove frames that are not fully contained in % both images if obj.Opts.cropFrames - % find frames fully visible in both images - bboxA = [1 1 imageASize(2)+1 imageASize(1)+1] ; - bboxB = [1 1 imageBSize(2)+1 imageBSize(1)+1] ; - - visibleFramesA = isEllipseInBBox(bboxA, framesA ) & ... - isEllipseInBBox(bboxB, reprojFramesA); - - visibleFramesB = isEllipseInBBox(bboxA, reprojFramesB) & ... - isEllipseInBBox(bboxB, framesB ); + [validFramesA validFramesB] = dataset.validateFrames(imgAId, imgBId, framesA, framesB); % Crop frames outside overlap region - framesA = framesA(:,visibleFramesA); - reprojFramesA = reprojFramesA(:,visibleFramesA); - framesB = framesB(:,visibleFramesB); - reprojFramesB = reprojFramesB(:,visibleFramesB); + framesA = framesA(:,validFramesA); + framesB = framesB(:,validFramesB); + if isempty(framesA) || isempty(framesB) matches = zeros(size(framesA,2)); reprojFrames = {}; return; end if matchDescriptors - descriptorsA = descriptorsA(:,visibleFramesA); - descriptorsB = descriptorsB(:,visibleFramesB); + descriptorsA = descriptorsA(:,validFramesA); + descriptorsB = descriptorsB(:,validFramesB); end end - if ~normFrames - % When frames are not normalised, account the descriptor region - magFactor = obj.Opts.magnification^2; - framesA = [framesA(1:2,:); framesA(3:5,:).*magFactor]; - reprojFramesB = [reprojFramesB(1:2,:); ... - reprojFramesB(3:5,:).*magFactor]; - end - - reprojFrames = {framesA,framesB,reprojFramesA,reprojFramesB}; + frameOverlaps = dataset.scoreFrameOverlaps(imgAId, imgBId, framesA, framesB); numFramesA = size(framesA,2); - numFramesB = size(reprojFramesB,2); - - % Find all ellipse overlaps (in one-to-n array) - frameOverlaps = fastEllipseOverlap(reprojFramesB, framesA, ... - 'NormaliseFrames',normFrames,'MinAreaRatio',overlapThresh,... - 'NormalisedScale',obj.Opts.normalisedScale); + numFramesB = size(framesB,2); matches = []; diff --git a/+datasets/DTURobotDataset.m b/+datasets/DTURobotDataset.m new file mode 100644 index 0000000..3384cf3 --- /dev/null +++ b/+datasets/DTURobotDataset.m @@ -0,0 +1,223 @@ +classdef DTURobotDataset < datasets.GenericCorrespondenceDataset ... + & helpers.Logger & helpers.GenericInstaller + + + properties (SetAccess=private, GetAccess=public) + Category = 'arc1'; % Dataset category + Viewpoints; + Lightings; + ReconstructionsDir; + CacheDir; + CamerasPath; + CodeDir; + DataDir; + end + + properties (Constant) + % All dataset categories + AllCategories = {'arc1', 'arc2', 'arc3', 'linear_path', ... + 'lighting_x', 'lighting_y'}; + end + + properties (Constant, Hidden) + % Names of the image transformations in particular categories + CategoryImageNames = {... + 'Arc 1, viewpoint angle',... + 'Arc 2, viewpoint angle',... + 'Arc 3, viewpoint angle',... + 'Linear path, camera distance',... + 'Lighting changes, x coordinate',... + 'Lighting changes, y coordinate',... + }; + % Image labels for particular categories (degree of transf.) + CategoryImageLabels = {... + round([[-40: 40/24: -1.6666666] [1.666666: 40/24: 40]]*10)/10,... % arc1 + round([-25: 50/29: 25]*10)/10,... % arc2 + round([-20: 40/24: 20]*10)/10,... % arc3 + round([.5214: .3/14: .8]*10)/10,... % linear_path + round([0: 180.8/8: 180.8]*10)/10,... % lighting_x + round([0: 80.3/6: 80.3]*10)/10,... % lighting_y + }; + % Viewpoint indices for each category. + CategoryViewpoints = {... + [1:24 26:49], ... + [65:94], ... + [95:119], ... + [51:64], ... + [12 25 60 87], ... + [12 25 60 87], ... + }; + % Lighting indices for each category. + CategoryLightings = {... + 0, ... + 0, ... + 0, ... + 0, ... + [20:28], ... + [29:35], ... + }; + % Size of search cells in the structured light grid. + CellRadius = 10; + % Acceptance threshold of point distance in 3D. 3e-3 = 3mm. + StrLBoxPad=3e-3; + % Acceptance threshold of backprojection error in pixels. + BackProjThresh=5; + % Acceptance threshold of scale difference after distance normalization. A + % value of 2 corresponds to an octave. + ScaleMargin = 2; + + ImgWidth = 1600; + ImgHeight = 1200; + + % Installation directory + RootInstallDir = fullfile('data','datasets','DTURobot'); + % URL for dataset tarballs + CodeUrl = 'http://roboimagedata.imm.dtu.dk/data/code.tar.gz'; + ReconstructionsUrl = 'http://roboimagedata.imm.dtu.dk/data/ground_truth.tar.gz'; + ScenesUrl = 'http://roboimagedata.imm.dtu.dk/data/condensed.tar.gz'; + end + + methods + function obj = DTURobotDataset(varargin) + import datasets.*; + import helpers.*; + opts.Category = obj.Category; + [opts varargin] = vl_argparse(opts,varargin); + [valid loc] = ismember(opts.Category,obj.AllCategories); + assert(valid,... + sprintf('Invalid category for DTURobot dataset: %s\n',opts.Category)); + obj.DatasetName = ['DTURobotDataset-' opts.Category]; + obj.Category= opts.Category; + obj.DataDir = fullfile(obj.RootInstallDir,'scenes'); + + obj.checkInstall(varargin); + obj.ImageNames = obj.CategoryImageLabels{loc}; + obj.ImageNamesLabel = obj.CategoryImageNames{loc}; + obj.Viewpoints = obj.CategoryViewpoints{loc}; + obj.Lightings = obj.CategoryLightings{loc}; + obj.ReconstructionsDir = fullfile(obj.RootInstallDir, 'reconstructions'); + obj.CamerasPath = fullfile(obj.RootInstallDir, 'cameras.mat'); + obj.CacheDir = fullfile(obj.RootInstallDir, 'cache'); + obj.CodeDir = fullfile(obj.RootInstallDir, 'code'); + obj.NumScenes = 60; + obj.NumLabels = numel(obj.ImageNames); + if strfind(obj.Category,'lighting') + obj.NumImages = obj.NumScenes * obj.NumLabels * numel(obj.CategoryViewpoints{loc}); + else + obj.NumImages = obj.NumScenes * obj.NumLabels; + end + addpath(obj.CodeDir) + end + + function imgPath = getImagePath(obj, imgId) + if isstruct(imgId) + imgPath = fullfile(obj.DataDir, sprintf('scene%.3d/%.3d_%.2d.png', ... + imgId.scene, imgId.viewpoint, imgId.lighting)); + else + labelNo = floor(imgId/obj.NumImages)+1; + sceneNo = mod(imgId,obj.NumImages)+1; + imgId = obj.getImageId(labelNo, sceneNo); + imgPath = obj.getImagePath(imgId); + end + end + + function imgId = getReferenceImageId(obj, labelNo, sceneNo) + imgId.scene = sceneNo; + imgId.viewpoint = 25; + imgId.lighting = 0; + end + + function imgId = getImageId(obj, labelNo, sceneNo) + if strfind(obj.Category,'lighting') + imgId.viewpoint = obj.Viewpoints(mod(sceneNo, numel(obj.Viewpoints))+1); + imgId.lighting = obj.Lightings(labelNo); + imgId.scene = floor(sceneNo/numel(obj.Viewpoints))+1; + else + imgId.viewpoint = obj.Viewpoints(labelNo); + imgId.lighting = 0; + imgId.scene = sceneNo; + end + end + + function [validFramesA validFramesB] = validateFrames(obj, ... + imgAId, imgBId, framesA, framesB) + validFramesA = logical(ones(1, size(framesA,2))); + validFramesB = logical(ones(1, size(framesB,2))); + end + + function overlaps = scoreFrameOverlaps(obj, imgAId, imgBId, framesA, framesB) + % Get 3D reconstruction + [Grid3D, Pts] = GenStrLightGrid(imgAId.viewpoint, ... + obj.ReconstructionsDir, obj.ImgHeight, obj.ImgWidth, ... + obj.CellRadius, imgAId.scene); + CamPair = GetCamPair(imgAId.viewpoint, imgBId.viewpoint); + + N = size(framesA,2); + neighs = cell(1,N); + scores = cell(1,N); + for f = 1:N + frame_ref = framesA(:, f)'; + [neighs{f}, scores{f}] = obj.overlap(Grid3D, Pts, CamPair, frame_ref, framesB'); + end + overlaps.neighs = neighs; + overlaps.scores = scores; + end + + + function [neighs, scores] = overlap(obj, Grid3D, Pts, Cams, frame_ref, frames) + KeyP = frame_ref(1:2); + KeyScale = frame_ref(3); + + % Project point onto 3D structure + [Mean,Var,IsEst] = Get3DGridEst(Grid3D,Pts,obj.CellRadius,KeyP(1),KeyP(2)); + + neighs = []; + scores = []; + if(IsEst) + Var = Var+obj.StrLBoxPad; + Q = Mean*ones(1,8)+[Var(1)*[-1 1 -1 1 -1 1 -1 1]; + Var(2)*[-1 -1 1 1 -1 -1 1 1]; + Var(3)*[-1 -1 -1 -1 1 1 1 1]]; + q = Cams(:,:,2)*[Q;ones(1,8)]; + depth = mean(q(3,:)); + q(1,:) = q(1,:)./q(3,:); + q(2,:) = q(2,:)./q(3,:); + q(3,:) = q(3,:)./q(3,:); + + kq = Cams(:,:,1)*[Q;ones(1,8)]; + kDepth = mean(kq(3,:)); + Scale = KeyScale*kDepth/depth; + + % Find neighbor frames within scale bounds + idx = find(frames(:,1)>min(q(1,:)) & frames(:,1)min(q(2,:)) & frames(:,2)Scale/obj.ScaleMargin & ... + frames(:,3)= 1 && imgNo <= obj.NumImages,'Out of bounds idx\n'); - imgPath = fullfile(obj.DataDir,sprintf('img%d.%s',imgNo,obj.ImgExt)); + function imgPath = getImagePath(obj,imgId) + assert(imgId >= 1 && imgId <= obj.NumImages,'Out of bounds idx\n'); + imgPath = fullfile(obj.DataDir,sprintf('img%d.%s',imgId,obj.ImgExt)); + end + + function imgId = getImageId(obj, labelNo, sceneNo) + assert(labelNo<=obj.NumLabels && sceneNo <= obj.NumScenes, 'Out of bounds idx\n'); + imgId = labelNo+1; + end + + function imgId = getReferenceImageId(obj, labelNo, sceneNo) + assert(labelNo<=obj.NumLabels && sceneNo <= obj.NumScenes, 'Out of bounds idx\n'); + imgId = 1; end function tfs = getTransformation(obj,imgIdx) @@ -101,6 +118,58 @@ textread(fullfile(obj.DataDir,sprintf('H1to%dp',imgIdx)),... '%f %f %f%*[^\n]'); end + + function [validFramesA validFramesB] = validateFrames(obj, ... + imgAId, imgBId, framesA, framesB) + import benchmarks.helpers.*; + imageASize = helpers.imageSize(obj.getImagePath(imgAId)); + imageBSize = helpers.imageSize(obj.getImagePath(imgBId)); + + % find frames fully visible in both images + bboxA = [1 1 imageASize(2)+1 imageASize(1)+1] ; + bboxB = [1 1 imageBSize(2)+1 imageBSize(1)+1] ; + + % map frames from image A to image B and viceversa + tf = obj.getTransformation(imgBId); + reprojFramesA = warpEllipse(tf, framesA,... + 'Method',obj.warpMethod) ; + reprojFramesB = warpEllipse(inv(tf), framesB,... + 'Method',obj.warpMethod) ; + + validFramesA = isEllipseInBBox(bboxA, framesA ) & ... + isEllipseInBBox(bboxB, reprojFramesA); + + validFramesB = isEllipseInBBox(bboxA, reprojFramesB) & ... + isEllipseInBBox(bboxB, framesB ); + end + + function overlaps = scoreFrameOverlaps(obj, imgAId, imgBId, framesA, framesB) + import benchmarks.helpers.*; + import helpers.*; + tf = getTransformation(obj, imgBId); + + % map frames from image A to image B and viceversa + reprojFramesA = warpEllipse(tf, framesA,... + 'Method',obj.warpMethod) ; + reprojFramesB = warpEllipse(inv(tf), framesB,... + 'Method',obj.warpMethod) ; + + + if ~obj.normaliseFrames + % When frames are not normalised, account the descriptor region + magFactor = obj.magnification^2; + framesA = [framesA(1:2,:); framesA(3:5,:).*magFactor]; + reprojFramesB = [reprojFramesB(1:2,:); ... + reprojFramesB(3:5,:).*magFactor]; + end + + % Find all ellipse overlaps (in one-to-n array) + overlapThresh = 1 - obj.overlapError; + overlaps = fastEllipseOverlap(reprojFramesB, framesA, ... + 'NormaliseFrames',obj.normaliseFrames,'MinAreaRatio',overlapThresh,... + 'NormalisedScale',obj.normalisedScale); + end + end methods (Access = protected) diff --git a/+localFeatures/HarrisScaleSelect.m b/+localFeatures/HarrisScaleSelect.m new file mode 100644 index 0000000..4a0c6e0 --- /dev/null +++ b/+localFeatures/HarrisScaleSelect.m @@ -0,0 +1,86 @@ +classdef HarrisScaleSelect < localFeatures.GenericLocalFeatureExtractor & ... + helpers.GenericInstaller + + properties (SetAccess=private, GetAccess=public) + Opts = struct( ... + 'localize', 0, ... + 'NoScales', 61, ... + 'sigmamin', 1.26, ... + 'sigmamax', -1, ... + 'Hthres', 1500 ... + ) + end + + properties (Constant, Hidden) + % Installation directory + RootInstallDir = fullfile('data','software'); + CodeDir = fullfile('data','software','diku-dtu_detectors'); + % URL for dataset tarballs + CodeUrl = 'http://imm.dtu.dk/~abll/files/diku-dtu_detectors.tar.gz'; + end + + + methods + function obj = HarrisScaleSelect(varargin) + import helpers.*; + obj.Name = 'Harris with scale selection'; + varargin = obj.checkInstall(varargin); + [obj.Opts varargin] = vl_argparse(obj.Opts, varargin); + obj.setup(); + end + + function [frames] = extractFeatures(obj, imagePath) + import helpers.*; + import localFeatures.*; + frames = obj.loadFeatures(imagePath,false); + if numel(frames) > 0; return; end; + startTime = tic; + obj.info('Computing frames of image %s.',getFileName(imagePath)); + + img = imread(imagePath); + if(size(img,3)>1), img = rgb2gray(img); end + img = im2uint8(img); % If not already in uint8, then convert + + frames = harrisscaleselect(img, obj.Opts)'; + frames([1,2],:) = frames([2,1],:); + timeElapsed = toc(startTime); + obj.debug('%d Frames from image %s computed in %gs',... + size(frames,2),getFileName(imagePath),timeElapsed); + obj.storeFeatures(imagePath, frames, []); + end + + function sign = getSignature(obj) + sign = [helpers.struct2str(obj.Opts),';',... + helpers.fileSignature(obj.CodeDir, 'harrisscaleselect.m'), ... + mfilename('fullpath')]; + end + + function setup(obj) + if(~exist('harrisscaleselect.m', 'file')), + fprintf('Adding HarrisScaleSelect to path.\n'); + addpath(obj.CodeDir) + end + end + + function unload(obj) + fprintf('Removing HarrisScaleSelect from path.\n'); + rmpath(obj.CodeDir) + end + + end + + methods (Access=protected, Hidden) + function [srclist flags] = getMexSources(obj) + srclist = {fullfile(obj.CodeDir, 'LocalMaxima3DFast.cpp')}; + flags = {''}; + end + end + + methods (Access = protected) + function [urls dstPaths] = getTarballsList(obj) + installDir = obj.RootInstallDir; + dstPaths = {fullfile(installDir)}; + urls = {obj.CodeUrl}; + end + end +end diff --git a/+localFeatures/LindebergScaleSelect.m b/+localFeatures/LindebergScaleSelect.m new file mode 100644 index 0000000..06f0a2e --- /dev/null +++ b/+localFeatures/LindebergScaleSelect.m @@ -0,0 +1,86 @@ +classdef LindebergScaleSelect < localFeatures.GenericLocalFeatureExtractor & ... + helpers.GenericInstaller + + properties (SetAccess=private, GetAccess=public) + Opts = struct( ... + 'localize', 0, ... + 'NoScales', 61, ... + 'sigmamin', 0.88, ... + 'sigmamax', -1, ... + 'Kthres', 10^(7.5) ... + ) + end + + properties (Constant, Hidden) + % Installation directory + RootInstallDir = fullfile('data','software'); + CodeDir = fullfile('data','software','diku-dtu_detectors'); + % URL for dataset tarballs + CodeUrl = 'http://imm.dtu.dk/~abll/files/diku-dtu_detectors.tar.gz'; + end + + + methods + function obj = LindebergScaleSelect(varargin) + import helpers.*; + obj.Name = 'Lindeberg with scale selection'; + varargin = obj.checkInstall(varargin); + [obj.Opts varargin] = vl_argparse(obj.Opts, varargin); + obj.setup(); + end + + function [frames] = extractFeatures(obj, imagePath) + import helpers.*; + import localFeatures.*; + frames = obj.loadFeatures(imagePath,false); + if numel(frames) > 0; return; end; + startTime = tic; + obj.info('Computing frames of image %s.',getFileName(imagePath)); + + img = imread(imagePath); + if(size(img,3)>1), img = rgb2gray(img); end + img = im2uint8(img); % If not already in uint8, then convert + + frames = lindebergcorner(img, obj.Opts)'; + frames([1,2],:) = frames([2,1],:); + timeElapsed = toc(startTime); + obj.debug('%d Frames from image %s computed in %gs',... + size(frames,2),getFileName(imagePath),timeElapsed); + obj.storeFeatures(imagePath, frames, []); + end + + function sign = getSignature(obj) + sign = [helpers.struct2str(obj.Opts),';',... + helpers.fileSignature(obj.CodeDir, 'lindebergcorner.m'), ... + mfilename('fullpath')]; + end + + function setup(obj) + if(~exist('lindebergcorner.m', 'file')), + fprintf('Adding LindebergScaleSelect to path.\n'); + addpath(obj.CodeDir) + end + end + + function unload(obj) + fprintf('Removing LindebergScaleSelect from path.\n'); + rmpath(obj.CodeDir) + end + + end + + methods (Access=protected, Hidden) + function [srclist flags] = getMexSources(obj) + srclist = {fullfile(obj.CodeDir, 'LocalMaxima3DFast.cpp')}; + flags = {''}; + end + end + + methods (Access = protected) + function [urls dstPaths] = getTarballsList(obj) + installDir = obj.RootInstallDir; + dstPaths = {fullfile(installDir)}; + urls = {obj.CodeUrl}; + end + end +end diff --git a/+localFeatures/VlFeatSift.m b/+localFeatures/VlFeatSift.m index 2d07446..9a5b717 100644 --- a/+localFeatures/VlFeatSift.m +++ b/+localFeatures/VlFeatSift.m @@ -64,7 +64,7 @@ % Get the input image img = imread(imagePath); imgSize = size(img); - if imgSize(3) > 1 + if numel(imgSize) > 2 && imgSize(3) > 1 img = rgb2gray(img); end img = single(img); diff --git a/+localFeatures/VlFeatSiftUnoriented.m b/+localFeatures/VlFeatSiftUnoriented.m new file mode 100644 index 0000000..64e68e9 --- /dev/null +++ b/+localFeatures/VlFeatSiftUnoriented.m @@ -0,0 +1,102 @@ +classdef VlFeatSiftUnoriented < localFeatures.GenericLocalFeatureExtractor & ... + helpers.GenericInstaller +% localFeatures.VlFeatSift VlFeat vl_sift wrapper +% localFeatures.VlFeatSift('OptionName',OptionValue,...) Creates new +% object which wraps around VLFeat covariant image frames detector. +% All given options defined in the constructor are passed directly +% to the vl_sift function when called. +% +% The options to the constructor are the same as that for vl_sift +% See help vl_sift to see those options and their default values. +% +% See also: vl_sift + +% Authors: Karel Lenc + +% AUTORIGHTS + properties (SetAccess=public, GetAccess=public) + Opts + VlSiftArguments + end + + methods + function obj = VlFeatSiftUnoriented(varargin) + % def. arguments + obj.Name = 'VLFeat SIFT Unoriented'; + varargin = obj.configureLogger(obj.Name,varargin); + obj.VlSiftArguments = obj.checkInstall(varargin); + obj.ExtractsDescriptors = true; + end + + function [frames descriptors] = extractFeatures(obj, imagePath) + import helpers.*; + [frames descriptors] = obj.loadFeatures(imagePath,nargout > 1); + if numel(frames) > 0; return; end; + img = imread(imagePath); + if(size(img,3)>1), img = rgb2gray(img); end + img = single(img); % If not already in uint8, then convert + startTime = tic; + if nargout == 1 + obj.info('Computing frames of image %s.',getFileName(imagePath)); + [frames] = vl_sift(img,obj.VlSiftArguments{:}); + else + obj.info('Computing frames and descriptors of image %s.',... + getFileName(imagePath)); + [frames descriptors] = vl_sift(img,obj.VlSiftArguments{:}); + end + timeElapsed = toc(startTime); + obj.debug('%d Frames from image %s computed in %gs',... + size(frames,2),getFileName(imagePath),timeElapsed); + obj.storeFeatures(imagePath, frames, descriptors); + end + + function [frames descriptors] = extractDescriptors(obj, imagePath, frames) + % extractDescriptor Extract SIFT descriptors of disc frames + % [DFRAMES DESCRIPTORS] = obj.extractDescriptor(IMG_PATH, + % FRAMES) Extracts SIFT descriptors DESCRIPTPORS of disc + % frames FRAMES from image defined by IMG_PATH. For the + % descriptor extraction, scale-space is used. Ellipses are + % converted to discs using their scale. The orientation of an + % oriented ellipse is dropped. + import localFeatures.helpers.*; + obj.info('Computing descriptors.'); + startTime = tic; + % Get the input image + img = imread(imagePath); + imgSize = size(img); + if imgSize(3) > 1 + img = rgb2gray(img); + end + img = single(img); + if size(frames,1) > 4 + % Convert frames to disks + frames = [frames(1,:); frames(2,:); getFrameScale(frames)]; + end + if size(frames,1) < 4 + % When no orientation, compute upright SIFT descriptors + frames = [frames; zeros(1,size(frames,2))]; + end + if size(frames,1) == 4 + % When no orientation, compute upright SIFT descriptors + frames(4,:) = zeros(1,size(frames,2)); + end + % Compute the descriptors (using scale space). + [frames, descriptors] = vl_sift(img,'Frames',frames,... + obj.VlSiftArguments{:}); + elapsedTime = toc(startTime); + obj.debug('Descriptors computed in %gs',elapsedTime); + end + + function sign = getSignature(obj) + sign = [helpers.VlFeatInstaller.getBinSignature('vl_sift'), ... + helpers.cell2str(obj.VlSiftArguments), ... + mfilename('fullpath')]; + end + end + + methods (Access=protected) + function deps = getDependencies(obj) + deps = {helpers.VlFeatInstaller('0.9.14')}; + end + end +end diff --git a/eval_mh.m b/eval_mh.m new file mode 100644 index 0000000..56d9597 --- /dev/null +++ b/eval_mh.m @@ -0,0 +1,208 @@ +function eval_mh() + +import datasets.*; +import benchmarks.*; +import localFeatures.*; + +set(0,'DefaultFigureVisible','off'); + + +%dataset_name = 'oxford'; +dataset_name = 'dtu'; + +if strcmp(dataset_name, 'oxford') + categories = datasets.VggAffineDataset.AllCategories; +elseif strcmp(dataset_name, 'dtu') + categories = datasets.DTURobotDataset.AllCategories; +else + error('invalid dataset name') +end + +for category_idx = 1:numel(categories) + category_name = categories{category_idx}; + if strcmp(dataset_name, 'oxford') + dataset = datasets.VggAffineDataset('Category', category_name); + else strcmp(dataset_name, 'dtu') + dataset = datasets.DTURobotDataset('Category','arc2'); + end + % -------------------------------------------------------------------- + % PART 1: Detector repeatability + % -------------------------------------------------------------------- + + vlcovdetDetector = VlFeatCovdet() + siftDetector = VlFeatSift(); + mhDetector = MultiscaleHarris(); + mhDetector.Opts.localization = 1; + mh_woDetector = MultiscaleHarris(); + mh_woDetector.Opts.localization = 0; + lcDetector = LindebergCorners(); + lcDetector.Opts.localization = 1; + lc_woDetector = LindebergCorners(); + lc_woDetector.Opts.localization = 0; + mser = VlFeatMser(); + + + repBenchmark = RepeatabilityBenchmark('Mode','Repeatability'); + + featExtractors = {vlcovdetDetector, siftDetector, mser, lcDetector, lc_woDetector, mhDetector, mh_woDetector}; + detectorNames = {'VLCovDet', 'VLSIFT', 'MSER', 'LC w. localization', 'LC w.o. localization', 'MH w. localization', 'MH w.o. localization'}; + +% featExtractors = {mh_woDetector}; +% detectorNames = {'MH w.o. localization'}; + + repeatability = []; + numCorresp = []; + + + for d = 1:numel(featExtractors) + % use a maximum of three scenes for this demo. + scenes = dataset.NumScenes; + for sceneNo = 1:scenes + for labelNo = 1:dataset.NumLabels + img_ref_id = dataset.getReferenceImageId(labelNo, sceneNo); + img_id = dataset.getImageId(labelNo, sceneNo); + [repeatability(d, labelNo, sceneNo) numCorresp(d, labelNo, sceneNo)] = ... + repBenchmark.testFeatureExtractor(featExtractors{d}, dataset, ... + img_ref_id, img_id); + end + end + end + + + printScores(detectorNames, 100 * repeatability, 'Repeatability'); + printScores(detectorNames, numCorresp, 'Number of correspondences'); + + figure(2); clf; + plotScores(detectorNames, dataset, 100 * repeatability, 'Repeatability'); + printFigure(['results_' dataset_name], [category_name '_repeatability']); + + figure(3); clf; + plotScores(detectorNames, dataset, numCorresp, 'Number of correspondences'); + printFigure(['results_' dataset_name], [category_name '_num-correspondences']); + + + % -------------------------------------------------------------------- + % PART 2: Detector matching score + % -------------------------------------------------------------------- + + vlcovdetWithSift = DescriptorAdapter(vlcovdetDetector, siftDetector); + mserWithSift = DescriptorAdapter(mser, siftDetector); + mhWithSift = DescriptorAdapter(mhDetector, siftDetector); + mh_woWithSift = DescriptorAdapter(mh_woDetector, siftDetector); + lcWithSift = DescriptorAdapter(lcDetector, siftDetector); + lc_woWithSift = DescriptorAdapter(lc_woDetector, siftDetector); + + featExtractors = {vlcovdetWithSift, siftDetector, mserWithSift, mhWithSift, mh_woWithSift, lcWithSift, lc_woWithSift}; + detectorNames = {'VLCovDet', 'SIFT', 'MSER', 'LC w. localization', 'LC w.o. localization', 'MH w. localization', 'MH w.o. localization'}; + + matchingBenchmark = RepeatabilityBenchmark('Mode','MatchingScore'); + + matchScore = []; + numMatches = []; + + for d = 1:numel(featExtractors) + % use a maximum of three scenes for this demo. + scenes = dataset.NumScenes; + for sceneNo = 1:scenes + for labelNo = 1:dataset.NumLabels + img_ref_id = dataset.getReferenceImageId(labelNo, sceneNo); + img_id = dataset.getImageId(labelNo, sceneNo); + [matchScore(d, labelNo, sceneNo) numMatches(d, labelNo, sceneNo)] = ... + matchingBenchmark.testFeatureExtractor(featExtractors{d}, ... + dataset, img_ref_id, img_id); + end + end + end + + printScores(detectorNames, matchScore*100, 'Match Score'); + printScores(detectorNames, numMatches, 'Number of matches') ; + + figure(5); clf; + plotScores(detectorNames, dataset, matchScore*100,'Matching Score (with SIFT description)'); + printFigure(['results_' dataset_name], [category_name '_matching_score']); + + figure(6); clf; + plotScores(detectorNames, dataset, numMatches,'Number of matches (with SIFT description)'); + printFigure(['results_' dataset_name], [category_name '_num-matches']); + +end % dataset category + + + +% -------------------------------------------------------------------- +% Helper functions +% -------------------------------------------------------------------- + +function printScores(detectorNames, scores, name) + numDetectors = numel(detectorNames); + maxNameLen = length('Method name'); + for k = 1:numDetectors + maxNameLen = max(maxNameLen,length(detectorNames{k})); + end + fprintf(['\n', name,':\n']); + formatString = ['%' sprintf('%d',maxNameLen) 's:']; + fprintf(formatString,'Method name'); + for k = 2:size(scores,2) + fprintf('\tImg#%02d',k); + end + fprintf('\n'); + for k = 1:numDetectors + fprintf(formatString,detectorNames{k}); + for l = 2:size(scores,2) + fprintf('\t%6s',sprintf('%.2f',scores(k,l))); + end + fprintf('\n'); + end +end + +function plotScores(detectorNames, dataset, score, titleText) + xstart = 1; + xend = size(score,2); + if ndims(score) == 2 + plot(xstart:xend,score(:,xstart:xend)','+-','linewidth', 2); hold on ; + else + score_std = std(score,0,3); + score = mean(score,3); + X = repmat(xstart:xend,[size(score, 1) 1])'; + Y = score(:,xstart:xend)'; + E = score_std(:,xstart:xend)'; + errorbar(X,Y,E,'+-','linewidth', 2); hold on ; + end + xLabel = dataset.ImageNamesLabel; + xTicks = dataset.ImageNames; + plot(xstart:xend,score(:,xstart:xend)','+-','linewidth', 2); hold on ; + ylabel(titleText) ; + xlabel(xLabel); + set(gca,'XTick',xstart:1:xend); + set(gca,'XTickLabel',xTicks); + title(titleText); + set(gca,'xtick',1:size(score,2)); + maxScore = max([max(max(score)) 1]); + meanEndValue = mean(score(:,xend)); + legendLocation = 'SouthEast'; + if meanEndValue < maxScore/2 + legendLocation = 'NorthEast'; + end + legend(detectorNames,'Location',legendLocation); + grid on ; + axis([xstart xend 0 maxScore]); +end +end + + +function printFigure(path, fileName, R, ext) +if isempty(path), return; end; +if ~exist(path, 'dir') + mkdir(path) ; +end +if ~exist('R','var') + R = 0.75; +end +vl_printsize(gcf, R) ; + +if ~exist('ext','var') + ext = 'pdf'; +end +filePath = fullfile(path, [fileName '.' ext]) ; +saveas(gcf, filePath, ext) +end diff --git a/repeatabilityDemo.m b/repeatabilityDemo.m index d1810ab..9c976e5 100644 --- a/repeatabilityDemo.m +++ b/repeatabilityDemo.m @@ -29,7 +29,7 @@ function repeatabilityDemo(resultsPath) % settings, one simply creates multiple instances of these objects. siftDetector = VlFeatSift(); -thrSiftDetector = VlFeatSift('PeakThresh',11); +thrSiftDetector = VlFeatSift('PeakThresh',10); % VLBenchmarks enables a simple access to a number of public % benchmakrs. It also provides simple facilities to generate test data @@ -73,6 +73,9 @@ function repeatabilityDemo(resultsPath) dataset = datasets.VggAffineDataset('Category','graf'); +% Uncomment this to demo the DTU Robot dataset instead of VGG Affine +%dataset = datasets.DTURobotDataset('Category','arc2'); + % Next, the benchmark is intialised by choosing various % parameters. The defaults correspond to the seetting in the original % publication (IJCV05). @@ -83,8 +86,18 @@ function repeatabilityDemo(resultsPath) % detects MSER image features. mser = VlFeatMser(); -featExtractors = {siftDetector, thrSiftDetector, mser}; +if strfind(dataset.DatasetName, 'DTURobotDataset') + % Set more restrictive thresholds to limit the number of interest points per + % image. At default thresholds, the detectors generate ~6000 interest points. + thrSiftDetector = VlFeatSift('PeakThresh',11); + thrMserDetector = VlFeatMser('Delta',15); + featExtractors = {thrSiftDetector, thrMserDetector}; + detectorNames = {'DoG(PT=11)', 'MSER(Delta=15)'}; +else + featExtractors = {siftDetector, thrSiftDetector, mser}; + detectorNames = {'SIFT','SIFT PT=10','MSER'}; +end % Now we are ready to run the repeatability test. We do this by fixing % a reference image A and looping through other images B in the % set. To this end we use the following information: @@ -105,21 +118,24 @@ function repeatabilityDemo(resultsPath) repeatability = []; numCorresp = []; -imageAPath = dataset.getImagePath(1); for d = 1:numel(featExtractors) - for i = 2:dataset.NumImages - [repeatability(d,i) numCorresp(d,i)] = ... - repBenchmark.testFeatureExtractor(featExtractors{d}, ... - dataset.getTransformation(i), ... - dataset.getImagePath(1), ... - dataset.getImagePath(i)); + % use a maximum of three scenes for this demo. + scenes = min(dataset.NumScenes, 3); + for sceneNo = 1:scenes + for labelNo = 1:dataset.NumLabels + img_ref_id = dataset.getReferenceImageId(labelNo, sceneNo); + img_id = dataset.getImageId(labelNo, sceneNo); + [repeatability(d, labelNo, sceneNo) numCorresp(d, labelNo, sceneNo)] = ... + repBenchmark.testFeatureExtractor(featExtractors{d}, dataset, ... + img_ref_id, img_id); + end end end + % The scores can now be prented, as well as visualized in a % graph. This uses two simple functions defined below in this file. -detectorNames = {'SIFT','SIFT PT=10','MSER'}; printScores(detectorNames, 100 * repeatability, 'Repeatability'); printScores(detectorNames, numCorresp, 'Number of correspondences'); @@ -138,24 +154,26 @@ function repeatabilityDemo(resultsPath) % We do this by running the repeatabiltiy score again. However, since % the results are cached, this is fast. -imageBIdx = 3; +% TODO: andersbll: I propose to implement a function in RepeatabilityBenchmark +% called 'matchFrames' and use it like the following. -[drop drop siftCorresps siftReprojFrames] = ... - repBenchmark.testFeatureExtractor(siftDetector, ... - dataset.getTransformation(imageBIdx), ... - dataset.getImagePath(1), ... - dataset.getImagePath(imageBIdx)); +%imageBIdx = 3; +%sceneNo = 1; +%img_ref_id = dataset.getReferenceImageId(imageBIdx, sceneNo); +%img_id = dataset.getImageId(imageBIdx, sceneNo); +%[siftCorresps siftReprojFrames] = ... +% repBenchmark.matchFrames(siftDetector, dataset, img_ref_id, img_id); -% And plot the feature frame correspondences +%% And plot the feature frame correspondences -figure(4); clf; -imshow(dataset.getImagePath(imageBIdx)); -benchmarks.helpers.plotFrameMatches(siftCorresps,... - siftReprojFrames,... - 'IsReferenceImage',false,... - 'PlotMatchLine',false,... - 'PlotUnmatched',false); -helpers.printFigure(resultsPath,'correspondences',0.75); +%figure(4); clf; +%imshow(dataset.getImagePath(imageBIdx)); +%benchmarks.helpers.plotFrameMatches(siftCorresps,... +% siftReprojFrames,... +% 'IsReferenceImage',false,... +% 'PlotMatchLine',false,... +% 'PlotUnmatched',false); +%helpers.printFigure(resultsPath,'correspondences',0.75); % -------------------------------------------------------------------- % PART 3: Detector matching score @@ -171,30 +189,54 @@ function repeatabilityDemo(resultsPath) % detector is used as descriptor form MSER. mserWithSift = DescriptorAdapter(mser, siftDetector); -featExtractors = {siftDetector, thrSiftDetector, mserWithSift}; +if strfind(dataset.DatasetName, 'DTURobotDataset') + thrSiftDetector = VlFeatSift('PeakThresh',11); + thrMserDetector = VlFeatMser('Delta',15); + mserWithSift = DescriptorAdapter(thrMserDetector, siftDetector); + featExtractors = {thrSiftDetector, mserWithSift}; + detectorNames = {'DoG(PT=11) with SIFT', 'MSER(Delta=15) with SIFT'} +else + featExtractors = {siftDetector, thrSiftDetector, mserWithSift}; + detectorNames = {'SIFT','SIFT PT=10','MSER with SIFT'} +end % We create a benchmark object and run the tests as before, but in % this case we request that descriptor-based matched should be tested. matchingBenchmark = RepeatabilityBenchmark('Mode','MatchingScore'); +%matchScore = []; +%numMatches = []; + +%for d = 1:numel(featExtractors) +% for i = 2:dataset.NumImages +% [matchScore(d,i) numMatches(d,i)] = ... +% matchingBenchmark.testFeatureExtractor(featExtractors{d}, ... +% dataset, ... +% 1, ... +% i); +% end +%end + matchScore = []; numMatches = []; for d = 1:numel(featExtractors) - for i = 2:dataset.NumImages - [matchScore(d,i) numMatches(d,i)] = ... - matchingBenchmark.testFeatureExtractor(featExtractors{d}, ... - dataset.getTransformation(i), ... - dataset.getImagePath(1), ... - dataset.getImagePath(i)); + % use a maximum of three scenes for this demo. + scenes = min(dataset.NumScenes, 3); + for sceneNo = 1:scenes + for labelNo = 1:dataset.NumLabels + img_ref_id = dataset.getReferenceImageId(labelNo, sceneNo); + img_id = dataset.getImageId(labelNo, sceneNo); + [matchScore(d, labelNo, sceneNo) numMatches(d, labelNo, sceneNo)] = ... + matchingBenchmark.testFeatureExtractor(featExtractors{d}, ... + dataset, img_ref_id, img_id); + end end end -% Print and plot the results - -detectorNames = {'SIFT','SIFT PT=10','MSER with SIFT'}; +% Print and plot the results printScores(detectorNames, matchScore*100, 'Match Score'); printScores(detectorNames, numMatches, 'Number of matches') ; @@ -209,21 +251,22 @@ function repeatabilityDemo(resultsPath) % Same as with the correspondences, we can plot the matches based on % feature frame descriptors. The code is nearly identical. -imageBIdx = 3; -[r nc siftCorresps siftReprojFrames] = ... - matchingBenchmark.testFeatureExtractor(siftDetector, ... - dataset.getTransformation(imageBIdx), ... - dataset.getImagePath(1), ... - dataset.getImagePath(imageBIdx)); - -figure(7); clf; -imshow(imread(dataset.getImagePath(imageBIdx))); -benchmarks.helpers.plotFrameMatches(siftCorresps,... - siftReprojFrames,... - 'IsReferenceImage',false,... - 'PlotMatchLine',false,... - 'PlotUnmatched',false); -helpers.printFigure(resultsPath,'matches',0.75); +% TODO, see above +%imageBIdx = 3; +%[r nc siftCorresps] = ... +% matchingBenchmark.testFeatureExtractor(siftDetector, ... +% dataset, ... +% 1, ... +% imageBIdx); + +%figure(7); clf; +%imshow(imread(dataset.getImagePath(imageBIdx))); +%benchmarks.helpers.plotFrameMatches(siftCorresps,... +% siftReprojFrames,... +% 'IsReferenceImage',false,... +% 'PlotMatchLine',false,... +% 'PlotUnmatched',false); +%helpers.printFigure(resultsPath,'matches',0.75); % -------------------------------------------------------------------- % Helper functions @@ -252,8 +295,18 @@ function printScores(detectorNames, scores, name) end function plotScores(detectorNames, dataset, score, titleText) - xstart = max([find(sum(score,1) == 0, 1) + 1 1]); + xstart = 1; xend = size(score,2); + if ndims(score) == 2 + plot(xstart:xend,score(:,xstart:xend)','+-','linewidth', 2); hold on ; + else + score_std = std(score,0,3); + score = mean(score,3); + X = repmat(xstart:xend,[size(score, 1) 1])'; + Y = score(:,xstart:xend)'; + E = score_std(:,xstart:xend)'; + errorbar(X,Y,E,'+-','linewidth', 2); hold on ; + end xLabel = dataset.ImageNamesLabel; xTicks = dataset.ImageNames; plot(xstart:xend,score(:,xstart:xend)','+-','linewidth', 2); hold on ; @@ -273,4 +326,5 @@ function plotScores(detectorNames, dataset, score, titleText) grid on ; axis([xstart xend 0 maxScore]); end -end \ No newline at end of file +end +