From 7d13b3d522823bd5c4077f69f99c18f34c596d07 Mon Sep 17 00:00:00 2001 From: Anders Boesen Lindbo Larsen Date: Thu, 21 Feb 2013 13:17:19 +0100 Subject: [PATCH 1/9] Initial support for the DTU Robot dataset. --- +benchmarks/DTURobotRepeatabilityBenchmark.m | 335 +++++++++++++++++++ +datasets/DTURobotDataset.m | 224 +++++++++++++ DTURobotrepeatabilityDemo.m | 284 ++++++++++++++++ 3 files changed, 843 insertions(+) create mode 100644 +benchmarks/DTURobotRepeatabilityBenchmark.m create mode 100644 +datasets/DTURobotDataset.m create mode 100644 DTURobotrepeatabilityDemo.m diff --git a/+benchmarks/DTURobotRepeatabilityBenchmark.m b/+benchmarks/DTURobotRepeatabilityBenchmark.m new file mode 100644 index 0000000..9961d96 --- /dev/null +++ b/+benchmarks/DTURobotRepeatabilityBenchmark.m @@ -0,0 +1,335 @@ +classdef DTURobotRepeatabilityBenchmark < benchmarks.GenericBenchmark ... + & helpers.Logger & helpers.GenericInstaller + + properties + Opts = struct(... + 'overlapError', 0.4,... + 'normaliseFrames', true,... + 'cropFrames', true,... + 'magnification', 3,... + 'warpMethod', 'linearise',... + 'mode', 'repeatability',... + 'descriptorsDistanceMetric', 'L2',... + 'normalisedScale', 30); + end + + properties(Constant, Hidden) + KeyPrefix = 'repeatability3d'; + Modes = {'repeatability','matchingscore','descmatchingscore'}; + ModesOpts = containers.Map(benchmarks.DTURobotRepeatabilityBenchmark.Modes,... + {struct('matchGeometry',true,'matchDescs',false),... + struct('matchGeometry',true,'matchDescs',true),... + struct('matchGeometry',false,'matchDescs',true)}); + end + + methods + function obj = DTURobotRepeatabilityBenchmark(varargin) + import benchmarks.*; + import helpers.*; + obj.BenchmarkName = 'repeatability'; + if numel(varargin) > 0 + [obj.Opts varargin] = vl_argparse(obj.Opts,varargin); + obj.Opts.mode = lower(obj.Opts.mode); + if ~ismember(obj.Opts.mode, obj.Modes) + error('Invalid mode %s.',obj.Opts.mode); + end + end + varargin = obj.configureLogger(obj.BenchmarkName,varargin); + obj.checkInstall(varargin); + end + + function [score numMatches bestMatches reprojFrames] = ... + testFeatureExtractor(obj, featExtractor, dataset, imageAId, imageBId) + % testFeatureExtractor Image feature extractor repeatability + % REPEATABILITY = obj.testFeatureExtractor(FEAT_EXTRACTOR, TF, + % IMAGEAPATH, IMAGEBPATH) computes the repeatability REP of a image + % feature extractor FEAT_EXTRACTOR and its frames extracted from + % images defined by their path IMAGEAPATH and IMAGEBPATH whose + % geometry is related by the homography transformation TF. TODO tf => gtc=ground truth correspondences + % FEAT_EXTRACTOR must be a subclass of + % localFeatures.GenericLocalFeatureExtractor. + % + % [REPEATABILITY, NUMMATCHES] = obj.testFeatureExtractor(...) + % returns also the total number of feature matches found. + % + % [REP, NUMMATCHES, REPR_FRAMES, MATCHES] = + % obj.testFeatureExtractor(...) returns cell array REPR_FRAMES which + % contains reprojected and eventually cropped frames in + % format: + % + % REPR_FRAMES = {CFRAMES_A,CFRAMES_B,REP_CFRAMES_A,REP_CFRAMES_B} + % + % where CFRAMES_A are (cropped) frames detected in the IMAGEAPATH + % image REP_CFRAMES_A are CFRAMES_A reprojected to the IMAGEBPATH + % image using homography TF. Same hold for frames from the secons + % image CFRAMES_B and REP_CFRAMES_B. + % MATCHES is an array of size [size(CFRAMES_A),1]. Two frames are + % CFRAMES_A(k) and CFRAMES_B(l) are matched when MATCHES(k) = l. + % When frame CFRAMES_A(k) is not matched, MATCHES(k) = 0. + % + % This method caches its results, so that calling it again will not + % recompute the repeatability score unless the cache is manually + % cleared. + % + % See also: benchmarks.RepeatabilityBenchmark(). + import benchmarks.*; + import helpers.*; + + imageAPath = dataset.getImagePath(imageAId); + imageBPath = dataset.getImagePath(imageBId); + obj.info('Comparing frames from det. %s and images %s and %s.',... + featExtractor.Name,getFileName(imageAPath),... + getFileName(imageBPath)); + + imageASign = helpers.fileSignature(imageAPath); + imageBSign = helpers.fileSignature(imageBPath); + imageASize = helpers.imageSize(imageAPath); + imageBSize = helpers.imageSize(imageBPath); + resultsKey = cell2str({obj.KeyPrefix, obj.getSignature(), ... + featExtractor.getSignature(), imageASign, imageBSign}); + cachedResults = obj.loadResults(resultsKey); + + % When detector does not cache results, do not use the cached data +% if isempty(cachedResults) || ~featExtractor.UseCache + if obj.ModesOpts(obj.Opts.mode).matchDescs + [framesA descriptorsA] = featExtractor.extractFeatures(imageAPath); + [framesB descriptorsB] = featExtractor.extractFeatures(imageBPath); + [score numMatches bestMatches reprojFrames] = obj.testFeatures(... + dataset, imageAId, imageBId, imageASize, imageBSize, framesA, framesB,... + descriptorsA, descriptorsB); + else + [framesA] = featExtractor.extractFeatures(imageAPath); + [framesB] = featExtractor.extractFeatures(imageBPath); + [score numMatches bestMatches reprojFrames] = ... + obj.testFeatures(dataset, imageAId, imageBId, imageASize, imageBSize, framesA, framesB); + end + if featExtractor.UseCache + results = {score numMatches bestMatches reprojFrames}; + obj.storeResults(results, resultsKey); + end +% else +% [score numMatches bestMatches reprojFrames] = cachedResults{:}; +% obj.debug('Results loaded from cache'); +% end + + end + + function [score numMatches matches reprojFrames] = ... + testFeatures(obj, dataset, imageAId, imageBId, imageASize, imageBSize, framesA, framesB, ... + descriptorsA, descriptorsB) + % testFeatures Compute repeatability of given image features + % [SCORE NUM_MATCHES] = obj.testFeatures(TF, IMAGE_A_SIZE, + % IMAGE_B_SIZE, FRAMES_A, FRAMES_B, DESCS_A, DESCS_B) Compute + % matching score SCORE between frames FRAMES_A and FRAMES_B + % and their descriptors DESCS_A and DESCS_B which were + % extracted from pair of images with sizes IMAGE_A_SIZE and + % IMAGE_B_SIZE which geometry is related by homography TF.TODO tf=> gtc + % NUM_MATHCES is number of matches which is calcuated + % according to object settings. + % + % [SCORE, NUM_MATCHES, REPR_FRAMES, MATCHES] = + % obj.testFeatures(...) returns cell array REPR_FRAMES which + % contains reprojected and eventually cropped frames in + % format: + % + % REPR_FRAMES = {CFRAMES_A,CFRAMES_B,REP_CFRAMES_A,REP_CFRAMES_B} + % + % where CFRAMES_A are (cropped) frames detected in the IMAGEAPATH + % image REP_CFRAMES_A are CFRAMES_A reprojected to the IMAGEBPATH + % image using homography TF. Same hold for frames from the secons + % image CFRAMES_B and REP_CFRAMES_B. + % MATCHES is an array of size [size(CFRAMES_A),1]. Two frames are + % CFRAMES_A(k) and CFRAMES_B(l) are matched when MATCHES(k) = l. + % When frame CFRAMES_A(k) is not matched, MATCHES(k) = 0. + import benchmarks.helpers.*; + import helpers.*; + + obj.info('Computing score between %d/%d frames.',... + size(framesA,2),size(framesB,2)); + matchGeometry = obj.ModesOpts(obj.Opts.mode).matchGeometry; + matchDescriptors = obj.ModesOpts(obj.Opts.mode).matchDescs; + + if isempty(framesA) || isempty(framesB) + matches = zeros(size(framesA,2)); reprojFrames = {}; + obj.info('Nothing to compute.'); + return; + end + if exist('descriptorsA','var') && exist('descriptorsB','var') + if size(framesA,2) ~= size(descriptorsA,2) ... + || size(framesB,2) ~= size(descriptorsB,2) + obj.error('Number of frames and descriptors must be the same.'); + end + elseif matchDescriptors + obj.error('Unable to match descriptors without descriptors.'); + end + + score = 0; numMatches = 0; + startTime = tic; + normFrames = obj.Opts.normaliseFrames; + overlapError = obj.Opts.overlapError; + overlapThresh = 1 - overlapError; + + % convert frames from any supported format to unortiented + % ellipses for uniformity + framesA = localFeatures.helpers.frameToEllipse(framesA) ; + framesB = localFeatures.helpers.frameToEllipse(framesB) ; + + % map frames from image A to image B and viceversa +% reprojFramesA = warpEllipse(tf, framesA,... +% 'Method',obj.Opts.warpMethod) ; +% reprojFramesB = warpEllipse(inv(tf), framesB,... +% 'Method',obj.Opts.warpMethod) ; +% reprojFrames = {framesA,framesB,reprojFramesA,reprojFramesB}; + reprojFrames = {framesA,framesB,framesB, framesA}; + +% % optionally remove frames that are not fully contained in +% % both images +% if obj.Opts.cropFrames +% % find frames fully visible in both images +% bboxA = [1 1 imageASize(2)+1 imageASize(1)+1] ; +% bboxB = [1 1 imageBSize(2)+1 imageBSize(1)+1] ; + +% visibleFramesA = isEllipseInBBox(bboxA, framesA ) & ... +% isEllipseInBBox(bboxB, reprojFramesA); + +% visibleFramesB = isEllipseInBBox(bboxA, reprojFramesB) & ... +% isEllipseInBBox(bboxB, framesB ); + +% % Crop frames outside overlap region +% framesA = framesA(:,visibleFramesA); +% reprojFramesA = reprojFramesA(:,visibleFramesA); +% framesB = framesB(:,visibleFramesB); +% reprojFramesB = reprojFramesB(:,visibleFramesB); +% if isempty(framesA) || isempty(framesB) +% matches = zeros(size(framesA,2)); reprojFrames = {}; +% return; +% end + +% if matchDescriptors +% descriptorsA = descriptorsA(:,visibleFramesA); +% descriptorsB = descriptorsB(:,visibleFramesB); +% end +% end + +% if ~normFrames +% % When frames are not normalised, account the descriptor region +% magFactor = obj.Opts.magnification^2; +% framesA = [framesA(1:2,:); framesA(3:5,:).*magFactor]; +% framesB = [framesB(1:2,:); framesB(3:5,:).*magFactor]; +%% reprojFramesB = [reprojFramesB(1:2,:); ... +%% reprojFramesB(3:5,:).*magFactor]; +% end + +% reprojFrames = {framesA,framesB,reprojFramesA,reprojFramesB}; +% numFramesA = size(framesA,2); +% numFramesB = size(reprojFramesB,2); + +% % Find all ellipse overlaps (in one-to-n array) +% frameOverlaps = fastEllipseOverlap(reprojFramesB, framesA, ... +% 'NormaliseFrames',normFrames,'MinAreaRatio',overlapThresh,... +% 'NormalisedScale',obj.Opts.normalisedScale); + + + frameOverlaps = dataset.getFrameOverlaps(imageAId, imageBId, framesA, framesB); + + numFramesA = size(framesA,2); + numFramesB = size(framesB,2); + + matches = []; + + if matchGeometry + % Create an edge between each feature in A and in B + % weighted by the overlap. Each edge is a candidate match. + corresp = cell(1,numFramesA); + for j=1:numFramesA + numNeighs = length(frameOverlaps.scores{j}); + if numNeighs > 0 + corresp{j} = [j *ones(1,numNeighs); ... + frameOverlaps.neighs{j}; ... + frameOverlaps.scores{j}]; + end + end + corresp = cat(2,corresp{:}) ; + if isempty(corresp) + score = 0; numMatches = 0; matches = zeros(1,numFramesA); return; + end + + % Remove edges (candidate matches) that have insufficient overlap + corresp = corresp(:,corresp(3,:) > overlapThresh) ; + if isempty(corresp) + score = 0; numMatches = 0; matches = zeros(1,numFramesA); return; + end + + % Sort the edgest by decrasing score + [drop, perm] = sort(corresp(3,:), 'descend'); + corresp = corresp(:, perm); + + % Approximate the best bipartite matching + obj.info('Matching frames geometry.'); + geometryMatches = greedyBipartiteMatching(numFramesA,... + numFramesB, corresp(1:2,:)'); + + matches = [matches ; geometryMatches]; + end + + if matchDescriptors + obj.info('Computing cross distances between all descriptors'); + dists = vl_alldist2(single(descriptorsA),single(descriptorsB),... + obj.Opts.descriptorsDistanceMetric); + obj.info('Sorting distances') + [dists, perm] = sort(dists(:),'ascend'); + + % Create list of edges in the bipartite graph + [aIdx bIdx] = ind2sub([numFramesA, numFramesB],perm(1:numel(dists))); + edges = [aIdx bIdx]; + + % Find one-to-one best matches + obj.info('Matching descriptors.'); + descMatches = greedyBipartiteMatching(numFramesA, numFramesB, edges); + + for aIdx=1:numFramesA + bIdx = descMatches(aIdx); + [hasCorresp bCorresp] = ismember(bIdx,frameOverlaps.neighs{aIdx}); + % Check whether found descriptor matches fulfill frame overlap + if ~hasCorresp || ... + ~frameOverlaps.scores{aIdx}(bCorresp) > overlapThresh + descMatches(aIdx) = 0; + end + end + matches = [matches ; descMatches]; + end + + % Combine collected matches, i.e. select only equal matches + validMatches = ... + prod(single(matches == repmat(matches(1,:),size(matches,1),1)),1); + matches = matches(1,:) .* validMatches; + + % Compute the score + numBestMatches = sum(matches ~= 0); + score = numBestMatches / min(size(framesA,2), size(framesB,2)); + numMatches = numBestMatches; + + obj.info('Score: %g \t Num matches: %g', ... + score,numMatches); + + obj.debug('Score between %d/%d frames comp. in %gs',size(framesA,2), ... + size(framesB,2),toc(startTime)); + end + + function signature = getSignature(obj) + signature = helpers.struct2str(obj.Opts); + end + end + + methods (Access = protected) + function deps = getDependencies(obj) + deps = {helpers.Installer(),helpers.VlFeatInstaller('0.9.14'),... + benchmarks.helpers.Installer()}; + end + + + + end % methods (Access = protected) + +end + diff --git a/+datasets/DTURobotDataset.m b/+datasets/DTURobotDataset.m new file mode 100644 index 0000000..8117af3 --- /dev/null +++ b/+datasets/DTURobotDataset.m @@ -0,0 +1,224 @@ +classdef DTURobotDataset < datasets.GenericDataset & helpers.Logger... + & helpers.GenericInstaller + + + properties (SetAccess=private, GetAccess=public) + Category = 'arc1'; % Dataset category + DataDir; % Image location + ImgExt; % Image extension + % Number of different perturbation configurations within the category. + NumPerturbationsConfigurations; + % Number of images for each perturbation configuration. + NumImagesPerPerturbation; + ImageNames; + ImageNamesLabel; + Viewpoints; + Lightings; + ReconstructionsDir; + CacheDir; + CamerasPath; + ImgWidth; + ImgHeight; + CodeDir; + end + + properties (Constant) + KeyPrefix = 'DTURobotDataset'; + % All dataset categories + AllCategories = {'arc1', 'arc2', 'arc3', 'linear_path', ... + 'lighting_x', 'lighting_y'}; + %=3mm + StrLBoxPad=3e-3; + %pixels + BackProjThresh=5; + %Margin for change in scale, corrected for distance to the secen, in orer for a correspondance to be accepted. A value of 2 corresponds to an octave. + ScaleMargin=2; + end + + properties (Constant, Hidden) + % Names of the image transformations in particular categories + CategoryImageNames = {... + 'Arc 1, viewpoint angle',... + 'Arc 2, viewpoint angle',... + 'Arc 3, viewpoint angle',... + 'Linear path, camera distance',... + 'Lighting changes, x coordinate',... + 'Lighting changes, y coordinate',... + }; + % Image labels for particular categories (degree of transf.) + CategoryImageLabels = {... + round([[-40: 40/24: -1.6666666] [1.666666: 40/24: 40]]*10)/10,... % arc1 + round([-25: 50/29: 25]*10)/10,... % arc2 + round([-20: 40/24: 20]*10)/10,... % arc3 + round([.5214: .3/14: .8]*10)/10,... % linear_path + round([0: 180.8/10: 180.8]*10)/10,... % lighting_x + round([0: 80.3/10: 80.3]*10)/10,... % lighting_y + }; + + CategoryViewpoints = {... + [1:24 26:49], ... + [65:94], ... + [95:119], ... + [51:64], ... + [12 25 56 64], ... + [12 25 56 64], ... + }; + + CategoryLightings = {... + 0, ... + 0, ... + 0, ... + 0, ... + [20:30], ... + [31:41], ... + }; + + NumScenes = 60; + %Size of search cells in the structured light grid. + CellRadius = 10; + % Installation directory + RootInstallDir = fullfile('data','datasets','DTURobot'); + % Root url for dataset tarballs +% % URL for code (+ some data) tarballs + CodeUrl = 'http://roboimagedata.imm.dtu.dk/code/RobotEvalCode.tar.gz'; +% DatasetUrl = 'http://roboimagedata.imm.dtu.dk/code/RobotEvalCode.tar.gz'; + end + + methods + function obj = DTURobotDataset(varargin) + import datasets.*; + import helpers.*; + opts.Category = obj.Category; + [opts varargin] = vl_argparse(opts,varargin); + [valid loc] = ismember(opts.Category,obj.AllCategories); + assert(valid,... + sprintf('Invalid category for DTURobot dataset: %s\n',opts.Category)); + obj.DatasetName = ['DTURobotDataset-' opts.Category]; + obj.Category= opts.Category; + obj.DataDir = fullfile(obj.RootInstallDir,'scenes'); + if strcmp(obj.Category,'lighting_x') || strcmp(obj.Category,'lighting_y') + obj.NumPerturbationsConfigurations = numel(obj.CategoryLightings{loc}) + obj.NumImagesPerPerturbation = obj.NumScenes * numel(obj.CategoryViewpoints{loc}); + else + obj.NumPerturbationsConfigurations = numel(obj.CategoryViewpoints{loc}); + obj.NumImagesPerPerturbation = obj.NumScenes; + end + + obj.checkInstall(varargin); + obj.ImageNames = obj.CategoryImageLabels{loc}; + obj.ImageNamesLabel = obj.CategoryImageNames{loc}; + obj.Viewpoints = obj.CategoryViewpoints{loc}; + obj.Lightings = obj.CategoryLightings{loc}; + obj.ReconstructionsDir = fullfile(obj.RootInstallDir, 'reconstructions'); + obj.CamerasPath = fullfile(obj.RootInstallDir, 'cameras.mat'); + obj.CacheDir = fullfile(obj.RootInstallDir, 'cache'); + obj.CodeDir = fullfile(obj.RootInstallDir, 'code'); + obj.ImgWidth = 1600; + obj.ImgHeight = 1200; + + addpath(obj.CodeDir) + end + + function imgPath = getImagePath(obj, imgId) + imgPath = fullfile(obj.DataDir, sprintf('scene%.3d/%.3d_%.2d.png', ... + imgId.scene, imgId.viewpoint, imgId.lighting)); + end + + function imgId = getReferenceImageId(obj, confNo, imgNo) + imgId.scene = imgNo; + imgId.viewpoint = 25; + imgId.lighting = 0; + end + + function imgId = getImageId(obj, confNo, imgNo) + if strcmp(obj.Category,'lighting_x') || strcmp(obj.Category,'lighting_y') + imgId.viewpoint = obj.Viewpoints(mod(imgNo, numel(obj.Viewpoints))+1); + imgId.lighting = obj.Lightings(confNo); + imgId.scene = floor(imgNo/numel(obj.Viewpoints)); + else + imgId.viewpoint = obj.Viewpoints(confNo); + imgId.lighting = 0; + imgId.scene = imgNo; + end + end + + function tfs = getTransformation(obj, imageAId, imageBId) + Cams = GetCamPair(imageAId.viewpoint, imageBId.viewpoint); + %TODO + end + + function frameOverlaps = getFrameOverlaps(obj, imageAId, imageBId, framesA, framesB) + % Get 3D reconstruction + [Grid3D, Pts] = GenStrLightGrid(imageAId.viewpoint, ... + obj.ReconstructionsDir, obj.ImgHeight, obj.ImgWidth, ... + obj.CellRadius, imageAId.scene); + CamPair = GetCamPair(imageAId.viewpoint, imageBId.viewpoint); + + N = size(framesA,2); + neighs = cell(1,N); + scores = cell(1,N); + for f = 1:N + frame_ref = framesA(:, f)'; + [neighs{f}, scores{f}] = obj.overlap(Grid3D, Pts, CamPair, frame_ref, framesB'); + end + frameOverlaps.neighs = neighs; + frameOverlaps.scores = scores; + end + + + function [neighs, scores] = overlap(obj, Grid3D, Pts, Cams, frame_ref, frames) + KeyP = frame_ref(1:2); + KeyScale = frame_ref(3); + + % Project point onto 3D structure + [Mean,Var,IsEst] = Get3DGridEst(Grid3D,Pts,obj.CellRadius,KeyP(1),KeyP(2)); + + neighs = []; + scores = []; + if(IsEst) + Var = Var+obj.StrLBoxPad; + Q = Mean*ones(1,8)+[Var(1)*[-1 1 -1 1 -1 1 -1 1]; + Var(2)*[-1 -1 1 1 -1 -1 1 1]; + Var(3)*[-1 -1 -1 -1 1 1 1 1]]; + q = Cams(:,:,2)*[Q;ones(1,8)]; + depth = mean(q(3,:)); + q(1,:) = q(1,:)./q(3,:); + q(2,:) = q(2,:)./q(3,:); + q(3,:) = q(3,:)./q(3,:); + + kq = Cams(:,:,1)*[Q;ones(1,8)]; + kDepth = mean(kq(3,:)); + Scale = KeyScale*kDepth/depth; + + % Find neighbor frames within scale bounds + idx = find(frames(:,1)>min(q(1,:)) & frames(:,1)min(q(2,:)) & frames(:,2)Scale/obj.ScaleMargin & ... + frames(:,3) Date: Thu, 7 Mar 2013 09:45:06 +0100 Subject: [PATCH 2/9] Program design changed to support datasets with different ground truth and datasets with multiple images per label. --- +benchmarks/DTURobotRepeatabilityBenchmark.m | 335 ------------------- +benchmarks/RepeatabilityBenchmark.m | 78 ++--- +datasets/DTURobotDataset.m | 153 +++++---- +datasets/GenericCorrespondenceDataset.m | 43 +++ +datasets/VggAffineDataset.m | 79 ++++- DTURobotrepeatabilityDemo.m | 284 ---------------- repeatabilityDemo.m | 158 ++++++--- 7 files changed, 323 insertions(+), 807 deletions(-) delete mode 100644 +benchmarks/DTURobotRepeatabilityBenchmark.m create mode 100644 +datasets/GenericCorrespondenceDataset.m delete mode 100644 DTURobotrepeatabilityDemo.m diff --git a/+benchmarks/DTURobotRepeatabilityBenchmark.m b/+benchmarks/DTURobotRepeatabilityBenchmark.m deleted file mode 100644 index 9961d96..0000000 --- a/+benchmarks/DTURobotRepeatabilityBenchmark.m +++ /dev/null @@ -1,335 +0,0 @@ -classdef DTURobotRepeatabilityBenchmark < benchmarks.GenericBenchmark ... - & helpers.Logger & helpers.GenericInstaller - - properties - Opts = struct(... - 'overlapError', 0.4,... - 'normaliseFrames', true,... - 'cropFrames', true,... - 'magnification', 3,... - 'warpMethod', 'linearise',... - 'mode', 'repeatability',... - 'descriptorsDistanceMetric', 'L2',... - 'normalisedScale', 30); - end - - properties(Constant, Hidden) - KeyPrefix = 'repeatability3d'; - Modes = {'repeatability','matchingscore','descmatchingscore'}; - ModesOpts = containers.Map(benchmarks.DTURobotRepeatabilityBenchmark.Modes,... - {struct('matchGeometry',true,'matchDescs',false),... - struct('matchGeometry',true,'matchDescs',true),... - struct('matchGeometry',false,'matchDescs',true)}); - end - - methods - function obj = DTURobotRepeatabilityBenchmark(varargin) - import benchmarks.*; - import helpers.*; - obj.BenchmarkName = 'repeatability'; - if numel(varargin) > 0 - [obj.Opts varargin] = vl_argparse(obj.Opts,varargin); - obj.Opts.mode = lower(obj.Opts.mode); - if ~ismember(obj.Opts.mode, obj.Modes) - error('Invalid mode %s.',obj.Opts.mode); - end - end - varargin = obj.configureLogger(obj.BenchmarkName,varargin); - obj.checkInstall(varargin); - end - - function [score numMatches bestMatches reprojFrames] = ... - testFeatureExtractor(obj, featExtractor, dataset, imageAId, imageBId) - % testFeatureExtractor Image feature extractor repeatability - % REPEATABILITY = obj.testFeatureExtractor(FEAT_EXTRACTOR, TF, - % IMAGEAPATH, IMAGEBPATH) computes the repeatability REP of a image - % feature extractor FEAT_EXTRACTOR and its frames extracted from - % images defined by their path IMAGEAPATH and IMAGEBPATH whose - % geometry is related by the homography transformation TF. TODO tf => gtc=ground truth correspondences - % FEAT_EXTRACTOR must be a subclass of - % localFeatures.GenericLocalFeatureExtractor. - % - % [REPEATABILITY, NUMMATCHES] = obj.testFeatureExtractor(...) - % returns also the total number of feature matches found. - % - % [REP, NUMMATCHES, REPR_FRAMES, MATCHES] = - % obj.testFeatureExtractor(...) returns cell array REPR_FRAMES which - % contains reprojected and eventually cropped frames in - % format: - % - % REPR_FRAMES = {CFRAMES_A,CFRAMES_B,REP_CFRAMES_A,REP_CFRAMES_B} - % - % where CFRAMES_A are (cropped) frames detected in the IMAGEAPATH - % image REP_CFRAMES_A are CFRAMES_A reprojected to the IMAGEBPATH - % image using homography TF. Same hold for frames from the secons - % image CFRAMES_B and REP_CFRAMES_B. - % MATCHES is an array of size [size(CFRAMES_A),1]. Two frames are - % CFRAMES_A(k) and CFRAMES_B(l) are matched when MATCHES(k) = l. - % When frame CFRAMES_A(k) is not matched, MATCHES(k) = 0. - % - % This method caches its results, so that calling it again will not - % recompute the repeatability score unless the cache is manually - % cleared. - % - % See also: benchmarks.RepeatabilityBenchmark(). - import benchmarks.*; - import helpers.*; - - imageAPath = dataset.getImagePath(imageAId); - imageBPath = dataset.getImagePath(imageBId); - obj.info('Comparing frames from det. %s and images %s and %s.',... - featExtractor.Name,getFileName(imageAPath),... - getFileName(imageBPath)); - - imageASign = helpers.fileSignature(imageAPath); - imageBSign = helpers.fileSignature(imageBPath); - imageASize = helpers.imageSize(imageAPath); - imageBSize = helpers.imageSize(imageBPath); - resultsKey = cell2str({obj.KeyPrefix, obj.getSignature(), ... - featExtractor.getSignature(), imageASign, imageBSign}); - cachedResults = obj.loadResults(resultsKey); - - % When detector does not cache results, do not use the cached data -% if isempty(cachedResults) || ~featExtractor.UseCache - if obj.ModesOpts(obj.Opts.mode).matchDescs - [framesA descriptorsA] = featExtractor.extractFeatures(imageAPath); - [framesB descriptorsB] = featExtractor.extractFeatures(imageBPath); - [score numMatches bestMatches reprojFrames] = obj.testFeatures(... - dataset, imageAId, imageBId, imageASize, imageBSize, framesA, framesB,... - descriptorsA, descriptorsB); - else - [framesA] = featExtractor.extractFeatures(imageAPath); - [framesB] = featExtractor.extractFeatures(imageBPath); - [score numMatches bestMatches reprojFrames] = ... - obj.testFeatures(dataset, imageAId, imageBId, imageASize, imageBSize, framesA, framesB); - end - if featExtractor.UseCache - results = {score numMatches bestMatches reprojFrames}; - obj.storeResults(results, resultsKey); - end -% else -% [score numMatches bestMatches reprojFrames] = cachedResults{:}; -% obj.debug('Results loaded from cache'); -% end - - end - - function [score numMatches matches reprojFrames] = ... - testFeatures(obj, dataset, imageAId, imageBId, imageASize, imageBSize, framesA, framesB, ... - descriptorsA, descriptorsB) - % testFeatures Compute repeatability of given image features - % [SCORE NUM_MATCHES] = obj.testFeatures(TF, IMAGE_A_SIZE, - % IMAGE_B_SIZE, FRAMES_A, FRAMES_B, DESCS_A, DESCS_B) Compute - % matching score SCORE between frames FRAMES_A and FRAMES_B - % and their descriptors DESCS_A and DESCS_B which were - % extracted from pair of images with sizes IMAGE_A_SIZE and - % IMAGE_B_SIZE which geometry is related by homography TF.TODO tf=> gtc - % NUM_MATHCES is number of matches which is calcuated - % according to object settings. - % - % [SCORE, NUM_MATCHES, REPR_FRAMES, MATCHES] = - % obj.testFeatures(...) returns cell array REPR_FRAMES which - % contains reprojected and eventually cropped frames in - % format: - % - % REPR_FRAMES = {CFRAMES_A,CFRAMES_B,REP_CFRAMES_A,REP_CFRAMES_B} - % - % where CFRAMES_A are (cropped) frames detected in the IMAGEAPATH - % image REP_CFRAMES_A are CFRAMES_A reprojected to the IMAGEBPATH - % image using homography TF. Same hold for frames from the secons - % image CFRAMES_B and REP_CFRAMES_B. - % MATCHES is an array of size [size(CFRAMES_A),1]. Two frames are - % CFRAMES_A(k) and CFRAMES_B(l) are matched when MATCHES(k) = l. - % When frame CFRAMES_A(k) is not matched, MATCHES(k) = 0. - import benchmarks.helpers.*; - import helpers.*; - - obj.info('Computing score between %d/%d frames.',... - size(framesA,2),size(framesB,2)); - matchGeometry = obj.ModesOpts(obj.Opts.mode).matchGeometry; - matchDescriptors = obj.ModesOpts(obj.Opts.mode).matchDescs; - - if isempty(framesA) || isempty(framesB) - matches = zeros(size(framesA,2)); reprojFrames = {}; - obj.info('Nothing to compute.'); - return; - end - if exist('descriptorsA','var') && exist('descriptorsB','var') - if size(framesA,2) ~= size(descriptorsA,2) ... - || size(framesB,2) ~= size(descriptorsB,2) - obj.error('Number of frames and descriptors must be the same.'); - end - elseif matchDescriptors - obj.error('Unable to match descriptors without descriptors.'); - end - - score = 0; numMatches = 0; - startTime = tic; - normFrames = obj.Opts.normaliseFrames; - overlapError = obj.Opts.overlapError; - overlapThresh = 1 - overlapError; - - % convert frames from any supported format to unortiented - % ellipses for uniformity - framesA = localFeatures.helpers.frameToEllipse(framesA) ; - framesB = localFeatures.helpers.frameToEllipse(framesB) ; - - % map frames from image A to image B and viceversa -% reprojFramesA = warpEllipse(tf, framesA,... -% 'Method',obj.Opts.warpMethod) ; -% reprojFramesB = warpEllipse(inv(tf), framesB,... -% 'Method',obj.Opts.warpMethod) ; -% reprojFrames = {framesA,framesB,reprojFramesA,reprojFramesB}; - reprojFrames = {framesA,framesB,framesB, framesA}; - -% % optionally remove frames that are not fully contained in -% % both images -% if obj.Opts.cropFrames -% % find frames fully visible in both images -% bboxA = [1 1 imageASize(2)+1 imageASize(1)+1] ; -% bboxB = [1 1 imageBSize(2)+1 imageBSize(1)+1] ; - -% visibleFramesA = isEllipseInBBox(bboxA, framesA ) & ... -% isEllipseInBBox(bboxB, reprojFramesA); - -% visibleFramesB = isEllipseInBBox(bboxA, reprojFramesB) & ... -% isEllipseInBBox(bboxB, framesB ); - -% % Crop frames outside overlap region -% framesA = framesA(:,visibleFramesA); -% reprojFramesA = reprojFramesA(:,visibleFramesA); -% framesB = framesB(:,visibleFramesB); -% reprojFramesB = reprojFramesB(:,visibleFramesB); -% if isempty(framesA) || isempty(framesB) -% matches = zeros(size(framesA,2)); reprojFrames = {}; -% return; -% end - -% if matchDescriptors -% descriptorsA = descriptorsA(:,visibleFramesA); -% descriptorsB = descriptorsB(:,visibleFramesB); -% end -% end - -% if ~normFrames -% % When frames are not normalised, account the descriptor region -% magFactor = obj.Opts.magnification^2; -% framesA = [framesA(1:2,:); framesA(3:5,:).*magFactor]; -% framesB = [framesB(1:2,:); framesB(3:5,:).*magFactor]; -%% reprojFramesB = [reprojFramesB(1:2,:); ... -%% reprojFramesB(3:5,:).*magFactor]; -% end - -% reprojFrames = {framesA,framesB,reprojFramesA,reprojFramesB}; -% numFramesA = size(framesA,2); -% numFramesB = size(reprojFramesB,2); - -% % Find all ellipse overlaps (in one-to-n array) -% frameOverlaps = fastEllipseOverlap(reprojFramesB, framesA, ... -% 'NormaliseFrames',normFrames,'MinAreaRatio',overlapThresh,... -% 'NormalisedScale',obj.Opts.normalisedScale); - - - frameOverlaps = dataset.getFrameOverlaps(imageAId, imageBId, framesA, framesB); - - numFramesA = size(framesA,2); - numFramesB = size(framesB,2); - - matches = []; - - if matchGeometry - % Create an edge between each feature in A and in B - % weighted by the overlap. Each edge is a candidate match. - corresp = cell(1,numFramesA); - for j=1:numFramesA - numNeighs = length(frameOverlaps.scores{j}); - if numNeighs > 0 - corresp{j} = [j *ones(1,numNeighs); ... - frameOverlaps.neighs{j}; ... - frameOverlaps.scores{j}]; - end - end - corresp = cat(2,corresp{:}) ; - if isempty(corresp) - score = 0; numMatches = 0; matches = zeros(1,numFramesA); return; - end - - % Remove edges (candidate matches) that have insufficient overlap - corresp = corresp(:,corresp(3,:) > overlapThresh) ; - if isempty(corresp) - score = 0; numMatches = 0; matches = zeros(1,numFramesA); return; - end - - % Sort the edgest by decrasing score - [drop, perm] = sort(corresp(3,:), 'descend'); - corresp = corresp(:, perm); - - % Approximate the best bipartite matching - obj.info('Matching frames geometry.'); - geometryMatches = greedyBipartiteMatching(numFramesA,... - numFramesB, corresp(1:2,:)'); - - matches = [matches ; geometryMatches]; - end - - if matchDescriptors - obj.info('Computing cross distances between all descriptors'); - dists = vl_alldist2(single(descriptorsA),single(descriptorsB),... - obj.Opts.descriptorsDistanceMetric); - obj.info('Sorting distances') - [dists, perm] = sort(dists(:),'ascend'); - - % Create list of edges in the bipartite graph - [aIdx bIdx] = ind2sub([numFramesA, numFramesB],perm(1:numel(dists))); - edges = [aIdx bIdx]; - - % Find one-to-one best matches - obj.info('Matching descriptors.'); - descMatches = greedyBipartiteMatching(numFramesA, numFramesB, edges); - - for aIdx=1:numFramesA - bIdx = descMatches(aIdx); - [hasCorresp bCorresp] = ismember(bIdx,frameOverlaps.neighs{aIdx}); - % Check whether found descriptor matches fulfill frame overlap - if ~hasCorresp || ... - ~frameOverlaps.scores{aIdx}(bCorresp) > overlapThresh - descMatches(aIdx) = 0; - end - end - matches = [matches ; descMatches]; - end - - % Combine collected matches, i.e. select only equal matches - validMatches = ... - prod(single(matches == repmat(matches(1,:),size(matches,1),1)),1); - matches = matches(1,:) .* validMatches; - - % Compute the score - numBestMatches = sum(matches ~= 0); - score = numBestMatches / min(size(framesA,2), size(framesB,2)); - numMatches = numBestMatches; - - obj.info('Score: %g \t Num matches: %g', ... - score,numMatches); - - obj.debug('Score between %d/%d frames comp. in %gs',size(framesA,2), ... - size(framesB,2),toc(startTime)); - end - - function signature = getSignature(obj) - signature = helpers.struct2str(obj.Opts); - end - end - - methods (Access = protected) - function deps = getDependencies(obj) - deps = {helpers.Installer(),helpers.VlFeatInstaller('0.9.14'),... - benchmarks.helpers.Installer()}; - end - - - - end % methods (Access = protected) - -end - diff --git a/+benchmarks/RepeatabilityBenchmark.m b/+benchmarks/RepeatabilityBenchmark.m index ce86cdd..61be5e5 100644 --- a/+benchmarks/RepeatabilityBenchmark.m +++ b/+benchmarks/RepeatabilityBenchmark.m @@ -139,13 +139,9 @@ properties Opts = struct(... 'overlapError', 0.4,... - 'normaliseFrames', true,... 'cropFrames', true,... - 'magnification', 3,... - 'warpMethod', 'linearise',... 'mode', 'repeatability',... - 'descriptorsDistanceMetric', 'L2',... - 'normalisedScale', 30); + 'descriptorsDistanceMetric', 'L2'); end properties(Constant, Hidden) @@ -174,8 +170,8 @@ obj.checkInstall(varargin); end - function [score numMatches bestMatches reprojFrames] = ... - testFeatureExtractor(obj, featExtractor, tf, imageAPath, imageBPath) + function [score numMatches bestMatches] = ... + testFeatureExtractor(obj, featExtractor, dataset, imgAId, imgBId) % testFeatureExtractor Image feature extractor repeatability % REPEATABILITY = obj.testFeatureExtractor(FEAT_EXTRACTOR, TF, % IMAGEAPATH, IMAGEBPATH) computes the repeatability REP of a image @@ -211,6 +207,8 @@ import benchmarks.*; import helpers.*; + imageAPath = dataset.getImagePath(imgAId); + imageBPath = dataset.getImagePath(imgBId); obj.info('Comparing frames from det. %s and images %s and %s.',... featExtractor.Name,getFileName(imageAPath),... getFileName(imageBPath)); @@ -228,29 +226,29 @@ if obj.ModesOpts(obj.Opts.mode).matchDescs [framesA descriptorsA] = featExtractor.extractFeatures(imageAPath); [framesB descriptorsB] = featExtractor.extractFeatures(imageBPath); - [score numMatches bestMatches reprojFrames] = obj.testFeatures(... - tf, imageASize, imageBSize, framesA, framesB,... + [score numMatches bestMatches] = obj.testFeatures(... + dataset, imgAId, imgBId, imageASize, imageBSize, framesA, framesB,... descriptorsA, descriptorsB); else [framesA] = featExtractor.extractFeatures(imageAPath); [framesB] = featExtractor.extractFeatures(imageBPath); - [score numMatches bestMatches reprojFrames] = ... - obj.testFeatures(tf,imageASize, imageBSize,framesA, framesB); + [score numMatches bestMatches] = ... + obj.testFeatures(dataset, imgAId, imgBId, imageASize, imageBSize,framesA, framesB); end if featExtractor.UseCache - results = {score numMatches bestMatches reprojFrames}; + results = {score numMatches bestMatches}; obj.storeResults(results, resultsKey); end else - [score numMatches bestMatches reprojFrames] = cachedResults{:}; + [score numMatches bestMatches] = cachedResults{:}; obj.debug('Results loaded from cache'); end end - function [score numMatches matches reprojFrames] = ... - testFeatures(obj, tf, imageASize, imageBSize, framesA, framesB, ... - descriptorsA, descriptorsB) + function [score numMatches matches] = ... + testFeatures(obj, dataset, imgAId, imgBId, imageASize, ... + imageBSize, framesA, framesB, descriptorsA, descriptorsB) % testFeatures Compute repeatability of given image features % [SCORE NUM_MATCHES] = obj.testFeatures(TF, IMAGE_A_SIZE, % IMAGE_B_SIZE, FRAMES_A, FRAMES_B, DESCS_A, DESCS_B) Compute @@ -284,7 +282,7 @@ matchDescriptors = obj.ModesOpts(obj.Opts.mode).matchDescs; if isempty(framesA) || isempty(framesB) - matches = zeros(size(framesA,2)); reprojFrames = {}; + matches = zeros(size(framesA,2)); obj.info('Nothing to compute.'); return; end @@ -299,7 +297,7 @@ score = 0; numMatches = 0; startTime = tic; - normFrames = obj.Opts.normaliseFrames; + overlapError = obj.Opts.overlapError; overlapThresh = 1 - overlapError; @@ -308,57 +306,29 @@ framesA = localFeatures.helpers.frameToEllipse(framesA) ; framesB = localFeatures.helpers.frameToEllipse(framesB) ; - % map frames from image A to image B and viceversa - reprojFramesA = warpEllipse(tf, framesA,... - 'Method',obj.Opts.warpMethod) ; - reprojFramesB = warpEllipse(inv(tf), framesB,... - 'Method',obj.Opts.warpMethod) ; - % optionally remove frames that are not fully contained in % both images if obj.Opts.cropFrames - % find frames fully visible in both images - bboxA = [1 1 imageASize(2)+1 imageASize(1)+1] ; - bboxB = [1 1 imageBSize(2)+1 imageBSize(1)+1] ; - - visibleFramesA = isEllipseInBBox(bboxA, framesA ) & ... - isEllipseInBBox(bboxB, reprojFramesA); - - visibleFramesB = isEllipseInBBox(bboxA, reprojFramesB) & ... - isEllipseInBBox(bboxB, framesB ); + [validFramesA validFramesB] = dataset.validateFrames(imgAId, imgBId, framesA, framesB); % Crop frames outside overlap region - framesA = framesA(:,visibleFramesA); - reprojFramesA = reprojFramesA(:,visibleFramesA); - framesB = framesB(:,visibleFramesB); - reprojFramesB = reprojFramesB(:,visibleFramesB); + framesA = framesA(:,validFramesA); + framesB = framesB(:,validFramesB); + if isempty(framesA) || isempty(framesB) matches = zeros(size(framesA,2)); reprojFrames = {}; return; end if matchDescriptors - descriptorsA = descriptorsA(:,visibleFramesA); - descriptorsB = descriptorsB(:,visibleFramesB); + descriptorsA = descriptorsA(:,validFramesA); + descriptorsB = descriptorsB(:,validFramesB); end end - if ~normFrames - % When frames are not normalised, account the descriptor region - magFactor = obj.Opts.magnification^2; - framesA = [framesA(1:2,:); framesA(3:5,:).*magFactor]; - reprojFramesB = [reprojFramesB(1:2,:); ... - reprojFramesB(3:5,:).*magFactor]; - end - - reprojFrames = {framesA,framesB,reprojFramesA,reprojFramesB}; + frameOverlaps = dataset.scoreFrameOverlaps(imgAId, imgBId, framesA, framesB); numFramesA = size(framesA,2); - numFramesB = size(reprojFramesB,2); - - % Find all ellipse overlaps (in one-to-n array) - frameOverlaps = fastEllipseOverlap(reprojFramesB, framesA, ... - 'NormaliseFrames',normFrames,'MinAreaRatio',overlapThresh,... - 'NormalisedScale',obj.Opts.normalisedScale); + numFramesB = size(framesB,2); matches = []; diff --git a/+datasets/DTURobotDataset.m b/+datasets/DTURobotDataset.m index 8117af3..3384cf3 100644 --- a/+datasets/DTURobotDataset.m +++ b/+datasets/DTURobotDataset.m @@ -1,38 +1,22 @@ -classdef DTURobotDataset < datasets.GenericDataset & helpers.Logger... - & helpers.GenericInstaller +classdef DTURobotDataset < datasets.GenericCorrespondenceDataset ... + & helpers.Logger & helpers.GenericInstaller properties (SetAccess=private, GetAccess=public) Category = 'arc1'; % Dataset category - DataDir; % Image location - ImgExt; % Image extension - % Number of different perturbation configurations within the category. - NumPerturbationsConfigurations; - % Number of images for each perturbation configuration. - NumImagesPerPerturbation; - ImageNames; - ImageNamesLabel; Viewpoints; Lightings; ReconstructionsDir; CacheDir; CamerasPath; - ImgWidth; - ImgHeight; CodeDir; + DataDir; end properties (Constant) - KeyPrefix = 'DTURobotDataset'; % All dataset categories AllCategories = {'arc1', 'arc2', 'arc3', 'linear_path', ... 'lighting_x', 'lighting_y'}; - %=3mm - StrLBoxPad=3e-3; - %pixels - BackProjThresh=5; - %Margin for change in scale, corrected for distance to the secen, in orer for a correspondance to be accepted. A value of 2 corresponds to an octave. - ScaleMargin=2; end properties (Constant, Hidden) @@ -51,37 +35,46 @@ round([-25: 50/29: 25]*10)/10,... % arc2 round([-20: 40/24: 20]*10)/10,... % arc3 round([.5214: .3/14: .8]*10)/10,... % linear_path - round([0: 180.8/10: 180.8]*10)/10,... % lighting_x - round([0: 80.3/10: 80.3]*10)/10,... % lighting_y + round([0: 180.8/8: 180.8]*10)/10,... % lighting_x + round([0: 80.3/6: 80.3]*10)/10,... % lighting_y }; - + % Viewpoint indices for each category. CategoryViewpoints = {... [1:24 26:49], ... [65:94], ... [95:119], ... [51:64], ... - [12 25 56 64], ... - [12 25 56 64], ... + [12 25 60 87], ... + [12 25 60 87], ... }; - + % Lighting indices for each category. CategoryLightings = {... 0, ... 0, ... 0, ... 0, ... - [20:30], ... - [31:41], ... + [20:28], ... + [29:35], ... }; - - NumScenes = 60; - %Size of search cells in the structured light grid. + % Size of search cells in the structured light grid. CellRadius = 10; + % Acceptance threshold of point distance in 3D. 3e-3 = 3mm. + StrLBoxPad=3e-3; + % Acceptance threshold of backprojection error in pixels. + BackProjThresh=5; + % Acceptance threshold of scale difference after distance normalization. A + % value of 2 corresponds to an octave. + ScaleMargin = 2; + + ImgWidth = 1600; + ImgHeight = 1200; + % Installation directory RootInstallDir = fullfile('data','datasets','DTURobot'); - % Root url for dataset tarballs -% % URL for code (+ some data) tarballs - CodeUrl = 'http://roboimagedata.imm.dtu.dk/code/RobotEvalCode.tar.gz'; -% DatasetUrl = 'http://roboimagedata.imm.dtu.dk/code/RobotEvalCode.tar.gz'; + % URL for dataset tarballs + CodeUrl = 'http://roboimagedata.imm.dtu.dk/data/code.tar.gz'; + ReconstructionsUrl = 'http://roboimagedata.imm.dtu.dk/data/ground_truth.tar.gz'; + ScenesUrl = 'http://roboimagedata.imm.dtu.dk/data/condensed.tar.gz'; end methods @@ -96,13 +89,6 @@ obj.DatasetName = ['DTURobotDataset-' opts.Category]; obj.Category= opts.Category; obj.DataDir = fullfile(obj.RootInstallDir,'scenes'); - if strcmp(obj.Category,'lighting_x') || strcmp(obj.Category,'lighting_y') - obj.NumPerturbationsConfigurations = numel(obj.CategoryLightings{loc}) - obj.NumImagesPerPerturbation = obj.NumScenes * numel(obj.CategoryViewpoints{loc}); - else - obj.NumPerturbationsConfigurations = numel(obj.CategoryViewpoints{loc}); - obj.NumImagesPerPerturbation = obj.NumScenes; - end obj.checkInstall(varargin); obj.ImageNames = obj.CategoryImageLabels{loc}; @@ -113,46 +99,58 @@ obj.CamerasPath = fullfile(obj.RootInstallDir, 'cameras.mat'); obj.CacheDir = fullfile(obj.RootInstallDir, 'cache'); obj.CodeDir = fullfile(obj.RootInstallDir, 'code'); - obj.ImgWidth = 1600; - obj.ImgHeight = 1200; - + obj.NumScenes = 60; + obj.NumLabels = numel(obj.ImageNames); + if strfind(obj.Category,'lighting') + obj.NumImages = obj.NumScenes * obj.NumLabels * numel(obj.CategoryViewpoints{loc}); + else + obj.NumImages = obj.NumScenes * obj.NumLabels; + end addpath(obj.CodeDir) end function imgPath = getImagePath(obj, imgId) - imgPath = fullfile(obj.DataDir, sprintf('scene%.3d/%.3d_%.2d.png', ... - imgId.scene, imgId.viewpoint, imgId.lighting)); + if isstruct(imgId) + imgPath = fullfile(obj.DataDir, sprintf('scene%.3d/%.3d_%.2d.png', ... + imgId.scene, imgId.viewpoint, imgId.lighting)); + else + labelNo = floor(imgId/obj.NumImages)+1; + sceneNo = mod(imgId,obj.NumImages)+1; + imgId = obj.getImageId(labelNo, sceneNo); + imgPath = obj.getImagePath(imgId); + end end - function imgId = getReferenceImageId(obj, confNo, imgNo) - imgId.scene = imgNo; + function imgId = getReferenceImageId(obj, labelNo, sceneNo) + imgId.scene = sceneNo; imgId.viewpoint = 25; imgId.lighting = 0; end - function imgId = getImageId(obj, confNo, imgNo) - if strcmp(obj.Category,'lighting_x') || strcmp(obj.Category,'lighting_y') - imgId.viewpoint = obj.Viewpoints(mod(imgNo, numel(obj.Viewpoints))+1); - imgId.lighting = obj.Lightings(confNo); - imgId.scene = floor(imgNo/numel(obj.Viewpoints)); + function imgId = getImageId(obj, labelNo, sceneNo) + if strfind(obj.Category,'lighting') + imgId.viewpoint = obj.Viewpoints(mod(sceneNo, numel(obj.Viewpoints))+1); + imgId.lighting = obj.Lightings(labelNo); + imgId.scene = floor(sceneNo/numel(obj.Viewpoints))+1; else - imgId.viewpoint = obj.Viewpoints(confNo); + imgId.viewpoint = obj.Viewpoints(labelNo); imgId.lighting = 0; - imgId.scene = imgNo; + imgId.scene = sceneNo; end end - function tfs = getTransformation(obj, imageAId, imageBId) - Cams = GetCamPair(imageAId.viewpoint, imageBId.viewpoint); - %TODO + function [validFramesA validFramesB] = validateFrames(obj, ... + imgAId, imgBId, framesA, framesB) + validFramesA = logical(ones(1, size(framesA,2))); + validFramesB = logical(ones(1, size(framesB,2))); end - function frameOverlaps = getFrameOverlaps(obj, imageAId, imageBId, framesA, framesB) + function overlaps = scoreFrameOverlaps(obj, imgAId, imgBId, framesA, framesB) % Get 3D reconstruction - [Grid3D, Pts] = GenStrLightGrid(imageAId.viewpoint, ... + [Grid3D, Pts] = GenStrLightGrid(imgAId.viewpoint, ... obj.ReconstructionsDir, obj.ImgHeight, obj.ImgWidth, ... - obj.CellRadius, imageAId.scene); - CamPair = GetCamPair(imageAId.viewpoint, imageBId.viewpoint); + obj.CellRadius, imgAId.scene); + CamPair = GetCamPair(imgAId.viewpoint, imgBId.viewpoint); N = size(framesA,2); neighs = cell(1,N); @@ -161,8 +159,8 @@ frame_ref = framesA(:, f)'; [neighs{f}, scores{f}] = obj.overlap(Grid3D, Pts, CamPair, frame_ref, framesB'); end - frameOverlaps.neighs = neighs; - frameOverlaps.scores = scores; + overlaps.neighs = neighs; + overlaps.scores = scores; end @@ -178,8 +176,8 @@ if(IsEst) Var = Var+obj.StrLBoxPad; Q = Mean*ones(1,8)+[Var(1)*[-1 1 -1 1 -1 1 -1 1]; - Var(2)*[-1 -1 1 1 -1 -1 1 1]; - Var(3)*[-1 -1 -1 -1 1 1 1 1]]; + Var(2)*[-1 -1 1 1 -1 -1 1 1]; + Var(3)*[-1 -1 -1 -1 1 1 1 1]]; q = Cams(:,:,2)*[Q;ones(1,8)]; depth = mean(q(3,:)); q(1,:) = q(1,:)./q(3,:); @@ -196,14 +194,12 @@ frames(:,3)>Scale/obj.ScaleMargin & ... frames(:,3)= 1 && imgNo <= obj.NumImages,'Out of bounds idx\n'); - imgPath = fullfile(obj.DataDir,sprintf('img%d.%s',imgNo,obj.ImgExt)); + function imgPath = getImagePath(obj,imgId) + assert(imgId >= 1 && imgId <= obj.NumImages,'Out of bounds idx\n'); + imgPath = fullfile(obj.DataDir,sprintf('img%d.%s',imgId,obj.ImgExt)); + end + + function imgId = getImageId(obj, labelNo, sceneNo) + assert(labelNo<=obj.NumLabels && sceneNo <= obj.NumScenes, 'Out of bounds idx\n'); + imgId = labelNo+1; + end + + function imgId = getReferenceImageId(obj, labelNo, sceneNo) + assert(labelNo<=obj.NumLabels && sceneNo <= obj.NumScenes, 'Out of bounds idx\n'); + imgId = 1; end function tfs = getTransformation(obj,imgIdx) @@ -101,6 +118,58 @@ textread(fullfile(obj.DataDir,sprintf('H1to%dp',imgIdx)),... '%f %f %f%*[^\n]'); end + + function [validFramesA validFramesB] = validateFrames(obj, ... + imgAId, imgBId, framesA, framesB) + import benchmarks.helpers.*; + imageASize = helpers.imageSize(obj.getImagePath(imgAId)); + imageBSize = helpers.imageSize(obj.getImagePath(imgBId)); + + % find frames fully visible in both images + bboxA = [1 1 imageASize(2)+1 imageASize(1)+1] ; + bboxB = [1 1 imageBSize(2)+1 imageBSize(1)+1] ; + + % map frames from image A to image B and viceversa + tf = obj.getTransformation(imgBId); + reprojFramesA = warpEllipse(tf, framesA,... + 'Method',obj.warpMethod) ; + reprojFramesB = warpEllipse(inv(tf), framesB,... + 'Method',obj.warpMethod) ; + + validFramesA = isEllipseInBBox(bboxA, framesA ) & ... + isEllipseInBBox(bboxB, reprojFramesA); + + validFramesB = isEllipseInBBox(bboxA, reprojFramesB) & ... + isEllipseInBBox(bboxB, framesB ); + end + + function overlaps = scoreFrameOverlaps(obj, imgAId, imgBId, framesA, framesB) + import benchmarks.helpers.*; + import helpers.*; + tf = getTransformation(obj, imgBId); + + % map frames from image A to image B and viceversa + reprojFramesA = warpEllipse(tf, framesA,... + 'Method',obj.warpMethod) ; + reprojFramesB = warpEllipse(inv(tf), framesB,... + 'Method',obj.warpMethod) ; + + + if ~obj.normaliseFrames + % When frames are not normalised, account the descriptor region + magFactor = obj.magnification^2; + framesA = [framesA(1:2,:); framesA(3:5,:).*magFactor]; + reprojFramesB = [reprojFramesB(1:2,:); ... + reprojFramesB(3:5,:).*magFactor]; + end + + % Find all ellipse overlaps (in one-to-n array) + overlapThresh = 1 - obj.overlapError; + overlaps = fastEllipseOverlap(reprojFramesB, framesA, ... + 'NormaliseFrames',obj.normaliseFrames,'MinAreaRatio',overlapThresh,... + 'NormalisedScale',obj.normalisedScale); + end + end methods (Access = protected) diff --git a/DTURobotrepeatabilityDemo.m b/DTURobotrepeatabilityDemo.m deleted file mode 100644 index a99c4cd..0000000 --- a/DTURobotrepeatabilityDemo.m +++ /dev/null @@ -1,284 +0,0 @@ -function repeatabilityDemo(resultsPath) -% REPEATABILITYDEMO Demonstrates how to run the repatability benchmark -% REPEATABILITYDEMO() Runs the repeatability demo. -% -% REPEATABILITYDEMO(RESULTS_PATH) Run the demo and save the results to -% path RESULTS_PATH. - -% Author: Karel Lenc and Andrea Vedaldi - -% AUTORIGHTS - -if nargin < 1, resultsPath = ''; end; - -% -------------------------------------------------------------------- -% PART 1: Image feature detectors -% -------------------------------------------------------------------- - -import datasets.*; -import benchmarks.*; -import localFeatures.*; - -% The feature detector/descriptor code is encapsualted in a corresponding -% class. For example, VLFeatSift() encapslate the SIFT implementation in -% VLFeat. -% -% In addition to wrapping the detector code, each object instance -% contains a specific setting of parameters (for example, the -% cornerness threshold). In order to compare different parameter -% settings, one simply creates multiple instances of these objects. - -siftDetector = VlFeatSift(); -thrSiftDetector = VlFeatSift('PeakThresh',11); - -% VLBenchmarks enables a simple access to a number of public -% benchmakrs. It also provides simple facilities to generate test data -% on the fly. Here we generate an image consiting of a number of -% Gaussian blobs and we save it to disk for use with the detectors. - -ellBlobs = datasets.helpers.genEllipticBlobs('Width',500,'Height',500,... - 'NumDeformations',4); -ellBlobsPath = 'ellBlobs.png'; -imwrite(ellBlobs,ellBlobsPath); - -% Next, we extract the features by running the detectors we -% prepared. -% -% VLBeanchmarks is smart. The detector output is cached (for each -% input image and parameter setting), so the next time the detector is -% called the output is read from disk rather than being comptued -% again. VLBenchmarks automatically checks whether the detector -% parameters, image, or code change based on their modification date -% and invalidates the cache if necessary. You can also invoke the -% disableCaching() method in each detector to prevent it from caching. - -siftFrames = siftDetector.extractFeatures(ellBlobsPath); -thrSiftFrames = thrSiftDetector.extractFeatures(ellBlobsPath); - -% Now show the frames -figure(1); clf; -imshow(ellBlobs); -siftHandle = vl_plotframe(siftFrames,'g'); -thrSiftHandle = vl_plotframe(thrSiftFrames,'r','LineWidth',1); -legend([siftHandle thrSiftHandle],'SIFT','SIFT PT=10','Location','SE'); -helpers.printFigure(resultsPath,'siftFrames',0.9); - -% -------------------------------------------------------------------- -% PART 2: Detector repeatability -% -------------------------------------------------------------------- - -% A detector repeatability is measured against a benchmark. In this -% case we create an instance of the VGG Affine Testbed (graffity -% sequence). - -dataset = datasets.DTURobotDataset('Category','arc1'); - -% Next, the benchmark is intialised by choosing various -% parameters. The defaults correspond to the seetting in the original -% publication (IJCV05). - -repBenchmark = DTURobotRepeatabilityBenchmark('Mode','Repeatability'); - -% Prepare three detectors, the two from PART 1 and a third one that -% detects MSER image features. - -mser = VlFeatMser(); -featExtractors = {thrSiftDetector}; %, siftDetector, mser}; -detectorNames = {'SIFT PT=10'};%,'SIFT','MSER'}; - -% Now we are ready to run the repeatability test. We do this by fixing -% a reference image A and looping through other images B in the -% set. To this end we use the following information: -% -% dataset.NumImages: -% Number of images in the dataset. -% -% dataset.getImagePath(i): -% Path to the i-th image. -% -% dataset.getTransformation(i): -% Transformation from the first (reference) image to image i. -% -% Like for the detector output (see PART 1), VLBenchmarks caches the -% output of the test. This can be disabled by calling -% repBenchmark.disableCaching(). - -repeatability = []; -numCorresp = []; - -for d = 1:numel(featExtractors) - for imgNo = 1:1 %dataset.NumImagesPerPerturbation - for confNo = 1:dataset.NumPerturbationsConfigurations - img_ref_id = dataset.getReferenceImageId(confNo, imgNo); - img_id = dataset.getImageId(confNo, imgNo); - - [repeatability(d, confNo, imgNo) numCorresp(d, confNo, imgNo)] = ... - repBenchmark.testFeatureExtractor(featExtractors{d}, dataset, ... - img_ref_id, img_id); - end - end -end - - -% The scores can now be prented, as well as visualized in a -% graph. This uses two simple functions defined below in this file. - -printScores(detectorNames, 100 * repeatability, 'Repeatability'); -printScores(detectorNames, numCorresp, 'Number of correspondences'); - -figure(2); clf; -plotScores(detectorNames, dataset, 100 * repeatability, 'Repeatability'); -helpers.printFigure(resultsPath,'repeatability',0.6); - -figure(3); clf; -plotScores(detectorNames, dataset, numCorresp, 'Number of correspondences'); -helpers.printFigure(resultsPath,'numCorresp',0.6); - -% Optionally, we can also see the matched frames itself. In this -% example we examine the matches between the reference and fourth -% image. -% -% We do this by running the repeatabiltiy score again. However, since -% the results are cached, this is fast. - -%imageBIdx = 3; - -%[drop drop siftCorresps siftReprojFrames] = ... -% repBenchmark.testFeatureExtractor(siftDetector, ... -% dataset.getTransformation(imageBIdx), ... -% dataset.getImagePath(1), ... -% dataset.getImagePath(imageBIdx)); - -%% And plot the feature frame correspondences - -%figure(4); clf; -%imshow(dataset.getImagePath(imageBIdx)); -%benchmarks.helpers.plotFrameMatches(siftCorresps,... -% siftReprojFrames,... -% 'IsReferenceImage',false,... -% 'PlotMatchLine',false,... -% 'PlotUnmatched',false); -%helpers.printFigure(resultsPath,'correspondences',0.75); - -% -------------------------------------------------------------------- -% PART 3: Detector matching score -% -------------------------------------------------------------------- - -% The matching score is similar to the repeatability score, but -% involves computing a descriptor. Detectors like SIFT bundle a -% descriptor as well. However, most of them (e.g. MSER) do not have an -% associated descriptor (e.g. MSER). In this case we can bind one of -% our choice by using the DescriptorAdapter class. -% -% In this particular example, the object encapsulating the SIFT -% detector is used as descriptor form MSER. - -mserWithSift = DescriptorAdapter(mser, siftDetector); -featExtractors = {thrSiftDetector};%, siftDetector, mserWithSift}; -detectorNames = {'SIFT PT=10'};%,'SIFT','MSER with SIFT'}; - -% We create a benchmark object and run the tests as before, but in -% this case we request that descriptor-based matched should be tested. - -matchingBenchmark = DTURobotRepeatabilityBenchmark('Mode','MatchingScore'); - -matchScore = []; -numMatches = []; - -for d = 1:numel(featExtractors) - for imgNo = 1:1%dataset.NumImagesPerPerturbation - for confNo = 1:dataset.NumPerturbationsConfigurations - img_ref_id = dataset.getReferenceImageId(confNo, imgNo); - img_id = dataset.getImageId(confNo, imgNo); - - [matchScore(d, confNo, imgNo) numMatches(d, confNo, imgNo)] = ... - matchingBenchmark.testFeatureExtractor(featExtractors{d}, ... - dataset, img_ref_id, img_id); - end - end -end - - - -% Print and plot the results - - -printScores(detectorNames, matchScore*100, 'Match Score'); -printScores(detectorNames, numMatches, 'Number of matches') ; - -figure(5); clf; -plotScores(detectorNames, dataset, matchScore*100,'Matching Score'); -helpers.printFigure(resultsPath,'matchingScore',0.6); - -figure(6); clf; -plotScores(detectorNames, dataset, numMatches,'Number of matches'); -helpers.printFigure(resultsPath,'numMatches',0.6); - -% Same as with the correspondences, we can plot the matches based on -% feature frame descriptors. The code is nearly identical. - -%imageBIdx = 3; -%[r nc siftCorresps siftReprojFrames] = ... -% matchingBenchmark.testFeatureExtractor(siftDetector, ... -% dataset.getTransformation(imageBIdx), ... -% dataset.getImagePath(1), ... -% dataset.getImagePath(imageBIdx)); - -%figure(7); clf; -%imshow(imread(dataset.getImagePath(imageBIdx))); -%benchmarks.helpers.plotFrameMatches(siftCorresps,... -% siftReprojFrames,... -% 'IsReferenceImage',false,... -% 'PlotMatchLine',false,... -% 'PlotUnmatched',false); -%helpers.printFigure(resultsPath,'matches',0.75); - -% -------------------------------------------------------------------- -% Helper functions -% -------------------------------------------------------------------- - -function printScores(detectorNames, scores, name) - numDetectors = numel(detectorNames); - maxNameLen = length('Method name'); - for k = 1:numDetectors - maxNameLen = max(maxNameLen,length(detectorNames{k})); - end - fprintf(['\n', name,':\n']); - formatString = ['%' sprintf('%d',maxNameLen) 's:']; - fprintf(formatString,'Method name'); - for k = 2:size(scores,2) - fprintf('\tImg#%02d',k); - end - fprintf('\n'); - for k = 1:numDetectors - fprintf(formatString,detectorNames{k}); - for l = 2:size(scores,2) - fprintf('\t%6s',sprintf('%.2f',scores(k,l))); - end - fprintf('\n'); - end -end - -function plotScores(detectorNames, dataset, score, titleText) - xstart = max([find(sum(score,1) == 0, 1) + 1 1]); - xend = size(score,2); - xLabel = dataset.ImageNamesLabel; - xTicks = dataset.ImageNames; - plot(xstart:xend,score(:,xstart:xend)','+-','linewidth', 2); hold on ; - ylabel(titleText) ; - xlabel(xLabel); - set(gca,'XTick',xstart:1:xend); - set(gca,'XTickLabel',xTicks); - title(titleText); - set(gca,'xtick',1:size(score,2)); - maxScore = max([max(max(score)) 1]); - meanEndValue = mean(score(:,xend)); - legendLocation = 'SouthEast'; - if meanEndValue < maxScore/2 - legendLocation = 'NorthEast'; - end - legend(detectorNames,'Location',legendLocation); - grid on ; - axis([xstart xend 0 maxScore]); -end -end diff --git a/repeatabilityDemo.m b/repeatabilityDemo.m index d1810ab..9c976e5 100644 --- a/repeatabilityDemo.m +++ b/repeatabilityDemo.m @@ -29,7 +29,7 @@ function repeatabilityDemo(resultsPath) % settings, one simply creates multiple instances of these objects. siftDetector = VlFeatSift(); -thrSiftDetector = VlFeatSift('PeakThresh',11); +thrSiftDetector = VlFeatSift('PeakThresh',10); % VLBenchmarks enables a simple access to a number of public % benchmakrs. It also provides simple facilities to generate test data @@ -73,6 +73,9 @@ function repeatabilityDemo(resultsPath) dataset = datasets.VggAffineDataset('Category','graf'); +% Uncomment this to demo the DTU Robot dataset instead of VGG Affine +%dataset = datasets.DTURobotDataset('Category','arc2'); + % Next, the benchmark is intialised by choosing various % parameters. The defaults correspond to the seetting in the original % publication (IJCV05). @@ -83,8 +86,18 @@ function repeatabilityDemo(resultsPath) % detects MSER image features. mser = VlFeatMser(); -featExtractors = {siftDetector, thrSiftDetector, mser}; +if strfind(dataset.DatasetName, 'DTURobotDataset') + % Set more restrictive thresholds to limit the number of interest points per + % image. At default thresholds, the detectors generate ~6000 interest points. + thrSiftDetector = VlFeatSift('PeakThresh',11); + thrMserDetector = VlFeatMser('Delta',15); + featExtractors = {thrSiftDetector, thrMserDetector}; + detectorNames = {'DoG(PT=11)', 'MSER(Delta=15)'}; +else + featExtractors = {siftDetector, thrSiftDetector, mser}; + detectorNames = {'SIFT','SIFT PT=10','MSER'}; +end % Now we are ready to run the repeatability test. We do this by fixing % a reference image A and looping through other images B in the % set. To this end we use the following information: @@ -105,21 +118,24 @@ function repeatabilityDemo(resultsPath) repeatability = []; numCorresp = []; -imageAPath = dataset.getImagePath(1); for d = 1:numel(featExtractors) - for i = 2:dataset.NumImages - [repeatability(d,i) numCorresp(d,i)] = ... - repBenchmark.testFeatureExtractor(featExtractors{d}, ... - dataset.getTransformation(i), ... - dataset.getImagePath(1), ... - dataset.getImagePath(i)); + % use a maximum of three scenes for this demo. + scenes = min(dataset.NumScenes, 3); + for sceneNo = 1:scenes + for labelNo = 1:dataset.NumLabels + img_ref_id = dataset.getReferenceImageId(labelNo, sceneNo); + img_id = dataset.getImageId(labelNo, sceneNo); + [repeatability(d, labelNo, sceneNo) numCorresp(d, labelNo, sceneNo)] = ... + repBenchmark.testFeatureExtractor(featExtractors{d}, dataset, ... + img_ref_id, img_id); + end end end + % The scores can now be prented, as well as visualized in a % graph. This uses two simple functions defined below in this file. -detectorNames = {'SIFT','SIFT PT=10','MSER'}; printScores(detectorNames, 100 * repeatability, 'Repeatability'); printScores(detectorNames, numCorresp, 'Number of correspondences'); @@ -138,24 +154,26 @@ function repeatabilityDemo(resultsPath) % We do this by running the repeatabiltiy score again. However, since % the results are cached, this is fast. -imageBIdx = 3; +% TODO: andersbll: I propose to implement a function in RepeatabilityBenchmark +% called 'matchFrames' and use it like the following. -[drop drop siftCorresps siftReprojFrames] = ... - repBenchmark.testFeatureExtractor(siftDetector, ... - dataset.getTransformation(imageBIdx), ... - dataset.getImagePath(1), ... - dataset.getImagePath(imageBIdx)); +%imageBIdx = 3; +%sceneNo = 1; +%img_ref_id = dataset.getReferenceImageId(imageBIdx, sceneNo); +%img_id = dataset.getImageId(imageBIdx, sceneNo); +%[siftCorresps siftReprojFrames] = ... +% repBenchmark.matchFrames(siftDetector, dataset, img_ref_id, img_id); -% And plot the feature frame correspondences +%% And plot the feature frame correspondences -figure(4); clf; -imshow(dataset.getImagePath(imageBIdx)); -benchmarks.helpers.plotFrameMatches(siftCorresps,... - siftReprojFrames,... - 'IsReferenceImage',false,... - 'PlotMatchLine',false,... - 'PlotUnmatched',false); -helpers.printFigure(resultsPath,'correspondences',0.75); +%figure(4); clf; +%imshow(dataset.getImagePath(imageBIdx)); +%benchmarks.helpers.plotFrameMatches(siftCorresps,... +% siftReprojFrames,... +% 'IsReferenceImage',false,... +% 'PlotMatchLine',false,... +% 'PlotUnmatched',false); +%helpers.printFigure(resultsPath,'correspondences',0.75); % -------------------------------------------------------------------- % PART 3: Detector matching score @@ -171,30 +189,54 @@ function repeatabilityDemo(resultsPath) % detector is used as descriptor form MSER. mserWithSift = DescriptorAdapter(mser, siftDetector); -featExtractors = {siftDetector, thrSiftDetector, mserWithSift}; +if strfind(dataset.DatasetName, 'DTURobotDataset') + thrSiftDetector = VlFeatSift('PeakThresh',11); + thrMserDetector = VlFeatMser('Delta',15); + mserWithSift = DescriptorAdapter(thrMserDetector, siftDetector); + featExtractors = {thrSiftDetector, mserWithSift}; + detectorNames = {'DoG(PT=11) with SIFT', 'MSER(Delta=15) with SIFT'} +else + featExtractors = {siftDetector, thrSiftDetector, mserWithSift}; + detectorNames = {'SIFT','SIFT PT=10','MSER with SIFT'} +end % We create a benchmark object and run the tests as before, but in % this case we request that descriptor-based matched should be tested. matchingBenchmark = RepeatabilityBenchmark('Mode','MatchingScore'); +%matchScore = []; +%numMatches = []; + +%for d = 1:numel(featExtractors) +% for i = 2:dataset.NumImages +% [matchScore(d,i) numMatches(d,i)] = ... +% matchingBenchmark.testFeatureExtractor(featExtractors{d}, ... +% dataset, ... +% 1, ... +% i); +% end +%end + matchScore = []; numMatches = []; for d = 1:numel(featExtractors) - for i = 2:dataset.NumImages - [matchScore(d,i) numMatches(d,i)] = ... - matchingBenchmark.testFeatureExtractor(featExtractors{d}, ... - dataset.getTransformation(i), ... - dataset.getImagePath(1), ... - dataset.getImagePath(i)); + % use a maximum of three scenes for this demo. + scenes = min(dataset.NumScenes, 3); + for sceneNo = 1:scenes + for labelNo = 1:dataset.NumLabels + img_ref_id = dataset.getReferenceImageId(labelNo, sceneNo); + img_id = dataset.getImageId(labelNo, sceneNo); + [matchScore(d, labelNo, sceneNo) numMatches(d, labelNo, sceneNo)] = ... + matchingBenchmark.testFeatureExtractor(featExtractors{d}, ... + dataset, img_ref_id, img_id); + end end end -% Print and plot the results - -detectorNames = {'SIFT','SIFT PT=10','MSER with SIFT'}; +% Print and plot the results printScores(detectorNames, matchScore*100, 'Match Score'); printScores(detectorNames, numMatches, 'Number of matches') ; @@ -209,21 +251,22 @@ function repeatabilityDemo(resultsPath) % Same as with the correspondences, we can plot the matches based on % feature frame descriptors. The code is nearly identical. -imageBIdx = 3; -[r nc siftCorresps siftReprojFrames] = ... - matchingBenchmark.testFeatureExtractor(siftDetector, ... - dataset.getTransformation(imageBIdx), ... - dataset.getImagePath(1), ... - dataset.getImagePath(imageBIdx)); - -figure(7); clf; -imshow(imread(dataset.getImagePath(imageBIdx))); -benchmarks.helpers.plotFrameMatches(siftCorresps,... - siftReprojFrames,... - 'IsReferenceImage',false,... - 'PlotMatchLine',false,... - 'PlotUnmatched',false); -helpers.printFigure(resultsPath,'matches',0.75); +% TODO, see above +%imageBIdx = 3; +%[r nc siftCorresps] = ... +% matchingBenchmark.testFeatureExtractor(siftDetector, ... +% dataset, ... +% 1, ... +% imageBIdx); + +%figure(7); clf; +%imshow(imread(dataset.getImagePath(imageBIdx))); +%benchmarks.helpers.plotFrameMatches(siftCorresps,... +% siftReprojFrames,... +% 'IsReferenceImage',false,... +% 'PlotMatchLine',false,... +% 'PlotUnmatched',false); +%helpers.printFigure(resultsPath,'matches',0.75); % -------------------------------------------------------------------- % Helper functions @@ -252,8 +295,18 @@ function printScores(detectorNames, scores, name) end function plotScores(detectorNames, dataset, score, titleText) - xstart = max([find(sum(score,1) == 0, 1) + 1 1]); + xstart = 1; xend = size(score,2); + if ndims(score) == 2 + plot(xstart:xend,score(:,xstart:xend)','+-','linewidth', 2); hold on ; + else + score_std = std(score,0,3); + score = mean(score,3); + X = repmat(xstart:xend,[size(score, 1) 1])'; + Y = score(:,xstart:xend)'; + E = score_std(:,xstart:xend)'; + errorbar(X,Y,E,'+-','linewidth', 2); hold on ; + end xLabel = dataset.ImageNamesLabel; xTicks = dataset.ImageNames; plot(xstart:xend,score(:,xstart:xend)','+-','linewidth', 2); hold on ; @@ -273,4 +326,5 @@ function plotScores(detectorNames, dataset, score, titleText) grid on ; axis([xstart xend 0 maxScore]); end -end \ No newline at end of file +end + From 659f293c7716503f8cf9d4f2fb7e85954cc3fa03 Mon Sep 17 00:00:00 2001 From: Anders Boesen Lindbo Larsen Date: Mon, 20 Jan 2014 11:17:26 +0100 Subject: [PATCH 3/9] Multiscale Harris detector + oxford evaluation added --- +localFeatures/MultiscaleHarris.m | 84 +++++++++++++++ eval_mh_oxford.m | 163 ++++++++++++++++++++++++++++++ 2 files changed, 247 insertions(+) create mode 100644 +localFeatures/MultiscaleHarris.m create mode 100644 eval_mh_oxford.m diff --git a/+localFeatures/MultiscaleHarris.m b/+localFeatures/MultiscaleHarris.m new file mode 100644 index 0000000..3967b6b --- /dev/null +++ b/+localFeatures/MultiscaleHarris.m @@ -0,0 +1,84 @@ +classdef MultiscaleHarris < localFeatures.GenericLocalFeatureExtractor & ... + helpers.GenericInstaller + + properties (SetAccess=private, GetAccess=public) + CodeDir; + Opts; + end + + properties (Constant, Hidden) + % Installation directory + RootInstallDir = fullfile('data','software','multiscale_harris'); + % URL for dataset tarballs + CodeUrl = 'http://imm.dtu.dk/~abll/files/multiscale_harris.tar.gz'; + end + + + methods + function obj = MultiscaleHarris(varargin) + obj.Name = 'Multiscale Harris'; + obj.CodeDir = fullfile(obj.RootInstallDir, 'code'); + obj.Opts = struct('localization', 1); + varargin = obj.checkInstall(varargin); + obj.setup(); + end + + function [frames] = extractFeatures(obj, imagePath) + import helpers.*; + import localFeatures.*; + frames = obj.loadFeatures(imagePath,false); + if numel(frames) > 0; return; end; + startTime = tic; + obj.info('Computing frames of image %s.',getFileName(imagePath)); + + img = imread(imagePath); + if(size(img,3)>1), img = rgb2gray(img); end + img = im2uint8(img); % If not already in uint8, then convert + + frames = multiscaleharris(img, obj.Opts.localization)'; + frames([1,2],:) = frames([2,1],:); + + timeElapsed = toc(startTime); + obj.debug('%d Frames from image %s computed in %gs',... + size(frames,2),getFileName(imagePath),timeElapsed); + obj.storeFeatures(imagePath, frames, []); + end + + function sign = getSignature(obj) + sign = [helpers.struct2str(obj.Opts),';',... + helpers.fileSignature(obj.CodeDir, 'lindebergcorner.cpp'), ... + helpers.fileSignature(obj.CodeDir, 'LocalMaxima3DFast.cpp'), ... + helpers.fileSignature(obj.CodeDir, 'multiscaleharris.cpp'), ... + helpers.fileSignature(obj.CodeDir, 'scale.cpp'), ... + mfilename('fullpath')]; + end + + function setup(obj) + if(~exist('multiscaleharris.m', 'file')), + fprintf('Adding MultiscaleHarris to path.\n'); + addpath(obj.CodeDir) + end + end + + function unload(obj) + fprintf('Removing MultiscaleHarris from path.\n'); + rmpath(obj.CodeDir) + end + + end + + methods (Access=protected, Hidden) + function [srclist flags] = getMexSources(obj) + srclist = {fullfile(obj.CodeDir, 'LocalMaxima3DFast.cpp')}; + flags = {''}; + end + end + + methods (Access = protected) + function [urls dstPaths] = getTarballsList(obj) + installDir = obj.RootInstallDir; + dstPaths = {fullfile(installDir)}; + urls = {obj.CodeUrl}; + end + end +end diff --git a/eval_mh_oxford.m b/eval_mh_oxford.m new file mode 100644 index 0000000..a05625f --- /dev/null +++ b/eval_mh_oxford.m @@ -0,0 +1,163 @@ +function eval_kimharris(resultsPath) +if nargin < 1, resultsPath = ''; end; + +import datasets.*; +import benchmarks.*; +import localFeatures.*; + +set(0,'DefaultFigureVisible','off'); + + +dataset_name = 'oxford'; +for category_idx = 1:numel(datasets.VggAffineDataset.AllCategories) + category_name = datasets.VggAffineDataset.AllCategories{category_idx}; + dataset = datasets.VggAffineDataset('Category', category_name); + + % -------------------------------------------------------------------- + % PART 1: Detector repeatability + % -------------------------------------------------------------------- + + siftDetector = VlFeatSift(); + mhDetector = MultiscaleHarris(); + mser = VlFeatMser(); + + + repBenchmark = RepeatabilityBenchmark('Mode','Repeatability'); + + featExtractors = {siftDetector, mser, mhDetector}; + detectorNames = {'SIFT', 'MSER', 'MH'}; + + repeatability = []; + numCorresp = []; + + + imageAPath = dataset.getImagePath(1); + for d = 1:numel(featExtractors) + for i = 2:dataset.NumImages + [repeatability(d,i) numCorresp(d,i)] = ... + repBenchmark.testFeatureExtractor(featExtractors{d}, ... + dataset.getTransformation(i), ... + dataset.getImagePath(1), ... + dataset.getImagePath(i)); + end + end + + printScores(detectorNames, 100 * repeatability, 'Repeatability'); + printScores(detectorNames, numCorresp, 'Number of correspondences'); + + figure(2); clf; + plotScores(detectorNames, dataset, 100 * repeatability, 'Repeatability'); + printFigure(['results_' dataset_name], [category_name '_repeatability']); + + figure(3); clf; + plotScores(detectorNames, dataset, numCorresp, 'Number of correspondences'); + printFigure(['results_' dataset_name], [category_name '_num-correspondences']); + + + % -------------------------------------------------------------------- + % PART 2: Detector matching score + % -------------------------------------------------------------------- + + mserWithSift = DescriptorAdapter(mser, siftDetector); + mhWithSift = DescriptorAdapter(mhDetector, siftDetector); + featExtractors = {siftDetector, mserWithSift, mhWithSift}; + detectorNames = {'SIFT', 'MSER with SIFT', 'MH with SIFT'}; + + matchingBenchmark = RepeatabilityBenchmark('Mode','MatchingScore'); + + matchScore = []; + numMatches = []; + + for d = 1:numel(featExtractors) + for i = 2:dataset.NumImages + [matchScore(d,i) numMatches(d,i)] = ... + matchingBenchmark.testFeatureExtractor(featExtractors{d}, ... + dataset.getTransformation(i), ... + dataset.getImagePath(1), ... + dataset.getImagePath(i)); + end + end + + + printScores(detectorNames, matchScore*100, 'Match Score'); + printScores(detectorNames, numMatches, 'Number of matches') ; + + figure(5); clf; + plotScores(detectorNames, dataset, matchScore*100,'Matching Score'); + printFigure(['results_' dataset_name], [category_name '_matching_score']); + + figure(6); clf; + plotScores(detectorNames, dataset, numMatches,'Number of matches'); + printFigure(['results_' dataset_name], [category_name '_num-matches']); + +end % datset category + + + +% -------------------------------------------------------------------- +% Helper functions +% -------------------------------------------------------------------- + +function printScores(detectorNames, scores, name) + numDetectors = numel(detectorNames); + maxNameLen = length('Method name'); + for k = 1:numDetectors + maxNameLen = max(maxNameLen,length(detectorNames{k})); + end + fprintf(['\n', name,':\n']); + formatString = ['%' sprintf('%d',maxNameLen) 's:']; + fprintf(formatString,'Method name'); + for k = 2:size(scores,2) + fprintf('\tImg#%02d',k); + end + fprintf('\n'); + for k = 1:numDetectors + fprintf(formatString,detectorNames{k}); + for l = 2:size(scores,2) + fprintf('\t%6s',sprintf('%.2f',scores(k,l))); + end + fprintf('\n'); + end +end + +function plotScores(detectorNames, dataset, score, titleText) + xstart = max([find(sum(score,1) == 0, 1) + 1 1]); + xend = size(score,2); + xLabel = dataset.ImageNamesLabel; + xTicks = dataset.ImageNames; + plot(xstart:xend,score(:,xstart:xend)','+-','linewidth', 2); hold on ; + ylabel(titleText) ; + xlabel(xLabel); + set(gca,'XTick',xstart:1:xend); + set(gca,'XTickLabel',xTicks); + title(titleText); + set(gca,'xtick',1:size(score,2)); + maxScore = max([max(max(score)) 1]); + meanEndValue = mean(score(:,xend)); + legendLocation = 'SouthEast'; + if meanEndValue < maxScore/2 + legendLocation = 'NorthEast'; + end + legend(detectorNames,'Location',legendLocation); + grid on ; + axis([xstart xend 0 maxScore]); +end +end + + +function printFigure(path, fileName, R, ext) +if isempty(path), return; end; +if ~exist(path, 'dir') + mkdir(path) ; +end +if ~exist('R','var') + R = 0.75; +end +vl_printsize(gcf, R) ; + +if ~exist('ext','var') + ext = 'pdf'; +end +filePath = fullfile(path, [fileName '.' ext]) ; +saveas(gcf, filePath, ext) +end From 47bb645f6169baa57afe5b8556eed4f971f5dbcd Mon Sep 17 00:00:00 2001 From: Anders Boesen Lindbo Larsen Date: Mon, 20 Jan 2014 13:39:46 +0100 Subject: [PATCH 4/9] Fix error when image is grayscale. --- +localFeatures/VlFeatSift.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/+localFeatures/VlFeatSift.m b/+localFeatures/VlFeatSift.m index 2d07446..9a5b717 100644 --- a/+localFeatures/VlFeatSift.m +++ b/+localFeatures/VlFeatSift.m @@ -64,7 +64,7 @@ % Get the input image img = imread(imagePath); imgSize = size(img); - if imgSize(3) > 1 + if numel(imgSize) > 2 && imgSize(3) > 1 img = rgb2gray(img); end img = single(img); From 9557c44d3187c4791ace036eeac74f7e56f71747 Mon Sep 17 00:00:00 2001 From: Anders Boesen Lindbo Larsen Date: Mon, 20 Jan 2014 13:41:34 +0100 Subject: [PATCH 5/9] Multi-scale Harris detector evaluation added. --- eval_mh.m | 193 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 eval_mh.m diff --git a/eval_mh.m b/eval_mh.m new file mode 100644 index 0000000..9954e79 --- /dev/null +++ b/eval_mh.m @@ -0,0 +1,193 @@ +function eval_mh_oxford(resultsPath) +if nargin < 1, resultsPath = ''; end; + +import datasets.*; +import benchmarks.*; +import localFeatures.*; + +set(0,'DefaultFigureVisible','off'); + + +dataset_name = 'oxford'; +%dataset_name = 'dtu'; + +if strcmp(dataset_name, 'oxford') + categories = datasets.VggAffineDataset.AllCategories; +elseif strcmp(dataset_name, 'dtu') + categories = datasets.DTURobotDataset.AllCategories; +else + error('invalid dataset name') +end + +for category_idx = 1:numel(categories) + category_name = categories{category_idx}; + if strcmp(dataset_name, 'oxford') + dataset = datasets.VggAffineDataset('Category', category_name); + else strcmp(dataset_name, 'dtu') + dataset = datasets.DTURobotDataset('Category','arc2'); + end + % -------------------------------------------------------------------- + % PART 1: Detector repeatability + % -------------------------------------------------------------------- + + siftDetector = VlFeatSift(); + mhDetector = MultiscaleHarris(); + mser = VlFeatMser(); + + + repBenchmark = RepeatabilityBenchmark('Mode','Repeatability'); + + featExtractors = {siftDetector, mser, mhDetector}; + detectorNames = {'SIFT', 'MSER', 'MH'}; + + repeatability = []; + numCorresp = []; + + + for d = 1:numel(featExtractors) + % use a maximum of three scenes for this demo. + scenes = dataset.NumScenes; + for sceneNo = 1:scenes + for labelNo = 1:dataset.NumLabels + img_ref_id = dataset.getReferenceImageId(labelNo, sceneNo); + img_id = dataset.getImageId(labelNo, sceneNo); + [repeatability(d, labelNo, sceneNo) numCorresp(d, labelNo, sceneNo)] = ... + repBenchmark.testFeatureExtractor(featExtractors{d}, dataset, ... + img_ref_id, img_id); + end + end + end + + + printScores(detectorNames, 100 * repeatability, 'Repeatability'); + printScores(detectorNames, numCorresp, 'Number of correspondences'); + + figure(2); clf; + plotScores(detectorNames, dataset, 100 * repeatability, 'Repeatability'); + printFigure(['results_mh_' dataset_name], [category_name '_repeatability']); + + figure(3); clf; + plotScores(detectorNames, dataset, numCorresp, 'Number of correspondences'); + printFigure(['results_mh_' dataset_name], [category_name '_num-correspondences']); + + + % -------------------------------------------------------------------- + % PART 2: Detector matching score + % -------------------------------------------------------------------- + + mserWithSift = DescriptorAdapter(mser, siftDetector); + mhWithSift = DescriptorAdapter(mhDetector, siftDetector); + featExtractors = {siftDetector, mserWithSift, mhWithSift}; + detectorNames = {'SIFT', 'MSER with SIFT', 'MH with SIFT'}; + + matchingBenchmark = RepeatabilityBenchmark('Mode','MatchingScore'); + + matchScore = []; + numMatches = []; + + for d = 1:numel(featExtractors) + % use a maximum of three scenes for this demo. + scenes = dataset.NumScenes; + for sceneNo = 1:scenes + for labelNo = 1:dataset.NumLabels + img_ref_id = dataset.getReferenceImageId(labelNo, sceneNo); + img_id = dataset.getImageId(labelNo, sceneNo); + [matchScore(d, labelNo, sceneNo) numMatches(d, labelNo, sceneNo)] = ... + matchingBenchmark.testFeatureExtractor(featExtractors{d}, ... + dataset, img_ref_id, img_id); + end + end + end + + printScores(detectorNames, matchScore*100, 'Match Score'); + printScores(detectorNames, numMatches, 'Number of matches') ; + + figure(5); clf; + plotScores(detectorNames, dataset, matchScore*100,'Matching Score'); + printFigure(['results_mh_' dataset_name], [category_name '_matching_score']); + + figure(6); clf; + plotScores(detectorNames, dataset, numMatches,'Number of matches'); + printFigure(['results_mh_' dataset_name], [category_name '_num-matches']); + +end % dataset category + + + +% -------------------------------------------------------------------- +% Helper functions +% -------------------------------------------------------------------- + +function printScores(detectorNames, scores, name) + numDetectors = numel(detectorNames); + maxNameLen = length('Method name'); + for k = 1:numDetectors + maxNameLen = max(maxNameLen,length(detectorNames{k})); + end + fprintf(['\n', name,':\n']); + formatString = ['%' sprintf('%d',maxNameLen) 's:']; + fprintf(formatString,'Method name'); + for k = 2:size(scores,2) + fprintf('\tImg#%02d',k); + end + fprintf('\n'); + for k = 1:numDetectors + fprintf(formatString,detectorNames{k}); + for l = 2:size(scores,2) + fprintf('\t%6s',sprintf('%.2f',scores(k,l))); + end + fprintf('\n'); + end +end + +function plotScores(detectorNames, dataset, score, titleText) + xstart = 1; + xend = size(score,2); + if ndims(score) == 2 + plot(xstart:xend,score(:,xstart:xend)','+-','linewidth', 2); hold on ; + else + score_std = std(score,0,3); + score = mean(score,3); + X = repmat(xstart:xend,[size(score, 1) 1])'; + Y = score(:,xstart:xend)'; + E = score_std(:,xstart:xend)'; + errorbar(X,Y,E,'+-','linewidth', 2); hold on ; + end + xLabel = dataset.ImageNamesLabel; + xTicks = dataset.ImageNames; + plot(xstart:xend,score(:,xstart:xend)','+-','linewidth', 2); hold on ; + ylabel(titleText) ; + xlabel(xLabel); + set(gca,'XTick',xstart:1:xend); + set(gca,'XTickLabel',xTicks); + title(titleText); + set(gca,'xtick',1:size(score,2)); + maxScore = max([max(max(score)) 1]); + meanEndValue = mean(score(:,xend)); + legendLocation = 'SouthEast'; + if meanEndValue < maxScore/2 + legendLocation = 'NorthEast'; + end + legend(detectorNames,'Location',legendLocation); + grid on ; + axis([xstart xend 0 maxScore]); +end +end + + +function printFigure(path, fileName, R, ext) +if isempty(path), return; end; +if ~exist(path, 'dir') + mkdir(path) ; +end +if ~exist('R','var') + R = 0.75; +end +vl_printsize(gcf, R) ; + +if ~exist('ext','var') + ext = 'pdf'; +end +filePath = fullfile(path, [fileName '.' ext]) ; +saveas(gcf, filePath, ext) +end From 37f88ce423cbf9f2722d94a1adaf1447a5bd8a7e Mon Sep 17 00:00:00 2001 From: Anders Boesen Lindbo Larsen Date: Mon, 20 Jan 2014 13:42:25 +0100 Subject: [PATCH 6/9] remove old evaluation script --- eval_mh_oxford.m | 163 ----------------------------------------------- 1 file changed, 163 deletions(-) delete mode 100644 eval_mh_oxford.m diff --git a/eval_mh_oxford.m b/eval_mh_oxford.m deleted file mode 100644 index a05625f..0000000 --- a/eval_mh_oxford.m +++ /dev/null @@ -1,163 +0,0 @@ -function eval_kimharris(resultsPath) -if nargin < 1, resultsPath = ''; end; - -import datasets.*; -import benchmarks.*; -import localFeatures.*; - -set(0,'DefaultFigureVisible','off'); - - -dataset_name = 'oxford'; -for category_idx = 1:numel(datasets.VggAffineDataset.AllCategories) - category_name = datasets.VggAffineDataset.AllCategories{category_idx}; - dataset = datasets.VggAffineDataset('Category', category_name); - - % -------------------------------------------------------------------- - % PART 1: Detector repeatability - % -------------------------------------------------------------------- - - siftDetector = VlFeatSift(); - mhDetector = MultiscaleHarris(); - mser = VlFeatMser(); - - - repBenchmark = RepeatabilityBenchmark('Mode','Repeatability'); - - featExtractors = {siftDetector, mser, mhDetector}; - detectorNames = {'SIFT', 'MSER', 'MH'}; - - repeatability = []; - numCorresp = []; - - - imageAPath = dataset.getImagePath(1); - for d = 1:numel(featExtractors) - for i = 2:dataset.NumImages - [repeatability(d,i) numCorresp(d,i)] = ... - repBenchmark.testFeatureExtractor(featExtractors{d}, ... - dataset.getTransformation(i), ... - dataset.getImagePath(1), ... - dataset.getImagePath(i)); - end - end - - printScores(detectorNames, 100 * repeatability, 'Repeatability'); - printScores(detectorNames, numCorresp, 'Number of correspondences'); - - figure(2); clf; - plotScores(detectorNames, dataset, 100 * repeatability, 'Repeatability'); - printFigure(['results_' dataset_name], [category_name '_repeatability']); - - figure(3); clf; - plotScores(detectorNames, dataset, numCorresp, 'Number of correspondences'); - printFigure(['results_' dataset_name], [category_name '_num-correspondences']); - - - % -------------------------------------------------------------------- - % PART 2: Detector matching score - % -------------------------------------------------------------------- - - mserWithSift = DescriptorAdapter(mser, siftDetector); - mhWithSift = DescriptorAdapter(mhDetector, siftDetector); - featExtractors = {siftDetector, mserWithSift, mhWithSift}; - detectorNames = {'SIFT', 'MSER with SIFT', 'MH with SIFT'}; - - matchingBenchmark = RepeatabilityBenchmark('Mode','MatchingScore'); - - matchScore = []; - numMatches = []; - - for d = 1:numel(featExtractors) - for i = 2:dataset.NumImages - [matchScore(d,i) numMatches(d,i)] = ... - matchingBenchmark.testFeatureExtractor(featExtractors{d}, ... - dataset.getTransformation(i), ... - dataset.getImagePath(1), ... - dataset.getImagePath(i)); - end - end - - - printScores(detectorNames, matchScore*100, 'Match Score'); - printScores(detectorNames, numMatches, 'Number of matches') ; - - figure(5); clf; - plotScores(detectorNames, dataset, matchScore*100,'Matching Score'); - printFigure(['results_' dataset_name], [category_name '_matching_score']); - - figure(6); clf; - plotScores(detectorNames, dataset, numMatches,'Number of matches'); - printFigure(['results_' dataset_name], [category_name '_num-matches']); - -end % datset category - - - -% -------------------------------------------------------------------- -% Helper functions -% -------------------------------------------------------------------- - -function printScores(detectorNames, scores, name) - numDetectors = numel(detectorNames); - maxNameLen = length('Method name'); - for k = 1:numDetectors - maxNameLen = max(maxNameLen,length(detectorNames{k})); - end - fprintf(['\n', name,':\n']); - formatString = ['%' sprintf('%d',maxNameLen) 's:']; - fprintf(formatString,'Method name'); - for k = 2:size(scores,2) - fprintf('\tImg#%02d',k); - end - fprintf('\n'); - for k = 1:numDetectors - fprintf(formatString,detectorNames{k}); - for l = 2:size(scores,2) - fprintf('\t%6s',sprintf('%.2f',scores(k,l))); - end - fprintf('\n'); - end -end - -function plotScores(detectorNames, dataset, score, titleText) - xstart = max([find(sum(score,1) == 0, 1) + 1 1]); - xend = size(score,2); - xLabel = dataset.ImageNamesLabel; - xTicks = dataset.ImageNames; - plot(xstart:xend,score(:,xstart:xend)','+-','linewidth', 2); hold on ; - ylabel(titleText) ; - xlabel(xLabel); - set(gca,'XTick',xstart:1:xend); - set(gca,'XTickLabel',xTicks); - title(titleText); - set(gca,'xtick',1:size(score,2)); - maxScore = max([max(max(score)) 1]); - meanEndValue = mean(score(:,xend)); - legendLocation = 'SouthEast'; - if meanEndValue < maxScore/2 - legendLocation = 'NorthEast'; - end - legend(detectorNames,'Location',legendLocation); - grid on ; - axis([xstart xend 0 maxScore]); -end -end - - -function printFigure(path, fileName, R, ext) -if isempty(path), return; end; -if ~exist(path, 'dir') - mkdir(path) ; -end -if ~exist('R','var') - R = 0.75; -end -vl_printsize(gcf, R) ; - -if ~exist('ext','var') - ext = 'pdf'; -end -filePath = fullfile(path, [fileName '.' ext]) ; -saveas(gcf, filePath, ext) -end From d4363d31eb52325931612cd77f5a89095e6b4fb4 Mon Sep 17 00:00:00 2001 From: Anders Boesen Lindbo Larsen Date: Mon, 20 Jan 2014 14:00:54 +0100 Subject: [PATCH 7/9] Path for results is no longer an option. --- eval_mh.m | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eval_mh.m b/eval_mh.m index 9954e79..d94bdef 100644 --- a/eval_mh.m +++ b/eval_mh.m @@ -1,5 +1,4 @@ -function eval_mh_oxford(resultsPath) -if nargin < 1, resultsPath = ''; end; +function eval_mh() import datasets.*; import benchmarks.*; From d3f143a61322ba279709d14c16933d07465dd9a9 Mon Sep 17 00:00:00 2001 From: Anders Boesen Lindbo Larsen Date: Mon, 16 Jun 2014 15:58:28 +0200 Subject: [PATCH 8/9] Lindeberg corners added --- +localFeatures/LindebergCorners.m | 85 +++++++++++++++++++++++++++++++ +localFeatures/MultiscaleHarris.m | 2 +- eval_mh.m | 36 +++++++++---- 3 files changed, 112 insertions(+), 11 deletions(-) create mode 100644 +localFeatures/LindebergCorners.m diff --git a/+localFeatures/LindebergCorners.m b/+localFeatures/LindebergCorners.m new file mode 100644 index 0000000..fa4a15a --- /dev/null +++ b/+localFeatures/LindebergCorners.m @@ -0,0 +1,85 @@ +classdef LindebergCorners < localFeatures.GenericLocalFeatureExtractor & ... + helpers.GenericInstaller + + properties (SetAccess=public, GetAccess=public) + CodeDir; + Opts; + end + + properties (Constant, Hidden) + % Installation directory + RootInstallDir = fullfile('data','software','multiscale_harris'); + % URL for dataset tarballs + CodeUrl = 'http://imm.dtu.dk/~abll/files/multiscale_harris.tar.gz'; + end + + + methods + function obj = LindebergCorners(varargin) + obj.Name = 'Lindeberg Corners'; + obj.CodeDir = fullfile(obj.RootInstallDir, 'code'); + obj.Opts = struct('localization', 1); + varargin = obj.checkInstall(varargin); + obj.setup(); + end + + function [frames] = extractFeatures(obj, imagePath) + import helpers.*; + import localFeatures.*; + frames = obj.loadFeatures(imagePath,false); + if numel(frames) > 0; return; end; + startTime = tic; + obj.info('Computing frames of image %s.',getFileName(imagePath)); + + img = imread(imagePath); + if(size(img,3)>1), img = rgb2gray(img); end + img = im2uint8(img); % If not already in uint8, then convert + + obj.Opts.localization + frames = lindebergcorner(img, obj.Opts.localization)'; + frames([1,2],:) = frames([2,1],:); + + timeElapsed = toc(startTime); + obj.debug('%d Frames from image %s computed in %gs',... + size(frames,2),getFileName(imagePath),timeElapsed); + obj.storeFeatures(imagePath, frames, []); + end + + function sign = getSignature(obj) + sign = [helpers.struct2str(obj.Opts),';',... + helpers.fileSignature(obj.CodeDir, 'lindebergcorner.cpp'), ... + helpers.fileSignature(obj.CodeDir, 'LocalMaxima3DFast.cpp'), ... + helpers.fileSignature(obj.CodeDir, 'multiscaleharris.cpp'), ... + helpers.fileSignature(obj.CodeDir, 'scale.cpp'), ... + mfilename('fullpath')]; + end + + function setup(obj) + if(~exist('lindebergcorner.m', 'file')), + fprintf('Adding LindebergCorners to path.\n'); + addpath(obj.CodeDir) + end + end + + function unload(obj) + fprintf('Removing LindebergCorners from path.\n'); + rmpath(obj.CodeDir) + end + + end + + methods (Access=protected, Hidden) + function [srclist flags] = getMexSources(obj) + srclist = {fullfile(obj.CodeDir, 'LocalMaxima3DFast.cpp')}; + flags = {''}; + end + end + + methods (Access = protected) + function [urls dstPaths] = getTarballsList(obj) + installDir = obj.RootInstallDir; + dstPaths = {fullfile(installDir)}; + urls = {obj.CodeUrl}; + end + end +end diff --git a/+localFeatures/MultiscaleHarris.m b/+localFeatures/MultiscaleHarris.m index 3967b6b..8a233b5 100644 --- a/+localFeatures/MultiscaleHarris.m +++ b/+localFeatures/MultiscaleHarris.m @@ -1,7 +1,7 @@ classdef MultiscaleHarris < localFeatures.GenericLocalFeatureExtractor & ... helpers.GenericInstaller - properties (SetAccess=private, GetAccess=public) + properties (SetAccess=public, GetAccess=public) CodeDir; Opts; end diff --git a/eval_mh.m b/eval_mh.m index d94bdef..4ef279e 100644 --- a/eval_mh.m +++ b/eval_mh.m @@ -29,15 +29,26 @@ function eval_mh() % PART 1: Detector repeatability % -------------------------------------------------------------------- + vlcovdetDetector = VlFeatCovdet() siftDetector = VlFeatSift(); mhDetector = MultiscaleHarris(); + mhDetector.Opts.localization = 1; + mh_woDetector = MultiscaleHarris(); + mh_woDetector.Opts.localization = 0; + lcDetector = LindebergCorners(); + lcDetector.Opts.localization = 1; + lc_woDetector = LindebergCorners(); + lc_woDetector.Opts.localization = 0; mser = VlFeatMser(); repBenchmark = RepeatabilityBenchmark('Mode','Repeatability'); - featExtractors = {siftDetector, mser, mhDetector}; - detectorNames = {'SIFT', 'MSER', 'MH'}; + featExtractors = {vlcovdetDetector, siftDetector, mser, lcDetector, lc_woDetector, mhDetector, mh_woDetector}; + detectorNames = {'VLCovDet', 'VLSIFT', 'MSER', 'LC w. localization', 'LC w.o. localization', 'MH w. localization', 'MH w.o. localization'}; + +% featExtractors = {mh_woDetector}; +% detectorNames = {'MH w.o. localization'}; repeatability = []; numCorresp = []; @@ -63,21 +74,26 @@ function eval_mh() figure(2); clf; plotScores(detectorNames, dataset, 100 * repeatability, 'Repeatability'); - printFigure(['results_mh_' dataset_name], [category_name '_repeatability']); + printFigure(['results_' dataset_name], [category_name '_repeatability']); figure(3); clf; plotScores(detectorNames, dataset, numCorresp, 'Number of correspondences'); - printFigure(['results_mh_' dataset_name], [category_name '_num-correspondences']); + printFigure(['results_' dataset_name], [category_name '_num-correspondences']); % -------------------------------------------------------------------- % PART 2: Detector matching score % -------------------------------------------------------------------- + vlcovdetWithSift = DescriptorAdapter(vlcovdetDetector, siftDetector); mserWithSift = DescriptorAdapter(mser, siftDetector); mhWithSift = DescriptorAdapter(mhDetector, siftDetector); - featExtractors = {siftDetector, mserWithSift, mhWithSift}; - detectorNames = {'SIFT', 'MSER with SIFT', 'MH with SIFT'}; + mh_woWithSift = DescriptorAdapter(mh_woDetector, siftDetector); + lcWithSift = DescriptorAdapter(lcDetector, siftDetector); + lc_woWithSift = DescriptorAdapter(lc_woDetector, siftDetector); + + featExtractors = {vlcovdetWithSift, siftDetector, mserWithSift, mhWithSift, mh_woWithSift, lcWithSift, lc_woWithSift}; + detectorNames = {'VLCovDet', 'SIFT', 'MSER', 'LC w. localization', 'LC w.o. localization', 'MH w. localization', 'MH w.o. localization'}; matchingBenchmark = RepeatabilityBenchmark('Mode','MatchingScore'); @@ -102,12 +118,12 @@ function eval_mh() printScores(detectorNames, numMatches, 'Number of matches') ; figure(5); clf; - plotScores(detectorNames, dataset, matchScore*100,'Matching Score'); - printFigure(['results_mh_' dataset_name], [category_name '_matching_score']); + plotScores(detectorNames, dataset, matchScore*100,'Matching Score (with SIFT description)'); + printFigure(['results_' dataset_name], [category_name '_matching_score']); figure(6); clf; - plotScores(detectorNames, dataset, numMatches,'Number of matches'); - printFigure(['results_mh_' dataset_name], [category_name '_num-matches']); + plotScores(detectorNames, dataset, numMatches,'Number of matches (with SIFT description)'); + printFigure(['results_' dataset_name], [category_name '_num-matches']); end % dataset category From 708512296daa31e5c2732104b9535a4a29c7efee Mon Sep 17 00:00:00 2001 From: Anders Boesen Lindbo Larsen Date: Thu, 3 Jul 2014 13:22:20 +0200 Subject: [PATCH 9/9] HarrisScaleSelect and LindebergScaleSelect detectors updated. --- +benchmarks/RepeatabilityBenchmark.m | 3 + ...MultiscaleHarris.m => HarrisScaleSelect.m} | 40 +++---- ...debergCorners.m => LindebergScaleSelect.m} | 39 +++---- +localFeatures/VlFeatSiftUnoriented.m | 102 ++++++++++++++++++ eval_mh.m | 4 +- 5 files changed, 148 insertions(+), 40 deletions(-) rename +localFeatures/{MultiscaleHarris.m => HarrisScaleSelect.m} (62%) rename +localFeatures/{LindebergCorners.m => LindebergScaleSelect.m} (63%) create mode 100644 +localFeatures/VlFeatSiftUnoriented.m diff --git a/+benchmarks/RepeatabilityBenchmark.m b/+benchmarks/RepeatabilityBenchmark.m index 61be5e5..6b758f8 100644 --- a/+benchmarks/RepeatabilityBenchmark.m +++ b/+benchmarks/RepeatabilityBenchmark.m @@ -284,6 +284,9 @@ if isempty(framesA) || isempty(framesB) matches = zeros(size(framesA,2)); obj.info('Nothing to compute.'); + score = 0; + numMatches = 0; + matches = []; return; end if exist('descriptorsA','var') && exist('descriptorsB','var') diff --git a/+localFeatures/MultiscaleHarris.m b/+localFeatures/HarrisScaleSelect.m similarity index 62% rename from +localFeatures/MultiscaleHarris.m rename to +localFeatures/HarrisScaleSelect.m index 8a233b5..4a0c6e0 100644 --- a/+localFeatures/MultiscaleHarris.m +++ b/+localFeatures/HarrisScaleSelect.m @@ -1,25 +1,31 @@ -classdef MultiscaleHarris < localFeatures.GenericLocalFeatureExtractor & ... +classdef HarrisScaleSelect < localFeatures.GenericLocalFeatureExtractor & ... helpers.GenericInstaller - properties (SetAccess=public, GetAccess=public) - CodeDir; - Opts; + properties (SetAccess=private, GetAccess=public) + Opts = struct( ... + 'localize', 0, ... + 'NoScales', 61, ... + 'sigmamin', 1.26, ... + 'sigmamax', -1, ... + 'Hthres', 1500 ... + ) end properties (Constant, Hidden) % Installation directory - RootInstallDir = fullfile('data','software','multiscale_harris'); + RootInstallDir = fullfile('data','software'); + CodeDir = fullfile('data','software','diku-dtu_detectors'); % URL for dataset tarballs - CodeUrl = 'http://imm.dtu.dk/~abll/files/multiscale_harris.tar.gz'; + CodeUrl = 'http://imm.dtu.dk/~abll/files/diku-dtu_detectors.tar.gz'; end methods - function obj = MultiscaleHarris(varargin) - obj.Name = 'Multiscale Harris'; - obj.CodeDir = fullfile(obj.RootInstallDir, 'code'); - obj.Opts = struct('localization', 1); + function obj = HarrisScaleSelect(varargin) + import helpers.*; + obj.Name = 'Harris with scale selection'; varargin = obj.checkInstall(varargin); + [obj.Opts varargin] = vl_argparse(obj.Opts, varargin); obj.setup(); end @@ -35,9 +41,8 @@ if(size(img,3)>1), img = rgb2gray(img); end img = im2uint8(img); % If not already in uint8, then convert - frames = multiscaleharris(img, obj.Opts.localization)'; + frames = harrisscaleselect(img, obj.Opts)'; frames([1,2],:) = frames([2,1],:); - timeElapsed = toc(startTime); obj.debug('%d Frames from image %s computed in %gs',... size(frames,2),getFileName(imagePath),timeElapsed); @@ -46,22 +51,19 @@ function sign = getSignature(obj) sign = [helpers.struct2str(obj.Opts),';',... - helpers.fileSignature(obj.CodeDir, 'lindebergcorner.cpp'), ... - helpers.fileSignature(obj.CodeDir, 'LocalMaxima3DFast.cpp'), ... - helpers.fileSignature(obj.CodeDir, 'multiscaleharris.cpp'), ... - helpers.fileSignature(obj.CodeDir, 'scale.cpp'), ... + helpers.fileSignature(obj.CodeDir, 'harrisscaleselect.m'), ... mfilename('fullpath')]; end function setup(obj) - if(~exist('multiscaleharris.m', 'file')), - fprintf('Adding MultiscaleHarris to path.\n'); + if(~exist('harrisscaleselect.m', 'file')), + fprintf('Adding HarrisScaleSelect to path.\n'); addpath(obj.CodeDir) end end function unload(obj) - fprintf('Removing MultiscaleHarris from path.\n'); + fprintf('Removing HarrisScaleSelect from path.\n'); rmpath(obj.CodeDir) end diff --git a/+localFeatures/LindebergCorners.m b/+localFeatures/LindebergScaleSelect.m similarity index 63% rename from +localFeatures/LindebergCorners.m rename to +localFeatures/LindebergScaleSelect.m index fa4a15a..06f0a2e 100644 --- a/+localFeatures/LindebergCorners.m +++ b/+localFeatures/LindebergScaleSelect.m @@ -1,25 +1,31 @@ -classdef LindebergCorners < localFeatures.GenericLocalFeatureExtractor & ... +classdef LindebergScaleSelect < localFeatures.GenericLocalFeatureExtractor & ... helpers.GenericInstaller - properties (SetAccess=public, GetAccess=public) - CodeDir; - Opts; + properties (SetAccess=private, GetAccess=public) + Opts = struct( ... + 'localize', 0, ... + 'NoScales', 61, ... + 'sigmamin', 0.88, ... + 'sigmamax', -1, ... + 'Kthres', 10^(7.5) ... + ) end properties (Constant, Hidden) % Installation directory - RootInstallDir = fullfile('data','software','multiscale_harris'); + RootInstallDir = fullfile('data','software'); + CodeDir = fullfile('data','software','diku-dtu_detectors'); % URL for dataset tarballs - CodeUrl = 'http://imm.dtu.dk/~abll/files/multiscale_harris.tar.gz'; + CodeUrl = 'http://imm.dtu.dk/~abll/files/diku-dtu_detectors.tar.gz'; end methods - function obj = LindebergCorners(varargin) - obj.Name = 'Lindeberg Corners'; - obj.CodeDir = fullfile(obj.RootInstallDir, 'code'); - obj.Opts = struct('localization', 1); + function obj = LindebergScaleSelect(varargin) + import helpers.*; + obj.Name = 'Lindeberg with scale selection'; varargin = obj.checkInstall(varargin); + [obj.Opts varargin] = vl_argparse(obj.Opts, varargin); obj.setup(); end @@ -35,10 +41,8 @@ if(size(img,3)>1), img = rgb2gray(img); end img = im2uint8(img); % If not already in uint8, then convert - obj.Opts.localization - frames = lindebergcorner(img, obj.Opts.localization)'; + frames = lindebergcorner(img, obj.Opts)'; frames([1,2],:) = frames([2,1],:); - timeElapsed = toc(startTime); obj.debug('%d Frames from image %s computed in %gs',... size(frames,2),getFileName(imagePath),timeElapsed); @@ -47,22 +51,19 @@ function sign = getSignature(obj) sign = [helpers.struct2str(obj.Opts),';',... - helpers.fileSignature(obj.CodeDir, 'lindebergcorner.cpp'), ... - helpers.fileSignature(obj.CodeDir, 'LocalMaxima3DFast.cpp'), ... - helpers.fileSignature(obj.CodeDir, 'multiscaleharris.cpp'), ... - helpers.fileSignature(obj.CodeDir, 'scale.cpp'), ... + helpers.fileSignature(obj.CodeDir, 'lindebergcorner.m'), ... mfilename('fullpath')]; end function setup(obj) if(~exist('lindebergcorner.m', 'file')), - fprintf('Adding LindebergCorners to path.\n'); + fprintf('Adding LindebergScaleSelect to path.\n'); addpath(obj.CodeDir) end end function unload(obj) - fprintf('Removing LindebergCorners from path.\n'); + fprintf('Removing LindebergScaleSelect from path.\n'); rmpath(obj.CodeDir) end diff --git a/+localFeatures/VlFeatSiftUnoriented.m b/+localFeatures/VlFeatSiftUnoriented.m new file mode 100644 index 0000000..64e68e9 --- /dev/null +++ b/+localFeatures/VlFeatSiftUnoriented.m @@ -0,0 +1,102 @@ +classdef VlFeatSiftUnoriented < localFeatures.GenericLocalFeatureExtractor & ... + helpers.GenericInstaller +% localFeatures.VlFeatSift VlFeat vl_sift wrapper +% localFeatures.VlFeatSift('OptionName',OptionValue,...) Creates new +% object which wraps around VLFeat covariant image frames detector. +% All given options defined in the constructor are passed directly +% to the vl_sift function when called. +% +% The options to the constructor are the same as that for vl_sift +% See help vl_sift to see those options and their default values. +% +% See also: vl_sift + +% Authors: Karel Lenc + +% AUTORIGHTS + properties (SetAccess=public, GetAccess=public) + Opts + VlSiftArguments + end + + methods + function obj = VlFeatSiftUnoriented(varargin) + % def. arguments + obj.Name = 'VLFeat SIFT Unoriented'; + varargin = obj.configureLogger(obj.Name,varargin); + obj.VlSiftArguments = obj.checkInstall(varargin); + obj.ExtractsDescriptors = true; + end + + function [frames descriptors] = extractFeatures(obj, imagePath) + import helpers.*; + [frames descriptors] = obj.loadFeatures(imagePath,nargout > 1); + if numel(frames) > 0; return; end; + img = imread(imagePath); + if(size(img,3)>1), img = rgb2gray(img); end + img = single(img); % If not already in uint8, then convert + startTime = tic; + if nargout == 1 + obj.info('Computing frames of image %s.',getFileName(imagePath)); + [frames] = vl_sift(img,obj.VlSiftArguments{:}); + else + obj.info('Computing frames and descriptors of image %s.',... + getFileName(imagePath)); + [frames descriptors] = vl_sift(img,obj.VlSiftArguments{:}); + end + timeElapsed = toc(startTime); + obj.debug('%d Frames from image %s computed in %gs',... + size(frames,2),getFileName(imagePath),timeElapsed); + obj.storeFeatures(imagePath, frames, descriptors); + end + + function [frames descriptors] = extractDescriptors(obj, imagePath, frames) + % extractDescriptor Extract SIFT descriptors of disc frames + % [DFRAMES DESCRIPTORS] = obj.extractDescriptor(IMG_PATH, + % FRAMES) Extracts SIFT descriptors DESCRIPTPORS of disc + % frames FRAMES from image defined by IMG_PATH. For the + % descriptor extraction, scale-space is used. Ellipses are + % converted to discs using their scale. The orientation of an + % oriented ellipse is dropped. + import localFeatures.helpers.*; + obj.info('Computing descriptors.'); + startTime = tic; + % Get the input image + img = imread(imagePath); + imgSize = size(img); + if imgSize(3) > 1 + img = rgb2gray(img); + end + img = single(img); + if size(frames,1) > 4 + % Convert frames to disks + frames = [frames(1,:); frames(2,:); getFrameScale(frames)]; + end + if size(frames,1) < 4 + % When no orientation, compute upright SIFT descriptors + frames = [frames; zeros(1,size(frames,2))]; + end + if size(frames,1) == 4 + % When no orientation, compute upright SIFT descriptors + frames(4,:) = zeros(1,size(frames,2)); + end + % Compute the descriptors (using scale space). + [frames, descriptors] = vl_sift(img,'Frames',frames,... + obj.VlSiftArguments{:}); + elapsedTime = toc(startTime); + obj.debug('Descriptors computed in %gs',elapsedTime); + end + + function sign = getSignature(obj) + sign = [helpers.VlFeatInstaller.getBinSignature('vl_sift'), ... + helpers.cell2str(obj.VlSiftArguments), ... + mfilename('fullpath')]; + end + end + + methods (Access=protected) + function deps = getDependencies(obj) + deps = {helpers.VlFeatInstaller('0.9.14')}; + end + end +end diff --git a/eval_mh.m b/eval_mh.m index 4ef279e..56d9597 100644 --- a/eval_mh.m +++ b/eval_mh.m @@ -7,8 +7,8 @@ function eval_mh() set(0,'DefaultFigureVisible','off'); -dataset_name = 'oxford'; -%dataset_name = 'dtu'; +%dataset_name = 'oxford'; +dataset_name = 'dtu'; if strcmp(dataset_name, 'oxford') categories = datasets.VggAffineDataset.AllCategories;