Skip to content

Commit e896fda

Browse files
emmanuelmathotemmanuelmathot
authored
Merge pull request #59 from Terradue/TERRAPI-27
TERRAPI-27: Assets Filtering for geosquare publication
2 parents 720ac86 + ac0d44e commit e896fda

14 files changed

+12748
-16
lines changed

‎src/Stars.Data.Tests/Harvesters/MetadataExtractorsTests.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ public async void TestExtractors(string key, string datadir, MetadataExtraction
4343
{
4444
StacRouter stacRouter = ServiceProvider.GetService<StacRouter>();
4545
TestItem testNode = new TestItem(datadir);
46-
IResource route = new ContainerNode(testNode, new Dictionary<string, IAsset>(testNode.Assets), "");
46+
IResource route = new ItemContainerNode(testNode, new Dictionary<string, IAsset>(testNode.Assets), "");
4747
IDestination destination = LocalFileDestination.Create(fileSystem.Directory.CreateDirectory("out/"), route);
4848
destination.PrepareDestination();
4949

‎src/Stars.Data.Tests/Resources/Translators/StacCollectionToAtomItemTests_EGMS_2018_2022.json

+12,552
Large diffs are not rendered by default.

‎src/Stars.Data.Tests/Translators/StacCollectionToAtomItemTests.cs

+48
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
using Terradue.ServiceModel.Ogc.Owc.AtomEncoding;
1212
using System;
1313
using System.IO;
14+
using Terradue.Stars.Services.Supplier;
15+
using Terradue.Stars.Services;
1416

1517
namespace Terradue.Data.Tests.Translators
1618
{
@@ -53,6 +55,52 @@ public async System.Threading.Tasks.Task EGMS()
5355
Assert.True(wmsIsPresent);
5456
}
5557

58+
[Fact]
59+
public async System.Threading.Tasks.Task EGMS_2018_2022()
60+
{
61+
string json = GetJson("Translators");
62+
63+
ValidateJson(json);
64+
65+
StacCollection stacCollection = StacConvert.Deserialize<StacCollection>(json);
66+
67+
StacCollectionToAtomItemTranslator stacCollectionToAtomItemTranslator = new StacCollectionToAtomItemTranslator(ServiceProvider);
68+
69+
StacCollectionNode stacCollectionNode = new StacCollectionNode(stacCollection, new System.Uri("https://api.terradue.com/timeseries/v1/ns/gep-egms/cs/EGMS-2018-2022"));
70+
71+
// Filter assets to remove the timeseries assets
72+
AssetFilters assetFilters = AssetFilters.CreateAssetFilters(
73+
new string[] { "{type}!application/csv" }
74+
);
75+
// Create a filtered asset container
76+
FilteredAssetContainer filteredAssetContainer = new FilteredAssetContainer(stacCollectionNode, assetFilters);
77+
// Create a container node with the filtered asset container
78+
CollectionContainerNode filteredNode = new CollectionContainerNode(stacCollectionNode, filteredAssetContainer.Assets, "filtered");
79+
StacCollection stacCollection1 = new StacCollection(stacCollection);
80+
stacCollection1.Assets.Clear();
81+
stacCollection1.Assets.AddRange(filteredAssetContainer.Assets.ToDictionary(asset => asset.Key, asset => (asset.Value as StacAssetAsset).StacAsset));
82+
StacCollectionNode stacCollectionNode1 = new StacCollectionNode(stacCollection1, new System.Uri("https://api.terradue.com/timeseries/v1/ns/gep-egms/cs/EGMS-2018-2022"));
83+
84+
AtomItemNode atomItemNode = await stacCollectionToAtomItemTranslator.TranslateAsync<AtomItemNode>(stacCollectionNode1, CancellationToken.None);
85+
86+
bool egmsIsPresent = false;
87+
if (atomItemNode.AtomItem.ElementExtensions != null && atomItemNode.AtomItem.ElementExtensions.Count > 0)
88+
{
89+
var offerings = atomItemNode.AtomItem.ElementExtensions.ReadElementExtensions<OwcOffering>("offering", OwcNamespaces.Owc, new System.Xml.Serialization.XmlSerializer(typeof(OwcOffering)));
90+
91+
foreach (var offering in offerings)
92+
{
93+
if(offering != null && offering.Code == "http://www.terradue.com/egms") egmsIsPresent = true;
94+
}
95+
}
96+
97+
Assert.True(egmsIsPresent);
98+
99+
// Check that there is no link with the relationship type "enclosure"
100+
Assert.DoesNotContain(atomItemNode.AtomItem.Links,
101+
l => l.MediaType == "application/csv" && l.RelationshipType == "enclosure");
102+
}
103+
56104
}
57105

58106
}

‎src/Stars.Data/Model/Metadata/Saocom1/Saocom1MetadataExtractor.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ protected override async Task<StacNode> ExtractMetadata(IItem item, string suffi
9999
{
100100
var mergedAssets = new Dictionary<string, IAsset>(item.Assets.ToDictionary(kvp => kvp.Key, kvp => kvp.Value));
101101
mergedAssets.MergeAssets(extractedAssets.Assets, false);
102-
innerStacItem = new ContainerNode(item, mergedAssets, "merged");
102+
innerStacItem = new ItemContainerNode(item, mergedAssets, "merged");
103103
}
104104

105105
IAsset metadataAsset = GetMetadataAsset(innerStacItem);

‎src/Stars.Data/ThirdParty/Publication/GeosquarePublicationModel.cs

+4
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ public GeosquarePublicationModel(IPublicationModel pubModel)
3838
Collection = pubModel.Collection;
3939
CatalogId = pubModel.CatalogId;
4040
Depth = pubModel.Depth;
41+
AssetsFilters = pubModel.AssetsFilters;
4142
}
4243

4344
public GeosquarePublicationModel(GeosquarePublicationModel publishCatalogModel)
@@ -51,6 +52,7 @@ public GeosquarePublicationModel(GeosquarePublicationModel publishCatalogModel)
5152
CustomLinkUpdater = publishCatalogModel.CustomLinkUpdater;
5253
CatalogId = publishCatalogModel.CatalogId;
5354
Depth = publishCatalogModel.Depth;
55+
AssetsFilters = publishCatalogModel.AssetsFilters;
5456
}
5557

5658
/// <summary>
@@ -111,6 +113,8 @@ internal void UpdateLink(SyndicationLink link, AtomItem item, IAssetsContainer a
111113
public bool ThrowPublicationException { get; set; } = true;
112114

113115
public string CatalogId { get; set; }
116+
117+
public List<string> AssetsFilters { get; set; }
114118
}
115119

116120
}

‎src/Stars.Data/ThirdParty/Publication/GeosquareService.cs

+39-3
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
using System.Collections.Specialized;
2626
using System.Web;
2727
using Terradue.ServiceModel.Ogc.Owc.AtomEncoding;
28+
using Terradue.Stars.Services.Supplier;
2829

2930
namespace Terradue.Stars.Data.ThirdParty.Geosquare
3031
{
@@ -100,7 +101,8 @@ private GeosquarePublicationModel CreateModelFromPublication(IPublicationModel p
100101
AdditionalLinks = publicationModel.AdditionalLinks,
101102
CreateIndex = true,
102103
SubjectsList = publicationModel.Subjects?.Select(s => new Subject(s)).ToList(),
103-
CatalogId = publicationModel.CatalogId ?? geosquareConfiguration.BaseUri.ToString()
104+
CatalogId = publicationModel.CatalogId ?? geosquareConfiguration.BaseUri.ToString(),
105+
AssetsFilters = publicationModel.AssetsFilters
104106
};
105107
}
106108

@@ -148,9 +150,26 @@ public async Task<object> PostCollectionToCatalog(Terradue.Stars.Interface.IColl
148150
{
149151
GeosquarePublicationState catalogPublicationState = state as GeosquarePublicationState;
150152
AtomItemNode atomItemNode = null;
153+
154+
// Filter assets
155+
// Create the asset filters based on the asset filters string from the catalog publication model
156+
AssetFilters assetFilters = AssetFilters.CreateAssetFilters(catalogPublicationState.GeosquarePublicationModel.AssetsFilters);
157+
// Create a filtered asset container
158+
FilteredAssetContainer filteredAssetContainer = new FilteredAssetContainer(collectionNode, assetFilters);
159+
// Create a container node with the filtered asset container
160+
ICollection filteredNode = new CollectionContainerNode(collectionNode, filteredAssetContainer.Assets, "filtered");
161+
// If the item is StacCollection, we recreate the StacCollection with the filtered assets
162+
if (collectionNode is StacCollectionNode stacCollectionNode)
163+
{
164+
StacCollection stacCollection = new StacCollection(stacCollectionNode.StacCollection);
165+
stacCollection.Assets.Clear();
166+
stacCollection.Assets.AddRange(filteredAssetContainer.Assets.ToDictionary(asset => asset.Key, asset => (asset.Value as StacAssetAsset).StacAsset));
167+
filteredNode = new StacCollectionNode(stacCollection, collectionNode.Uri);
168+
}
169+
151170
try
152171
{
153-
atomItemNode = await translatorManager.TranslateAsync<AtomItemNode>(collectionNode, ct);
172+
atomItemNode = await translatorManager.TranslateAsync<AtomItemNode>(filteredNode, ct);
154173
}
155174
catch (Exception e)
156175
{
@@ -176,9 +195,26 @@ public async Task<object> PostItemToCatalog(IItem itemNode, IRouter router, obje
176195
{
177196
GeosquarePublicationState catalogPublicationState = state as GeosquarePublicationState;
178197
AtomItemNode atomItemNode = null;
198+
199+
// Filter assets
200+
// Create the asset filters based on the asset filters string from the catalog publication model
201+
AssetFilters assetFilters = AssetFilters.CreateAssetFilters(catalogPublicationState.GeosquarePublicationModel.AssetsFilters);
202+
// Create a filtered asset container
203+
FilteredAssetContainer filteredAssetContainer = new FilteredAssetContainer(itemNode, assetFilters);
204+
// Create a container node with the filtered asset container
205+
IItem filteredNode = new ItemContainerNode(itemNode, filteredAssetContainer.Assets, "filtered");
206+
// If the item is StacItem, we recreate the StacItem with the filtered assets
207+
if (itemNode is StacItemNode stacItemNode)
208+
{
209+
StacItem stacItem = new StacItem(stacItemNode.StacItem);
210+
stacItem.Assets.Clear();
211+
stacItem.Assets.AddRange(filteredAssetContainer.Assets.ToDictionary(asset => asset.Key, asset => (asset.Value as StacAssetAsset).StacAsset));
212+
filteredNode = new StacItemNode(stacItem, itemNode.Uri);
213+
}
214+
179215
try
180216
{
181-
atomItemNode = await translatorManager.TranslateAsync<AtomItemNode>(itemNode, ct);
217+
atomItemNode = await translatorManager.TranslateAsync<AtomItemNode>(filteredNode, ct);
182218
}
183219
catch (Exception e)
184220
{

‎src/Stars.Data/Translators/StacItemToAtomItemTranslator.cs

+3-1
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,10 @@ public async Task<T> TranslateAsync<T>(IResource node, CancellationToken ct) whe
4343

4444
private StarsAtomItem CreateAtomItem(StacItemNode stacItemNode)
4545
{
46+
StacItem item = stacItemNode.StacItem;
47+
4648
// First, let's create our atomItem
47-
StarsAtomItem atomItem = StarsAtomItem.Create(stacItemNode.StacItem, stacItemNode.Uri);
49+
StarsAtomItem atomItem = StarsAtomItem.Create(item, stacItemNode.Uri);
4850

4951
// Add EO Profile if possible
5052
atomItem.TryAddEarthObservationProfile(stacItemNode.StacItem);

‎src/Stars.Interface/ICollection.cs

+4-1
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,15 @@
66
using System.Net.Mime;
77
using System.Threading.Tasks;
88
using Stac;
9+
using Stac.Collection;
910
using Terradue.Stars.Interface.Router;
1011

1112
namespace Terradue.Stars.Interface
1213
{
1314
public interface ICollection : ICatalog, IAssetsContainer
1415
{
15-
16+
StacExtent Extent { get; }
17+
18+
IDictionary<string, object> Properties { get; }
1619
}
1720
}

‎src/Stars.Interface/IPublicationModel.cs

+2
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,7 @@ public interface IPublicationModel
1616
string Collection { get; }
1717

1818
int Depth { get; }
19+
20+
List<string> AssetsFilters { get; }
1921
}
2022
}

‎src/Stars.Services/Model/Stac/StacCollectionNode.cs

+5
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.Linq;
44
using System.Net;
55
using Stac;
6+
using Stac.Collection;
67
using Stars.Services.Model.Stac;
78
using Terradue.Stars.Interface;
89

@@ -17,5 +18,9 @@ public StacCollectionNode(StacCollection stacCollection, Uri uri) : base(stacCol
1718
public IReadOnlyDictionary<string, IAsset> Assets => (stacObject as StacCollection).Assets.ToDictionary(asset => asset.Key, asset => (IAsset)new StacAssetAsset(asset.Value, this));
1819

1920
public StacCollection StacCollection => stacObject as StacCollection;
21+
22+
public StacExtent Extent => StacCollection.Extent;
23+
24+
public IDictionary<string, object> Properties => StacCollection.Properties;
2025
}
2126
}

‎src/Stars.Services/Processing/ExtractArchiveAction.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ public async Task<IResource> ProcessAsync(IResource route, IDestination destinat
119119

120120
if (newAssets == null || newAssets.Count == 0) return route;
121121

122-
return new ContainerNode(route as IItem, newAssets, suffix);
122+
return new ItemContainerNode(route as IItem, newAssets, suffix);
123123

124124
}
125125

‎src/Stars.Services/Supplier/AssetFilters.cs

+21-3
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ public override string ToString()
2424
return Count == 0 ? "No filter" : string.Join(", ", this.Select(af => af.ToString()));
2525
}
2626

27-
public static AssetFilters CreateAssetFilters(string[] assetFiltersStr)
27+
public static AssetFilters CreateAssetFilters(IEnumerable<string> assetFiltersStr)
2828
{
2929
AssetFilters assetFilters = new AssetFilters();
3030
if (assetFiltersStr == null)
@@ -153,17 +153,35 @@ public bool IsMatch(KeyValuePair<string, IAsset> asset)
153153
public class ContentTypeAssetFilter : IAssetFilter
154154
{
155155
private readonly string mediaType;
156+
157+
private readonly bool negated = false;
158+
156159
private readonly Dictionary<string, string> parameters;
157160

158161
public ContentTypeAssetFilter(string mediaType, Dictionary<string, string> parameters)
159162
{
160-
this.mediaType = mediaType;
163+
if( mediaType.StartsWith("!") )
164+
{
165+
this.mediaType = mediaType.Substring(1);
166+
this.negated = true;
167+
}
168+
else
169+
{
170+
this.mediaType = mediaType;
171+
}
161172
this.parameters = parameters;
162173
}
163174

164175
public bool IsMatch(KeyValuePair<string, IAsset> asset)
165176
{
166-
return
177+
return negated ?
178+
( string.IsNullOrEmpty(mediaType) ||
179+
!asset.Value.ContentType.MediaType.Equals(mediaType, System.StringComparison.InvariantCultureIgnoreCase)
180+
) &&
181+
( parameters == null ||
182+
parameters.Any(p => !asset.Value.ContentType.Parameters.ContainsKey(p.Key) ||
183+
asset.Value.ContentType.Parameters[p.Key] != p.Value)
184+
) :
167185
( string.IsNullOrEmpty(mediaType) ||
168186
asset.Value.ContentType.MediaType.Equals(mediaType, System.StringComparison.InvariantCultureIgnoreCase)
169187
) &&

‎src/Stars.Services/Supplier/CollectionContainerNode.cs

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Collections.ObjectModel;
4+
using System.IO;
5+
using System.Net.Mime;
6+
using System.Threading.Tasks;
7+
using GeoJSON.Net.Geometry;
8+
using Itenso.TimePeriod;
9+
using Stac.Collection;
10+
using Terradue.Stars.Interface;
11+
using Terradue.Stars.Interface.Router;
12+
using Terradue.Stars.Interface.Supplier.Destination;
13+
14+
namespace Terradue.Stars.Services.Supplier
15+
{
16+
public class CollectionContainerNode : IAssetsContainer, ICollection
17+
{
18+
private readonly ICollection collection;
19+
private readonly IReadOnlyDictionary<string, IAsset> assets;
20+
private readonly string suffix;
21+
22+
public CollectionContainerNode(ICollection collection, IReadOnlyDictionary<string, IAsset> assets, string suffix)
23+
{
24+
this.collection = collection;
25+
this.assets = assets;
26+
this.suffix = suffix;
27+
}
28+
29+
public IResource Node => collection;
30+
31+
public Uri Uri => collection.Uri;
32+
33+
public ContentType ContentType => collection.ContentType;
34+
35+
public ResourceType ResourceType => collection.ResourceType;
36+
37+
public ulong ContentLength => collection.ContentLength;
38+
39+
public string Title => collection.Title;
40+
41+
public string Id => collection.Id + suffix;
42+
43+
public ContentDisposition ContentDisposition => collection.ContentDisposition;
44+
45+
public StacExtent Geometry => collection.Extent;
46+
47+
public IDictionary<string, object> Properties => collection.Properties;
48+
49+
public IReadOnlyDictionary<string, IAsset> Assets => assets;
50+
51+
public StacExtent Extent => throw new NotImplementedException();
52+
53+
public IReadOnlyList<IResourceLink> GetLinks()
54+
{
55+
return collection.GetLinks();
56+
}
57+
58+
public IReadOnlyList<IResource> GetRoutes(IRouter router)
59+
{
60+
return collection.GetRoutes(router);
61+
}
62+
}
63+
}

‎src/Stars.Services/Supplier/ContainerNode.cs ‎src/Stars.Services/Supplier/ItemContainerNode.cs

+4-5
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,13 @@
1212

1313
namespace Terradue.Stars.Services.Supplier
1414
{
15-
public class ContainerNode : IAssetsContainer, IItem
15+
public class ItemContainerNode : IAssetsContainer, IItem
1616
{
1717
private readonly IItem item;
18-
private readonly IDictionary<string, IAsset> assets;
18+
private readonly IReadOnlyDictionary<string, IAsset> assets;
1919
private readonly string suffix;
2020

21-
public ContainerNode(IItem item, IDictionary<string, IAsset> assets, string suffix)
21+
public ItemContainerNode(IItem item, IReadOnlyDictionary<string, IAsset> assets, string suffix)
2222
{
2323
this.item = item;
2424
this.assets = assets;
@@ -45,11 +45,10 @@ public ContainerNode(IItem item, IDictionary<string, IAsset> assets, string suff
4545

4646
public IDictionary<string, object> Properties => item.Properties;
4747

48-
public IReadOnlyDictionary<string, IAsset> Assets => new ReadOnlyDictionary<string, IAsset>(assets);
48+
public IReadOnlyDictionary<string, IAsset> Assets => assets;
4949

5050
public ITimePeriod DateTime => item.DateTime;
5151

52-
5352
public IReadOnlyList<IResourceLink> GetLinks()
5453
{
5554
return item.GetLinks();

0 commit comments

Comments
 (0)