Skip to content

Commit

Permalink
New search index (#2099)
Browse files Browse the repository at this point in the history 
* Main commit to use the content of the new indexs.

* Add source info to index.

* Use the new source, which is a single file

* Added the usage of the new indexs in client, removed some code related to features saved in the indexes.

* Removal of old zip file creation

* Fix lint

* Fix lint

* Get point data from OSM, remove get point by id from database.

* Allow testing the new search API

* Align planet-search and this code with latest changes.

* Added more icons inference

* Add more missing POI tagging

* Use new indices for places and exact match search

* Move get closest point to use new index

* Remove unneeded method.

* Update more places to use new index.

* Build external file from OSM file and external sources repo.

* Added `stream_pool` to icon matching

* Added coverage and more guards

* Added coverage for path
  • Loading branch information
@HarelM
HarelM authored Feb 9, 2025
1 parent e5839fa commit b8f9ed0
Show file tree
Hide file tree
Showing 35 changed files with 1,453 additions and 2,198 deletions.
27 changes: 7 additions & 20 deletions IsraelHiking.API/Controllers/OsmTracesController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,28 +166,15 @@ await gateway.CreateTrace(new GpxFile

private async Task<string> GetDescriptionByArea(string language, List<Coordinate> allPoints, string defaultDescription)
{
var containersStart = await _searchRepository.GetContainers(allPoints.First());
var containersEnd = await _searchRepository.GetContainers(allPoints.Last());
var containers = containersStart.Concat(containersEnd)
.GroupBy(f => f.GetId())
.Select(g => g.First())
.OrderBy(c => c.Geometry.Area)
.ToList();
foreach (var container in containers)
var containerName = await _searchRepository.GetContainerName(allPoints.ToArray(), language);
if (string.IsNullOrEmpty(containerName))
{
var pointsInside = allPoints.Count(c => container.Geometry.Contains(new Point(c)));
if (pointsInside * 100.0 / allPoints.Count <= 20.0)
{
continue;
}
container.SetTitles();
var replacementTarget = language == "he"
? "מסלול ב" + container.GetTitle(language)
: "A route in " + container.GetTitle(language);
return defaultDescription.Replace("מסלול", replacementTarget).Replace("Route", replacementTarget).Replace("Recorded using IHM at", replacementTarget);
return defaultDescription;
}

return defaultDescription;
var replacementTarget = language == "he"
? "מסלול ב" + containerName
: "A route in " + containerName;
return defaultDescription.Replace("מסלול", replacementTarget).Replace("Route", replacementTarget).Replace("Recorded using IHM at", replacementTarget);
}

/// <summary>
Expand Down
28 changes: 7 additions & 21 deletions IsraelHiking.API/Controllers/PointsOfInterestController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,17 +81,11 @@ public IEnumerable<Category> GetCategoriesByGroup(string categoriesGroup)
/// <returns>A list of GeoJSON features</returns>
[Route("")]
[HttpGet]
public async Task<IFeature[]> GetPointsOfInterest(string northEast, string southWest, string categories,
[Obsolete("Should be removed by 6.2025")]
public IFeature[] GetPointsOfInterest(string northEast, string southWest, string categories,
string language = "")
{
if (string.IsNullOrWhiteSpace(categories))
{
return Array.Empty<IFeature>();
}
var categoriesArray = categories.Split(',').Select(f => f.Trim()).ToArray();
var northEastCoordinate = northEast.ToCoordinate();
var southWestCoordinate = southWest.ToCoordinate();
return await _pointsOfInterestProvider.GetFeatures(northEastCoordinate, southWestCoordinate, categoriesArray, language);
return Array.Empty<IFeature>();
}

/// <summary>
Expand Down Expand Up @@ -141,21 +135,14 @@ await AddSimplePoint(new AddSimplePointOfInterestRequest
});
return Ok();
}
var mappedId = _persistentCache.GetString(feature.GetId());
if (!string.IsNullOrEmpty(mappedId))
if (!string.IsNullOrEmpty(_persistentCache.GetString(feature.GetId())))
{
var featureFromDatabase = await _pointsOfInterestProvider.GetFeatureById(feature.Attributes[FeatureAttributes.POI_SOURCE].ToString(), mappedId);
if (featureFromDatabase == null)
{
return BadRequest("Feature is still in process please try again later...");
}
return Ok(featureFromDatabase);
return BadRequest("Feature creation was already requested, ignoring request.");
}
_persistentCache.SetString(feature.GetId(), "In process", new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = TimeSpan.FromDays(30) });

var osmGateway = OsmAuthFactoryWrapper.ClientFromUser(User, _clientsFactory);
var newFeature = await _pointsOfInterestProvider.AddFeature(feature, osmGateway, language);
_persistentCache.SetString(feature.GetId(), newFeature.Attributes[FeatureAttributes.ID].ToString(), new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = TimeSpan.FromDays(30) });
return Ok(newFeature);
}

Expand Down Expand Up @@ -221,14 +208,13 @@ private string ValidateFeature(IFeature feature, string language)
/// Gets the closest point to a given location.
/// </summary>
/// <param name="location">The location string "lat,lon" to search around</param>
/// <param name="source">Optional, if given this is the only source this methosd will use</param>
/// <param name="language">Optional, if given this is the only language this method will use</param>
/// <returns></returns>
[HttpGet]
[Route("closest")]
public Task<IFeature> GetClosestPoint(string location, string source, string language)
public Task<IFeature> GetClosestPoint(string location, string language)
{
return _pointsOfInterestProvider.GetClosestPoint(location.ToCoordinate(), source, language);
return _pointsOfInterestProvider.GetClosestPoint(location.ToCoordinate(), language);
}

/// <summary>
Expand Down
30 changes: 9 additions & 21 deletions IsraelHiking.API/Controllers/SearchController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
namespace IsraelHiking.API.Controllers
{
/// <summary>
/// This contoller allows search of geo-locations
/// This controller allows search of geo-locations
/// </summary>
[Route("api/[controller]")]
public class SearchController : ControllerBase
Expand Down Expand Up @@ -41,25 +41,22 @@ public SearchController(ISearchRepository searchRepository)
public async Task<IEnumerable<SearchResultsPointOfInterest>> GetSearchResults(string term, string language)
{
if ((term.StartsWith("\"") || term.StartsWith("״")) &&
(term.EndsWith("\"") || term.StartsWith("״")))
(term.EndsWith("\"") || term.EndsWith("״")))
{
var exactFeatures = await _searchRepository.SearchExact(term.Substring(1, term.Length - 2), language);
return await Task.WhenAll(exactFeatures.ToList().Select(f => ConvertFromFeature(f, language)));
}

if (term.Count(c => c == ',') == 1)
{
var split = term.Split(',');
var place = split.Last().Trim();
term = split.First().Trim();
var placesFeatures = await _searchRepository.SearchPlaces(place, language);
if (placesFeatures.Any())
var featuresWithinPlaces = await _searchRepository.SearchPlaces(term, language);
if (featuresWithinPlaces.Count != 0)
{
var envelope = placesFeatures.First().Geometry.EnvelopeInternal;
var featuresWithinPlaces = await _searchRepository.SearchByLocation(
new Coordinate(envelope.MaxX, envelope.MaxY), new Coordinate(envelope.MinX, envelope.MinY), term, language);
return await Task.WhenAll(featuresWithinPlaces.ToList().Select(f => ConvertFromFeature(f,language)));
return await Task.WhenAll(featuresWithinPlaces.ToList().Select(f => ConvertFromFeature(f, language)));
}
term = term.Split(",").First().Trim();
}

var features = await _searchRepository.Search(term, language);
return await Task.WhenAll(features.ToList().Select(f => ConvertFromFeature(f, language)));
}
Expand Down Expand Up @@ -90,20 +87,11 @@ private async Task<SearchResultsPointOfInterest> ConvertFromFeature(IFeature fea
private async Task<string> GetDisplayName(IFeature feature, string language, string title)
{
var displayName = title;
var containers = await _searchRepository.GetContainers(feature.Geometry.Coordinate);
var geometries = Enumerable.Range(0, feature.Geometry.NumGeometries).Select(i => feature.Geometry.GetGeometryN(i)).ToArray();
var container = containers.Where(c =>
c.Attributes[FeatureAttributes.ID] != feature.Attributes[FeatureAttributes.ID] &&
geometries.All(g => c.Geometry.Covers(g)) &&
c.Geometry.EqualsTopologically(feature.Geometry) == false)
.OrderBy(c => c.Geometry.Area)
.FirstOrDefault();
var containerTitle = container?.GetTitle(language);
var containerTitle = await _searchRepository.GetContainerName([feature.Geometry.Coordinate], language);
if (!string.IsNullOrWhiteSpace(containerTitle))
{
displayName += ", " + containerTitle;
}

return displayName;
}
}
Expand Down
1 change: 0 additions & 1 deletion IsraelHiking.API/RegisterApi.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ public static void AddIHMApi(this IServiceCollection services)
// registration here is what determines the order of which to merge points:
services.AddTransient<IPointsOfInterestAdapter, NakebPointsOfInterestAdapter>();
services.AddTransient<IPointsOfInterestAdapter, INaturePointsOfInterestAdapter>();
services.AddTransient<IPointsOfInterestAdapter, WikipediaPointsOfInterestAdapter>();
services.AddTransient<IPointsOfInterestAdapter, WikidataPointsOfInterestAdapter>();
services.AddTransient<CsvPointsOfInterestAdapter>();
services.AddSingleton<IPointsOfInterestAdapterFactory, PointsOfInterestAdapterFactory>();
Expand Down
79 changes: 37 additions & 42 deletions IsraelHiking.API/Services/Osm/DatabasesUpdaterService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@
using IsraelHiking.API.Services.Poi;
using IsraelHiking.Common;
using IsraelHiking.Common.Api;
using IsraelHiking.Common.Extensions;
using IsraelHiking.DataAccessInterfaces;
using IsraelHiking.DataAccessInterfaces.Repositories;
using Microsoft.Extensions.Logging;
using OsmSharp;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using NetTopologySuite.Features;

namespace IsraelHiking.API.Services.Osm
{
Expand All @@ -22,14 +22,10 @@ public class DatabasesUpdaterService : IDatabasesUpdaterService
private readonly IOsmGeoJsonPreprocessorExecutor _osmGeoJsonPreprocessorExecutor;
private readonly IOsmRepository _osmRepository;
private readonly IPointsOfInterestAdapterFactory _pointsOfInterestAdapterFactory;
private readonly IPointsOfInterestProvider _pointsOfInterestProvider;
private readonly IFeaturesMergeExecutor _featuresMergeExecutor;
private readonly IOsmLatestFileGateway _osmLatestFileGateway;
private readonly IPointsOfInterestFilesCreatorExecutor _pointsOfInterestFilesCreatorExecutor;
private readonly IImagesUrlsStorageExecutor _imagesUrlsStorageExecutor;
private readonly IExternalSourceUpdaterExecutor _externalSourceUpdaterExecutor;
private readonly IElevationGateway _elevationGateway;
private readonly IUnauthorizedImageUrlsRemover _unauthorizedImageUrlsRemover;
private readonly IElevationSetterExecutor _elevationSetterExecutor;
private readonly ILogger _logger;

Expand All @@ -42,14 +38,10 @@ public class DatabasesUpdaterService : IDatabasesUpdaterService
/// <param name="osmGeoJsonPreprocessorExecutor"></param>
/// <param name="osmRepository"></param>
/// <param name="pointsOfInterestAdapterFactory"></param>
/// <param name="featuresMergeExecutor"></param>
/// <param name="latestFileGateway"></param>
/// <param name="pointsOfInterestFilesCreatorExecutor"></param>
/// <param name="imagesUrlsStorageExecutor"></param>
/// <param name="pointsOfInterestProvider"></param>
/// <param name="externalSourceUpdaterExecutor"></param>
/// <param name="elevationGateway"></param>
/// <param name="unauthorizedImageUrlsRemover"></param>
/// <param name="elevationSetterExecutor"></param>
/// <param name="logger"></param>
public DatabasesUpdaterService(IExternalSourcesRepository externalSourcesRepository,
Expand All @@ -58,14 +50,10 @@ public DatabasesUpdaterService(IExternalSourcesRepository externalSourcesReposit
IOsmGeoJsonPreprocessorExecutor osmGeoJsonPreprocessorExecutor,
IOsmRepository osmRepository,
IPointsOfInterestAdapterFactory pointsOfInterestAdapterFactory,
IFeaturesMergeExecutor featuresMergeExecutor,
IOsmLatestFileGateway latestFileGateway,
IPointsOfInterestFilesCreatorExecutor pointsOfInterestFilesCreatorExecutor,
IImagesUrlsStorageExecutor imagesUrlsStorageExecutor,
IPointsOfInterestProvider pointsOfInterestProvider,
IExternalSourceUpdaterExecutor externalSourceUpdaterExecutor,
IElevationGateway elevationGateway,
IUnauthorizedImageUrlsRemover unauthorizedImageUrlsRemover,
IElevationSetterExecutor elevationSetterExecutor,
ILogger logger)
{
Expand All @@ -76,13 +64,9 @@ public DatabasesUpdaterService(IExternalSourcesRepository externalSourcesReposit
_osmRepository = osmRepository;
_pointsOfInterestAdapterFactory = pointsOfInterestAdapterFactory;
_pointsOfInterestFilesCreatorExecutor = pointsOfInterestFilesCreatorExecutor;
_featuresMergeExecutor = featuresMergeExecutor;
_osmLatestFileGateway = latestFileGateway;
_pointsOfInterestProvider = pointsOfInterestProvider;
_imagesUrlsStorageExecutor = imagesUrlsStorageExecutor;
_externalSourceUpdaterExecutor = externalSourceUpdaterExecutor;
_elevationGateway = elevationGateway;
_unauthorizedImageUrlsRemover = unauthorizedImageUrlsRemover;
_elevationSetterExecutor = elevationSetterExecutor;
_logger = logger;
}
Expand All @@ -107,10 +91,6 @@ public async Task Rebuild(UpdateRequest request)
{
await RebuildHighways();
}
if (request.PointsOfInterest)
{
await RebuildPointsOfInterest(rebuildContext);
}
if (request.Images)
{
await RebuildImages();
Expand All @@ -121,7 +101,7 @@ public async Task Rebuild(UpdateRequest request)
}
if (request.OfflinePoisFile)
{
await RebuildOfflinePoisFile(rebuildContext);
await RebuildOfflineFiles();
}
}
catch (Exception ex)
Expand All @@ -137,20 +117,6 @@ public async Task Rebuild(UpdateRequest request)

}

private async Task RebuildPointsOfInterest(RebuildContext rebuildContext)
{
_logger.LogInformation("Starting rebuilding POIs database.");
var osmFeaturesTask = _pointsOfInterestProvider.GetAll();
var sources = _pointsOfInterestAdapterFactory.GetAll().Select(s => s.Source);
var externalFeatures = sources.Select(s => _externalSourcesRepository.GetExternalPoisBySource(s)).SelectMany(t => t.Result).ToList();
var features = _featuresMergeExecutor.Merge(osmFeaturesTask.Result, externalFeatures);
_unauthorizedImageUrlsRemover.RemoveImages(features);
await _pointsOfInterestRepository.StorePointsOfInterestDataToSecondaryIndex(features);
_logger.LogInformation("Finished storing all features " + features.Count);
await _pointsOfInterestRepository.SwitchPointsOfInterestIndices();
_logger.LogInformation("Finished rebuilding POIs database.");
}

private async Task RebuildHighways()
{
_logger.LogInformation("Starting rebuilding highways database.");
Expand Down Expand Up @@ -185,12 +151,41 @@ private async Task RebuildSiteMap()
_logger.LogInformation("Finished rebuilding sitemap.");
}

private async Task RebuildOfflinePoisFile(RebuildContext context)
private async Task RebuildOfflineFiles()
{
_logger.LogInformation($"Starting rebuilding offline pois file for date: {context.StartTime.ToInvariantString()}");
var features = await _pointsOfInterestRepository.GetAllPointsOfInterest();
_elevationSetterExecutor.GeometryTo3D(features);
_pointsOfInterestFilesCreatorExecutor.CreateOfflinePoisFile(features);
_logger.LogInformation($"Starting rebuilding offline files.");
await using var stream = await _osmLatestFileGateway.Get();
var references = await _osmRepository.GetExternalReferences(stream);
var sources = _pointsOfInterestAdapterFactory.GetAll().Select(s => s.Source);
var externalFeatures = new List<IFeature>();
foreach (var source in sources)
{
var features = await _externalSourcesRepository.GetExternalPoisBySource(source);
if (!references.TryGetValue(source, out var reference))
{
externalFeatures.AddRange(features);
continue;
}
var referencesNames = reference.ToHashSet();
_logger.LogInformation($"Got {referencesNames.Count} references from OSM file for {source}.");
foreach (var feature in features)
{
if (feature.Attributes.GetNames().Any(n => n == FeatureAttributes.NAME) &&
referencesNames.Contains(feature.Attributes[FeatureAttributes.NAME]))
{
continue;
}
if (feature.Attributes.GetNames().Any(n => n == FeatureAttributes.ID) &&
referencesNames.Contains(feature.Attributes[FeatureAttributes.ID]))
{
continue;
}
externalFeatures.Add(feature);
}
}
_logger.LogInformation($"Starting rebuilding offline files with {externalFeatures.Count} features.");
_elevationSetterExecutor.GeometryTo3D(externalFeatures);
_pointsOfInterestFilesCreatorExecutor.CreateOfflinePoisFile(externalFeatures);
_logger.LogInformation("Finished rebuilding offline pois file.");
}

Expand Down
Loading

0 comments on commit b8f9ed0

Please sign in to comment.