Skip to content

Commit

Permalink
Merge pull request Framstag#1514 from Framstag/improve_area_index_gen…
Browse files Browse the repository at this point in the history 
…erator

Improvement: See issue Framstag#1506
  • Loading branch information
@Framstag
Framstag authored Oct 3, 2023
2 parents e4867f0 + 9dc13dd commit 0c2dcd8
Show file tree
Hide file tree
Showing 8 changed files with 106 additions and 94 deletions.
102 changes: 57 additions & 45 deletions libosmscout-import/include/osmscoutimport/AreaIndexGenerator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

#include <list>
#include <map>
#include <utility>

#include <osmscout/Pixel.h>

Expand Down Expand Up @@ -66,16 +67,17 @@ namespace osmscout {
}
};

private:
std::string typeName;
std::string typeNamePlural;
std::string dataFile;
std::string indexFile;

protected:
AreaIndexGenerator(const std::string &typeName,
const std::string &typeNamePlural,
const std::string &dataFile,
const std::string &indexFile):
AreaIndexGenerator(const std::string& typeName,
const std::string& typeNamePlural,
const std::string& dataFile,
const std::string& indexFile):
typeName(typeName),
typeNamePlural(typeNamePlural),
dataFile(dataFile),
Expand Down Expand Up @@ -118,9 +120,8 @@ namespace osmscout {
* @param typeInfo
* @param typeData
* @param typeCellOffsets
* @return
*/
bool WriteBitmap(Progress& progress,
void WriteBitmap(Progress& progress,
FileWriter& writer,
const TypeInfo& typeInfo,
const TypeData& typeData,
Expand All @@ -135,12 +136,12 @@ namespace osmscout {
Progress& progress,
const std::vector<TypeInfoRef> &types,
const MagnificationLevel &areaIndexMinMag,
const MagnificationLevel &areaIndexMaxLevel,
const MagnificationLevel &areaIndexMaxMag,
bool useMmap);
};

template <typename Object>
bool AreaIndexGenerator<Object>::WriteBitmap(Progress& progress,
void AreaIndexGenerator<Object>::WriteBitmap(Progress& progress,
FileWriter& writer,
const TypeInfo& typeInfo,
const TypeData& typeData,
Expand Down Expand Up @@ -187,9 +188,7 @@ namespace osmscout {
std::to_string(typeData.indexEntries/typeData.indexCells)+"/cell"+
")");

FileOffset bitmapOffset;

bitmapOffset=writer.GetPos();
FileOffset bitmapOffset=writer.GetPos();

assert(typeData.indexOffset!=0);

Expand All @@ -208,21 +207,18 @@ namespace osmscout {
dataOffsetBytes);
}

FileOffset dataStartOffset;

dataStartOffset=writer.GetPos();
FileOffset dataStartOffset=writer.GetPos();

// Now write the list of offsets of objects for every cell with content
for (const auto& cell : typeCellOffsets) {
FileOffset bitmapCellOffset=bitmapOffset+
((cell.first.GetY()-typeData.tileBox.GetMinY())*typeData.tileBox.GetWidth()+
cell.first.GetX()-typeData.tileBox.GetMinX())*(FileOffset)dataOffsetBytes;
FileOffset previousOffset=0;
FileOffset cellOffset;

assert(bitmapCellOffset>=bitmapOffset);

cellOffset=writer.GetPos();
FileOffset cellOffset=writer.GetPos();

writer.SetPos(bitmapCellOffset);

Expand All @@ -245,8 +241,6 @@ namespace osmscout {
previousOffset=offset;
}
}

return true;
}

template <typename Object>
Expand All @@ -267,37 +261,59 @@ namespace osmscout {
}

// Average number of entries per tile cell
double average=overallCount*1.0/cellFillCount.size();
double average=double(overallCount)/double(cellFillCount.size());

size_t emptyCount=0;
size_t toLowCount=0;
size_t toHighCount=0;
size_t inCount=0;
size_t tooLowCount=0;
size_t tooHighCount=0;
size_t muchTooHighCount=0;
size_t okCount=0;
size_t allCount=0;

size_t tooLowValue=4*average/10;
size_t tooHighValue=64+32;
size_t muchTooHighValue=128+64;

for (const auto& cell : cellFillCount) {
allCount++;

if (cell.second==0) {
emptyCount++;
}
else if (cell.second<0.4*average) {
toLowCount++;
else if (cell.second<tooLowValue) {
tooLowCount++;
}
else if (cell.second>128){
toHighCount++;
else if (cell.second>muchTooHighValue) {
muchTooHighCount++;
}
else if (cell.second>tooHighValue) {
tooHighCount++;
}
else {
inCount++;
okCount++;
}
}

allCount++;
progress.Info(typeInfo.GetName()+" "+
std::to_string(emptyCount)+" | "+
std::to_string(tooLowCount)+" < "+
std::to_string(okCount)+" < "+
std::to_string(tooHighCount)+" *"+
std::to_string(muchTooHighCount)+"* - "+
std::to_string(allCount));

if (double(muchTooHighCount) / double(allCount) >= 0.01) {
progress.Warning(typeInfo.GetName() + " has more than 1% cells with much too high entry count, will use smaller tile size");
return false;
}

if (toHighCount*1.0/allCount>=0.05) {
if (double(tooHighCount) / double(allCount) >= 0.05) {
progress.Warning(typeInfo.GetName() + " has more than 5% cells with too high entry count, will use smaller tile size");
return false;
}

if (toLowCount*1.0/allCount>=0.2) {
progress.Warning(typeInfo.GetName()+" has more than 20% cells with <40% of average filling ("+std::to_string(toLowCount)+"/"+std::to_string(allCount)+")");
if (double(tooLowCount) / double(allCount) >= 0.2) {
progress.Warning(typeInfo.GetName() + " has more than 20% cells with <40% of average filling");
}

/*
Expand All @@ -323,7 +339,7 @@ namespace osmscout {
Progress& progress,
const std::vector<TypeInfoRef> &types,
const MagnificationLevel &areaIndexMinMag,
const MagnificationLevel &areaIndexMaxLevel,
const MagnificationLevel &areaIndexMaxMag,
bool useMmap)
{
using namespace std::string_literals;
Expand All @@ -347,7 +363,7 @@ namespace osmscout {
types,
typeData,
areaIndexMinMag,
areaIndexMaxLevel,
areaIndexMaxMag,
useMmap,
maxLevel)) {
return false;
Expand Down Expand Up @@ -400,7 +416,7 @@ namespace osmscout {
FileScanner::Sequential,
useMmap);

for (MagnificationLevel l=areaIndexMinMag; l<=maxLevel; l++) {
for (MagnificationLevel l=areaIndexMinMag; l <= maxLevel; l++) {
Magnification magnification(l);
TypeInfoSet indexTypes(*typeConfig);

Expand Down Expand Up @@ -428,9 +444,7 @@ namespace osmscout {
for (uint32_t w=1; w <= objectCount; w++) {
progress.SetProgress(w, objectCount);

FileOffset offset;

offset=scanner.GetPos();
FileOffset offset=scanner.GetPos();

obj.Read(*typeConfig,
scanner);
Expand All @@ -449,13 +463,11 @@ namespace osmscout {
for (const auto &type : indexTypes) {
size_t index=type->GetIndex();

if (!WriteBitmap(progress,
writer,
*typeConfig->GetTypeInfo(index),
typeData[index],
typeCellOffsets[index])) {
return false;
}
WriteBitmap(progress,
writer,
*typeConfig->GetTypeInfo(index),
typeData[index],
typeCellOffsets[index]);
}
}

Expand Down Expand Up @@ -570,7 +582,7 @@ namespace osmscout {
currentObjectTypes.Remove(type);
}
else {
progress.Warning(typeConfig.GetTypeInfo(typeIndex)->GetName()+" has too many index cells, that area filled over the limit");
progress.Warning(typeConfig.GetTypeInfo(typeIndex)->GetName()+" still does not fit good index criteria");
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion libosmscout-import/include/osmscoutimport/GenAreaRouteIndex.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ namespace osmscout {

public:
AreaRouteIndexGenerator();
virtual ~AreaRouteIndexGenerator() = default;
~AreaRouteIndexGenerator() override = default;

void GetDescription(const ImportParameter& parameter,
ImportModuleDescription& description) const override;
Expand Down
2 changes: 1 addition & 1 deletion libosmscout-import/include/osmscoutimport/GenAreaWayIndex.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ namespace osmscout {

public:
AreaWayIndexGenerator();
virtual ~AreaWayIndexGenerator() = default;
~AreaWayIndexGenerator() override = default;

void GetDescription(const ImportParameter& parameter,
ImportModuleDescription& description) const override;
Expand Down
30 changes: 15 additions & 15 deletions libosmscout-import/include/osmscoutimport/ImportParameter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,14 @@ class OSMSCOUT_IMPORT_API ImportParameter CLASS_FINAL

using RouterRef = std::shared_ptr<Router>;

enum class AssumeLandStrategy
enum class AssumeLandStrategy : std::uint8_t
{
disable = 0, // disable land detection by db objects
enable = 1, // enable land detection
automatic = 2, // disable land detection when data polygon is known
};

enum class TextIndexVariant
enum class TextIndexVariant : std::uint8_t
{
original = 0, // store original names
transliterate = 1, // store transliterated form of names
Expand Down Expand Up @@ -161,11 +161,11 @@ class OSMSCOUT_IMPORT_API ImportParameter CLASS_FINAL
MagnificationLevel areaNodeBitmapMaxMag; //<! Maximum Magnification level for bitmap index
uint16_t areaNodeBitmapLimit; //<! All cells must have less entries for a given zoom level

MagnificationLevel areaWayMinMag; //<! Minimum magnification of index for individual type
MagnificationLevel areaWayIndexMaxLevel; //<! Maximum zoom level for area way index bitmap
MagnificationLevel areaWayIndexMinMag; //<! Minimum magnification of index for individual type
MagnificationLevel areaWayIndexMaxMag; //<! Maximum zoom level for area way index bitmap

MagnificationLevel areaRouteMinMag; //<! Minimum magnification of index for individual type
MagnificationLevel areaRouteIndexMaxLevel; //<! Maximum zoom level for area route index bitmap
MagnificationLevel areaRouteIndexMinMag; //<! Minimum magnification of index for individual type
MagnificationLevel areaRouteIndexMaxMag; //<! Maximum zoom level for area route index bitmap

uint32_t waterIndexMinMag; //<! Minimum level of the generated water index
uint32_t waterIndexMaxMag; //<! Maximum level of the generated water index
Expand Down Expand Up @@ -252,11 +252,11 @@ class OSMSCOUT_IMPORT_API ImportParameter CLASS_FINAL
MagnificationLevel GetAreaNodeBitmapMaxMag() const;
uint16_t GetAreaNodeBitmapLimit() const;

MagnificationLevel GetAreaWayMinMag() const;
MagnificationLevel GetAreaWayIndexMaxLevel() const;
MagnificationLevel GetAreaWayIndexMinMag() const;
MagnificationLevel GetAreaWayIndexMaxMag() const;

MagnificationLevel GetAreaRouteMinMag() const;
MagnificationLevel GetAreaRouteIndexMaxLevel() const;
MagnificationLevel GetAreaRouteIndexMinMag() const;
MagnificationLevel GetAreaRouteIndexMaxMag() const;

size_t GetAreaAreaIndexMaxMag() const;

Expand Down Expand Up @@ -338,11 +338,11 @@ class OSMSCOUT_IMPORT_API ImportParameter CLASS_FINAL
void SetAreaNodeBitmapMaxMag(const MagnificationLevel& areaNodeBitmapMaxMag);
void SetAreaNodeBitmapLimit(uint16_t areaNodeBitmapLimit);

void SetAreaWayMinMag(MagnificationLevel areaWayMinMag);
void SetAreaWayIndexMaxMag(MagnificationLevel areaWayIndexMaxLevel);
void SetAreaWayIndexMinMag(MagnificationLevel areaWayIndexMinMag);
void SetAreaWayIndexMaxMag(MagnificationLevel areaWayIndexMaxMag);

void SetAreaRouteMinMag(MagnificationLevel areaRouteMinMag);
void SetAreaRouteIndexMaxMag(MagnificationLevel areaRouteIndexMaxLevel);
void SetAreaRouteIndexMinMag(MagnificationLevel areaRouteIndexMinMag);
void SetAreaRouteIndexMaxMag(MagnificationLevel areaRouteIndexMaxMag);

void SetWaterIndexMinMag(uint32_t waterIndexMinMag);
void SetWaterIndexMaxMag(uint32_t waterIndexMaxMag);
Expand Down Expand Up @@ -374,7 +374,7 @@ class OSMSCOUT_IMPORT_API ImportParameter CLASS_FINAL
std::unique_ptr<Preprocessor> GetPreprocessor(const std::string& filename,
PreprocessorCallback& callback) const;

void SetAreaWayIndexMaxLevel(const MagnificationLevel& areaWayIndexMaxLevel);
void SetAreaWayIndexMaxMag(const MagnificationLevel& areaWayIndexMaxLevel);

void SetTextIndexVariant(TextIndexVariant textIndexVariant);
TextIndexVariant GetTextIndexVariant() const;
Expand Down
6 changes: 3 additions & 3 deletions libosmscout-import/src/osmscoutimport/GenAreaRouteIndex.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ namespace osmscout {

AreaRouteIndexGenerator::AreaRouteIndexGenerator():
AreaIndexGenerator<Route>("route",
"rotues",
"routes",
RouteDataFile::ROUTE_DAT,
AreaRouteIndex::AREA_ROUTE_IDX)
{}
Expand Down Expand Up @@ -58,8 +58,8 @@ namespace osmscout {
parameter,
progress,
typeConfig->GetRouteTypes(),
parameter.GetAreaRouteMinMag(),
parameter.GetAreaRouteIndexMaxLevel(),
parameter.GetAreaRouteIndexMinMag(),
parameter.GetAreaRouteIndexMaxMag(),
parameter.GetWayDataMemoryMaped());
}

Expand Down
4 changes: 2 additions & 2 deletions libosmscout-import/src/osmscoutimport/GenAreaWayIndex.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ namespace osmscout {
parameter,
progress,
typeConfig->GetWayTypes(),
parameter.GetAreaWayMinMag(),
parameter.GetAreaWayIndexMaxLevel(),
parameter.GetAreaWayIndexMinMag(),
parameter.GetAreaWayIndexMaxMag(),
parameter.GetWayDataMemoryMaped());
}

Expand Down
9 changes: 7 additions & 2 deletions libosmscout-import/src/osmscoutimport/Import.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,13 @@ namespace osmscout {

bool Importer::ValidateParameter(Progress& progress)
{
if (parameter.GetAreaWayMinMag()<=parameter.GetOptimizationMaxMag()) {
progress.Error("Area way index minimum magnification is <= than optimization max magnification");
if (parameter.GetAreaWayIndexMinMag() > parameter.GetAreaWayIndexMaxMag()) {
progress.Error("Area way index minimum magnification is > than area way index max magnification");
return false;
}

if (parameter.GetAreaRouteIndexMinMag() > parameter.GetAreaRouteIndexMaxMag()) {
progress.Error("Area route index minimum magnification is > than area route index max magnification");
return false;
}

Expand Down
Loading

0 comments on commit 0c2dcd8

Please sign in to comment.