Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Warn on unknown cfg, CI cleanup #461

Merged
merged 2 commits into from
Oct 24, 2022
Merged

Warn on unknown cfg, CI cleanup #461

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8276431
Warn on unknown cfg, CI cleanup
nyurik Oct 23, 2022
a47c650
crash psql on error
nyurik Oct 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
190 changes: 95 additions & 95 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,15 +61,15 @@ jobs:
- name: Setup database
run: |
sudo apt-get install postgresql-client
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -f tests/fixtures/TileBBox.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -f tests/fixtures/table_source.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -f tests/fixtures/table_source_multiple_geom.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -f tests/fixtures/function_source.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -f tests/fixtures/function_source_query_params.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -f tests/fixtures/points1_source.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -f tests/fixtures/points2_source.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -f tests/fixtures/points3857_source.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -f tests/fixtures/points_empty_srid_source.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -v ON_ERROR_STOP=1 -f tests/fixtures/TileBBox.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -v ON_ERROR_STOP=1 -f tests/fixtures/table_source.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -v ON_ERROR_STOP=1 -f tests/fixtures/table_source_multiple_geom.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -v ON_ERROR_STOP=1 -f tests/fixtures/function_source.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -v ON_ERROR_STOP=1 -f tests/fixtures/function_source_query_params.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -v ON_ERROR_STOP=1 -f tests/fixtures/points1_source.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -v ON_ERROR_STOP=1 -f tests/fixtures/points2_source.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -v ON_ERROR_STOP=1 -f tests/fixtures/points3857_source.sql
psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d db -v ON_ERROR_STOP=1 -f tests/fixtures/points_empty_srid_source.sql
env:
POSTGRES_HOST: localhost
POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}
Expand Down Expand Up @@ -100,91 +100,91 @@ jobs:

- name: Test server response for table source
run: |
curl "localhost:3000/index.json" | jq -e
curl "localhost:3000/public.table_source/0/0/0.pbf" > table_source.pbf
./tests/vtzero-check table_source.pbf
./tests/vtzero-show table_source.pbf
curl "localhost:3000/public.table_source/6/38/20.pbf" > table_source.pbf
./tests/vtzero-check table_source.pbf
./tests/vtzero-show table_source.pbf
curl "localhost:3000/public.table_source/12/2476/1280.pbf" > table_source.pbf
./tests/vtzero-check table_source.pbf
./tests/vtzero-show table_source.pbf
curl "localhost:3000/public.table_source/13/4952/2560.pbf" > table_source.pbf
./tests/vtzero-check table_source.pbf
./tests/vtzero-show table_source.pbf
curl "localhost:3000/public.table_source/14/9904/5121.pbf" > table_source.pbf
./tests/vtzero-check table_source.pbf
./tests/vtzero-show table_source.pbf
curl "localhost:3000/public.table_source/20/633856/327787.pbf" > table_source.pbf
./tests/vtzero-check table_source.pbf
./tests/vtzero-show table_source.pbf
curl "localhost:3000/public.table_source/21/1267712/655574.pbf" > table_source.pbf
./tests/vtzero-check table_source.pbf
./tests/vtzero-show table_source.pbf
curl -sS "localhost:3000/index.json" | jq -e
curl -sS "localhost:3000/public.table_source/0/0/0.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/public.table_source/6/38/20.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/public.table_source/12/2476/1280.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/public.table_source/13/4952/2560.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/public.table_source/14/9904/5121.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/public.table_source/20/633856/327787.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/public.table_source/21/1267712/655574.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf

- name: Test server response for composite source
run: |
curl "localhost:3000/public.table_source,public.points1,public.points2/0/0/0.pbf" > composite_source.pbf
./tests/vtzero-check composite_source.pbf
./tests/vtzero-show composite_source.pbf
curl "localhost:3000/public.table_source,public.points1,public.points2/6/38/20.pbf" > composite_source.pbf
./tests/vtzero-check composite_source.pbf
./tests/vtzero-show composite_source.pbf
curl "localhost:3000/public.table_source,public.points1,public.points2/12/2476/1280.pbf" > composite_source.pbf
./tests/vtzero-check composite_source.pbf
./tests/vtzero-show composite_source.pbf
curl "localhost:3000/public.table_source,public.points1,public.points2/13/4952/2560.pbf" > composite_source.pbf
./tests/vtzero-check composite_source.pbf
./tests/vtzero-show composite_source.pbf
curl "localhost:3000/public.table_source,public.points1,public.points2/14/9904/5121.pbf" > composite_source.pbf
./tests/vtzero-check composite_source.pbf
./tests/vtzero-show composite_source.pbf
curl "localhost:3000/public.table_source,public.points1,public.points2/20/633856/327787.pbf" > composite_source.pbf
./tests/vtzero-check composite_source.pbf
./tests/vtzero-show composite_source.pbf
curl "localhost:3000/public.table_source,public.points1,public.points2/21/1267712/655574.pbf" > composite_source.pbf
./tests/vtzero-check composite_source.pbf
./tests/vtzero-show composite_source.pbf
curl -sS "localhost:3000/public.table_source,public.points1,public.points2/0/0/0.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/public.table_source,public.points1,public.points2/6/38/20.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/public.table_source,public.points1,public.points2/12/2476/1280.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/public.table_source,public.points1,public.points2/13/4952/2560.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/public.table_source,public.points1,public.points2/14/9904/5121.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/public.table_source,public.points1,public.points2/20/633856/327787.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/public.table_source,public.points1,public.points2/21/1267712/655574.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf

- name: Test server response for function source
run: |
curl "localhost:3000/rpc/public.function_source/0/0/0.pbf" > function_source0.pbf
./tests/vtzero-check function_source0.pbf
./tests/vtzero-show function_source0.pbf
curl "localhost:3000/rpc/public.function_source/6/38/20.pbf" > function_source6.pbf
./tests/vtzero-check function_source6.pbf
./tests/vtzero-show function_source6.pbf
curl "localhost:3000/rpc/public.function_source/12/2476/1280.pbf" > function_source12.pbf
./tests/vtzero-check function_source12.pbf
./tests/vtzero-show function_source12.pbf
curl "localhost:3000/rpc/public.function_source/13/4952/2560.pbf" > function_source13.pbf
./tests/vtzero-check function_source13.pbf
./tests/vtzero-show function_source13.pbf
curl "localhost:3000/rpc/public.function_source/14/9904/5121.pbf" > function_source14.pbf
./tests/vtzero-check function_source14.pbf
./tests/vtzero-show function_source14.pbf
curl "localhost:3000/rpc/public.function_source/20/633856/327787.pbf" > function_source20.pbf
./tests/vtzero-check function_source20.pbf
./tests/vtzero-show function_source20.pbf
curl "localhost:3000/rpc/public.function_source/21/1267712/655574.pbf" > function_source21.pbf
./tests/vtzero-check function_source21.pbf
./tests/vtzero-show function_source21.pbf
curl "localhost:3000/rpc/public.function_source_query_params/0/0/0.pbf?token=martin" > function_source_query_params.pbf
./tests/vtzero-check function_source_query_params.pbf
./tests/vtzero-show function_source_query_params.pbf
curl -sS "localhost:3000/rpc/public.function_source/0/0/0.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/rpc/public.function_source/6/38/20.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/rpc/public.function_source/12/2476/1280.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/rpc/public.function_source/13/4952/2560.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/rpc/public.function_source/14/9904/5121.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/rpc/public.function_source/20/633856/327787.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/rpc/public.function_source/21/1267712/655574.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/rpc/public.function_source_query_params/0/0/0.pbf?token=martin" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf

- name: Test server response for table source with different SRID
run: |
curl "localhost:3000/public.points3857/0/0/0.pbf" > table_source.pbf
./tests/vtzero-check table_source.pbf
./tests/vtzero-show table_source.pbf
curl -sS "localhost:3000/public.points3857/0/0/0.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf

- name: Test server response for table source with empty SRID
run: |
curl "localhost:3000/public.points_empty_srid/0/0/0.pbf" > table_source.pbf
./tests/vtzero-check table_source.pbf
./tests/vtzero-show table_source.pbf
curl -sS "localhost:3000/public.points_empty_srid/0/0/0.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf

- name: Prepare config
run: sed -i 's/5432/${{ job.services.postgres.ports[5432] }}/g' ./tests/config.yaml
Expand All @@ -194,19 +194,19 @@ jobs:

- name: Test server response
run: |
curl "localhost:3000/index.json" | jq -e
curl "localhost:3000/public.table_source/0/0/0.pbf" > table_source.pbf
./tests/vtzero-check table_source.pbf
./tests/vtzero-show table_source.pbf
curl "localhost:3000/public.points1,public.points2/0/0/0.pbf" > composite_source.pbf
./tests/vtzero-check composite_source.pbf
./tests/vtzero-show composite_source.pbf
curl "localhost:3000/rpc/public.function_source/0/0/0.pbf" > function_source.pbf
./tests/vtzero-check function_source.pbf
./tests/vtzero-show function_source.pbf
curl "localhost:3000/rpc/public.function_source_query_params/0/0/0.pbf?token=martin" > function_source_query_params.pbf
./tests/vtzero-check function_source_query_params.pbf
./tests/vtzero-show function_source_query_params.pbf
curl -sS "localhost:3000/index.json" | jq -e
curl -sS "localhost:3000/public.table_source/0/0/0.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/public.points1,public.points2/0/0/0.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/rpc/public.function_source/0/0/0.pbf" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf
curl -sS "localhost:3000/rpc/public.function_source_query_params/0/0/0.pbf?token=martin" > tmp.pbf
./tests/vtzero-check tmp.pbf
./tests/vtzero-show tmp.pbf

build:
needs: [test]
Expand Down
2 changes: 2 additions & 0 deletions benches/sources.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ fn mock_table_source(schema: &str, table: &str) -> TableSource {
clip_geom: Some(true),
geometry_type: None,
properties: HashMap::new(),
unrecognized: HashMap::new(),
}
}

Expand All @@ -33,6 +34,7 @@ fn mock_function_source(schema: &str, function: &str) -> FunctionSource {
minzoom: None,
maxzoom: None,
bounds: None,
unrecognized: HashMap::new(),
}
}

Expand Down
2 changes: 2 additions & 0 deletions src/bin/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use martin::pg::config::{PgArgs, PgConfigBuilder};
use martin::pg::db::configure_db_sources;
use martin::srv::config::{SrvArgs, SrvConfigBuilder};
use martin::srv::server;
use std::collections::HashMap;
use std::{env, io};

const VERSION: &str = env!("CARGO_PKG_VERSION");
Expand Down Expand Up @@ -41,6 +42,7 @@ impl From<Args> for ConfigBuilder {
ConfigBuilder {
srv: SrvConfigBuilder::from(args.srv),
pg: PgConfigBuilder::from((args.pg, args.connection)),
unrecognized: HashMap::new(),
}
}
}
Expand Down
15 changes: 15 additions & 0 deletions src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
use crate::pg::config::{PgConfig, PgConfigBuilder};
use crate::prettify_error;
use crate::srv::config::{SrvConfig, SrvConfigBuilder};
use log::warn;
use serde::{Deserialize, Serialize};
use serde_yaml::Value;
use std::collections::HashMap;
use std::fs::File;
use std::io;
use std::io::prelude::*;
Expand All @@ -20,6 +23,8 @@ pub struct ConfigBuilder {
pub srv: SrvConfigBuilder,
#[serde(flatten)]
pub pg: PgConfigBuilder,
#[serde(flatten)]
pub unrecognized: HashMap<String, Value>,
}

/// Update empty option in place with a non-empty value from the second option.
Expand All @@ -33,18 +38,26 @@ impl ConfigBuilder {
pub fn merge(&mut self, other: ConfigBuilder) -> &mut Self {
self.srv.merge(other.srv);
self.pg.merge(other.pg);
self.unrecognized.extend(other.unrecognized);
self
}

/// Apply defaults to the config, and validate if there is a connection string
pub fn finalize(self) -> io::Result<Config> {
report_unrecognized_config("", &self.unrecognized);
Ok(Config {
srv: self.srv.finalize()?,
pg: self.pg.finalize()?,
})
}
}

pub fn report_unrecognized_config(prefix: &str, unrecognized: &HashMap<String, Value>) {
for key in unrecognized.keys() {
warn!("Unrecognized config key: {prefix}{key}");
}
}

/// Read config from a file
pub fn read_config(file_name: &str) -> io::Result<ConfigBuilder> {
let mut file = File::open(file_name)
Expand Down Expand Up @@ -142,6 +155,7 @@ mod tests {
clip_geom: Some(true),
geometry_type: Some("GEOMETRY".to_string()),
properties: HashMap::from([("gid".to_string(), "int4".to_string())]),
unrecognized: HashMap::new(),
}),
)]),
function_sources: HashMap::from([(
Expand All @@ -158,6 +172,7 @@ mod tests {
right: 180.0,
top: 90.0,
}),
unrecognized: HashMap::new(),
}),
)]),
},
Expand Down
14 changes: 13 additions & 1 deletion src/pg/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use crate::config::set_option;
use crate::config::{report_unrecognized_config, set_option};
use crate::pg::function_source::FunctionSources;
use crate::pg::table_source::TableSources;
use serde::{Deserialize, Serialize};
Expand All @@ -25,8 +25,10 @@ pub struct PgArgs {
#[derive(Clone, Debug, Serialize, PartialEq)]
pub struct PgConfig {
pub connection_string: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub ca_root_file: Option<String>,
pub danger_accept_invalid_certs: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub default_srid: Option<i32>,
pub pool_size: u32,
pub use_dynamic_sources: bool,
Expand Down Expand Up @@ -62,6 +64,16 @@ impl PgConfigBuilder {

/// Apply defaults to the config, and validate if there is a connection string
pub fn finalize(self) -> io::Result<PgConfig> {
if let Some(ref ts) = self.table_sources {
for (k, v) in ts {
report_unrecognized_config(&format!("table_sources.{}.", k), &v.unrecognized);
}
}
if let Some(ref fs) = self.function_sources {
for (k, v) in fs {
report_unrecognized_config(&format!("function_sources.{}.", k), &v.unrecognized);
}
}
let connection_string = self.connection_string.ok_or_else(|| {
io::Error::new(
io::ErrorKind::Other,
Expand Down
Loading