From 387b467958a702168c443941a91ce640481343f3 Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Fri, 6 Sep 2024 14:55:27 +0200 Subject: [PATCH 1/3] updates --- Cargo.lock | 584 +++++++++++------- datatypes/Cargo.toml | 10 +- .../src/collections/feature_collection.rs | 9 +- datatypes/src/primitives/circle.rs | 2 +- datatypes/tests/example-arrow.rs | 4 +- expression/Cargo.toml | 2 +- operators/Cargo.toml | 11 +- .../circle_merging_quadtree/node.rs | 2 +- operators/src/source/gdal_source/mod.rs | 8 +- operators/src/util/gdal.rs | 11 +- .../src/util/raster_stream_to_geotiff.rs | 108 ++-- rust-toolchain.toml | 2 +- services/Cargo.toml | 30 +- services/build.rs | 25 +- services/src/api/model/services.rs | 2 +- services/src/bin/main.rs | 19 +- .../src/datasets/external/netcdfcf/error.rs | 5 + .../datasets/external/netcdfcf/overviews.rs | 63 +- services/src/datasets/listing.rs | 4 +- services/src/datasets/storage.rs | 4 +- services/src/layers/layer.rs | 2 +- services/src/projects/project.rs | 5 +- services/src/tasks/mod.rs | 2 +- 23 files changed, 512 insertions(+), 402 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e39862d2..a6b26913c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -562,9 +562,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "50.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa285343fba4d829d49985bdc541e3789cf6000ed0e84be7c039438df4a4e78c" +checksum = "45aef0d9cf9a039bf6cd1acc451b137aca819977b0928dece52bd92811b640ba" dependencies = [ "arrow-arith", "arrow-array", @@ -583,9 +583,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "50.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "753abd0a5290c1bcade7c6623a556f7d1659c5f4148b140b5b63ce7bd1a45705" +checksum = "03675e42d1560790f3524800e41403b40d0da1c793fe9528929fde06d8c7649a" dependencies = [ "arrow-array", "arrow-buffer", @@ -598,9 +598,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "50.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609" +checksum = "cd2bf348cf9f02a5975c5962c7fa6dee107a2009a7b41ac5fb1a027e12dc033f" dependencies = [ "ahash", "arrow-buffer", @@ -614,9 +614,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "50.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69615b061701bcdffbc62756bc7e85c827d5290b472b580c972ebbbf690f5aa4" +checksum = "3092e37715f168976012ce52273c3989b5793b0db5f06cbaa246be25e5f0924d" dependencies = [ "bytes", "half", @@ -625,27 +625,29 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "50.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e448e5dd2f4113bf5b74a1f26531708f5edcacc77335b7066f9398f4bcf4cdef" +checksum = "7ce1018bb710d502f9db06af026ed3561552e493e989a79d0d0f5d9cf267a785" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", - "base64 0.21.7", + "atoi", + "base64 0.22.1", "chrono", "half", "lexical-core", "num", + "ryu", ] [[package]] name = "arrow-csv" -version = "50.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46af72211f0712612f5b18325530b9ad1bfbdc87290d5fbfd32a7da128983781" +checksum = "fd178575f45624d045e4ebee714e246a05d9652e41363ee3f57ec18cca97f740" dependencies = [ "arrow-array", "arrow-buffer", @@ -662,9 +664,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "50.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67d644b91a162f3ad3135ce1184d0a31c28b816a581e08f29e8e9277a574c64e" +checksum = "4e4ac0c4ee79150afe067dc4857154b3ee9c1cd52b5f40d59a77306d0ed18d65" dependencies = [ "arrow-buffer", "arrow-schema", @@ -674,9 +676,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "50.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03dea5e79b48de6c2e04f03f62b0afea7105be7b77d134f6c5414868feefb80d" +checksum = "bb307482348a1267f91b0912e962cd53440e5de0f7fb24c5f7b10da70b38c94a" dependencies = [ "arrow-array", "arrow-buffer", @@ -690,9 +692,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "50.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8950719280397a47d37ac01492e3506a8a724b3fb81001900b866637a829ee0f" +checksum = "d24805ba326758effdd6f2cbdd482fcfab749544f21b134701add25b33f474e6" dependencies = [ "arrow-array", "arrow-buffer", @@ -710,9 +712,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "50.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ed9630979034077982d8e74a942b7ac228f33dd93a93b615b4d02ad60c260be" +checksum = "644046c479d80ae8ed02a7f1e1399072ea344ca6a7b0e293ab2d5d9ed924aa3b" dependencies = [ "arrow-array", "arrow-buffer", @@ -725,9 +727,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "50.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "007035e17ae09c4e8993e4cb8b5b96edf0afb927cd38e2dff27189b274d83dcf" +checksum = "a29791f8eb13b340ce35525b723f5f0df17ecb955599e11f65c2a94ab34e2efb" dependencies = [ "ahash", "arrow-array", @@ -735,20 +737,19 @@ dependencies = [ "arrow-data", "arrow-schema", "half", - "hashbrown 0.14.5", ] [[package]] name = "arrow-schema" -version = "50.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ff3e9c01f7cd169379d269f926892d0e622a704960350d09d331be3ec9e0029" +checksum = "c85320a3a2facf2b2822b57aa9d6d9d55edb8aee0b6b5d3b8df158e503d10858" [[package]] name = "arrow-select" -version = "50.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c" +checksum = "9cc7e6b582e23855fd1625ce46e51647aa440c20ea2e71b1d748e0839dd73cba" dependencies = [ "ahash", "arrow-array", @@ -760,15 +761,16 @@ dependencies = [ [[package]] name = "arrow-string" -version = "50.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f3b37f2aeece31a2636d1b037dabb69ef590e03bdc7eb68519b51ec86932a7" +checksum = "0775b6567c66e56ded19b87a954b6b1beffbdd784ef95a3a2b03f59570c1d230" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "memchr", "num", "regex", "regex-syntax 0.8.4", @@ -780,12 +782,12 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92c7de98d1e35610d7768dad99ba004769def30538d500d93725a912d6f0deb5" dependencies = [ - "prost", + "prost 0.12.6", "prost-types", "prost-wkt-types", "serde", "serde_json", - "tonic", + "tonic 0.11.0", "tonic-build", ] @@ -818,7 +820,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc65048dd435533bb1baf2ed9956b9a278fbfdcf90301b39ee117f06c0199d37" dependencies = [ "anstyle", - "bstr 1.10.0", + "bstr", "doc-comment", "predicates", "predicates-core", @@ -859,6 +861,15 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-polyfill" version = "1.0.3" @@ -921,7 +932,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.3.4", "bitflags 1.3.2", "bytes", "futures-util", @@ -942,6 +953,33 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +dependencies = [ + "async-trait", + "axum-core 0.4.3", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 1.0.1", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.3.4" @@ -959,6 +997,26 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 0.1.2", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.73" @@ -1137,17 +1195,6 @@ dependencies = [ "alloc-stdlib", ] -[[package]] -name = "bstr" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" -dependencies = [ - "lazy_static", - "memchr", - "regex-automata 0.1.10", -] - [[package]] name = "bstr" version = "1.10.0" @@ -2396,9 +2443,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" -version = "23.5.26" +version = "24.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -2417,9 +2464,9 @@ dependencies = [ [[package]] name = "flexi_logger" -version = "0.28.5" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cca927478b3747ba47f98af6ba0ac0daea4f12d12f55e9104071b3dc00276310" +checksum = "a250587a211932896a131f214a4f64c047b826ce072d2018764e5ff5141df8fa" dependencies = [ "chrono", "crossbeam-channel", @@ -2591,9 +2638,9 @@ dependencies = [ [[package]] name = "gdal" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6639365794fa1f35f36f8cd3e2ea21c94c9b04aa8d9ee0d0f3324621efeb0800" +checksum = "f9ceebb762ff36ae45feb3e636752bcdada8e5379d90d73019763aabd3342fcd" dependencies = [ "bitflags 2.6.0", "chrono", @@ -2607,9 +2654,9 @@ dependencies = [ [[package]] name = "gdal-sys" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab73a6c9d0d620fb5b07e02ffbb8ae08f053299cbe5db0c1bbe1d58cb33e3f38" +checksum = "18ad5d608ee6726efcf6e1d91261eb6dec7da3ee7db6bda984cdfb8a7d65ebf9" dependencies = [ "libc", "pkg-config", @@ -2675,6 +2722,23 @@ dependencies = [ "spade", ] +[[package]] +name = "geo" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f811f663912a69249fa620dcd2a005db7254529da2d8a0b23942e81f47084501" +dependencies = [ + "earcutr", + "float_next_after", + "geo-types", + "geographiclib-rs", + "log", + "num-traits", + "robust 1.1.0", + "rstar 0.12.0", + "spade", +] + [[package]] name = "geo-rand" version = "0.3.0" @@ -2694,6 +2758,7 @@ dependencies = [ "approx 0.5.1", "num-traits", "rstar 0.11.0", + "rstar 0.12.0", "rstar 0.8.4", "serde", ] @@ -2711,7 +2776,7 @@ dependencies = [ "fallible-iterator", "float-cmp", "gdal", - "geo 0.27.0", + "geo 0.28.0", "geojson", "image", "num", @@ -2737,7 +2802,7 @@ name = "geoengine-expression" version = "0.8.0" dependencies = [ "cargo", - "geo 0.27.0", + "geo 0.28.0", "geoengine-expression-deps", "libloading", "log", @@ -2784,11 +2849,11 @@ dependencies = [ "futures", "gdal", "gdal-sys", - "geo 0.27.0", + "geo 0.28.0", "geo-rand", "geoengine-datatypes", "geoengine-expression", - "httptest 0.16.1", + "httptest", "itertools 0.13.0", "libloading", "log", @@ -2834,6 +2899,7 @@ dependencies = [ "actix-web-actors", "actix-web-httpauth", "aes-gcm", + "anyhow", "aruna-rust-api", "assert-json-diff", "assert_cmd", @@ -2852,12 +2918,12 @@ dependencies = [ "futures-util", "gdal", "gdal-sys", - "geo 0.27.0", + "geo 0.28.0", "geoengine-datatypes", "geoengine-macros", "geoengine-operators", "geojson", - "httptest 0.15.5", + "httptest", "itertools 0.13.0", "log", "mime", @@ -2866,14 +2932,16 @@ dependencies = [ "oauth2", "openidconnect", "opentelemetry", - "opentelemetry-jaeger", + "opentelemetry-appender-tracing", + "opentelemetry-otlp", + "opentelemetry_sdk", "ordered-float 4.2.2", "pbkdf2", "postgres-types", "pretty_assertions", "proj", "proj-sys", - "prost", + "prost 0.12.6", "pwhash", "rand", "rayon", @@ -2891,7 +2959,7 @@ dependencies = [ "time", "tokio", "tokio-postgres", - "tonic", + "tonic 0.11.0", "tracing", "tracing-actix-web", "tracing-opentelemetry", @@ -2903,6 +2971,7 @@ dependencies = [ "uuid", "validator", "vergen", + "vergen-gitcl", "walkdir", "xml-rs", "zip 2.1.5", @@ -2944,6 +3013,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getset" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e45727250e75cc04ff2846a66397da8ef2b3db8e40e0cef4df67950a07621eb9" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "ghash" version = "0.5.1" @@ -3057,7 +3138,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eadca029ef716b4378f7afb19f7ee101fde9e58ba1f1445971315ac866db417" dependencies = [ - "bstr 1.10.0", + "bstr", "btoi", "gix-date", "itoa", @@ -3071,7 +3152,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f395469d38c76ec47cd1a6c5a53fbc3f13f737b96eaf7535f4e6b367e643381" dependencies = [ - "bstr 1.10.0", + "bstr", "gix-glob", "gix-path", "gix-quote", @@ -3106,7 +3187,7 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c576cfbf577f72c097b5f88aedea502cd62952bdc1fb3adcab4531d5525a4c7" dependencies = [ - "bstr 1.10.0", + "bstr", ] [[package]] @@ -3115,7 +3196,7 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85a7007ba021f059803afaf6f8a48872422abc20550ac12ede6ddea2936cec36" dependencies = [ - "bstr 1.10.0", + "bstr", "gix-chunk", "gix-features 0.36.1", "gix-hash", @@ -3129,7 +3210,7 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cae98c6b4c66c09379bc35274b172587d6b0ac369a416c39128ad8c6454f9bb" dependencies = [ - "bstr 1.10.0", + "bstr", "gix-config-value", "gix-features 0.36.1", "gix-glob", @@ -3151,7 +3232,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b328997d74dd15dc71b2773b162cb4af9a25c424105e4876e6d0686ab41c383e" dependencies = [ "bitflags 2.6.0", - "bstr 1.10.0", + "bstr", "gix-path", "libc", "thiserror", @@ -3163,7 +3244,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c5c5d74069b842a1861e581027ac6b7ad9ff66f5911c89b9f45484d7ebda6a4" dependencies = [ - "bstr 1.10.0", + "bstr", "gix-command", "gix-config-value", "gix-path", @@ -3179,7 +3260,7 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9eed6931f21491ee0aeb922751bd7ec97b4b2fe8fbfedcb678e2a2dce5f3b8c0" dependencies = [ - "bstr 1.10.0", + "bstr", "itoa", "thiserror", "time", @@ -3202,7 +3283,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a45d5cf0321178883e38705ab2b098f625d609a7d4c391b33ac952eff2c490f2" dependencies = [ - "bstr 1.10.0", + "bstr", "dunce", "gix-hash", "gix-path", @@ -3249,7 +3330,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92f674d3fdb6b1987b04521ec9a5b7be8650671f2c4bbd17c3c81e2a364242ff" dependencies = [ - "bstr 1.10.0", + "bstr", "encoding_rs", "gix-attributes", "gix-command", @@ -3279,7 +3360,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5db19298c5eeea2961e5b3bf190767a2d1f09b8802aeb5f258e42276350aff19" dependencies = [ "bitflags 2.6.0", - "bstr 1.10.0", + "bstr", "gix-features 0.36.1", "gix-path", ] @@ -3311,7 +3392,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a215cc8cf21645bca131fcf6329d3ebd46299c47dbbe27df71bb1ca9e328b879" dependencies = [ - "bstr 1.10.0", + "bstr", "gix-glob", "gix-path", "unicode-bom", @@ -3324,7 +3405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c83a4fcc121b2f2e109088f677f89f85e7a8ebf39e8e6659c0ae54d4283b1650" dependencies = [ "bitflags 2.6.0", - "bstr 1.10.0", + "bstr", "btoi", "filetime", "gix-bitmap", @@ -3384,7 +3465,7 @@ version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "740f2a44267f58770a1cb3a3d01d14e67b089c7136c48d4bddbb3cfd2bf86a51" dependencies = [ - "bstr 1.10.0", + "bstr", "btoi", "gix-actor", "gix-date", @@ -3442,7 +3523,7 @@ version = "0.16.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a8384b1e964151aff0d5632dd9b191059d07dff358b96bd940f1b452600d7ab" dependencies = [ - "bstr 1.10.0", + "bstr", "faster-hex 0.8.1", "thiserror", ] @@ -3453,7 +3534,7 @@ version = "0.16.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8395f7501c84d6a1fe902035fdfd8cd86d89e2dd6be0200ec1a72fd3c92d39" dependencies = [ - "bstr 1.10.0", + "bstr", "faster-hex 0.8.1", "thiserror", ] @@ -3464,7 +3545,7 @@ version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d23d5bbda31344d8abc8de7c075b3cf26e5873feba7c4a15d916bce67382bd9" dependencies = [ - "bstr 1.10.0", + "bstr", "gix-trace", "home", "once_cell", @@ -3478,7 +3559,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dbbb92f75a38ef043c8bb830b339b38d0698d7f3746968b5fcbade7a880494d" dependencies = [ "bitflags 2.6.0", - "bstr 1.10.0", + "bstr", "gix-attributes", "gix-config-value", "gix-glob", @@ -3505,7 +3586,7 @@ version = "0.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "391e3feabdfa5f90dad6673ce59e3291ac28901b2ff248d86c5a7fbde0391e0e" dependencies = [ - "bstr 1.10.0", + "bstr", "btoi", "gix-credentials", "gix-date", @@ -3523,7 +3604,7 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbff4f9b9ea3fa7a25a70ee62f545143abef624ac6aa5884344e70c8b0a1d9ff" dependencies = [ - "bstr 1.10.0", + "bstr", "gix-utils", "thiserror", ] @@ -3555,7 +3636,7 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccb0974cc41dbdb43a180c7f67aa481e1c1e160fcfa8f4a55291fd1126c1a6e7" dependencies = [ - "bstr 1.10.0", + "bstr", "gix-hash", "gix-revision", "gix-validate", @@ -3569,7 +3650,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ca97ac73459a7f3766aa4a5638a6e37d56d4c7962bc1986fbaf4883d0772588" dependencies = [ - "bstr 1.10.0", + "bstr", "gix-date", "gix-hash", "gix-hashtable", @@ -3612,7 +3693,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bba78c8d12aa24370178453ec3a472ff08dfaa657d116229f57f2c9cd469a1c2" dependencies = [ - "bstr 1.10.0", + "bstr", "gix-config", "gix-path", "gix-pathspec", @@ -3647,7 +3728,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f209a93364e24f20319751bc11092272e2f3fe82bb72592b2822679cf5be752" dependencies = [ "base64 0.21.7", - "bstr 1.10.0", + "bstr", "curl", "gix-command", "gix-credentials", @@ -3681,7 +3762,7 @@ version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c427a1a11ccfa53a4a2da47d9442c2241deee63a154bc15cc14b8312fbc4005" dependencies = [ - "bstr 1.10.0", + "bstr", "gix-features 0.36.1", "gix-path", "home", @@ -3705,7 +3786,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82c27dd34a49b1addf193c92070bcbf3beaf6e10f16a78544de6372e146a0acf" dependencies = [ - "bstr 1.10.0", + "bstr", "thiserror", ] @@ -3715,7 +3796,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddaf79e721dba64fe726a42f297a3c8ed42e55cdc0d81ca68452f2def3c2d7fd" dependencies = [ - "bstr 1.10.0", + "bstr", "gix-attributes", "gix-features 0.36.1", "gix-fs", @@ -3740,7 +3821,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" dependencies = [ "aho-corasick", - "bstr 1.10.0", + "bstr", "log", "regex-automata 0.4.7", "regex-syntax 0.8.4", @@ -3824,6 +3905,15 @@ dependencies = [ "byteorder", ] +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -3871,6 +3961,16 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.4.1" @@ -4024,35 +4124,13 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "httptest" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8b44a11846bda8c9fe9194f9924db7132c34635c7ce020f180f6c5d46d2308f" -dependencies = [ - "bstr 0.2.17", - "bytes", - "crossbeam-channel", - "form_urlencoded", - "futures", - "http 0.2.12", - "hyper 0.14.30", - "log", - "once_cell", - "regex", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", -] - [[package]] name = "httptest" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae0fc8d140f1f0f3e7f821c8eff55cd13966db7a3370b2d9a7b08e9ec8ee8786" dependencies = [ - "bstr 1.10.0", + "bstr", "bytes", "crossbeam-channel", "form_urlencoded", @@ -4165,6 +4243,19 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "hyper-timeout" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +dependencies = [ + "hyper 1.4.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -4227,16 +4318,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.5.0" @@ -4253,12 +4334,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9007da9cacbd3e6343da136e98b0d2df013f553d35bdec8b518f07bea768e19c" -[[package]] -name = "if_chain" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed" - [[package]] name = "ignore" version = "0.4.22" @@ -4379,12 +4454,6 @@ dependencies = [ "generic-array 0.14.7", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "interpolate_name" version = "0.2.4" @@ -5210,16 +5279,6 @@ dependencies = [ "libm", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi 0.3.9", - "libc", -] - [[package]] name = "num_enum" version = "0.7.3" @@ -5303,7 +5362,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c62dcb6174f9cb326eac248f07e955d5d559c272730b6c03e396b443b562788" dependencies = [ - "bstr 1.10.0", + "bstr", "normpath", "winapi", ] @@ -5386,61 +5445,76 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.21.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" +checksum = "4c365a63eec4f55b7efeceb724f1336f26a9cf3427b70e59e2cd2a5b947fba96" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.2.6", "js-sys", "once_cell", "pin-project-lite", "thiserror", - "urlencoding", ] [[package]] -name = "opentelemetry-jaeger" -version = "0.20.0" +name = "opentelemetry-appender-tracing" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e617c66fd588e40e0dbbd66932fdc87393095b125d4459b1a3a10feb1712f8a1" +checksum = "b84de945cb3a6f1e0d6317cbd998bbd0519ab00f4b790db67e0ff4fdcf7cedb6" +dependencies = [ + "opentelemetry", + "tracing", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b925a602ffb916fb7421276b86756027b37ee708f9dce2dbdcc51739f07e727" dependencies = [ "async-trait", "futures-core", - "futures-util", + "http 1.1.0", "opentelemetry", - "opentelemetry-semantic-conventions", + "opentelemetry-proto", "opentelemetry_sdk", - "thrift", + "prost 0.13.2", + "thiserror", + "tokio", + "tonic 0.12.2", ] [[package]] -name = "opentelemetry-semantic-conventions" -version = "0.13.0" +name = "opentelemetry-proto" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" +checksum = "30ee9f20bff9c984511a02f082dc8ede839e4a9bf15cc2487c8d6fea5ad850d9" dependencies = [ "opentelemetry", + "opentelemetry_sdk", + "prost 0.13.2", + "tonic 0.12.2", ] [[package]] name = "opentelemetry_sdk" -version = "0.21.2" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" +checksum = "692eac490ec80f24a17828d49b40b60f5aeaccdfe6a503f939713afd22bc28df" dependencies = [ "async-trait", - "crossbeam-channel", "futures-channel", "futures-executor", "futures-util", "glob", "once_cell", "opentelemetry", - "ordered-float 4.2.2", "percent-encoding", "rand", + "serde_json", "thiserror", ] @@ -6094,7 +6168,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.12.6", +] + +[[package]] +name = "prost" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2ecbe40f08db5c006b5764a2645f7f3f141ce756412ac9e1dd6087e6d32995" +dependencies = [ + "bytes", + "prost-derive 0.13.2", ] [[package]] @@ -6111,7 +6195,7 @@ dependencies = [ "once_cell", "petgraph", "prettyplease", - "prost", + "prost 0.12.6", "prost-types", "regex", "syn 2.0.72", @@ -6131,13 +6215,26 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "prost-derive" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf0c195eebb4af52c752bec4f52f645da98b6e92077a04110c7f349477ae5ac" +dependencies = [ + "anyhow", + "itertools 0.13.0", + "proc-macro2", + "quote", + "syn 2.0.72", +] + [[package]] name = "prost-types" version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" dependencies = [ - "prost", + "prost 0.12.6", ] [[package]] @@ -6148,7 +6245,7 @@ checksum = "5fb7ec2850c138ebaa7ab682503b5d08c3cb330343e9c94776612928b6ddb53f" dependencies = [ "chrono", "inventory", - "prost", + "prost 0.12.6", "serde", "serde_derive", "serde_json", @@ -6162,7 +6259,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "598b7365952c2ed4e32902de0533653aafbe5ae3da436e8e2335c7d375a1cef3" dependencies = [ "heck 0.5.0", - "prost", + "prost 0.12.6", "prost-build", "prost-types", "quote", @@ -6175,7 +6272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a8eadc2381640a49c1fbfb9f4a857794b4e5bf5a2cbc2d858cfdb74f64dcd22" dependencies = [ "chrono", - "prost", + "prost 0.12.6", "prost-build", "prost-types", "prost-wkt", @@ -6657,6 +6754,17 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rstar" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133315eb94c7b1e8d0cb097e5a710d850263372fd028fff18969de708afc7008" +dependencies = [ + "heapless 0.8.0", + "num-traits", + "smallvec", +] + [[package]] name = "rust-embed" version = "8.5.0" @@ -7574,28 +7682,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "threadpool" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" -dependencies = [ - "num_cpus", -] - -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "log", - "ordered-float 2.10.1", - "threadpool", -] - [[package]] name = "tiff" version = "0.9.1" @@ -7871,17 +7957,17 @@ checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" dependencies = [ "async-stream", "async-trait", - "axum", + "axum 0.6.20", "base64 0.21.7", "bytes", "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.30", - "hyper-timeout", + "hyper-timeout 0.4.1", "percent-encoding", "pin-project", - "prost", + "prost 0.12.6", "rustls-native-certs", "rustls-pemfile 2.1.2", "rustls-pki-types", @@ -7894,6 +7980,36 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f6ba989e4b2c58ae83d862d3a3e27690b6e3ae630d0deb59f3697f32aa88ad" +dependencies = [ + "async-stream", + "async-trait", + "axum 0.7.5", + "base64 0.22.1", + "bytes", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.4.1", + "hyper-timeout 0.5.1", + "hyper-util", + "percent-encoding", + "pin-project", + "prost 0.13.2", + "socket2", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tonic-build" version = "0.11.0" @@ -7998,9 +8114,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.22.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" +checksum = "a9784ed4da7d921bc8df6963f8c80a0e4ce34ba6ba76668acadd3edbd985ff3b" dependencies = [ "js-sys", "once_cell", @@ -8203,17 +8319,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna", "percent-encoding", "serde", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - [[package]] name = "utf8parse" version = "0.2.2" @@ -8295,12 +8405,12 @@ checksum = "4e8257fbc510f0a46eb602c10215901938b5c2a7d5e70fc11483b1d3c9b5b18c" [[package]] name = "validator" -version = "0.16.1" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b92f40481c04ff1f4f61f304d61793c7b56ff76ac1469f1beb199b1445b253bd" +checksum = "db79c75af171630a3148bd3e6d7c4f42b6a9a014c2945bc5ed0020cbb8d9478e" dependencies = [ - "idna 0.4.0", - "lazy_static", + "idna", + "once_cell", "regex", "serde", "serde_derive", @@ -8311,28 +8421,16 @@ dependencies = [ [[package]] name = "validator_derive" -version = "0.16.0" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc44ca3088bb3ba384d9aecf40c6a23a676ce23e09bdaca2073d99c207f864af" +checksum = "df0bcf92720c40105ac4b2dda2a4ea3aa717d4d6a862cc217da653a4bd5c6b10" dependencies = [ - "if_chain", - "lazy_static", + "darling", + "once_cell", "proc-macro-error", "proc-macro2", "quote", - "regex", - "syn 1.0.109", - "validator_types", -] - -[[package]] -name = "validator_types" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "111abfe30072511849c5910134e8baf8dc05de4c0e5903d681cbd5c9c4d611e3" -dependencies = [ - "proc-macro2", - "syn 1.0.109", + "syn 2.0.72", ] [[package]] @@ -8355,16 +8453,44 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "vergen" -version = "8.3.2" +version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566" +checksum = "c32e7318e93a9ac53693b6caccfb05ff22e04a44c7cf8a279051f24c09da286f" dependencies = [ "anyhow", "cargo_metadata", - "cfg-if", + "derive_builder", + "getset", "regex", "rustversion", "time", + "vergen-lib", +] + +[[package]] +name = "vergen-gitcl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bbdc9746577cb4767f218d320ee0b623d415e8130332f8f562b910b61cc2c4e" +dependencies = [ + "anyhow", + "derive_builder", + "rustversion", + "time", + "vergen", + "vergen-lib", +] + +[[package]] +name = "vergen-lib" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e06bee42361e43b60f363bad49d63798d0f42fb1768091812270eca00c784720" +dependencies = [ + "anyhow", + "derive_builder", + "getset", + "rustversion", ] [[package]] @@ -8510,9 +8636,9 @@ dependencies = [ [[package]] name = "web-time" -version = "0.2.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/datatypes/Cargo.toml b/datatypes/Cargo.toml index 26dfe35a3..9c4f0a6d0 100644 --- a/datatypes/Cargo.toml +++ b/datatypes/Cargo.toml @@ -13,15 +13,15 @@ repository.workspace = true pro = [] [dependencies] -arrow = { version = "50.0", features = ["ipc_compression"] } -arrow-array = "50.0" -arrow-ord = "50.0" +arrow = { version = "53.0", features = ["ipc_compression"] } +arrow-array = "53.0" +arrow-ord = "53.0" bytes = "1.5" # for postgres-types impls chrono = "0.4" fallible-iterator = "0.2" # only for postgres-protocol float-cmp = "0.9" -gdal = "0.16" -geo = "0.27" +gdal = "0.17" +geo = "0.28" geojson = "0.24" image = "0.25" num = "0.4" diff --git a/datatypes/src/collections/feature_collection.rs b/datatypes/src/collections/feature_collection.rs index 22485603d..ac778afca 100644 --- a/datatypes/src/collections/feature_collection.rs +++ b/datatypes/src/collections/feature_collection.rs @@ -1768,6 +1768,8 @@ mod tests { #[test] fn it_does_not_json_serialize() { + use arrow::record_batch::RecordBatch; + let collection = FeatureCollection::::from_data( vec![(0.0, 0.1).into()], vec![TimeInterval::new(0, 1).unwrap(); 1], @@ -1783,9 +1785,12 @@ mod tests { .unwrap(); let struct_array = collection.table; - let array: Arc = Arc::new(struct_array); // TODO: if this stops failing, change the strange custom byte serialization to use JSON - arrow::json::writer::array_to_json_array(&array).unwrap_err(); + let json = Vec::new(); + let mut json_writer = arrow::json::ArrayWriter::new(json); + json_writer + .write(&RecordBatch::from(struct_array)) + .unwrap_err(); } } diff --git a/datatypes/src/primitives/circle.rs b/datatypes/src/primitives/circle.rs index 3e9e6dafc..655e582b2 100644 --- a/datatypes/src/primitives/circle.rs +++ b/datatypes/src/primitives/circle.rs @@ -1,4 +1,4 @@ -use geo::intersects::Intersects; +use geo::Intersects; use num_traits::abs; use crate::operations::Contains; diff --git a/datatypes/tests/example-arrow.rs b/datatypes/tests/example-arrow.rs index 1fe5d643b..53c0f98c7 100755 --- a/datatypes/tests/example-arrow.rs +++ b/datatypes/tests/example-arrow.rs @@ -445,7 +445,7 @@ fn multipoints() { false, )))) .len(2) // number of multipoints - .add_buffer(Buffer::from(&[0_i32, 2, 5].to_byte_slice())) + .add_buffer(Buffer::from([0_i32, 2, 5].to_byte_slice())) .add_child_data( ArrayData::builder(DataType::FixedSizeList( Arc::new(Field::new("", DataType::Float64, false)), @@ -456,7 +456,7 @@ fn multipoints() { ArrayData::builder(DataType::Float64) .len(10) // number of floats .add_buffer(Buffer::from( - &[ + [ 1_f64, 2., 11., 12., 21., 22., 31., 32., 41., 42., 51., 52., 61., 62., 71., 72., 81., 82., 91., 92., ] diff --git a/expression/Cargo.toml b/expression/Cargo.toml index 11185e018..fb0a99b3a 100644 --- a/expression/Cargo.toml +++ b/expression/Cargo.toml @@ -23,7 +23,7 @@ syn = "2.0" tempfile = "3.10" [dev-dependencies] -geo = "0.27.0" +geo = "0.28" pretty_assertions = "1.4" [lints] diff --git a/operators/Cargo.toml b/operators/Cargo.toml index c54572509..2d606f814 100644 --- a/operators/Cargo.toml +++ b/operators/Cargo.toml @@ -13,7 +13,7 @@ repository.workspace = true pro = ["geoengine-datatypes/pro"] [dependencies] -arrow = { version = "50.0" } +arrow = { version = "53.0" } async-trait = "0.1" bb8-postgres = "0.8" bytes = "1.5" # for postgres-types impls @@ -21,9 +21,9 @@ chrono = "0.4" csv = "1.3" float-cmp = "0.9" futures = "0.3" -gdal = "0.16" -gdal-sys = "0.9" -geo = "0.27" +gdal = "0.17" +gdal-sys = "0.10" +geo = "0.28" geoengine-datatypes = { path = "../datatypes" } geoengine-expression = { path = "../expression" } itertools = "0.13" @@ -31,7 +31,7 @@ libloading = "0.8" log = "0.4" lru = "0.12" lz4_flex = { version = "0.11" } -ndarray = { version = "0.15", features = ["approx"] } +ndarray = { version = "0.15", features = ["approx"] } # has to match with `ort` num-traits = "0.2" num = "0.4" ouroboros = "0.18" @@ -70,7 +70,6 @@ async-stream = "0.3" approx = "0.5" geo-rand = { git = "https://github.com/lelongg/geo-rand", tag = "v0.3.0" } httptest = "0.16.1" -ndarray = "0.15" tracing-subscriber = { version = "0.3", features = ["json"] } rand = { version = "0.8", features = ["small_rng"] } diff --git a/operators/src/processing/circle_merging_quadtree/node.rs b/operators/src/processing/circle_merging_quadtree/node.rs index f3b570048..f68061a79 100644 --- a/operators/src/processing/circle_merging_quadtree/node.rs +++ b/operators/src/processing/circle_merging_quadtree/node.rs @@ -1,4 +1,4 @@ -use geo::algorithm::intersects::Intersects; +use geo::Intersects; use geoengine_datatypes::{operations::Contains, primitives::BoundingBox2D}; use super::circle_of_points::CircleOfPoints; diff --git a/operators/src/source/gdal_source/mod.rs b/operators/src/source/gdal_source/mod.rs index 12bf0c04d..ab17dba81 100644 --- a/operators/src/source/gdal_source/mod.rs +++ b/operators/src/source/gdal_source/mod.rs @@ -939,7 +939,8 @@ where gdal_out_shape, // requested raster size None, // sampling mode )?; - let data_grid = Grid::new(out_shape.clone(), buffer.data)?; + let (_, buffer_data) = buffer.into_shape_and_vec(); + let data_grid = Grid::new(out_shape.clone(), buffer_data)?; let data_grid = if flip_y_axis { data_grid.reversed_y_axis_grid() @@ -981,7 +982,8 @@ where gdal_out_shape, // requested raster size None, // sampling mode )?; - let mask_grid = Grid::new(out_shape, mask_buffer.data)?.map_elements(|p: u8| p > 0); + let (_, mask_buffer_data) = mask_buffer.into_shape_and_vec(); + let mask_grid = Grid::new(out_shape, mask_buffer_data)?.map_elements(|p: u8| p > 0); let mask_grid = if flip_y_axis { mask_grid.reversed_y_axis_grid() @@ -1107,7 +1109,7 @@ fn read_raster_tile_with_properties Result> { - let rasterband = dataset.rasterband(dataset_params.rasterband_channel as isize)?; + let rasterband = dataset.rasterband(dataset_params.rasterband_channel)?; let result_grid = read_grid_and_handle_edges(tile_info, dataset, &rasterband, dataset_params)?; diff --git a/operators/src/util/gdal.rs b/operators/src/util/gdal.rs index bd6d88424..e2104baf0 100644 --- a/operators/src/util/gdal.rs +++ b/operators/src/util/gdal.rs @@ -233,7 +233,7 @@ pub fn gdal_open_dataset_ex(path: &Path, dataset_options: DatasetOptions) -> Res /// type is a complex floating point type, an error is returned pub fn raster_descriptor_from_dataset( dataset: &Dataset, - band: isize, + band: usize, ) -> Result { let rasterband = &dataset.rasterband(band)?; @@ -260,7 +260,7 @@ pub fn raster_descriptor_from_dataset( // a version of `raster_descriptor_from_dataset` that does not read the sref from the dataset but takes it as an argument pub fn raster_descriptor_from_dataset_and_sref( dataset: &Dataset, - band: isize, + band: usize, spatial_ref: SpatialReference, ) -> Result { let rasterband = &dataset.rasterband(band)?; @@ -284,7 +284,7 @@ pub fn raster_descriptor_from_dataset_and_sref( } // TODO: use https://github.com/georust/gdal/pull/271 when merged and released -fn measurement_from_rasterband(dataset: &Dataset, band: isize) -> Result { +fn measurement_from_rasterband(dataset: &Dataset, band: usize) -> Result { unsafe fn _string(raw_ptr: *const std::os::raw::c_char) -> String { let c_str = std::ffi::CStr::from_ptr(raw_ptr); c_str.to_string_lossy().into_owned() @@ -301,7 +301,8 @@ fn measurement_from_rasterband(dataset: &Dataset, band: isize) -> Result Result` - let c_band = gdal_sys::GDALGetRasterBand(dataset.c_dataset(), band as std::os::raw::c_int); + let c_band = + gdal_sys::GDALGetRasterBand(dataset.c_dataset(), band as isize as std::os::raw::c_int); if c_band.is_null() { Err(_last_null_pointer_err("GDALGetRasterBand"))?; } @@ -329,7 +330,7 @@ pub fn gdal_parameters_from_dataset( band_out: Option, open_options: Option>, ) -> Result { - let rasterband = &dataset.rasterband(band as isize)?; + let rasterband = &dataset.rasterband(band)?; Ok(GdalDatasetParameters { file_path: PathBuf::from(path), diff --git a/operators/src/util/raster_stream_to_geotiff.rs b/operators/src/util/raster_stream_to_geotiff.rs index 89194ffac..fcf6a60d0 100644 --- a/operators/src/util/raster_stream_to_geotiff.rs +++ b/operators/src/util/raster_stream_to_geotiff.rs @@ -10,7 +10,7 @@ use crate::{ }; use futures::future::BoxFuture; use futures::{StreamExt, TryFutureExt}; -use gdal::raster::{Buffer, GdalType, RasterBand, RasterCreationOption}; +use gdal::raster::{Buffer, GdalType, RasterBand, RasterCreationOptions}; use gdal::{Dataset, DriverManager, Metadata}; use geoengine_datatypes::primitives::{ AxisAlignedRectangle, BandSelection, DateTimeParseFormat, QueryRectangle, RasterQueryRectangle, @@ -161,7 +161,7 @@ where &gdal_compression_num_threads, gdal_tiff_options.as_cog, use_big_tiff, - ); + )?; let driver = DriverManager::get_driver_by_name("GTiff")?; @@ -183,9 +183,9 @@ where let mut dataset = driver.create_with_band_type_with_options::( &file_path, - width as isize, - height as isize, - num_timesteps as isize, + width, + height, + num_timesteps, &options, )?; dataset.set_spatial_ref(&gdal_tiff_metadata.spatial_reference.try_into()?)?; @@ -471,7 +471,7 @@ struct IntermediateDataset { #[derive(Debug)] struct IntermediateDatasetMetadata { - raster_band_index: isize, + raster_band_index: usize, width: u32, height: u32, use_big_tiff: bool, @@ -601,14 +601,14 @@ impl GdalDatasetHolder

{ &compression_num_threads, gdal_tiff_options.as_cog, intermediate_dataset_metadata.use_big_tiff, - ); + )?; let mut dataset = driver.create_with_band_type_with_options::( &intermediate_dataset_metadata .intermediate_dataset_parameters .file_path, - intermediate_dataset_metadata.width as isize, - intermediate_dataset_metadata.height as isize, + intermediate_dataset_metadata.width as usize, + intermediate_dataset_metadata.height as usize, 1, &options, )?; @@ -838,9 +838,9 @@ impl GdalDatasetWriter

{ let out_no_data_value_p: P = P::from_(no_data_value); let no_data_value_grid = NoDataValueGrid::from_masked_grid(grid_array, out_no_data_value_p); - let buffer = Buffer::new(window_size, no_data_value_grid.inner_grid.data); // TODO: also write mask! + let mut buffer = Buffer::new(window_size, no_data_value_grid.inner_grid.data); // TODO: also write mask! - raster_band.write(window, window_size, &buffer)?; + raster_band.write(window, window_size, &mut buffer)?; Ok(()) } @@ -851,9 +851,9 @@ impl GdalDatasetWriter

{ mut raster_band: RasterBand, ) -> Result<()> { // Write the MaskedGrid data and mask if no no-data value is set. - let data_buffer = Buffer::new(window_size, masked_grid.inner_grid.data); + let mut data_buffer = Buffer::new(window_size, masked_grid.inner_grid.data); - raster_band.write(window, window_size, &data_buffer)?; + raster_band.write(window, window_size, &mut data_buffer)?; // No-data masks are described by the rasterio docs as: // "One is the the valid data mask from GDAL, an unsigned byte array with the same number of rows and columns as the dataset in which non-zero elements (typically 255) indicate that the corresponding data elements are valid. Other elements are invalid, or nodata elements." @@ -862,10 +862,10 @@ impl GdalDatasetWriter

{ masked_grid .validity_mask .map_elements(|is_valid| if is_valid { 255_u8 } else { 0 }); // TODO: investigate if we can transmute the vec of bool to u8. - let mask_buffer = Buffer::new(window_size, mask_grid_gdal_values.data); + let mut mask_buffer = Buffer::new(window_size, mask_grid_gdal_values.data); let mut mask_band = raster_band.open_mask_band()?; - mask_band.write(window, window_size, &mask_buffer)?; + mask_band.write(window, window_size, &mut mask_buffer)?; Ok(()) } @@ -899,47 +899,25 @@ fn create_gdal_tiff_options( compression_num_threads: &str, as_cog: bool, as_big_tiff: bool, -) -> Vec> { - let mut options = vec![ - RasterCreationOption { - key: "COMPRESS", - value: COMPRESSION_FORMAT, - }, - RasterCreationOption { - key: "TILED", - value: "YES", - }, - RasterCreationOption { - key: "ZLEVEL", - value: COMPRESSION_LEVEL, - }, - RasterCreationOption { - key: "NUM_THREADS", - value: compression_num_threads, - }, - RasterCreationOption { - key: "INTERLEAVE", - value: "BAND", - }, - ]; +) -> Result { + let mut options = RasterCreationOptions::new(); + options.add_name_value("COMPRESS", COMPRESSION_FORMAT)?; + options.add_name_value("TILED", "YES")?; + options.add_name_value("ZLEVEL", COMPRESSION_LEVEL)?; + options.add_name_value("NUM_THREADS", compression_num_threads)?; + options.add_name_value("INTERLEAVE", "BAND")?; + if as_cog { // COGs require a block size of 512x512, so we enforce it now so that we do the work only once. - options.push(RasterCreationOption { - key: "BLOCKXSIZE", - value: COG_BLOCK_SIZE, - }); - options.push(RasterCreationOption { - key: "BLOCKYSIZE", - value: COG_BLOCK_SIZE, - }); + options.add_name_value("BLOCKXSIZE", COG_BLOCK_SIZE)?; + options.add_name_value("BLOCKYSIZE", COG_BLOCK_SIZE)?; } + if as_big_tiff { - options.push(RasterCreationOption { - key: "BIGTIFF", - value: "YES", - }); + options.add_name_value("BIGTIFF", "YES")?; } - options + + Ok(options) } #[derive(Debug, Clone, Copy, Serialize, Deserialize)] @@ -1009,30 +987,14 @@ fn geotiff_to_cog( let output_driver = DriverManager::get_driver_by_name("COG")?; let num_threads = &compression_num_threads.to_string(); - let mut options = vec![ - RasterCreationOption { - key: "COMPRESS", - value: COMPRESSION_FORMAT, - }, - RasterCreationOption { - key: "LEVEL", - value: COMPRESSION_LEVEL, - }, - RasterCreationOption { - key: "NUM_THREADS", - value: num_threads, - }, - RasterCreationOption { - key: "BLOCKSIZE", - value: COG_BLOCK_SIZE, - }, - ]; + let mut options = RasterCreationOptions::new(); + options.add_name_value("COMPRESS", COMPRESSION_FORMAT)?; + options.add_name_value("TILED", "YES")?; + options.add_name_value("NUM_THREADS", num_threads)?; + options.add_name_value("BLOCKSIZE", COG_BLOCK_SIZE)?; if as_big_tiff { - options.push(RasterCreationOption { - key: "BIGTIFF", - value: "YES", - }); + options.add_name_value("BIGTIFF", "YES")?; } input_dataset.create_copy(&output_driver, output_file_path, &options)?; diff --git a/rust-toolchain.toml b/rust-toolchain.toml index a99fc7064..fd7d663d3 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.80.0" +channel = "1.80.1" components = ["cargo", "rustfmt", "rust-src", "clippy", "llvm-tools"] diff --git a/services/Cargo.toml b/services/Cargo.toml index 42f4fc6d8..7055f0b99 100644 --- a/services/Cargo.toml +++ b/services/Cargo.toml @@ -40,13 +40,13 @@ chrono = { version = "0.4", features = ["serde"] } config = "0.14" derive_builder = "0.20" erased-serde = "0.4" -flexi_logger = { version = "0.28", features = ["trc"] } +flexi_logger = { version = "0.29", features = ["trc"] } float-cmp = "0.9" futures = "0.3" futures-util = "0.3" -gdal = "0.16" -gdal-sys = "0.9" -geo = "0.27" +gdal = "0.17" +gdal-sys = "0.10" +geo = "0.28" geoengine-datatypes = { path = "../datatypes" } geoengine-macros = { path = "../macros" } geoengine-operators = { path = "../operators" } @@ -56,11 +56,11 @@ log = "0.4" mime = "0.3" oauth2 = "4.4.2" openidconnect = "3.4" -# `opentelemetry` version has to be compatible with `tracing-opentelemetry` & `opentelemetry-jaeger` -opentelemetry = { version = "0.21", default-features = false, features = [ - "trace", -] } -opentelemetry-jaeger = "0.20" +# `opentelemetry` version has to be compatible with `tracing-opentelemetry` & `opentelemetry-*` +opentelemetry = { version = "0.24", default-features = false, features = ["trace"] } +opentelemetry-appender-tracing = "0.5" +opentelemetry-otlp = "0.17" +opentelemetry_sdk = "0.24" ordered-float = { version = "4.2", features = ["serde"] } num-traits = "0.2" pbkdf2 = "0.12.2" @@ -70,7 +70,7 @@ proj-sys = "0.19" # needs to stay fixed to use fixed proj version pwhash = "1.0" rand = "0.8" rayon = "1.8" -reqwest = { version = "0.11", features = ["json", "multipart", "stream"] } +reqwest = { version = "0.11", features = ["json", "multipart", "stream"] } # has to match with `oauth` serde = { version = "1.0", features = ["derive", "rc"] } serde_json = "1.0" serde_urlencoded = "0.7" @@ -93,7 +93,7 @@ tonic = { version = "0.11.0", features = [ ] } # must be compatible with `aruna-rust-api` tracing = "0.1" tracing-actix-web = "0.7" -tracing-opentelemetry = "0.22" +tracing-opentelemetry = "0.25" tracing-subscriber = { version = "0.3", features = [ "local-time", "ansi", @@ -113,7 +113,7 @@ uuid = { version = "1.7", features = [ "v4", "v5", ] } # must be compatible with `bb8-postgres` -validator = { version = "0.16", features = ["derive"] } +validator = { version = "0.18", features = ["derive"] } walkdir = "2.4" zip = "2.1" assert-json-diff = "2.0.2" @@ -125,7 +125,7 @@ nix = { version = "0.29", features = ["socket"] } [dev-dependencies] assert_cmd = "2.0" -httptest = "0.15" +httptest = "0.16" pretty_assertions = "1.4" prost = "0.12.3" # must be compatbile with aruna-rust-api serial_test = "3.0" @@ -133,7 +133,9 @@ tempfile = "3.10" xml-rs = "0.8" [build-dependencies] -vergen = { version = "8", features = ["build", "cargo", "git", "gitcl"] } +anyhow = "1.0" +vergen = { version = "9.0", features = ["build", "cargo", "emit_and_set"] } +vergen-gitcl = { version = "1.0", features = ["build"] } [[bench]] name = "quota_check" diff --git a/services/build.rs b/services/build.rs index c0c95f62d..52fa61d6b 100644 --- a/services/build.rs +++ b/services/build.rs @@ -1,8 +1,21 @@ -fn main() { - vergen::EmitBuilder::builder() - .build_date() // `VERGEN_BUILD_DATE` - .git_sha(true) // `VERGEN_GIT_SHA` - .cargo_features() // `VERGEN_CARGO_FEATURES` +use anyhow::Context; +use vergen::{BuildBuilder, CargoBuilder, Emitter}; +use vergen_gitcl::GitclBuilder; + +fn main() -> anyhow::Result<()> { + Emitter::default() + .add_instructions( + // `VERGEN_BUILD_DATE` + &BuildBuilder::default().build_date(true).build()?, + )? + .add_instructions( + // `VERGEN_CARGO_FEATURES` + &CargoBuilder::all_cargo()?, + )? + .add_instructions( + // `VERGEN_GIT_SHA` + &GitclBuilder::default().sha(true).build()?, + )? .emit_and_set() - .expect("Unable to generate version info"); + .context("Unable to generate version info") } diff --git a/services/src/api/model/services.rs b/services/src/api/model/services.rs index 5315fc8bd..3a54ccf18 100644 --- a/services/src/api/model/services.rs +++ b/services/src/api/model/services.rs @@ -142,7 +142,7 @@ pub struct UpdateDataset { #[validate(length(min = 1))] pub display_name: String, pub description: String, - #[validate(custom = "validate_tags")] + #[validate(custom(function = "validate_tags"))] pub tags: Vec, } diff --git a/services/src/bin/main.rs b/services/src/bin/main.rs index b004b5e46..7a61ab5f9 100644 --- a/services/src/bin/main.rs +++ b/services/src/bin/main.rs @@ -132,15 +132,26 @@ fn open_telemetry_layer( where S: Subscriber + for<'a> LookupSpan<'a>, { - use opentelemetry_jaeger::config::agent::AgentPipeline; + use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge; + use opentelemetry_otlp::WithExportConfig; use tracing_opentelemetry::OpenTelemetryLayer; - let tracer = AgentPipeline::default() - .with_endpoint(open_telemetry_config.endpoint.to_string()) - .with_service_name("Geo Engine") + let exporter = opentelemetry_otlp::new_exporter() + .tonic() + .with_endpoint(open_telemetry_config.endpoint.to_string()); + let tracer = opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter(exporter) + .with_trace_config(opentelemetry_sdk::trace::Config::default().with_resource( + opentelemetry_sdk::Resource::new(vec![opentelemetry::KeyValue::new( + "service.name", + "Geo Engine", + )]), + )) .install_simple()?; let opentelemetry: OpenTelemetryLayer = tracing_opentelemetry::layer().with_tracer(tracer); Ok(opentelemetry) + // Ok(OpenTelemetryTracingBridge::new(tracer)) } fn console_layer_with_filter + 'static>(filter: F) -> impl Layer diff --git a/services/src/datasets/external/netcdfcf/error.rs b/services/src/datasets/external/netcdfcf/error.rs index f77b1a6ff..0c49f8eac 100644 --- a/services/src/datasets/external/netcdfcf/error.rs +++ b/services/src/datasets/external/netcdfcf/error.rs @@ -277,4 +277,9 @@ pub enum NetCdfCf4DProviderError { UnexpectedExecution { source: Box, }, + + #[snafu(display("Unexpected GDAL error during creation of dataset"))] + OpeningDatasetForWriting { + source: GdalError, + }, } diff --git a/services/src/datasets/external/netcdfcf/overviews.rs b/services/src/datasets/external/netcdfcf/overviews.rs index 2e17c14a1..4785c6c41 100644 --- a/services/src/datasets/external/netcdfcf/overviews.rs +++ b/services/src/datasets/external/netcdfcf/overviews.rs @@ -9,10 +9,11 @@ use crate::{ }; use gdal::{ cpl::CslStringList, + errors::GdalError, programs::raster::{ multi_dim_translate, MultiDimTranslateDestination, MultiDimTranslateOptions, }, - raster::{Group, RasterCreationOption}, + raster::{Group, RasterCreationOptions}, Dataset, }; use geoengine_datatypes::{ @@ -71,7 +72,7 @@ struct ConversionMetadataEntity { pub base: Arc, pub time_coverage: Arc, pub entity: usize, - pub raster_creation_options: Arc, + pub raster_creation_options: Arc, } /// Metadata for converting a `NetCDF` cube slice to a COG @@ -285,7 +286,7 @@ pub async fn create_overviews< let number_of_conversions = conversion_metadata.len(); let mut stats_for_group = HashMap::::new(); let raster_creation_options = - Arc::new(CogRasterCreationOptions::new(options.resampling_method)?); + Arc::new(CogRasterCreationOptionss::new(options.resampling_method)?); let mut loading_info_metadatas = Vec::with_capacity(number_of_conversions); for (conversion_index, conversion) in conversion_metadata.into_iter().enumerate() { @@ -570,13 +571,9 @@ fn _create_subdataset_tiff( let input_sref = subdataset_sref(subdataset, conversion)?; - for raster_creation_option in conversion.entity.raster_creation_options.options() { + for raster_creation_option in &conversion.entity.raster_creation_options.options()? { options.push("-co".to_string()); - options.push(format!( - "{key}={value}", - key = raster_creation_option.key, - value = raster_creation_option.value - )); + options.push(raster_creation_option.to_string()); } let overview_dataset = multi_dim_translate( &[subdataset], @@ -619,14 +616,14 @@ fn subdataset_sref( } #[derive(Debug, Clone)] -struct CogRasterCreationOptions { +struct CogRasterCreationOptionss { compression_format: String, compression_level: String, num_threads: String, resampling_method: String, } -impl CogRasterCreationOptions { +impl CogRasterCreationOptionss { fn new(resampling_method: Option) -> Result { const COMPRESSION_FORMAT: &str = "LZW"; // this is the GDAL default const DEFAULT_COMPRESSION_LEVEL: u8 = 6; // this is the GDAL default @@ -657,36 +654,24 @@ impl CogRasterCreationOptions { } } -impl CogRasterCreationOptions { - fn options(&self) -> Vec> { +impl CogRasterCreationOptionss { + fn options(&self) -> Result { const COG_BLOCK_SIZE: &str = "512"; - vec![ - RasterCreationOption { - key: "COMPRESS", - value: &self.compression_format, - }, - RasterCreationOption { - key: "LEVEL", - value: &self.compression_level, - }, - RasterCreationOption { - key: "NUM_THREADS", - value: &self.num_threads, - }, - RasterCreationOption { - key: "BLOCKSIZE", - value: COG_BLOCK_SIZE, - }, - RasterCreationOption { - key: "BIGTIFF", - value: "IF_SAFER", // TODO: test if this suffices - }, - RasterCreationOption { - key: "RESAMPLING", - value: &self.resampling_method, - }, - ] + fn _options(this: &CogRasterCreationOptionss) -> Result { + let mut options = RasterCreationOptions::new(); + options.add_name_value("COMPRESS", &this.compression_format)?; + options.add_name_value("TILED", "YES")?; + options.add_name_value("LEVEL", &this.compression_level)?; + options.add_name_value("NUM_THREADS", &this.num_threads)?; + options.add_name_value("BLOCKSIZE", COG_BLOCK_SIZE)?; + options.add_name_value("BIGTIFF", "IF_SAFER")?; + options.add_name_value("RESAMPLING", &this.resampling_method)?; // TODO: test if this suffices + + Ok(options) + } + + _options(self).context(error::OpeningDatasetForWriting) } } diff --git a/services/src/datasets/listing.rs b/services/src/datasets/listing.rs index cc63046b7..4c6f788e2 100644 --- a/services/src/datasets/listing.rs +++ b/services/src/datasets/listing.rs @@ -43,10 +43,10 @@ pub struct DatasetListOptions { #[param(example = 0)] pub offset: u32, #[param(example = 2)] - #[validate(custom = "validate_list_limit")] + #[validate(custom(function = "validate_list_limit"))] pub limit: u32, #[param(example = "['tag1', 'tag2']")] - #[validate(custom = "validate_tags")] + #[validate(custom(function = "validate_tags"))] pub tags: Option>, } diff --git a/services/src/datasets/storage.rs b/services/src/datasets/storage.rs index 5576a5e7b..e59b909cd 100755 --- a/services/src/datasets/storage.rs +++ b/services/src/datasets/storage.rs @@ -86,10 +86,10 @@ pub struct AutoCreateDataset { #[validate(length(min = 1))] pub dataset_name: String, pub dataset_description: String, - #[validate(custom = "validate_main_file")] + #[validate(custom(function = "validate_main_file"))] pub main_file: String, pub layer_name: Option, - #[validate(custom = "validate_tags")] + #[validate(custom(function = "validate_tags"))] pub tags: Option>, } diff --git a/services/src/layers/layer.rs b/services/src/layers/layer.rs index 4d23ca53c..9f6e66151 100644 --- a/services/src/layers/layer.rs +++ b/services/src/layers/layer.rs @@ -261,7 +261,7 @@ pub struct LayerCollectionListOptions { #[param(example = 0)] pub offset: u32, #[param(example = 20)] - #[validate(custom = "validate_list_limit")] + #[validate(custom(function = "validate_list_limit"))] pub limit: u32, } diff --git a/services/src/projects/project.rs b/services/src/projects/project.rs index dca56ed74..f68e998f5 100644 --- a/services/src/projects/project.rs +++ b/services/src/projects/project.rs @@ -452,8 +452,7 @@ impl From<&Project> for ProjectListing { } }))] pub struct CreateProject { - #[validate(length(min = 1))] - #[validate(length(max = 256))] + #[validate(length(min = 1, max = 256))] pub name: String, #[validate(length(min = 1))] pub description: String, @@ -558,7 +557,7 @@ pub struct ProjectListOptions { #[param(example = 0)] pub offset: u32, #[param(example = 2)] - #[validate(custom = "validate_list_limit")] + #[validate(custom(function = "validate_list_limit"))] pub limit: u32, } diff --git a/services/src/tasks/mod.rs b/services/src/tasks/mod.rs index 48a8ddc13..153bea1da 100644 --- a/services/src/tasks/mod.rs +++ b/services/src/tasks/mod.rs @@ -412,7 +412,7 @@ pub struct TaskListOptions { pub offset: u32, #[serde(default = "task_list_limit_default")] #[param(example = 20)] - #[validate(custom = "validate_list_limit")] + #[validate(custom(function = "validate_list_limit"))] pub limit: u32, } From 92d90e8adfdcf759011828a86d3814a374d22723 Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Tue, 10 Sep 2024 10:54:27 +0200 Subject: [PATCH 2/3] update oltp --- Cargo.lock | 13 ------------- services/Cargo.toml | 1 - services/src/bin/main.rs | 9 ++++----- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a6b26913c..70dd7a83d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2932,7 +2932,6 @@ dependencies = [ "oauth2", "openidconnect", "opentelemetry", - "opentelemetry-appender-tracing", "opentelemetry-otlp", "opentelemetry_sdk", "ordered-float 4.2.2", @@ -5457,18 +5456,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "opentelemetry-appender-tracing" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84de945cb3a6f1e0d6317cbd998bbd0519ab00f4b790db67e0ff4fdcf7cedb6" -dependencies = [ - "opentelemetry", - "tracing", - "tracing-core", - "tracing-subscriber", -] - [[package]] name = "opentelemetry-otlp" version = "0.17.0" diff --git a/services/Cargo.toml b/services/Cargo.toml index 7055f0b99..fc7a39b1f 100644 --- a/services/Cargo.toml +++ b/services/Cargo.toml @@ -58,7 +58,6 @@ oauth2 = "4.4.2" openidconnect = "3.4" # `opentelemetry` version has to be compatible with `tracing-opentelemetry` & `opentelemetry-*` opentelemetry = { version = "0.24", default-features = false, features = ["trace"] } -opentelemetry-appender-tracing = "0.5" opentelemetry-otlp = "0.17" opentelemetry_sdk = "0.24" ordered-float = { version = "4.2", features = ["serde"] } diff --git a/services/src/bin/main.rs b/services/src/bin/main.rs index 7a61ab5f9..a17a6fcf5 100644 --- a/services/src/bin/main.rs +++ b/services/src/bin/main.rs @@ -132,9 +132,8 @@ fn open_telemetry_layer( where S: Subscriber + for<'a> LookupSpan<'a>, { - use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge; + use opentelemetry::trace::TracerProvider; use opentelemetry_otlp::WithExportConfig; - use tracing_opentelemetry::OpenTelemetryLayer; let exporter = opentelemetry_otlp::new_exporter() .tonic() .with_endpoint(open_telemetry_config.endpoint.to_string()); @@ -147,9 +146,9 @@ where "Geo Engine", )]), )) - .install_simple()?; - let opentelemetry: OpenTelemetryLayer = - tracing_opentelemetry::layer().with_tracer(tracer); + .install_simple()? + .tracer("Geo Engine"); + let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer); Ok(opentelemetry) // Ok(OpenTelemetryTracingBridge::new(tracer)) } From ca2d2a61524d949cfdd05935409c1ab9b22311af Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Thu, 12 Sep 2024 11:51:31 +0200 Subject: [PATCH 3/3] serde features update --- Cargo.lock | 4 + datatypes/Cargo.toml | 1 + .../src/collections/feature_collection.rs | 109 +++++++++++++-- .../mock/mock_feature_collection_source.rs | 132 ++++++++++-------- 4 files changed, 180 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 70dd7a83d..4e725c3bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -744,6 +744,9 @@ name = "arrow-schema" version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c85320a3a2facf2b2822b57aa9d6d9d55edb8aee0b6b5d3b8df158e503d10858" +dependencies = [ + "serde", +] [[package]] name = "arrow-select" @@ -2770,6 +2773,7 @@ dependencies = [ "arrow", "arrow-array", "arrow-ord", + "arrow-schema", "bytes", "chrono", "criterion", diff --git a/datatypes/Cargo.toml b/datatypes/Cargo.toml index 9c4f0a6d0..9cdfe4b57 100644 --- a/datatypes/Cargo.toml +++ b/datatypes/Cargo.toml @@ -16,6 +16,7 @@ pro = [] arrow = { version = "53.0", features = ["ipc_compression"] } arrow-array = "53.0" arrow-ord = "53.0" +arrow-schema = { version = "53", features = ["serde"] } bytes = "1.5" # for postgres-types impls chrono = "0.4" fallible-iterator = "0.2" # only for postgres-protocol diff --git a/datatypes/src/collections/feature_collection.rs b/datatypes/src/collections/feature_collection.rs index ac778afca..728bb5a4e 100644 --- a/datatypes/src/collections/feature_collection.rs +++ b/datatypes/src/collections/feature_collection.rs @@ -1515,7 +1515,6 @@ mod struct_serde { use serde::de::{SeqAccess, Visitor}; use serde::ser::Error; use serde::{Deserializer, Serializer}; - use std::fmt::Formatter; use std::io::Cursor; @@ -1599,6 +1598,97 @@ mod struct_serde { self.visit_byte_buf(bytes) } } + + #[cfg(test)] + pub fn serialize_json(struct_array: &StructArray, serializer: S) -> Result + where + S: Serializer, + { + use serde::ser::SerializeStruct; + + let batch = RecordBatch::from(struct_array); + + let mut json_writer = arrow::json::ArrayWriter::new(Vec::::new()); + json_writer.write(&batch).map_err(S::Error::custom)?; + json_writer.finish().map_err(S::Error::custom)?; + let json: serde_json::Value = + serde_json::from_slice(&json_writer.into_inner()).map_err(S::Error::custom)?; + + let mut struct_serializer = serializer.serialize_struct("FeatureCollection", 2)?; + struct_serializer.serialize_field("schema", batch.schema_ref())?; + struct_serializer.serialize_field("data", &json)?; + struct_serializer.end() + } + + #[cfg(test)] + pub fn deserialize_json<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + use std::io::BufReader; + + struct StructArrayJsonDeserializer; + + impl<'de> Visitor<'de> for StructArrayJsonDeserializer { + type Value = StructArray; + + fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { + formatter.write_str("an Arrow StructArray") + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + use serde::de::Error; + + let (mut schema, mut data) = (None, None); + + while let Some(key) = map.next_key()? { + match key { + "schema" => { + schema = Some(map.next_value::()?); + } + "data" => { + let value = map.next_value::()?; + data = Some(serde_json::to_vec(&value).map_err(A::Error::custom)?); + } + other => return Err(A::Error::custom(format!("Unexpected field {other}"))), + } + } + + let (Some(schema), Some(data)) = (schema, data) else { + return Err(A::Error::custom("Missing fields `schema` & `data`")); + }; + + let mut reader = arrow::json::ReaderBuilder::new(Arc::new(schema)) + .build(BufReader::new(Cursor::new(&data))) + .map_err(A::Error::custom)?; + + let batch = reader + .next() + .ok_or_else(|| { + A::Error::custom( + "Must be one batch inside the serialized data. Found none.", + ) + })? + .map_err(A::Error::custom)?; + if reader.next().is_some() { + return Err(A::Error::custom( + "Must be one batch inside the serialized data. Found more.", + )); + } + + Ok(batch.into()) + } + } + + deserializer.deserialize_struct( + "FeatureCollection", + &["schema", "data"], + StructArrayJsonDeserializer, + ) + } } impl Reproject

for FeatureCollection @@ -1766,10 +1856,9 @@ mod tests { .is_err()); } + /// If this test fails, change serialization to JSON (cf. methods below) instead of IPC. #[test] fn it_does_not_json_serialize() { - use arrow::record_batch::RecordBatch; - let collection = FeatureCollection::::from_data( vec![(0.0, 0.1).into()], vec![TimeInterval::new(0, 1).unwrap(); 1], @@ -1786,11 +1875,13 @@ mod tests { let struct_array = collection.table; - // TODO: if this stops failing, change the strange custom byte serialization to use JSON - let json = Vec::new(); - let mut json_writer = arrow::json::ArrayWriter::new(json); - json_writer - .write(&RecordBatch::from(struct_array)) - .unwrap_err(); + let serialized_struct_array = + struct_serde::serialize_json(&struct_array, serde_json::value::Serializer) + .unwrap() + .to_string(); + + let mut deserializer = serde_json::Deserializer::from_str(&serialized_struct_array); + + assert!(struct_serde::deserialize_json(&mut deserializer).is_err()); } } diff --git a/operators/src/mock/mock_feature_collection_source.rs b/operators/src/mock/mock_feature_collection_source.rs index 5594bcdcc..4950693cc 100644 --- a/operators/src/mock/mock_feature_collection_source.rs +++ b/operators/src/mock/mock_feature_collection_source.rs @@ -251,6 +251,7 @@ mod tests { use geoengine_datatypes::{collections::MultiPointCollection, primitives::SpatialResolution}; #[test] + #[allow(clippy::too_many_lines)] fn serde() { let cache_hint = CacheHint::default(); let collection = MultiPointCollection::from_data( @@ -272,64 +273,81 @@ mod tests { let serialized = serde_json::to_value(&source).unwrap(); let collection_bytes = [ - 65, 82, 82, 79, 87, 49, 0, 0, 255, 255, 255, 255, 184, 1, 0, 0, 16, 0, 0, 0, 0, 0, 10, - 0, 12, 0, 10, 0, 9, 0, 4, 0, 10, 0, 0, 0, 16, 0, 0, 0, 0, 1, 4, 0, 8, 0, 8, 0, 0, 0, 4, - 0, 8, 0, 0, 0, 4, 0, 0, 0, 3, 0, 0, 0, 180, 0, 0, 0, 56, 0, 0, 0, 4, 0, 0, 0, 52, 255, - 255, 255, 16, 0, 0, 0, 24, 0, 0, 0, 0, 0, 1, 2, 20, 0, 0, 0, 172, 255, 255, 255, 64, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 102, 111, 111, 98, 97, 114, 0, 0, 152, 255, - 255, 255, 24, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 16, 76, 0, 0, 0, 1, 0, 0, 0, 12, 0, 0, 0, - 82, 255, 255, 255, 2, 0, 0, 0, 136, 255, 255, 255, 24, 0, 0, 0, 32, 0, 0, 0, 0, 0, 1, - 2, 28, 0, 0, 0, 8, 0, 12, 0, 4, 0, 11, 0, 8, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 4, 0, 0, 0, 105, 116, 101, 109, 0, 0, 0, 0, 6, 0, 0, 0, 95, 95, 116, 105, 109, 101, - 0, 0, 16, 0, 20, 0, 16, 0, 0, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, 0, 28, 0, 0, 0, 12, - 0, 0, 0, 0, 0, 0, 12, 160, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 4, 0, 4, 0, 4, 0, 0, 0, - 16, 0, 20, 0, 16, 0, 14, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, 0, 32, 0, 0, 0, 12, 0, - 0, 0, 0, 0, 1, 16, 96, 0, 0, 0, 1, 0, 0, 0, 36, 0, 0, 0, 0, 0, 6, 0, 8, 0, 4, 0, 6, 0, - 0, 0, 2, 0, 0, 0, 16, 0, 22, 0, 16, 0, 14, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, 0, 24, - 0, 0, 0, 28, 0, 0, 0, 0, 0, 1, 3, 24, 0, 0, 0, 0, 0, 6, 0, 8, 0, 6, 0, 6, 0, 0, 0, 0, - 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 105, 116, 101, 109, 0, 0, 0, 0, 4, 0, 0, 0, 105, 116, - 101, 109, 0, 0, 0, 0, 10, 0, 0, 0, 95, 95, 103, 101, 111, 109, 101, 116, 114, 121, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 120, - 1, 0, 0, 16, 0, 0, 0, 12, 0, 26, 0, 24, 0, 23, 0, 4, 0, 8, 0, 12, 0, 0, 0, 32, 0, 0, 0, - 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 10, 0, 20, 0, 12, 0, 8, 0, 4, - 0, 10, 0, 0, 0, 116, 0, 0, 0, 12, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, - 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 32, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, - 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, - 0, 1, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 152, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, + 65, 82, 82, 79, 87, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 2, 0, 0, 0, 3, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 154, 153, 153, 153, 153, 153, 185, 63, 0, 0, 0, 0, 0, 0, 240, 63, 154, - 153, 153, 153, 153, 153, 241, 63, 0, 0, 0, 0, 0, 0, 0, 64, 205, 204, 204, 204, 204, - 204, 8, 64, 255, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0, 16, 0, 0, 0, - 12, 0, 20, 0, 18, 0, 12, 0, 8, 0, 4, 0, 12, 0, 0, 0, 152, 1, 0, 0, 176, 1, 0, 0, 16, 0, - 0, 0, 0, 0, 4, 0, 8, 0, 8, 0, 0, 0, 4, 0, 8, 0, 0, 0, 4, 0, 0, 0, 3, 0, 0, 0, 180, 0, - 0, 0, 56, 0, 0, 0, 4, 0, 0, 0, 52, 255, 255, 255, 16, 0, 0, 0, 24, 0, 0, 0, 0, 0, 1, 2, - 20, 0, 0, 0, 172, 255, 255, 255, 64, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 102, - 111, 111, 98, 97, 114, 0, 0, 152, 255, 255, 255, 24, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 16, - 76, 0, 0, 0, 1, 0, 0, 0, 12, 0, 0, 0, 82, 255, 255, 255, 2, 0, 0, 0, 136, 255, 255, - 255, 24, 0, 0, 0, 32, 0, 0, 0, 0, 0, 1, 2, 28, 0, 0, 0, 8, 0, 12, 0, 4, 0, 11, 0, 8, 0, - 0, 0, 64, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 105, 116, 101, 109, 0, 0, 0, 0, - 6, 0, 0, 0, 95, 95, 116, 105, 109, 101, 0, 0, 16, 0, 20, 0, 16, 0, 0, 0, 15, 0, 4, 0, - 0, 0, 8, 0, 16, 0, 0, 0, 28, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 12, 160, 0, 0, 0, 1, 0, 0, - 0, 28, 0, 0, 0, 4, 0, 4, 0, 4, 0, 0, 0, 16, 0, 20, 0, 16, 0, 14, 0, 15, 0, 4, 0, 0, 0, - 8, 0, 16, 0, 0, 0, 32, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 16, 96, 0, 0, 0, 1, 0, 0, 0, 36, - 0, 0, 0, 0, 0, 6, 0, 8, 0, 4, 0, 6, 0, 0, 0, 2, 0, 0, 0, 16, 0, 22, 0, 16, 0, 14, 0, - 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, 0, 24, 0, 0, 0, 28, 0, 0, 0, 0, 0, 1, 3, 24, 0, 0, - 0, 0, 0, 6, 0, 8, 0, 6, 0, 6, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 105, 116, - 101, 109, 0, 0, 0, 0, 4, 0, 0, 0, 105, 116, 101, 109, 0, 0, 0, 0, 10, 0, 0, 0, 95, 95, - 103, 101, 111, 109, 101, 116, 114, 121, 0, 0, 1, 0, 0, 0, 200, 1, 0, 0, 0, 0, 0, 0, - 128, 1, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 1, 0, 0, 65, 82, - 82, 79, 87, 49, + 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 184, 1, 0, 0, 16, 0, 0, 0, 0, 0, 10, 0, 12, + 0, 10, 0, 9, 0, 4, 0, 10, 0, 0, 0, 16, 0, 0, 0, 0, 1, 4, 0, 8, 0, 8, 0, 0, 0, 4, 0, 8, + 0, 0, 0, 4, 0, 0, 0, 3, 0, 0, 0, 180, 0, 0, 0, 56, 0, 0, 0, 4, 0, 0, 0, 52, 255, 255, + 255, 16, 0, 0, 0, 24, 0, 0, 0, 0, 0, 1, 2, 20, 0, 0, 0, 172, 255, 255, 255, 64, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 102, 111, 111, 98, 97, 114, 0, 0, 152, 255, 255, + 255, 24, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 16, 76, 0, 0, 0, 1, 0, 0, 0, 12, 0, 0, 0, 82, + 255, 255, 255, 2, 0, 0, 0, 136, 255, 255, 255, 24, 0, 0, 0, 32, 0, 0, 0, 0, 0, 1, 2, + 28, 0, 0, 0, 8, 0, 12, 0, 4, 0, 11, 0, 8, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, + 4, 0, 0, 0, 105, 116, 101, 109, 0, 0, 0, 0, 6, 0, 0, 0, 95, 95, 116, 105, 109, 101, 0, + 0, 16, 0, 20, 0, 16, 0, 0, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, 0, 28, 0, 0, 0, 12, 0, + 0, 0, 0, 0, 0, 12, 160, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 4, 0, 4, 0, 4, 0, 0, 0, 16, + 0, 20, 0, 16, 0, 14, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, 0, 32, 0, 0, 0, 12, 0, 0, 0, + 0, 0, 1, 16, 96, 0, 0, 0, 1, 0, 0, 0, 36, 0, 0, 0, 0, 0, 6, 0, 8, 0, 4, 0, 6, 0, 0, 0, + 2, 0, 0, 0, 16, 0, 22, 0, 16, 0, 14, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, 0, 24, 0, 0, + 0, 28, 0, 0, 0, 0, 0, 1, 3, 24, 0, 0, 0, 0, 0, 6, 0, 8, 0, 6, 0, 6, 0, 0, 0, 0, 0, 2, + 0, 0, 0, 0, 0, 4, 0, 0, 0, 105, 116, 101, 109, 0, 0, 0, 0, 4, 0, 0, 0, 105, 116, 101, + 109, 0, 0, 0, 0, 10, 0, 0, 0, 95, 95, 103, 101, 111, 109, 101, 116, 114, 121, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 120, 1, 0, + 0, 16, 0, 0, 0, 12, 0, 26, 0, 24, 0, 23, 0, 4, 0, 8, 0, 12, 0, 0, 0, 32, 0, 0, 0, 128, + 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 10, 0, 20, 0, 12, 0, 8, 0, 4, 0, 10, + 0, 0, 0, 116, 0, 0, 0, 12, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, + 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, + 0, 0, 0, 0, 64, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 128, 1, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 192, 1, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 64, 2, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, + 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 153, 153, 153, 153, 153, 185, + 63, 0, 0, 0, 0, 0, 0, 240, 63, 154, 153, 153, 153, 153, 153, 241, 63, 0, 0, 0, 0, 0, 0, + 0, 64, 205, 204, 204, 204, 204, 204, 8, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0, 16, 0, 0, 0, 12, 0, 20, 0, 18, 0, 12, 0, + 8, 0, 4, 0, 12, 0, 0, 0, 152, 1, 0, 0, 176, 1, 0, 0, 16, 0, 0, 0, 0, 0, 4, 0, 8, 0, 8, + 0, 0, 0, 4, 0, 8, 0, 0, 0, 4, 0, 0, 0, 3, 0, 0, 0, 180, 0, 0, 0, 56, 0, 0, 0, 4, 0, 0, + 0, 52, 255, 255, 255, 16, 0, 0, 0, 24, 0, 0, 0, 0, 0, 1, 2, 20, 0, 0, 0, 172, 255, 255, + 255, 64, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 102, 111, 111, 98, 97, 114, 0, 0, + 152, 255, 255, 255, 24, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 16, 76, 0, 0, 0, 1, 0, 0, 0, 12, + 0, 0, 0, 82, 255, 255, 255, 2, 0, 0, 0, 136, 255, 255, 255, 24, 0, 0, 0, 32, 0, 0, 0, + 0, 0, 1, 2, 28, 0, 0, 0, 8, 0, 12, 0, 4, 0, 11, 0, 8, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 4, 0, 0, 0, 105, 116, 101, 109, 0, 0, 0, 0, 6, 0, 0, 0, 95, 95, 116, 105, + 109, 101, 0, 0, 16, 0, 20, 0, 16, 0, 0, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, 0, 28, 0, + 0, 0, 12, 0, 0, 0, 0, 0, 0, 12, 160, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 4, 0, 4, 0, 4, + 0, 0, 0, 16, 0, 20, 0, 16, 0, 14, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, 0, 32, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 1, 16, 96, 0, 0, 0, 1, 0, 0, 0, 36, 0, 0, 0, 0, 0, 6, 0, 8, 0, 4, 0, + 6, 0, 0, 0, 2, 0, 0, 0, 16, 0, 22, 0, 16, 0, 14, 0, 15, 0, 4, 0, 0, 0, 8, 0, 16, 0, 0, + 0, 24, 0, 0, 0, 28, 0, 0, 0, 0, 0, 1, 3, 24, 0, 0, 0, 0, 0, 6, 0, 8, 0, 6, 0, 6, 0, 0, + 0, 0, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 105, 116, 101, 109, 0, 0, 0, 0, 4, 0, 0, 0, 105, + 116, 101, 109, 0, 0, 0, 0, 10, 0, 0, 0, 95, 95, 103, 101, 111, 109, 101, 116, 114, 121, + 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 128, 1, 0, 0, 0, 0, 0, 0, 128, 2, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 204, 1, 0, 0, 65, 82, 82, 79, 87, 49, ] .to_vec(); assert_eq!(