diff --git a/Cargo.lock b/Cargo.lock index 14ec5fdc27..07986bbb8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -934,6 +934,30 @@ dependencies = [ "riscv", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset 0.9.0", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.8.15" @@ -3033,6 +3057,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -3158,7 +3191,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "memoffset", + "memoffset 0.6.5", ] [[package]] @@ -3170,7 +3203,7 @@ dependencies = [ "bitflags 1.3.2", "cfg-if", "libc", - "memoffset", + "memoffset 0.6.5", ] [[package]] @@ -3387,22 +3420,34 @@ version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" +[[package]] +name = "osm-reader" +version = "0.1.0" +source = "git+https://github.com/a-b-street/osm-reader#ae39050497fff871bba37559ee67918781dc6838" +dependencies = [ + "anyhow", + "osmpbf", + "roxmltree 0.19.0", + "serde", +] + [[package]] name = "osm2lanes" version = "0.1.0" -source = "git+https://github.com/a-b-street/osm2streets#bb4d1cdafd2138b1b50698ac64798cda01115e3b" +source = "git+https://github.com/a-b-street/osm2streets#6303322c6846aece1e9f6a0b1e2cf23cfb84a67e" dependencies = [ "abstutil", "anyhow", "enumset", "geom", + "osm-reader", "serde", ] [[package]] name = "osm2streets" version = "0.1.0" -source = "git+https://github.com/a-b-street/osm2streets#bb4d1cdafd2138b1b50698ac64798cda01115e3b" +source = "git+https://github.com/a-b-street/osm2streets#6303322c6846aece1e9f6a0b1e2cf23cfb84a67e" dependencies = [ "abstutil", "anyhow", @@ -3455,7 +3500,7 @@ dependencies = [ "derive_builder", "flate2", "iter-progress", - "protobuf", + "protobuf 2.8.2", "quick-xml", "rusqlite", "separator", @@ -3464,6 +3509,20 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "osmpbf" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "131587696d8b76086a68158d70c873a0ab23970f46ea1df7c279fc3410a71d81" +dependencies = [ + "byteorder", + "flate2", + "memmap2", + "protobuf 3.3.0", + "protobuf-codegen", + "rayon", +] + [[package]] name = "pad" version = "0.1.6" @@ -3789,6 +3848,57 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70731852eec72c56d11226c8a5f96ad5058a3dab73647ca5f7ee351e464f2571" +[[package]] +name = "protobuf" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b65f4a8ec18723a734e5dc09c173e0abf9690432da5340285d536edcb4dac190" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror", +] + +[[package]] +name = "protobuf-codegen" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e85514a216b1c73111d9032e26cc7a5ecb1bb3d4d9539e91fb72a4395060f78" +dependencies = [ + "anyhow", + "once_cell", + "protobuf 3.3.0", + "protobuf-parse", + "regex", + "tempfile", + "thiserror", +] + +[[package]] +name = "protobuf-parse" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77d6fbd6697c9e531873e81cec565a85e226b99a0f10e1acc079be057fe2fcba" +dependencies = [ + "anyhow", + "indexmap 1.9.2", + "log", + "protobuf 3.3.0", + "protobuf-support", + "tempfile", + "thiserror", + "which", +] + +[[package]] +name = "protobuf-support" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6872f4d4f4b98303239a2b5838f5bbbb77b01ffc892d627957f37a22d7cfe69c" +dependencies = [ + "thiserror", +] + [[package]] name = "quick-xml" version = "0.30.0" @@ -3881,6 +3991,26 @@ dependencies = [ "strum_macros", ] +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rctree" version = "0.5.0" @@ -4607,15 +4737,15 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "streets_reader" version = "0.1.0" -source = "git+https://github.com/a-b-street/osm2streets#bb4d1cdafd2138b1b50698ac64798cda01115e3b" +source = "git+https://github.com/a-b-street/osm2streets#6303322c6846aece1e9f6a0b1e2cf23cfb84a67e" dependencies = [ "abstutil", "anyhow", "country-geocoder", "geom", "log", + "osm-reader", "osm2streets", - "xmlparser", ] [[package]] diff --git a/cli/src/one_step_import.rs b/cli/src/one_step_import.rs index e93f1f57bf..50666d9dbc 100644 --- a/cli/src/one_step_import.rs +++ b/cli/src/one_step_import.rs @@ -43,7 +43,7 @@ pub async fn run( } else { println!("Figuring out what Geofabrik file contains your boundary"); let (url, pbf) = importer::pick_geofabrik(geojson_path.clone()).await?; - osm = city.input_path(format!("osm/{}.osm", name)); + osm = city.input_path(format!("osm/{}.osm.pbf", name)); fs_err::create_dir_all(std::path::Path::new(&pbf).parent().unwrap()) .expect("Creating parent dir failed"); fs_err::create_dir_all(std::path::Path::new(&osm).parent().unwrap()) diff --git a/convert_osm/src/extract.rs b/convert_osm/src/extract.rs index 0b0b61baec..762bc7de7f 100644 --- a/convert_osm/src/extract.rs +++ b/convert_osm/src/extract.rs @@ -31,9 +31,9 @@ pub fn extract_osm( opts: &Options, timer: &mut Timer, ) -> Extract { - let osm_xml = fs_err::read_to_string(osm_input_path).unwrap(); + let osm_input_bytes = fs_err::read(osm_input_path).unwrap(); let mut doc = streets_reader::osm_reader::Document::read( - &osm_xml, + &osm_input_bytes, clip_pts.as_ref().map(|pts| GPSBounds::from(pts.clone())), timer, ) diff --git a/importer/src/lib.rs b/importer/src/lib.rs index ccfff4fd38..db07fc68bf 100644 --- a/importer/src/lib.rs +++ b/importer/src/lib.rs @@ -45,7 +45,7 @@ pub async fn regenerate_everything(shard_num: usize, num_shards: usize) { } } -/// Transforms a .osm file to a map in one step. +/// Transforms a .osm.xml or .pbf file to a map in one step. pub async fn oneshot( osm_path: String, clip: Option, diff --git a/importer/src/soundcast/popdat.rs b/importer/src/soundcast/popdat.rs index 0bb1a2174e..9b4bea5fd0 100644 --- a/importer/src/soundcast/popdat.rs +++ b/importer/src/soundcast/popdat.rs @@ -209,7 +209,7 @@ fn import_parcels( attributes.insert("parking".to_string(), offstreet_parking_spaces.to_string()); } if let Some(b) = osm_building { - attributes.insert("osm_bldg".to_string(), b.inner().to_string()); + attributes.insert("osm_bldg".to_string(), b.inner_id().to_string()); } shapes.insert( id, diff --git a/importer/src/utils.rs b/importer/src/utils.rs index 8ddb7d8c52..b8b3dd2126 100644 --- a/importer/src/utils.rs +++ b/importer/src/utils.rs @@ -85,8 +85,8 @@ pub async fn download_kml( fs_err::rename(tmp, output.replace(".bin", ".kml")).unwrap(); } -/// Uses osmium to clip the input .osm (or .pbf) against a polygon and produce some output. Skips -/// if the output exists. +/// Uses osmium to clip the input .osm.xml or osm.pbf against a polygon and produce some output pbf +/// file. Skips if the output exists. pub fn osmium( input: String, clipping_polygon: String, @@ -114,7 +114,7 @@ pub fn osmium( .arg(output) .arg("-f") // Smaller files without author, timestamp, version - .arg("osm,add_metadata=false"), + .arg("pbf,add_metadata=false"), ); } @@ -144,12 +144,12 @@ pub async fn osm_to_raw( osmium( local_osm_file, boundary_polygon.clone(), - name.city.input_path(format!("osm/{}.osm", name.map)), + name.city.input_path(format!("osm/{}.osm.pbf", name.map)), config, ); let map = convert_osm::convert( - name.city.input_path(format!("osm/{}.osm", name.map)), + name.city.input_path(format!("osm/{}.osm.pbf", name.map)), name.clone(), Some(boundary_polygon), opts, diff --git a/map_model/src/make/buildings.rs b/map_model/src/make/buildings.rs index c9599e5e60..9be4809327 100644 --- a/map_model/src/make/buildings.rs +++ b/map_model/src/make/buildings.rs @@ -62,7 +62,7 @@ pub fn make_all_buildings( let id = BuildingID(results.len()); - let mut rng = XorShiftRng::seed_from_u64(orig_id.inner() as u64); + let mut rng = XorShiftRng::seed_from_u64(orig_id.inner_id() as u64); // TODO is it worth using height or building:height as an alternative if not tagged? let levels = b .osm_tags