From 46fad245382d2d75aef24dfaf527ab09aac9254b Mon Sep 17 00:00:00 2001 From: Lionel Sambuc Date: Tue, 14 Jan 2020 14:25:45 +0100 Subject: [PATCH] Adding support for XYZ files --- .gitignore | 5 +++- Cargo.lock | 2 -- README.md | 9 +++----- src/main.rs | 66 +++++++++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 69 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 786d2d8..6fa7e6d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,11 @@ .DS_Store .* *~ -1*k.* test* test* +*.index +*.bin +*.json +*.xyz !.gitignore diff --git a/Cargo.lock b/Cargo.lock index 7cdbf49..c124f00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,9 +199,7 @@ dependencies = [ "ironsea_index_sfc_dbc 0.1.0", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "measure_time 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "pretty_env_logger 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/README.md b/README.md index 259842b..66f5164 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Mercator Indexer -Small tool to generate indexes for the Mercator Spatial Index service. +Tool to generate indexes for the Mercator Spatial Index service. ## Mercator: Spatial Index @@ -47,7 +47,7 @@ For 3 datasets, `setA`, `setB`, `setC`, the following files are expected to be i Run (and build if necessary) the indexer: ```sh -cargo run --release -- setA setB setC +cargo run --release -- -f json setA setB setC ``` This will produce the following files: @@ -63,12 +63,11 @@ This will produce the following files: - setC.objects.bin - setC.spaces.bin - setC.index - By default, each dataset will have a version set to the empty string, if you want to specify the dataset version you can like this: ```sh -cargo run --release -- setA:v0.1 setB setC:MyAwesomeVersion +cargo run --release -- -f json setA:v0.1 setB setC:MyAwesomeVersion ``` With the above, `setA` will have its version set to `v0.1`, `setB` to the empty string and `setC` to `MyAwesomeVersion`. @@ -79,8 +78,6 @@ For more options, please refer to the online help: cargo run --release -- --help ``` - - ## Acknowledgements This open source software code was developed in part or in whole in the diff --git a/src/main.rs b/src/main.rs index 275808a..31f292d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,10 @@ #[macro_use] extern crate measure_time; -use mercator_db::json::storage; +use std::io::Error; + +use mercator_db::storage; +use mercator_db::storage::model; use structopt::StructOpt; #[derive(StructOpt, Debug)] @@ -14,15 +17,55 @@ struct Opt { /// Optional hint to be used when you wish to generate extra, coarser /// indices. This argument is ignored when `scales` is also provided. + #[allow(clippy::option_option)] #[structopt(long, short)] max_elements: Option>, + /// Storage format of the input data. Either `xyz` or `json`. + #[structopt(long, short)] + format: String, + /// List of datasets to index, with the following syntax per dataset: /// name[:version]: where name is the basename of the input files, and /// `version` a string to add to the dataset description datasets: Vec, } +enum StorageFormat { + Json, + XYZ, +} + +impl StorageFormat { + pub fn convert(&self, title: &str) -> Result<(), Error> { + match self { + StorageFormat::Json => { + storage::json::from::>(&format!("{}.spaces", title))?; + storage::json::from::>(&format!( + "{}.objects", + title + ))?; + } + StorageFormat::XYZ => { + storage::json::from::>(&format!("{}.spaces", title))?; + storage::xyz::from(&format!("{}.objects", title))?; + } + } + + Ok(()) + } +} + +impl From<&str> for StorageFormat { + fn from(name: &str) -> Self { + match name { + "json" => StorageFormat::Json, + "xyz" => StorageFormat::XYZ, + _ => panic!("Unknown input format: {}", name), + } + } +} + fn main() { // If RUST_LOG is unset, set it to INFO, otherwise keep it as-is. if std::env::var("RUST_LOG").is_err() { @@ -31,6 +74,9 @@ fn main() { pretty_env_logger::init(); let opt = Opt::from_args(); + + let format = StorageFormat::from(opt.format.as_str()); + let scales = match opt.scales { None => None, Some(v) => { @@ -68,14 +114,26 @@ fn main() { // Convert to binary the JSON data: { - info_time!("Converting to binary JSON data"); - storage::convert(&title); + info_time!("Converting to binary data"); + match format.convert(title) { + Err(e) => { + warn!("Error converting input files: {:?}, skipping.", e); + continue; + } + Ok(()) => (), + } } // Build a Database Index: { info_time!("Building database index"); - storage::build(&title, version, scales.clone(), max_elements); + match storage::bincode::build(&title, version, scales.clone(), max_elements) { + Err(e) => { + warn!("Error building index: {:?}, skipping.", e); + continue; + } + Ok(()) => (), + } } } }