Adding support for XYZ files

.gitignore

@@ -3,8 +3,11 @@
 .DS_Store
 .*
 *~
-1*k.*
 test*
 test*
+*.index
+*.bin
+*.json
+*.xyz
 
 !.gitignore

Cargo.lock

@@ -199,9 +199,7 @@ dependencies = [
  "ironsea_index_sfc_dbc 0.1.0",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "measure_time 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "pretty_env_logger 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",

README.md

@@ -1,6 +1,6 @@
 # Mercator Indexer
 
-Small tool to generate indexes for the Mercator Spatial Index service.
+Tool to generate indexes for the Mercator Spatial Index service.
 
 ## Mercator: Spatial Index
 
@@ -47,7 +47,7 @@ For 3 datasets, `setA`, `setB`, `setC`, the following files are expected to be i
 Run (and build if necessary) the indexer:
 
 ```sh
-cargo run --release -- setA setB setC
+cargo run --release -- -f json setA setB setC
 ```
 
 This will produce the following files:
@@ -63,12 +63,11 @@ This will produce the following files:
    - setC.objects.bin
    - setC.spaces.bin
    - setC.index
-   
 
 By default, each dataset will have a version set to the empty string, if you want to specify the dataset version you can like this:
 
 ```sh
-cargo run --release -- setA:v0.1 setB setC:MyAwesomeVersion
+cargo run --release -- -f json setA:v0.1 setB setC:MyAwesomeVersion
 ```
 
 With the above, `setA` will have its version set to `v0.1`, `setB` to the empty string and `setC` to `MyAwesomeVersion`. 
@@ -79,8 +78,6 @@ For more options, please refer to the online help:
 cargo run --release -- --help
 ```
 
-
-
 ## Acknowledgements
 
 This open source software code was developed in part or in whole in the

src/main.rs

@@ -1,7 +1,10 @@
 #[macro_use]
 extern crate measure_time;
 
-use mercator_db::json::storage;
+use std::io::Error;
+
+use mercator_db::storage;
+use mercator_db::storage::model;
 use structopt::StructOpt;
 
 #[derive(StructOpt, Debug)]
@@ -14,15 +17,55 @@ struct Opt {
 
     /// Optional hint to be used when you wish to generate extra, coarser
     /// indices. This argument is ignored when `scales` is also provided.
+    #[allow(clippy::option_option)]
     #[structopt(long, short)]
     max_elements: Option<Option<usize>>,
 
+    /// Storage format of the input data. Either `xyz` or `json`.
+    #[structopt(long, short)]
+    format: String,
+
     /// List of datasets to index, with the following syntax per dataset:
     /// name[:version]: where name is the basename of the input files, and
     /// `version` a string to add to the dataset description
     datasets: Vec<String>,
 }
 
+enum StorageFormat {
+    Json,
+    XYZ,
+}
+
+impl StorageFormat {
+    pub fn convert(&self, title: &str) -> Result<(), Error> {
+        match self {
+            StorageFormat::Json => {
+                storage::json::from::<Vec<model::Space>>(&format!("{}.spaces", title))?;
+                storage::json::from::<Vec<model::v1::SpatialObject>>(&format!(
+                    "{}.objects",
+                    title
+                ))?;
+            }
+            StorageFormat::XYZ => {
+                storage::json::from::<Vec<model::Space>>(&format!("{}.spaces", title))?;
+                storage::xyz::from(&format!("{}.objects", title))?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl From<&str> for StorageFormat {
+    fn from(name: &str) -> Self {
+        match name {
+            "json" => StorageFormat::Json,
+            "xyz" => StorageFormat::XYZ,
+            _ => panic!("Unknown input format: {}", name),
+        }
+    }
+}
+
 fn main() {
     // If RUST_LOG is unset, set it to INFO, otherwise keep it as-is.
     if std::env::var("RUST_LOG").is_err() {
@@ -31,6 +74,9 @@ fn main() {
     pretty_env_logger::init();
 
     let opt = Opt::from_args();
+
+    let format = StorageFormat::from(opt.format.as_str());
+
     let scales = match opt.scales {
         None => None,
         Some(v) => {
@@ -68,14 +114,26 @@ fn main() {
 
         // Convert to binary the JSON data:
         {
-            info_time!("Converting to binary JSON data");
+            info_time!("Converting to binary data");
-            storage::convert(&title);
+            match format.convert(title) {
+                Err(e) => {
+                    warn!("Error converting input files: {:?}, skipping.", e);
+                    continue;
+                }
+                Ok(()) => (),
+            }
         }
 
         // Build a Database Index:
         {
             info_time!("Building database index");
-            storage::build(&title, version, scales.clone(), max_elements);
+            match storage::bincode::build(&title, version, scales.clone(), max_elements) {
+                Err(e) => {
+                    warn!("Error building index: {:?}, skipping.", e);
+                    continue;
+                }
+                Ok(()) => (),
+            }
         }
     }
 }