From c45f9c145de69367f8096523acbaff98057bd75b Mon Sep 17 00:00:00 2001 From: Lionel Sambuc Date: Tue, 14 Jan 2020 11:04:13 +0100 Subject: [PATCH] Reorganising storage layer --- src/json/mod.rs | 2 -- src/lib.rs | 2 +- src/main.rs | 10 +++--- src/{json/storage.rs => storage/bincode.rs} | 39 ++------------------- src/storage/json.rs | 39 +++++++++++++++++++++ src/storage/mod.rs | 3 ++ src/{json => storage}/model.rs | 4 ++- 7 files changed, 53 insertions(+), 46 deletions(-) delete mode 100644 src/json/mod.rs rename src/{json/storage.rs => storage/bincode.rs} (55%) create mode 100644 src/storage/json.rs create mode 100644 src/storage/mod.rs rename src/{json => storage}/model.rs (99%) diff --git a/src/json/mod.rs b/src/json/mod.rs deleted file mode 100644 index 7acca0a..0000000 --- a/src/json/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod model; -pub mod storage; diff --git a/src/lib.rs b/src/lib.rs index fab7dbe..8046d0e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,6 @@ extern crate arrayref; extern crate serde_derive; mod database; -pub mod json; +pub mod storage; pub use database::*; diff --git a/src/main.rs b/src/main.rs index 5b3d0f9..691615f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,8 @@ #[macro_use] extern crate measure_time; -use mercator_db::json::storage; use mercator_db::space::Shape; +use mercator_db::storage; use mercator_db::CoreQueryParameters; use mercator_db::DataBase; @@ -16,14 +16,14 @@ fn main() { // Convert to binary the JSON data: if true { info_time!("Converting to binary JSON data"); - storage::convert::>("10k.spaces"); - storage::convert::>("10k.objects"); + storage::json::from::>("10k.spaces"); + storage::json::from::>("10k.objects"); } // Build a Database Index: if true { info_time!("Building database index"); - storage::build("10k", "v0.1", None, None); + storage::bincode::build("10k", "v0.1", None, None); } // Load a Database: @@ -69,7 +69,7 @@ fn main() { let r = core.get_by_label(&c, id).unwrap(); println!("get_by_label {}: {}", id, r.len()); if !r.is_empty() { - println!("{}: {:?}\n", id, r[0].1[0]); + println!("{}: {:?}\n", id, r); // no overlaping point, so no results } let lower = space.encode(&[0.2, 0.2, 0.2]).unwrap(); diff --git a/src/json/storage.rs b/src/storage/bincode.rs similarity index 55% rename from src/json/storage.rs rename to src/storage/bincode.rs index a5fc0c3..492140a 100644 --- a/src/json/storage.rs +++ b/src/storage/bincode.rs @@ -5,31 +5,8 @@ use memmap::Mmap; use serde::de::DeserializeOwned; use serde::Serialize; -use crate::json::model; +use super::model; -pub fn from_json(from: &str, to: &str) -where - T: Serialize + DeserializeOwned, -{ - let file_in = - File::open(from).unwrap_or_else(|e| panic!("Unable to read file: {}: {}", from, e)); - let file_out = - File::create(to).unwrap_or_else(|e| panic!("Unable to create file: {}: {}", to, e)); - - // We create a buffered writer from the file we get - let writer = BufWriter::new(&file_out); - - let mmap = unsafe { - Mmap::map(&file_in) - .unwrap_or_else(|e| panic!("Unable to map in memory the file: {}: {}", from, e)) - }; - let v: T = serde_json::from_slice(&mmap[..]) - .unwrap_or_else(|e| panic!("Unable to parse the json data from: {}: {}", from, e)); - - bincode::serialize_into(writer, &v).unwrap(); -} - -//FIXME: Move to ironsea_store? pub fn load(from: &str) -> T where T: DeserializeOwned, @@ -46,7 +23,6 @@ where .unwrap_or_else(|e| panic!("Unable to parse the json data from: {}: {}", from, e)) } -//FIXME: Move to ironsea_store? pub fn store(data: T, to: &str) where T: Serialize, @@ -60,17 +36,6 @@ where bincode::serialize_into(writer, &data).unwrap(); } -pub fn convert(name: &str) -where - T: Serialize + DeserializeOwned, -{ - // Convert definitions from json to bincode - let fn_in = format!("{}.json", name); - let fn_out = format!("{}.bin", name); - - from_json::(&fn_in, &fn_out); -} - pub fn build( name: &str, version: &str, @@ -86,7 +51,7 @@ pub fn build( .map(|s| s.into()) .collect::>(); - let objects = load::>(&fn_objects); + let objects = load::>(&fn_objects); let core = model::build_index(name, version, &spaces, &objects, scales, max_elements); diff --git a/src/storage/json.rs b/src/storage/json.rs new file mode 100644 index 0000000..da7cada --- /dev/null +++ b/src/storage/json.rs @@ -0,0 +1,39 @@ +use std::fs::File; +use std::io::BufWriter; + +use memmap::Mmap; +use serde::de::DeserializeOwned; +use serde::Serialize; + +fn convert(from: &str, to: &str) +where + T: Serialize + DeserializeOwned, +{ + let file_in = + File::open(from).unwrap_or_else(|e| panic!("Unable to read file: {}: {}", from, e)); + let file_out = + File::create(to).unwrap_or_else(|e| panic!("Unable to create file: {}: {}", to, e)); + + // We create a buffered writer from the file we get + let writer = BufWriter::new(&file_out); + + let mmap = unsafe { + Mmap::map(&file_in) + .unwrap_or_else(|e| panic!("Unable to map in memory the file: {}: {}", from, e)) + }; + let v: T = serde_json::from_slice(&mmap[..]) + .unwrap_or_else(|e| panic!("Unable to parse the json data from: {}: {}", from, e)); + + bincode::serialize_into(writer, &v).unwrap(); +} + +pub fn from(name: &str) +where + T: Serialize + DeserializeOwned, +{ + // Convert definitions from json to bincode + let fn_in = format!("{}.json", name); + let fn_out = format!("{}.bin", name); + + convert::(&fn_in, &fn_out); +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs new file mode 100644 index 0000000..bd931a9 --- /dev/null +++ b/src/storage/mod.rs @@ -0,0 +1,3 @@ +pub mod bincode; +pub mod json; +pub mod model; diff --git a/src/json/model.rs b/src/storage/model.rs similarity index 99% rename from src/json/model.rs rename to src/storage/model.rs index 9d5319a..8920b3c 100644 --- a/src/json/model.rs +++ b/src/storage/model.rs @@ -233,11 +233,13 @@ impl From<&&database::Properties> for Properties { } } +pub use v1::SpatialObject; + pub fn build_index( name: &str, version: &str, spaces: &[space::Space], - objects: &[v1::SpatialObject], + objects: &[SpatialObject], scales: Option>>, max_elements: Option, ) -> Core {