From 57e2710915935d53f8aadc4643aee95d01c56a22 Mon Sep 17 00:00:00 2001 From: Lionel Sambuc Date: Fri, 4 Oct 2019 15:43:51 +0200 Subject: [PATCH] Add dataset versionning & multi-core support --- src/database/mod.rs | 55 +++++++++++++++++++------ src/database/space/axis.rs | 6 +-- src/database/space/coordinate_system.rs | 2 +- src/database/space/mod.rs | 2 +- src/json/model.rs | 15 ++++--- src/json/storage.rs | 10 ++--- src/main.rs | 4 +- 7 files changed, 62 insertions(+), 32 deletions(-) diff --git a/src/database/mod.rs b/src/database/mod.rs index 908af32..775ac29 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -3,6 +3,7 @@ pub mod space; mod space_db; mod space_index; +use std::collections::HashMap; use std::fs::File; use std::hash::Hash; use std::hash::Hasher; @@ -94,26 +95,56 @@ impl DataBase { } } - pub fn load(name: S) -> Result - where - S: Into, - { - let name = name.into(); - let fn_index = format!("{}.index", name); + pub fn load(indices: &[&str]) -> Result { + let mut spaces = HashMap::new(); + let mut cores = vec![]; - let file_in = match File::open(fn_index) { + for index in indices.iter() { + let (core_spaces, core) = DataBase::load_core(index)?; + for core_space in core_spaces { + if let Some(space) = spaces.get(core_space.name()) { + // Space is already registered, but with a different definitions. + if space != &core_space { + return Err(format!( + "Reference Space ID `{}` defined two times, but differently\n{:?}\n VS \n{:?}", + core_space.name(), + spaces.get(core_space.name()), + core_space + )); + } + } else { + spaces.insert(core_space.name().clone(), core_space); + } + } + + cores.push(core); + } + + let spaces = spaces.drain().map(|(_, v)| v).collect(); + + Ok(DataBase::new(spaces, cores)) + } + + fn mmap_file(filename: &str) -> Result { + let file_in = match File::open(filename) { Err(e) => return Err(format!("{:?}", e)), Ok(file) => file, }; - let mmap = match unsafe { Mmap::map(&file_in) } { - Err(e) => return Err(format!("{:?}", e)), - Ok(mmap) => mmap, - }; + match unsafe { Mmap::map(&file_in) } { + Err(e) => Err(format!("{:?}", e)), + Ok(mmap) => Ok(mmap), + } + } + + pub fn load_core(name: &str) -> Result<(Vec, Core), String> { + let fn_index = format!("{}.index", name); + + let mmap = DataBase::mmap_file(&fn_index)?; match bincode::deserialize(&mmap[..]) { Err(e) => Err(format!("Index deserialization error: {:?}", e)), - Ok(db) => Ok(db), + Ok(index) => Ok(index), } } diff --git a/src/database/space/axis.rs b/src/database/space/axis.rs index 9624c55..28d4a23 100644 --- a/src/database/space/axis.rs +++ b/src/database/space/axis.rs @@ -1,7 +1,7 @@ use super::coordinate::Coordinate; use super::position::Position; -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub enum NumberSet { N, Z, @@ -34,7 +34,7 @@ impl From for String { } } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Graduation { pub set: NumberSet, pub minimum: f64, @@ -56,7 +56,7 @@ impl Graduation { } // TODO: In the future this might become an Enum with AffineAxis, ArbitraryAxis, etc... -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Axis { measurement_unit: String, graduation: Graduation, diff --git a/src/database/space/coordinate_system.rs b/src/database/space/coordinate_system.rs index 9a3fb62..c76d894 100644 --- a/src/database/space/coordinate_system.rs +++ b/src/database/space/coordinate_system.rs @@ -3,7 +3,7 @@ use super::coordinate::Coordinate; use super::position::Position; use super::MAX_K; -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub enum CoordinateSystem { Universe, // Coordinates in Universe, expressed in f64, and in the Universe number of dimensions. diff --git a/src/database/space/mod.rs b/src/database/space/mod.rs index b06a6b0..27c5329 100644 --- a/src/database/space/mod.rs +++ b/src/database/space/mod.rs @@ -24,7 +24,7 @@ lazy_static! { }; } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Space { name: String, system: CoordinateSystem, diff --git a/src/json/model.rs b/src/json/model.rs index ebdc4d3..2a4ad9e 100644 --- a/src/json/model.rs +++ b/src/json/model.rs @@ -157,7 +157,12 @@ pub fn to_spatial_objects(db: &DataBase, list: Vec) -> Vec Vec { +pub fn build_index( + name: &str, + version: &str, + spaces: &[space::Space], + objects: &[SpatialObject], +) -> Core { let mut properties = vec![]; let mut space_set_objects = vec![]; @@ -205,11 +210,5 @@ pub fn build_index(name: &str, spaces: &[space::Space], objects: &[SpatialObject object.set_value(value.into()); }); - vec![Core::new( - name, - "v0.1", - spaces, - properties, - space_set_objects, - )] + Core::new(name, version, spaces, properties, space_set_objects) } diff --git a/src/json/storage.rs b/src/json/storage.rs index 0ea998c..2612678 100644 --- a/src/json/storage.rs +++ b/src/json/storage.rs @@ -5,7 +5,6 @@ use memmap::Mmap; use serde::de::DeserializeOwned; use serde::Serialize; -use crate::database::DataBase; use crate::json::model; pub fn from_json(from: &str, to: &str) @@ -75,7 +74,7 @@ pub fn convert(name: &str) { from_json::>(&fn_in, &fn_out); } -pub fn build(name: &str) { +pub fn build(name: &str, version: &str) { let fn_spaces = format!("{}.spaces.bin", name); let fn_objects = format!("{}.objects.bin", name); let fn_index = format!("{}.index", name); @@ -85,11 +84,12 @@ pub fn build(name: &str) { .map(|s| s.into()) .collect::>(); - let cores = model::build_index( - &name, + let core = model::build_index( + name, + version, &spaces, &load::>(&fn_objects), ); - store(DataBase::new(spaces, cores), &fn_index); + store((spaces, core), &fn_index); } diff --git a/src/main.rs b/src/main.rs index 67be888..0c52f81 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,14 +22,14 @@ fn main() { // Build a Database Index: if true { info_time!("Building database index"); - storage::build("10k"); + storage::build("10k", "v0.1"); } // Load a Database: let db; { info_time!("Loading database index"); - db = DataBase::load("10k").unwrap(); + db = DataBase::load(&["10k"]).unwrap(); } if true {