Add dataset versionning & multi-core support

src/database/mod.rs

@@ -3,6 +3,7 @@ pub mod space;
 mod space_db;
 mod space_index;
 
+use std::collections::HashMap;
 use std::fs::File;
 use std::hash::Hash;
 use std::hash::Hasher;
@@ -94,26 +95,56 @@ impl DataBase {
         }
     }
 
-    pub fn load<S>(name: S) -> Result<Self, String>
+    pub fn load(indices: &[&str]) -> Result<Self, String> {
-    where
+        let mut spaces = HashMap::new();
-        S: Into<String>,
+        let mut cores = vec![];
-    {
-        let name = name.into();
-        let fn_index = format!("{}.index", name);
 
-        let file_in = match File::open(fn_index) {
+        for index in indices.iter() {
+            let (core_spaces, core) = DataBase::load_core(index)?;
+            for core_space in core_spaces {
+                if let Some(space) = spaces.get(core_space.name()) {
+                    // Space is already registered, but with a different definitions.
+                    if space != &core_space {
+                        return Err(format!(
+                            "Reference Space ID `{}` defined two times, but differently\n{:?}\n VS \n{:?}",
+                            core_space.name(),
+                            spaces.get(core_space.name()),
+                            core_space
+                        ));
+                    }
+                } else {
+                    spaces.insert(core_space.name().clone(), core_space);
+                }
+            }
+
+            cores.push(core);
+        }
+
+        let spaces = spaces.drain().map(|(_, v)| v).collect();
+
+        Ok(DataBase::new(spaces, cores))
+    }
+
+    fn mmap_file(filename: &str) -> Result<Mmap, String> {
+        let file_in = match File::open(filename) {
             Err(e) => return Err(format!("{:?}", e)),
             Ok(file) => file,
         };
 
-        let mmap = match unsafe { Mmap::map(&file_in) } {
+        match unsafe { Mmap::map(&file_in) } {
-            Err(e) => return Err(format!("{:?}", e)),
+            Err(e) => Err(format!("{:?}", e)),
-            Ok(mmap) => mmap,
+            Ok(mmap) => Ok(mmap),
-        };
+        }
+    }
+
+    pub fn load_core(name: &str) -> Result<(Vec<Space>, Core), String> {
+        let fn_index = format!("{}.index", name);
+
+        let mmap = DataBase::mmap_file(&fn_index)?;
 
         match bincode::deserialize(&mmap[..]) {
             Err(e) => Err(format!("Index deserialization error: {:?}", e)),
-            Ok(db) => Ok(db),
+            Ok(index) => Ok(index),
         }
     }
 

src/database/space/axis.rs

@@ -1,7 +1,7 @@
 use super::coordinate::Coordinate;
 use super::position::Position;
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
 pub enum NumberSet {
     N,
     Z,
@@ -34,7 +34,7 @@ impl From<NumberSet> for String {
     }
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
 pub struct Graduation {
     pub set: NumberSet,
     pub minimum: f64,
@@ -56,7 +56,7 @@ impl Graduation {
 }
 
 // TODO: In the future this might become an Enum with AffineAxis, ArbitraryAxis, etc...
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
 pub struct Axis {
     measurement_unit: String,
     graduation: Graduation,

src/database/space/coordinate_system.rs

@@ -3,7 +3,7 @@ use super::coordinate::Coordinate;
 use super::position::Position;
 use super::MAX_K;
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
 pub enum CoordinateSystem {
     Universe,
     // Coordinates in Universe, expressed in f64, and in the Universe number of dimensions.

src/database/space/mod.rs

@@ -24,7 +24,7 @@ lazy_static! {
     };
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
 pub struct Space {
     name: String,
     system: CoordinateSystem,

src/json/model.rs

@@ -157,7 +157,12 @@ pub fn to_spatial_objects(db: &DataBase, list: Vec<SpaceObject>) -> Vec<SpatialO
     results
 }
 
-pub fn build_index(name: &str, spaces: &[space::Space], objects: &[SpatialObject]) -> Vec<Core> {
+pub fn build_index(
+    name: &str,
+    version: &str,
+    spaces: &[space::Space],
+    objects: &[SpatialObject],
+) -> Core {
     let mut properties = vec![];
     let mut space_set_objects = vec![];
 
@@ -205,11 +210,5 @@ pub fn build_index(name: &str, spaces: &[space::Space], objects: &[SpatialObject
         object.set_value(value.into());
     });
 
-    vec![Core::new(
+    Core::new(name, version, spaces, properties, space_set_objects)
-        name,
-        "v0.1",
-        spaces,
-        properties,
-        space_set_objects,
-    )]
 }

src/json/storage.rs

@@ -5,7 +5,6 @@ use memmap::Mmap;
 use serde::de::DeserializeOwned;
 use serde::Serialize;
 
-use crate::database::DataBase;
 use crate::json::model;
 
 pub fn from_json<T>(from: &str, to: &str)
@@ -75,7 +74,7 @@ pub fn convert(name: &str) {
     from_json::<Vec<model::SpatialObject>>(&fn_in, &fn_out);
 }
 
-pub fn build(name: &str) {
+pub fn build(name: &str, version: &str) {
     let fn_spaces = format!("{}.spaces.bin", name);
     let fn_objects = format!("{}.objects.bin", name);
     let fn_index = format!("{}.index", name);
@@ -85,11 +84,12 @@ pub fn build(name: &str) {
         .map(|s| s.into())
         .collect::<Vec<_>>();
 
-    let cores = model::build_index(
+    let core = model::build_index(
-        &name,
+        name,
+        version,
         &spaces,
         &load::<Vec<model::SpatialObject>>(&fn_objects),
     );
 
-    store(DataBase::new(spaces, cores), &fn_index);
+    store((spaces, core), &fn_index);
 }

src/main.rs

@@ -22,14 +22,14 @@ fn main() {
     // Build a Database Index:
     if true {
         info_time!("Building database index");
-        storage::build("10k");
+        storage::build("10k", "v0.1");
     }
 
     // Load a Database:
     let db;
     {
         info_time!("Loading database index");
-        db = DataBase::load("10k").unwrap();
+        db = DataBase::load(&["10k"]).unwrap();
     }
 
     if true {