Mercator/mercator_db
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Adapt to Mercator JSON API formats

Browse Source
This commit is contained in:
Lionel Sambuc
2019-09-03 17:07:09 +02:00
parent 0a24bb441e
commit 142646f7e4
13 changed files with 364 additions and 100263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.gitignore vendored
View File

@@ -3,9 +3,10 @@
.DS_Store
.*
*~
test.bin
test.index
*k.json
*.bin
*.index
*k..*.json
!.gitignore
!100k.json
!10k.spaces.json
!10k.objects.json

100002
100k.json
View File

File diff suppressed because it is too large Load Diff

1
10k.objects.json Normal file
View File

File diff suppressed because one or more lines are too long

36
10k.spaces.json Normal file
View File

@@ -0,0 +1,36 @@
[ {
"name": "std",
"origin": [ 0.0, 0.0, 0.0 ],
"axes": [
{
"measurement_unit": "mm",
"graduation": {
"set": "N",
"minimum": 0.0,
"maximum": 1.0,
"steps": 1000000000
},
"unit_vector": [ 1.0, 0.0, 0.0 ]
},
{
"measurement_unit": "mm",
"graduation": {
"set": "N",
"minimum": 0.0,
"maximum": 1.0,
"steps": 1000000000
},
"unit_vector": [ 0.0, 1.0, 0.0 ]
},
{
"measurement_unit": "mm",
"graduation": {
"set": "N",
"minimum": 0.0,
"maximum": 1.0,
"steps": 1000000000
},
"unit_vector": [ 0.0, 0.0, 1.0 ]
}
]
} ]

1
src/database/mod.rs
View File

@@ -73,6 +73,7 @@ impl PartialEq for SpaceObject {
impl Eq for SpaceObject {}
impl Hash for SpaceObject {
//FIXME: ADD HASHING IMPLEMENTATION, REQUIRED FOR distinct()!
fn hash<H: Hasher>(&self, state: &mut H) {
unimplemented!()
}

5
src/database/space_index.rs
View File

@@ -22,11 +22,6 @@ impl SpaceSetObject {
}
}
/*
pub fn eval(&self, _predicate: &Predicate) -> bool {
false
}
*/
pub fn id(&self) -> &Coordinate {
&self.value
}

2
src/json/mod.rs Normal file
View File

@@ -0,0 +1,2 @@
pub mod model;
pub mod storage;

215
src/json/model.rs Normal file
View File

@@ -0,0 +1,215 @@
use std::collections::HashMap;
use crate::database;
use database::space;
use database::Core;
use database::DataBase;
use database::SpaceObject;
use database::SpaceSetObject;
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct Space {
pub name: String,
pub origin: Vec<f64>,
pub axes: Vec<Axis>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct Axis {
pub measurement_unit: String,
pub graduation: Graduation,
pub unit_vector: Vec<f64>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct Graduation {
pub set: String,
pub minimum: f64,
pub maximum: f64,
pub steps: u64,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct SpatialObject {
pub properties: Properties,
pub shapes: Vec<Shape>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct Shape {
#[serde(rename = "type")]
pub type_name: String,
#[serde(rename = "space")]
pub reference_space: String,
pub vertices: Vec<Point>,
}
type Point = Vec<f64>;
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct Properties {
#[serde(rename = "type")]
pub type_name: String,
pub id: String,
}
impl From<&space::Graduation> for Graduation {
fn from(g: &space::Graduation) -> Self {
Graduation {
set: g.set.clone().into(),
minimum: g.minimum,
maximum: g.maximum,
steps: g.steps,
}
}
}
impl From<Axis> for space::Axis {
fn from(axis: Axis) -> Self {
let g = axis.graduation;
space::Axis::new(
axis.measurement_unit,
axis.unit_vector,
g.set.into(),
g.minimum,
g.maximum,
g.steps,
)
.unwrap_or_else(|e| panic!("Unable to create Axis as defined: {}", e))
}
}
impl From<&space::Axis> for Axis {
fn from(axis: &space::Axis) -> Self {
Axis {
measurement_unit: axis.measurement_unit().clone(),
graduation: axis.graduation().into(),
unit_vector: axis.unit_vector().into(),
}
}
}
impl From<&Space> for space::Space {
fn from(space: &Space) -> Self {
let axes = space
.axes
.iter()
.map(|a| a.clone().into())
.collect::<Vec<_>>();
let system = space::CoordinateSystem::new(space.origin.clone(), axes);
space::Space::new(&space.name, system)
}
}
impl From<&space::Space> for Space {
fn from(space: &space::Space) -> Self {
let axes = space.axes().iter().map(|a| a.into()).collect::<Vec<_>>();
Space {
name: space.name().clone(),
origin: space.origin().into(),
axes,
}
}
}
pub fn to_spatial_objects(db: &DataBase, list: Vec<SpaceObject>) -> Vec<SpatialObject> {
// Filter per Properties, in order to regroup by it, then build a single SpatialObject per Properties.
let mut properties = HashMap::new();
for object in list {
let k = object.value.id().clone();
properties.entry(k).or_insert_with(|| vec![]).push(object);
}
let mut results = vec![];
for (k, v) in properties.iter() {
// Group by spaces, to collect points shapes together
let shapes = v
.iter()
.filter_map(|o| match db.space(&o.space_id) {
Err(_) => None,
Ok(space) => {
if let Ok(vertices) = space.decode(&o.position) {
Some(Shape {
type_name: "Point".to_string(),
reference_space: o.space_id.clone(),
vertices: vec![vertices],
})
} else {
None
}
}
})
.collect();
results.push(SpatialObject {
properties: Properties {
type_name: "Feature".to_string(),
id: k.to_string(),
},
shapes,
});
}
results
}
pub fn build_index(name: &str, spaces: &[space::Space], objects: &[SpatialObject]) -> Vec<Core> {
let mut properties = vec![];
let mut space_set_objects = vec![];
let mut properties_ref = vec![];
{
let mut properties_hm = HashMap::new();
for object in objects {
let value = match properties_hm.get(object.properties.id.as_str()) {
Some(_) => {
properties_ref.push(object.properties.id.as_str());
properties_ref.len() - 1
}
None => {
properties_hm.insert(
object.properties.id.as_str(),
database::Properties::Feature(object.properties.id.clone()),
);
properties_ref.push(object.properties.id.as_str());
properties_ref.len() - 1
}
};
for point in &object.shapes {
assert_eq!(point.type_name, "Point");
space_set_objects.push(SpaceSetObject::new(
&point.reference_space,
point.vertices[0].clone().into(),
value.into(),
))
}
}
properties.append(&mut properties_hm.drain().map(|(_, v)| v).collect::<Vec<_>>());
}
properties.sort_unstable_by_key(|p| p.id().clone());
space_set_objects.iter_mut().for_each(|object| {
let id = properties_ref[object.value().u64() as usize];
let value = properties.binary_search_by_key(&id, |p| p.id()).unwrap();
object.set_value(value.into());
});
vec![Core::new(
name,
"v0.1",
spaces,
properties,
space_set_objects,
)]
}

95
src/json/storage.rs Normal file
View File

@@ -0,0 +1,95 @@
use std::fs::File;
use std::io::BufWriter;
use memmap::Mmap;
use serde::de::DeserializeOwned;
use serde::Serialize;
use crate::database::DataBase;
use crate::json::model;
pub fn from_json<T>(from: &str, to: &str)
where
T: Serialize + DeserializeOwned,
{
let file_in =
File::open(from).unwrap_or_else(|e| panic!("Unable to read file: {}: {}", from, e));
let file_out =
File::create(to).unwrap_or_else(|e| panic!("Unable to create file: {}: {}", to, e));
// We create a buffered writer from the file we get
let writer = BufWriter::new(&file_out);
let mmap = unsafe {
Mmap::map(&file_in)
.unwrap_or_else(|e| panic!("Unable to map in memory the file: {}: {}", from, e))
};
let v: T = serde_json::from_slice(&mmap[..])
.unwrap_or_else(|e| panic!("Unable to parse the json data from: {}: {}", from, e));
bincode::serialize_into(writer, &v).unwrap();
}
//FIXME: Move to ironsea_store?
pub fn load<T>(from: &str) -> T
where
T: DeserializeOwned,
{
let file_in =
File::open(from).unwrap_or_else(|e| panic!("Unable to read file: {}: {}", from, e));
let mmap = unsafe {
Mmap::map(&file_in)
.unwrap_or_else(|e| panic!("Unable to map in memory the file: {}: {}", from, e))
};
bincode::deserialize(&mmap[..])
.unwrap_or_else(|e| panic!("Unable to parse the json data from: {}: {}", from, e))
}
//FIXME: Move to ironsea_store?
pub fn store<T>(data: T, to: &str)
where
T: Serialize,
{
let file_out =
File::create(to).unwrap_or_else(|e| panic!("Unable to create file: {}: {}", to, e));
// We create a buffered writer from the file we get
let writer = BufWriter::new(&file_out);
bincode::serialize_into(writer, &data).unwrap();
}
pub fn convert(name: &str) {
// Convert Reference Space definitions
let fn_in = format!("{}.spaces.json", name);
let fn_out = format!("{}.spaces.bin", name);
from_json::<Vec<model::Space>>(&fn_in, &fn_out);
// Convert Spatial Objects
let fn_in = format!("{}.objects.json", name);
let fn_out = format!("{}.objects.bin", name);
from_json::<Vec<model::SpatialObject>>(&fn_in, &fn_out);
}
pub fn build(name: &str) {
let fn_spaces = format!("{}.spaces.bin", name);
let fn_objects = format!("{}.objects.bin", name);
let fn_index = format!("{}.index", name);
let spaces = load::<Vec<model::Space>>(&fn_spaces)
.iter()
.map(|s| s.into())
.collect::<Vec<_>>();
let cores = model::build_index(
&name,
&spaces,
&load::<Vec<model::SpatialObject>>(&fn_objects),
);
store(DataBase::new(spaces, cores), &fn_index);
}

1
src/lib.rs
View File

@@ -8,5 +8,6 @@ extern crate arrayref;
extern crate serde_derive;
mod database;
pub mod json;
pub use database::*;

21
src/main.rs
View File

@@ -1,14 +1,11 @@
#[macro_use]
extern crate measure_time;
#[macro_use]
extern crate arrayref;
#[macro_use]
extern crate serde_derive;
mod storage;
use mercator_db::json::model;
use mercator_db::json::storage;
use mercator_db::space::Shape;
use mercator_db::DataBase;
@@ -22,28 +19,26 @@ fn main() {
// Convert to binary the JSON data:
if true {
info_time!("Converting to binary JSON data");
storage::convert("test");
storage::convert("10k");
}
// Build a Database Index:
if true {
info_time!("Building database index");
storage::build("test");
storage::build("10k");
}
// Load a Database:
let db;
{
info_time!("Loading database index");
db = DataBase::load("test").unwrap();
db = DataBase::load("10k").unwrap();
}
if true {
let core = db.core("test").unwrap();
// 100k
let space = db.space("space0.146629817062").unwrap();
//let id = "oid0.606846546049";
let id = "oid0.732128500546";
let core = db.core("10k").unwrap();
let space = db.space("std").unwrap();
let id = "oid0.5793259558369925";
let r = core.get_by_id(&db, id, None, std::f64::MAX).unwrap();
println!("get_by_id {}: {}", id, r.len());

238
src/storage.rs
View File

@@ -1,238 +0,0 @@
use memmap::Mmap;
use serde::Deserialize;
use std::fs::File;
use std::io::BufWriter;
const K: usize = 3;
#[derive(Serialize, Deserialize, Debug)]
pub struct Properties {
pub id: String,
}
#[derive(Serialize, Deserialize, Debug)]
// Geometry is parametric as we have a specific deserializer for the JSON format.
pub struct Shape<'a, G> {
#[serde(rename = "type")]
pub type_name: &'a str,
pub geometry: G,
pub properties: Properties,
}
pub mod json {
use super::*;
use serde::Deserializer;
#[derive(Serialize, Deserialize, Debug)]
pub struct Geometry<'a> {
#[serde(rename = "type")]
pub type_name: &'a str,
#[serde(rename = "referenceSpace")]
pub reference_space: &'a str,
#[serde(deserialize_with = "deserialize_coordinates")]
pub coordinates: Vec<[f64; K]>,
}
fn deserialize_coordinates<'de, D>(deserializer: D) -> Result<Vec<[f64; K]>, D::Error>
where
D: Deserializer<'de>,
{
// Retrieve from the deserializer a vector of Strings, it is important to specify both the type
// of elements in the vector and use `Vec::` to obtain a vector from the json input.
// Vec<String> corresponds to ["0.1,0.1,0.1", ...] of the input.
let strings: Vec<String> = Vec::deserialize(deserializer)?;
let mut shape_coords = vec![];
// For each string, decompose into a fixed point float. A string might have multiple dimensions,
// we are generic in this regards, although we do not check for each point to be have a constant
// number of dimensions.
for pos_string in &strings {
// split the string on the `,`, convert each part to float, and store the vector.
let pos_float: Vec<f64> = pos_string
.split(',')
.map(move |a| a.parse::<f64>().unwrap())
.collect();
assert_eq!(pos_float.len(), K);
shape_coords.push(*array_ref![pos_float, 0, K])
}
Ok(shape_coords)
}
pub fn convert(from: &str, to: &str) {
let file_in = File::open(from).unwrap();
let file_out = File::create(to).expect("Unable to create file");
// We create a buffered writer from the file we get
let writer = BufWriter::new(&file_out);
let mmap = unsafe { Mmap::map(&file_in).unwrap() };
let v: Vec<Shape<Geometry>> = serde_json::from_slice(&mmap[..]).unwrap();
bincode::serialize_into(writer, &v).unwrap();
}
}
pub mod bin {
use super::*;
use mercator_db::space;
use mercator_db::Core;
use mercator_db::DataBase;
use mercator_db::Properties;
use mercator_db::SpaceSetObject;
use std::collections::HashMap;
#[derive(Serialize, Deserialize, Debug)]
pub struct Geometry<'a> {
pub type_name: &'a str,
pub reference_space: &'a str,
pub coordinates: Vec<[f64; K]>,
}
pub fn build(from: &str, to: &str) {
let file_in = File::open(from).unwrap();
let file_out = File::create(to).expect("Unable to create file");
// We create a buffered writer from the file we get
let writer = BufWriter::new(&file_out);
let mmap = unsafe { Mmap::map(&file_in).unwrap() };
let v: Vec<Shape<Geometry>> = bincode::deserialize(&mmap[..]).unwrap();
let mut spaces = vec![];
let mut properties = vec![];
let mut space_set_objects = Vec::with_capacity(v.len());
{
let mut properties_hm = HashMap::new();
let mut space_ids = HashMap::new();
let mut properties_ref = Vec::with_capacity(v.len());
// What to write in binary, a vec of json::shape or a Vec of SpaceShape?
for shape in &v {
assert!(shape.type_name == "Feature");
assert!(shape.geometry.type_name == "Point");
space_ids.insert(shape.geometry.reference_space, 1u8);
// Check if a properties Object exists, if not create it, keep an
// offset to a reference to that Properties.
// We store a new reference into a reference list, so that, we can
// later on build a deduplicated list and keep stable references.
// FIXME: Comment unclear
let value = match properties_hm.get(shape.properties.id.as_str()) {
Some(_) => {
properties_ref.push(shape.properties.id.as_str());
properties_ref.len() - 1
}
None => {
properties_hm.insert(
shape.properties.id.as_str(),
Properties::Feature(shape.properties.id.clone()),
);
properties_ref.push(shape.properties.id.as_str());
properties_ref.len() - 1
}
};
space_set_objects.push(SpaceSetObject::new(
shape.geometry.reference_space,
shape.geometry.coordinates[0].to_vec().into(),
value.into(),
));
}
properties.append(&mut properties_hm.drain().map(|(_, v)| v).collect::<Vec<_>>());
spaces.append(
&mut space_ids
.keys()
.map(|&space_name| {
space::Space::new(
space_name,
space::CoordinateSystem::new(
vec![0f64, 0f64, 0f64],
vec![
space::Axis::new(
"m",
vec![1f64, 0f64, 0f64],
space::NumberSet::N,
0.0,
1.0,
1E9 as u64,
)
.unwrap(),
space::Axis::new(
"m",
vec![0f64, 1f64, 0f64],
space::NumberSet::N,
0.0,
1.0,
1E9 as u64,
)
.unwrap(),
space::Axis::new(
"m",
vec![0f64, 0f64, 1f64],
space::NumberSet::N,
0.0,
1.0,
1E9 as u64,
)
.unwrap(),
],
),
)
})
.collect::<Vec<_>>(),
);
properties.sort_unstable_by_key(|p| p.id().clone());
space_set_objects.iter_mut().for_each(|object| {
let id = properties_ref[object.value().u64() as usize];
let value = properties.binary_search_by_key(&id, |p| p.id()).unwrap();
object.set_value(value.into());
});
}
let cores = vec![Core::new(
"test",
"v0.1",
&spaces,
properties,
space_set_objects,
)];
let db = DataBase::new(spaces, cores);
bincode::serialize_into(writer, &db).unwrap();
}
}
pub fn convert<S>(name: S)
where
S: Into<String>,
{
let name = name.into();
let fn_in = format!("{}.json", name);
let fn_out = format!("{}.bin", name);
json::convert(&fn_in, &fn_out);
}
pub fn build<S>(name: S)
where
S: Into<String>,
{
let name = name.into();
let fn_in = format!("{}.bin", name);
let fn_out = format!("{}.index", name);
bin::build(&fn_in, &fn_out);
}

1
test.json
View File

@@ -1 +0,0 @@
100k.json