Implement multiple backing stores for Rkv

Cargo.toml

@@ -14,8 +14,9 @@ categories = ["database"]
 exclude = ["/tests/envs/*"]
 
 [features]
-default = []
+default = ["with-safe-mode"]
 backtrace = ["failure/backtrace", "failure/std"]
+with-safe-mode = ["log", "uuid/v4", "uuid/serde", "serde/derive", "serde/rc"]
 with-asan = ["lmdb-rkv/with-asan"]
 with-fuzzer = ["lmdb-rkv/with-fuzzer"]
 with-fuzzer-no-link = ["lmdb-rkv/with-fuzzer-no-link"]
@@ -27,6 +28,7 @@ bitflags = "1"
 byteorder = "1"
 lazy_static = "1.0"
 lmdb-rkv = "0.12.3"
+log = { version = "0.4", optional = true }
 ordered-float = "1.0"
 uuid = "0.7"
 serde = "1.0"

examples/iterator.rs

@@ -7,6 +7,16 @@
 //!
 //!     cargo run --example iterator
 
+use std::fs;
+use std::str;
+
+use tempfile::Builder;
+
+use rkv::backend::{
+    Lmdb,
+    LmdbDatabase,
+    LmdbEnvironment,
+};
 use rkv::{
     Manager,
     Rkv,
@@ -15,21 +25,17 @@ use rkv::{
     StoreOptions,
     Value,
 };
-use tempfile::Builder;
-
-use std::fs;
-use std::str;
 
 fn main() {
     let root = Builder::new().prefix("iterator").tempdir().unwrap();
     fs::create_dir_all(root.path()).unwrap();
     let p = root.path();
 
-    let created_arc = Manager::singleton().write().unwrap().get_or_create(p, Rkv::new).unwrap();
+    let created_arc = Manager::singleton().write().unwrap().get_or_create(p, Rkv::new::<Lmdb>).unwrap();
     let k = created_arc.read().unwrap();
     let store = k.open_single("store", StoreOptions::create()).unwrap();
 
-    populate_store(&k, store).unwrap();
+    populate_store(&k, &store).unwrap();
 
     let reader = k.read().unwrap();
 
@@ -58,7 +64,7 @@ fn main() {
     }
 }
 
-fn populate_store(k: &Rkv, store: SingleStore) -> Result<(), StoreError> {
+fn populate_store(k: &Rkv<LmdbEnvironment>, store: &SingleStore<LmdbDatabase>) -> Result<(), StoreError> {
     let mut writer = k.write()?;
     for (country, city) in vec![
         ("Canada", Value::Str("Ottawa")),

examples/simple-store.rs

@@ -7,19 +7,27 @@
 //!
 //!     cargo run --example simple-store
 
+use std::fs;
+
+use tempfile::Builder;
+
+use rkv::backend::{
+    BackendStat,
+    Lmdb,
+    LmdbDatabase,
+    LmdbRwTransaction,
+};
 use rkv::{
     Manager,
-    MultiStore,
     Rkv,
     StoreOptions,
     Value,
-    Writer,
 };
-use tempfile::Builder;
 
-use std::fs;
+type MultiStore = rkv::MultiStore<LmdbDatabase>;
+type Writer<'env> = rkv::Writer<LmdbRwTransaction<'env>>;
 
-fn getput<'env, 's>(store: MultiStore, writer: &'env mut Writer, ids: &'s mut Vec<String>) {
+fn getput<'env, 's>(store: &MultiStore, writer: &'env mut Writer, ids: &'s mut Vec<String>) {
     let keys = vec!["str1", "str2", "str3"];
     // we convert the writer into a cursor so that we can safely read
     for k in keys.iter() {
@@ -38,7 +46,7 @@ fn getput<'env, 's>(store: MultiStore, writer: &'env mut Writer, ids: &'s mut Ve
     }
 }
 
-fn delete(store: MultiStore, writer: &mut Writer) {
+fn delete(store: &MultiStore, writer: &mut Writer) {
     let keys = vec!["str1", "str2", "str3"];
     let vals = vec!["string uno", "string quatro", "string siete"];
     // we convert the writer into a cursor so that we can safely read
@@ -53,12 +61,11 @@ fn main() {
     let p = root.path();
 
     // The manager enforces that each process opens the same lmdb environment at most once
-    let created_arc = Manager::singleton().write().unwrap().get_or_create(p, Rkv::new).unwrap();
+    let created_arc = Manager::singleton().write().unwrap().get_or_create(p, Rkv::new::<Lmdb>).unwrap();
     let k = created_arc.read().unwrap();
 
     // Creates a store called "store"
     let store = k.open_single("store", StoreOptions::create()).unwrap();
-
     let multistore = k.open_multi("multistore", StoreOptions::create()).unwrap();
 
     println!("Inserting data...");
@@ -89,12 +96,13 @@ fn main() {
         multistore.put(&mut writer, "str3", &Value::Str("string siete")).unwrap();
         multistore.put(&mut writer, "str3", &Value::Str("string ocho")).unwrap();
         multistore.put(&mut writer, "str3", &Value::Str("string nueve")).unwrap();
-        getput(multistore, &mut writer, &mut ids);
+        getput(&multistore, &mut writer, &mut ids);
         writer.commit().unwrap();
         let mut writer = k.write().unwrap();
-        delete(multistore, &mut writer);
+        delete(&multistore, &mut writer);
         writer.commit().unwrap();
     }
+
     println!("Looking up keys...");
     {
         // Use a reader to query the store
@@ -179,5 +187,6 @@ fn main() {
         println!("Get from store value: {:?}", store.get(&reader, "foo").unwrap());
         println!("Get from another store value: {:?}", another_store.get(&reader, "foo").unwrap());
     }
+
     println!("Environment statistics: btree depth = {}", k.stat().unwrap().depth());
 }

src/backend.rs

@@ -0,0 +1,33 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+mod common;
+mod impl_lmdb;
+mod impl_safe;
+mod traits;
+
+pub use common::*;
+pub use traits::*;
+
+pub use impl_lmdb::DatabaseImpl as LmdbDatabase;
+pub use impl_lmdb::EnvironmentBuilderImpl as Lmdb;
+pub use impl_lmdb::EnvironmentImpl as LmdbEnvironment;
+pub use impl_lmdb::ErrorImpl as LmdbError;
+pub use impl_lmdb::RoCursorImpl as LmdbRoCursor;
+pub use impl_lmdb::RoTransactionImpl as LmdbRoTransaction;
+pub use impl_lmdb::RwTransactionImpl as LmdbRwTransaction;
+
+pub use impl_safe::DatabaseImpl as SafeModeDatabase;
+pub use impl_safe::EnvironmentBuilderImpl as SafeMode;
+pub use impl_safe::EnvironmentImpl as SafeModeEnvironment;
+pub use impl_safe::ErrorImpl as SafeModeError;
+pub use impl_safe::RoCursorImpl as SafeModeRoCursor;
+pub use impl_safe::RoTransactionImpl as SafeModeRoTransaction;
+pub use impl_safe::RwTransactionImpl as SafeModeRwTransaction;

src/backend/common.rs

@@ -0,0 +1,41 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+#![allow(non_camel_case_types)]
+
+pub enum EnvironmentFlags {
+    FIXED_MAP,
+    NO_SUB_DIR,
+    WRITE_MAP,
+    READ_ONLY,
+    NO_META_SYNC,
+    NO_SYNC,
+    MAP_ASYNC,
+    NO_TLS,
+    NO_LOCK,
+    NO_READAHEAD,
+    NO_MEM_INIT,
+}
+
+pub enum DatabaseFlags {
+    REVERSE_KEY,
+    DUP_SORT,
+    INTEGER_KEY,
+    DUP_FIXED,
+    INTEGER_DUP,
+    REVERSE_DUP,
+}
+
+pub enum WriteFlags {
+    NO_OVERWRITE,
+    NO_DUP_DATA,
+    CURRENT,
+    APPEND,
+    APPEND_DUP,
+}

src/backend/impl_lmdb.rs

@@ -0,0 +1,42 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+mod cursor;
+mod database;
+mod environment;
+mod error;
+mod flags;
+mod info;
+mod iter;
+mod stat;
+mod transaction;
+
+pub use cursor::{
+    RoCursorImpl,
+    RwCursorImpl,
+};
+pub use database::DatabaseImpl;
+pub use environment::{
+    EnvironmentBuilderImpl,
+    EnvironmentImpl,
+};
+pub use error::ErrorImpl;
+pub use flags::{
+    DatabaseFlagsImpl,
+    EnvironmentFlagsImpl,
+    WriteFlagsImpl,
+};
+pub use info::InfoImpl;
+pub use iter::IterImpl;
+pub use stat::StatImpl;
+pub use transaction::{
+    RoTransactionImpl,
+    RwTransactionImpl,
+};

src/backend/impl_lmdb/cursor.rs

@@ -0,0 +1,64 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use lmdb::Cursor;
+
+use super::IterImpl;
+use crate::backend::traits::BackendRoCursor;
+
+#[derive(Debug)]
+pub struct RoCursorImpl<'env>(pub(crate) lmdb::RoCursor<'env>);
+
+impl<'env> BackendRoCursor<'env> for RoCursorImpl<'env> {
+    type Iter = IterImpl<'env>;
+
+    fn iter(&mut self) -> Self::Iter {
+        IterImpl(self.0.iter())
+    }
+
+    fn iter_from<K>(&mut self, key: K) -> Self::Iter
+    where
+        K: AsRef<[u8]>,
+    {
+        IterImpl(self.0.iter_from(key))
+    }
+
+    fn iter_dup_of<K>(&mut self, key: K) -> Self::Iter
+    where
+        K: AsRef<[u8]>,
+    {
+        IterImpl(self.0.iter_dup_of(key))
+    }
+}
+
+#[derive(Debug)]
+pub struct RwCursorImpl<'env>(pub(crate) lmdb::RwCursor<'env>);
+
+impl<'env> BackendRoCursor<'env> for RwCursorImpl<'env> {
+    type Iter = IterImpl<'env>;
+
+    fn iter(&mut self) -> Self::Iter {
+        IterImpl(self.0.iter())
+    }
+
+    fn iter_from<K>(&mut self, key: K) -> Self::Iter
+    where
+        K: AsRef<[u8]>,
+    {
+        IterImpl(self.0.iter_from(key))
+    }
+
+    fn iter_dup_of<K>(&mut self, key: K) -> Self::Iter
+    where
+        K: AsRef<[u8]>,
+    {
+        IterImpl(self.0.iter_dup_of(key))
+    }
+}

src/backend/impl_lmdb/database.rs

@@ -0,0 +1,16 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use crate::backend::traits::BackendDatabase;
+
+#[derive(Debug)]
+pub struct DatabaseImpl(pub(crate) lmdb::Database);
+
+impl BackendDatabase for DatabaseImpl {}