From 09391f45932bd417bacd4135e16a5453f82bb574 Mon Sep 17 00:00:00 2001 From: Caleb Moore Date: Tue, 23 Apr 2024 14:34:44 +1000 Subject: [PATCH 01/12] This changes destreaming to use serde. Also changed batch read to be a bit faster. --- .github/workflows/build.yml | 1 + .gitignore | 3 +- Cargo.toml | 6 +- aerospike-core/Cargo.toml | 1 + aerospike-core/src/batch/batch_executor.rs | 100 +- aerospike-core/src/batch/batch_read.rs | 10 +- aerospike-core/src/bin.rs | 57 +- aerospike-core/src/client.rs | 85 +- aerospike-core/src/cluster/mod.rs | 22 + .../src/commands/batch_read_command.rs | 68 +- aerospike-core/src/commands/buffer.rs | 138 +- aerospike-core/src/commands/delete_command.rs | 1 + .../src/commands/execute_udf_command.rs | 21 +- aerospike-core/src/commands/exists_command.rs | 1 + aerospike-core/src/commands/mod.rs | 3 +- .../src/commands/operate_command.rs | 100 +- aerospike-core/src/commands/particle_type.rs | 52 +- aerospike-core/src/commands/query_command.rs | 19 +- aerospike-core/src/commands/read_command.rs | 169 +- aerospike-core/src/commands/scan_command.rs | 19 +- aerospike-core/src/commands/single_command.rs | 15 +- aerospike-core/src/commands/stream_command.rs | 62 +- aerospike-core/src/commands/touch_command.rs | 1 + aerospike-core/src/commands/write_command.rs | 14 +- aerospike-core/src/derive/mod.rs | 19 + aerospike-core/src/derive/readable.rs | 1657 +++++++++++++++++ aerospike-core/src/derive/writable.rs | 291 +++ aerospike-core/src/errors.rs | 18 +- aerospike-core/src/expressions/mod.rs | 40 +- aerospike-core/src/lib.rs | 7 +- aerospike-core/src/msgpack/decoder.rs | 216 --- aerospike-core/src/msgpack/encoder.rs | 45 +- aerospike-core/src/msgpack/mod.rs | 3 +- aerospike-core/src/net/connection.rs | 28 + aerospike-core/src/operations/scalar.rs | 18 +- aerospike-core/src/query/recordset.rs | 17 +- aerospike-core/src/record.rs | 43 +- aerospike-core/src/traits.rs | 586 ++++++ aerospike-core/src/value.rs | 217 ++- aerospike-macro/src/lib.rs | 113 +- aerospike-macro/src/traits/mod.rs | 2 + aerospike-macro/src/traits/readable.rs | 184 ++ aerospike-macro/src/traits/writable.rs | 400 ++++ aerospike-rt/src/lib.rs | 1 + benches/client_server.rs | 83 +- tests/common/mod.rs | 2 +- tests/src/batch.rs | 6 +- tests/src/cdt_bitwise.rs | 83 +- tests/src/cdt_list.rs | 256 ++- tests/src/cdt_map.rs | 108 +- tests/src/derive.rs | 278 +++ tests/src/exp.rs | 45 +- tests/src/exp_bitwise.rs | 47 +- tests/src/exp_hll.rs | 26 +- tests/src/exp_list.rs | 10 +- tests/src/exp_map.rs | 8 +- tests/src/exp_op.rs | 13 +- tests/src/hll.rs | 71 +- tests/src/index.rs | 9 +- tests/src/kv.rs | 25 +- tests/src/mod.rs | 1 + tests/src/query.rs | 52 +- tests/src/scan.rs | 7 +- tests/src/serialization.rs | 7 +- 64 files changed, 4943 insertions(+), 1067 deletions(-) create mode 100644 aerospike-core/src/derive/mod.rs create mode 100644 aerospike-core/src/derive/readable.rs create mode 100644 aerospike-core/src/derive/writable.rs delete mode 100644 aerospike-core/src/msgpack/decoder.rs create mode 100644 aerospike-core/src/traits.rs create mode 100644 aerospike-macro/src/traits/mod.rs create mode 100644 aerospike-macro/src/traits/readable.rs create mode 100644 aerospike-macro/src/traits/writable.rs create mode 100644 tests/src/derive.rs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 66554e31..e8ee3d27 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,7 @@ on: pull_request: branches: - master + workflow_dispatch: env: CARGO_TERM_COLOR: always diff --git a/.gitignore b/.gitignore index c794e179..9ebe4fbe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target /Cargo.lock *.todo - +aerospike-rust-client.sublime-project +aerospike-rust-client.sublime-workspace diff --git a/Cargo.toml b/Cargo.toml index d7c5f91a..15b64158 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,9 +43,11 @@ harness = false members = ["tools/benchmark", "aerospike-core", "aerospike-rt", "aerospike-sync", "aerospike-macro"] [dev-dependencies] -env_logger = "0.9" +log = "0.4" +env_logger = "0.10.0" hex = "0.4" -bencher = "0.1" +criterion = { version = "0.5.1", features = ["async_tokio", "async_futures", "async"]} +serde = "1.0" serde_json = "1.0" rand = "0.8" lazy_static = "1.4" diff --git a/aerospike-core/Cargo.toml b/aerospike-core/Cargo.toml index d20a598d..56248c56 100644 --- a/aerospike-core/Cargo.toml +++ b/aerospike-core/Cargo.toml @@ -19,6 +19,7 @@ serde = { version = "1.0", features = ["derive"], optional = true } aerospike-rt = {path = "../aerospike-rt"} futures = {version = "0.3.16" } async-trait = "0.1.51" +num = "0.4.0" [features] serialization = ["serde"] diff --git a/aerospike-core/src/batch/batch_executor.rs b/aerospike-core/src/batch/batch_executor.rs index 053c677f..d2195e7f 100644 --- a/aerospike-core/src/batch/batch_executor.rs +++ b/aerospike-core/src/batch/batch_executor.rs @@ -36,29 +36,45 @@ impl BatchExecutor { } - pub async fn execute_batch_read( + pub async fn execute_batch_read( &self, policy: &BatchPolicy, - batch_reads: Vec, - ) -> Result> { - let batch_nodes = self.get_batch_nodes(&batch_reads, policy.replica)?; - let mut jobs = Vec::::new(); - for (node, node_jobs) in batch_nodes { - for node_chunk in node_jobs.chunks(MAX_BATCH_REQUEST_SIZE) { - jobs.push( BatchReadCommand::new(policy, node.clone(), node_chunk.to_vec()) ); + batch_reads: Vec>, + ) -> Result>> { + let total = batch_reads.len(); + let jobs = self.get_batch_nodes(policy, batch_reads)?; + let reads = self.execute_batch_jobs::(jobs, policy.concurrency).await?; + + let mut as_iter = reads.into_iter(); + if let Some(BatchReadCommand { mut batch_reads, mut original_indexes, .. }) = as_iter.next() { + // Reserve enough to make the first element the return value + batch_reads.reserve_exact(total - batch_reads.len()); + original_indexes.reserve_exact(total - original_indexes.len()); + // Shove everything into the same list + for another_job in as_iter { + batch_reads.extend(another_job.batch_reads); + original_indexes.extend(another_job.original_indexes); + } + + // Put records back where it belongs... this is 0(n) because everything is swapped into its correct position + for i in 0..batch_reads.len() { + while original_indexes[i] != i { + let to = original_indexes[i]; + batch_reads.swap(i, to); + original_indexes.swap(i, to); + } } + Ok(batch_reads) + } else { + Ok(Default::default()) } - let reads = self.execute_batch_jobs(jobs, policy.concurrency).await?; - let mut all_results: Vec<_> = reads.into_iter().flat_map(|cmd|cmd.batch_reads).collect(); - all_results.sort_by_key(|(_, i)|*i); - Ok(all_results.into_iter().map(|(b, _)|b).collect()) } - async fn execute_batch_jobs( + async fn execute_batch_jobs( &self, - jobs: Vec, + jobs: Vec>, concurrency: Concurrency, - ) -> Result> { + ) -> Result>> { let handles = jobs.into_iter().map(|job|job.execute(self.cluster.clone())); match concurrency { Concurrency::Sequential => futures::future::join_all(handles).await.into_iter().collect(), @@ -66,19 +82,51 @@ impl BatchExecutor { } } - fn get_batch_nodes( + fn get_batch_nodes<'l, T: serde::de::DeserializeOwned + Send>( &self, - batch_reads: &[BatchRead], - replica: crate::policy::Replica, - ) -> Result, Vec<(BatchRead, usize)>>> { - let mut map = HashMap::new(); - for (index, batch_read) in batch_reads.iter().enumerate() { - let node = self.node_for_key(&batch_read.key, replica)?; - map.entry(node) - .or_insert_with(Vec::new) - .push((batch_read.clone(), index)); + policy: &BatchPolicy, + batch_reads: Vec>, + ) -> Result>> { + let mut map: HashMap, (Vec>, Vec)> = HashMap::new(); + let mut vec = Vec::new(); + let choices = batch_reads.first().map(|read|self.cluster.n_nodes_for_policy(&read.key.namespace, policy.replica)).unwrap_or_default(); + vec.reserve(choices); + let estimate = batch_reads.len() / (choices.max(2) - 1); + + for (index, batch_read) in batch_reads.into_iter().enumerate() { + let node = self.node_for_key(&batch_read.key, policy.replica)?; + let (reads, indexes) = map.entry(node) + .or_insert_with(||{ + let mut reads = Vec::new(); + let mut indexes = Vec::new(); + if estimate > MAX_BATCH_REQUEST_SIZE { + reads.reserve_exact(MAX_BATCH_REQUEST_SIZE); + indexes.reserve_exact(MAX_BATCH_REQUEST_SIZE); + } else { + reads.reserve(estimate); + indexes.reserve(estimate); + } + (reads, indexes) + }); + + // Enough reads, make a new one. + if reads.len() >= MAX_BATCH_REQUEST_SIZE { + // To avoid copying node above, we just re-do node from key when it's needed. + let node = self.node_for_key(&batch_read.key, policy.replica)?; + vec.push(BatchReadCommand::new(policy, node, std::mem::take(reads), std::mem::take(indexes))); + // If we're blowing out buffers, we'll probably do it again. + reads.reserve_exact(MAX_BATCH_REQUEST_SIZE); + indexes.reserve_exact(MAX_BATCH_REQUEST_SIZE); + } + reads.push(batch_read); + indexes.push(index); + } + + vec.reserve_exact(map.len()); + for (node, (reads, indexes)) in map { + vec.push(BatchReadCommand::new(policy, node, reads, indexes)); } - Ok(map) + Ok(vec) } fn node_for_key(&self, key: &Key, replica: crate::policy::Replica) -> Result> { diff --git a/aerospike-core/src/batch/batch_read.rs b/aerospike-core/src/batch/batch_read.rs index 420f634a..ede32392 100644 --- a/aerospike-core/src/batch/batch_read.rs +++ b/aerospike-core/src/batch/batch_read.rs @@ -22,7 +22,7 @@ use serde::Serialize; /// Key and bin names used in batch read commands where variable bins are needed for each key. #[cfg_attr(feature = "serialization", derive(Serialize))] #[derive(Debug, Clone)] -pub struct BatchRead { +pub struct BatchRead { /// Key. pub key: Key, @@ -30,10 +30,10 @@ pub struct BatchRead { pub bins: Bins, /// Will contain the record after the batch read operation. - pub record: Option, + pub record: Option>, } -impl BatchRead { +impl BatchRead { /// Create a new `BatchRead` instance for the given key and bin selector. pub const fn new(key: Key, bins: Bins) -> Self { BatchRead { @@ -44,11 +44,11 @@ impl BatchRead { } #[doc(hidden)] - pub fn match_header(&self, other: &BatchRead, match_set: bool) -> bool { + pub fn match_header(&self, other: &BatchRead, match_set: bool) -> bool { let key = &self.key; let other_key = &other.key; (key.namespace == other_key.namespace) && (match_set && (key.set_name == other_key.set_name)) && (self.bins == other.bins) } -} +} \ No newline at end of file diff --git a/aerospike-core/src/bin.rs b/aerospike-core/src/bin.rs index dc84ac85..7e34abab 100644 --- a/aerospike-core/src/bin.rs +++ b/aerospike-core/src/bin.rs @@ -20,22 +20,22 @@ use std::convert::From; /// Container object for a record bin, comprising a name and a value. #[derive(Clone)] -pub struct Bin<'a> { +pub struct Bin { /// Bin name - pub name: &'a str, + pub name: String, /// Bin value pub value: Value, } -impl<'a> Bin<'a> { +impl Bin { /// Construct a new bin given a name and a value. - pub const fn new(name: &'a str, val: Value) -> Self { + pub const fn new(name: String, val: Value) -> Self { Bin { name, value: val } } } -impl<'a> AsRef> for Bin<'a> { +impl AsRef for Bin { fn as_ref(&self) -> &Self { self } @@ -45,10 +45,10 @@ impl<'a> AsRef> for Bin<'a> { #[macro_export] macro_rules! as_bin { ($bin_name:expr, None) => {{ - $crate::Bin::new($bin_name, $crate::Value::Nil) + $crate::Bin::new($bin_name.into(), $crate::Value::Nil) }}; ($bin_name:expr, $val:expr) => {{ - $crate::Bin::new($bin_name, $crate::Value::from($val)) + $crate::Bin::new($bin_name.into(), $crate::Value::from($val)) }}; } @@ -78,50 +78,15 @@ impl Bins { } } -impl<'a> From<&'a [&'a str]> for Bins { - fn from(bins: &'a [&'a str]) -> Self { +impl From<&[&str]> for Bins { + fn from(bins: &[&str]) -> Self { let bins = bins.iter().copied().map(String::from).collect(); Bins::Some(bins) } } -impl<'a> From<[&'a str; 1]> for Bins { - fn from(bins: [&'a str; 1]) -> Self { - let bins = bins.iter().copied().map(String::from).collect(); - Bins::Some(bins) - } -} - -impl<'a> From<[&'a str; 2]> for Bins { - fn from(bins: [&'a str; 2]) -> Self { - let bins = bins.iter().copied().map(String::from).collect(); - Bins::Some(bins) - } -} - -impl<'a> From<[&'a str; 3]> for Bins { - fn from(bins: [&'a str; 3]) -> Self { - let bins = bins.iter().copied().map(String::from).collect(); - Bins::Some(bins) - } -} - -impl<'a> From<[&'a str; 4]> for Bins { - fn from(bins: [&'a str; 4]) -> Self { - let bins = bins.iter().copied().map(String::from).collect(); - Bins::Some(bins) - } -} - -impl<'a> From<[&'a str; 5]> for Bins { - fn from(bins: [&'a str; 5]) -> Self { - let bins = bins.iter().copied().map(String::from).collect(); - Bins::Some(bins) - } -} - -impl<'a> From<[&'a str; 6]> for Bins { - fn from(bins: [&'a str; 6]) -> Self { +impl From<[&str; COUNT]> for Bins { + fn from(bins: [&str; COUNT]) -> Self { let bins = bins.iter().copied().map(String::from).collect(); Bins::Some(bins) } diff --git a/aerospike-core/src/client.rs b/aerospike-core/src/client.rs index 7b3bf313..70bf75d3 100644 --- a/aerospike-core/src/client.rs +++ b/aerospike-core/src/client.rs @@ -13,6 +13,7 @@ // License for the specific language governing permissions and limitations under // the License. +use std::collections::HashMap; use std::path::Path; use std::str; use std::sync::Arc; @@ -20,19 +21,19 @@ use std::vec::Vec; use crate::batch::BatchExecutor; use crate::cluster::{Cluster, Node}; -use crate::commands::operate_command::OperateRecord; use crate::commands::{ DeleteCommand, ExecuteUDFCommand, ExistsCommand, OperateCommand, QueryCommand, ReadCommand, ScanCommand, TouchCommand, WriteCommand, }; +use crate::derive::writable::WritableBins; use crate::errors::{ErrorKind, Result, ResultExt}; use crate::net::ToHosts; use crate::operations::{Operation, OperationType}; use crate::policy::{BatchPolicy, ClientPolicy, QueryPolicy, ReadPolicy, ScanPolicy, WritePolicy}; use crate::task::{IndexTask, RegisterTask}; use crate::{ - BatchRead, Bin, Bins, CollectionIndexType, IndexType, Key, Record, Recordset, ResultCode, - Statement, UDFLang, Value, + BatchRead, Bins, CollectionIndexType, IndexType, Key, Record, Recordset, ResultCode, Statement, + UDFLang, Value, }; use aerospike_rt::fs::File; #[cfg(all(any(feature = "rt-tokio"), not(feature = "rt-async-std")))] @@ -179,7 +180,12 @@ impl Client { /// /// # Panics /// Panics if the return is invalid - pub async fn get(&self, policy: &ReadPolicy, key: &Key, bins: T) -> Result + pub async fn get( + &self, + policy: &ReadPolicy, + key: &Key, + bins: T, + ) -> Result> where T: Into + Send + Sync + 'static, { @@ -223,11 +229,11 @@ impl Client { /// => println!("Error executing batch request: {}", err), /// } /// ``` - pub async fn batch_get( + pub async fn batch_get( &self, policy: &BatchPolicy, - batch_reads: Vec, - ) -> Result> { + batch_reads: Vec>, + ) -> Result>> { let executor = BatchExecutor::new(self.cluster.clone()); executor.execute_batch_read(policy, batch_reads).await } @@ -268,11 +274,11 @@ impl Client { /// Err(err) => println!("Error writing record: {}", err), /// } /// ``` - pub async fn put<'a, 'b>( + pub async fn put( &self, - policy: &'a WritePolicy, - key: &'a Key, - bins: &'a [Bin<'b>], + policy: &WritePolicy, + key: &Key, + bins: &T, ) -> Result<()> { let mut command = WriteCommand::new( policy, @@ -306,11 +312,11 @@ impl Client { /// Err(err) => println!("Error writing record: {}", err), /// } /// ``` - pub async fn add<'a, 'b>( + pub async fn add( &self, - policy: &'a WritePolicy, - key: &'a Key, - bins: &'a [Bin<'b>], + policy: &WritePolicy, + key: &Key, + bins: &T, ) -> Result<()> { let mut command = WriteCommand::new(policy, self.cluster.clone(), key, bins, OperationType::Incr); @@ -320,11 +326,11 @@ impl Client { /// Append bin string values to existing record bin values. The policy specifies the /// transaction timeout, record expiration and how the transaction is handled when the record /// already exists. This call only works for string values. - pub async fn append<'a, 'b>( + pub async fn append( &self, - policy: &'a WritePolicy, - key: &'a Key, - bins: &'a [Bin<'b>], + policy: &WritePolicy, + key: &Key, + bins: &T, ) -> Result<()> { let mut command = WriteCommand::new( policy, @@ -339,11 +345,11 @@ impl Client { /// Prepend bin string values to existing record bin values. The policy specifies the /// transaction timeout, record expiration and how the transaction is handled when the record /// already exists. This call only works for string values. - pub async fn prepend<'a, 'b>( + pub async fn prepend( &self, - policy: &'a WritePolicy, - key: &'a Key, - bins: &'a [Bin<'b>], + policy: &WritePolicy, + key: &Key, + bins: &T, ) -> Result<()> { let mut command = WriteCommand::new( policy, @@ -441,15 +447,14 @@ impl Client { /// ``` /// # Panics /// Panics if the return is invalid - pub async fn operate( + pub async fn operate( &self, policy: &WritePolicy, key: &Key, ops: &[Operation<'_>], - ) -> Result { + ) -> Result> { let mut command = OperateCommand::new(policy, self.cluster.clone(), key, ops); - command.execute().await?; - Ok(command.record) + command.execute().await } /// Register a package containing user-defined functions (UDF) with the cluster. This @@ -576,7 +581,7 @@ impl Client { function_name: &str, args: Option<&[Value]>, ) -> Result> { - let mut command = ExecuteUDFCommand::new( + let mut command: ExecuteUDFCommand> = ExecuteUDFCommand::new( policy, self.cluster.clone(), key, @@ -636,13 +641,13 @@ impl Client { /// /// # Panics /// Panics if the async block fails - pub async fn scan( + pub async fn scan( &self, policy: &ScanPolicy, namespace: &str, set_name: &str, bins: T, - ) -> Result> + ) -> Result>> where T: Into + Send + Sync + 'static, { @@ -658,7 +663,7 @@ impl Client { let set_name = set_name.to_owned(); let bins = bins.clone(); - aerospike_rt::spawn(async move { + let _ = aerospike_rt::spawn(async move { let mut command = ScanCommand::new( &policy, node, &namespace, &set_name, bins, recordset, partitions, ); @@ -677,14 +682,14 @@ impl Client { /// /// # Panics /// panics if the async block fails - pub async fn scan_node( + pub async fn scan_node( &self, policy: &ScanPolicy, node: Arc, namespace: &str, set_name: &str, bins: T, - ) -> Result> + ) -> Result>> where T: Into + Send + Sync + 'static, { @@ -696,7 +701,7 @@ impl Client { let namespace = namespace.to_owned(); let set_name = set_name.to_owned(); - aerospike_rt::spawn(async move { + let _ = aerospike_rt::spawn(async move { let mut command = ScanCommand::new( &policy, node, @@ -738,11 +743,11 @@ impl Client { /// /// # Panics /// Panics if the async block fails - pub async fn query( + pub async fn query( &self, policy: &QueryPolicy, statement: Statement, - ) -> Result> { + ) -> Result>> { statement.validate()?; let statement = Arc::new(statement); @@ -756,7 +761,7 @@ impl Client { let t_recordset = recordset.clone(); let policy = policy.clone(); let statement = statement.clone(); - aerospike_rt::spawn(async move { + let _ = aerospike_rt::spawn(async move { let mut command = QueryCommand::new(&policy, node, statement, t_recordset, partitions); command.execute().await.unwrap(); @@ -772,12 +777,12 @@ impl Client { /// /// # Panics /// Panics when the async block fails - pub async fn query_node( + pub async fn query_node( &self, policy: &QueryPolicy, node: Arc, statement: Statement, - ) -> Result> { + ) -> Result>> { statement.validate()?; let recordset = Arc::new(Recordset::new(policy.record_queue_size, 1)); @@ -788,7 +793,7 @@ impl Client { .cluster .node_partitions(node.as_ref(), &statement.namespace); - aerospike_rt::spawn(async move { + let _ = aerospike_rt::spawn(async move { let mut command = QueryCommand::new(&policy, node, statement, t_recordset, partitions); command.execute().await.unwrap(); }) diff --git a/aerospike-core/src/cluster/mod.rs b/aerospike-core/src/cluster/mod.rs index 62c812d7..d23ff217 100644 --- a/aerospike-core/src/cluster/mod.rs +++ b/aerospike-core/src/cluster/mod.rs @@ -561,6 +561,28 @@ impl Cluster { namespace.get_node(self, partition, replica, last_tried) } + pub fn n_nodes_for_policy(&self, namespace: &str, replica: crate::policy::Replica) -> usize { + let partitions = self.partition_write_map.lock().unwrap(); + + let Some(partition) = partitions + .get(namespace) else { + return 0; + }; + + match replica { + crate::policy::Replica::Master | crate::policy::Replica::Sequence => { + partition.nodes.iter().flat_map(|(_, node)|node.as_ref()).count() + }, + crate::policy::Replica::PreferRack => { + let Some(rack_ids) = self.client_policy.rack_ids.as_ref() else { + return 0; + }; + partition.nodes.iter().flat_map(|(_, node)|node.as_ref()).filter(|node|node.is_in_rack(namespace, rack_ids)) + .count() + }, + } + } + pub fn get_random_node(&self) -> Result> { let node_array = self.nodes(); let length = node_array.len() as isize; diff --git a/aerospike-core/src/commands/batch_read_command.rs b/aerospike-core/src/commands/batch_read_command.rs index 732cea10..6c4a7e49 100644 --- a/aerospike-core/src/commands/batch_read_command.rs +++ b/aerospike-core/src/commands/batch_read_command.rs @@ -13,7 +13,6 @@ // limitations under the License. use aerospike_rt::time::Instant; -use std::collections::HashMap; use std::sync::Arc; use crate::cluster::{Node, Cluster}; @@ -22,27 +21,29 @@ use crate::commands; use crate::errors::{ErrorKind, Result, ResultExt}; use crate::net::Connection; use crate::policy::{BatchPolicy, Policy, PolicyLike, Replica}; -use crate::{value, BatchRead, Record, ResultCode, Value}; +use crate::{BatchRead, Record, ResultCode}; use aerospike_rt::sleep; -struct BatchRecord { +struct BatchRecord { batch_index: usize, - record: Option, + record: Option>, } #[derive(Clone, Debug)] -pub struct BatchReadCommand { +pub struct BatchReadCommand { policy: BatchPolicy, pub node: Arc, - pub batch_reads: Vec<(BatchRead, usize)>, + pub batch_reads: Vec>, + pub original_indexes: Vec, } -impl BatchReadCommand { - pub fn new(policy: &BatchPolicy, node: Arc, batch_reads: Vec<(BatchRead, usize)>) -> Self { +impl BatchReadCommand { + pub fn new(policy: &BatchPolicy, node: Arc, batch_reads: Vec>, original_indexes: Vec) -> Self { BatchReadCommand { policy: policy.clone(), node, batch_reads, + original_indexes, } } @@ -55,15 +56,18 @@ impl BatchReadCommand { // Execute command until successful, timed out or maximum iterations have been reached. loop { - let success = if iterations & 1 == 0 || matches!(self.policy.replica, Replica::Master) { + if iterations & 1 == 0 || matches!(self.policy.replica, Replica::Master) { // For even iterations, we request all keys from the same node for efficiency. - Self::request_group(&mut self.batch_reads, &self.policy, self.node.clone(), deadline).await? + if Self::request_group(&mut self.batch_reads, &self.policy, self.node.clone(), deadline).await? { + // command has completed successfully. Exit method. + return Ok(self); + } } else { // However, for odd iterations try the second choice for each. Instead of re-sharding the batch (as the second choice may not correspond to the first), just try each by itself. let mut all_successful = true; for individual_read in self.batch_reads.chunks_mut(1) { // Find somewhere else to try. - let partition = Partition::new_by_key(&individual_read[0].0.key); + let partition = Partition::new_by_key(&individual_read[0].key); let node = cluster.get_node(&partition, self.policy.replica, Arc::downgrade(&self.node))?; if !Self::request_group(individual_read, &self.policy, node, deadline).await? { @@ -71,13 +75,12 @@ impl BatchReadCommand { break; } } - all_successful - }; - if success { - // command has completed successfully. Exit method. - return Ok(self); - } + if all_successful { + // command has completed successfully. Exit method. + return Ok(self); + } + }; iterations += 1; @@ -105,7 +108,7 @@ impl BatchReadCommand { } } - async fn request_group(batch_reads: &mut [(BatchRead, usize)], policy: &BatchPolicy, node: Arc, deadline: Option) -> Result { + async fn request_group(batch_reads: &mut [BatchRead], policy: &BatchPolicy, node: Arc, deadline: Option) -> Result { let mut conn = match crate::commands::single_command::try_with_timeout(deadline, node.get_connection()).await { Ok(conn) => conn, Err(err) => { @@ -144,7 +147,7 @@ impl BatchReadCommand { } } - async fn parse_group(batch_reads: &mut [(BatchRead, usize)], conn: &mut Connection, size: usize) -> Result { + async fn parse_group(batch_reads: &mut [BatchRead], conn: &mut Connection, size: usize) -> Result { while conn.bytes_read() < size { conn.read_buffer(commands::buffer::MSG_REMAINING_HEADER_SIZE as usize) .await?; @@ -154,14 +157,14 @@ impl BatchReadCommand { let batch_read = batch_reads .get_mut(batch_record.batch_index) .expect("Invalid batch index"); - batch_read.0.record = batch_record.record; + batch_read.record = batch_record.record; } } } Ok(true) } - async fn parse_record(conn: &mut Connection) -> Result> { + async fn parse_record(conn: &mut Connection) -> Result>> { // if cmd is the end marker of the response, do not proceed further let info3 = conn.buffer.read_u8(Some(3)); if info3 & commands::buffer::INFO3_LAST == commands::buffer::INFO3_LAST { @@ -181,27 +184,12 @@ impl BatchReadCommand { let field_count = conn.buffer.read_u16(None) as usize; // almost certainly 0 let op_count = conn.buffer.read_u16(None) as usize; - let key = commands::StreamCommand::parse_key(conn, field_count).await?; + let key = commands::StreamCommand::::parse_key(conn, field_count).await?; let record = if found_key { - let mut bins: HashMap = HashMap::with_capacity(op_count); - - for _ in 0..op_count { - conn.read_buffer(8).await?; - let op_size = conn.buffer.read_u32(None) as usize; - conn.buffer.skip(1); - let particle_type = conn.buffer.read_u8(None); - conn.buffer.skip(1); - let name_size = conn.buffer.read_u8(None) as usize; - conn.read_buffer(name_size).await?; - let name = conn.buffer.read_str(name_size)?; - let particle_bytes_size = op_size - (4 + name_size); - conn.read_buffer(particle_bytes_size).await?; - let value = - value::bytes_to_particle(particle_type, &mut conn.buffer, particle_bytes_size)?; - bins.insert(name, value); - } + let reader = crate::derive::readable::BinsDeserializer{ bins: conn.pre_parse_stream_bins(op_count).await?.into() }; + let bins = T::deserialize(reader)?; Some(Record::new(Some(key), bins, generation, expiration)) } else { None @@ -212,7 +200,7 @@ impl BatchReadCommand { })) } - async fn parse_result(batch_reads: &mut [(BatchRead, usize)], conn: &mut Connection) -> Result<()> { + async fn parse_result(batch_reads: &mut [BatchRead], conn: &mut Connection) -> Result<()> { loop { conn.read_buffer(8).await?; let size = conn.buffer.read_msg_size(None); diff --git a/aerospike-core/src/commands/buffer.rs b/aerospike-core/src/commands/buffer.rs index 1dbf0d13..88542e25 100644 --- a/aerospike-core/src/commands/buffer.rs +++ b/aerospike-core/src/commands/buffer.rs @@ -26,7 +26,7 @@ use crate::policy::{ BatchPolicy, CommitLevel, ConsistencyLevel, GenerationPolicy, QueryPolicy, BasePolicy, RecordExistsAction, ScanPolicy, WritePolicy, }; -use crate::{BatchRead, Bin, Bins, CollectionIndexType, Key, Statement, Value}; +use crate::{BatchRead, Bins, CollectionIndexType, Key, Statement, Value}; // Contains a read operation. const INFO1_READ: u8 = 1; @@ -95,15 +95,16 @@ const AS_MSG_TYPE: u8 = 3; // LDT elements in your queries. const MAX_BUFFER_SIZE: usize = 16 * 1024 * 1024 + 8; // 16 MB + header -// Holds data buffer for the command -#[derive(Debug, Default)] +/// Aerospike Wire Buffer. This holds the raw communication Buffer for the commands to read and write. +#[derive(Debug, Default, Clone)] pub struct Buffer { pub data_buffer: Vec, pub data_offset: usize, - pub reclaim_threshold: usize, + pub(crate) reclaim_threshold: usize, } impl Buffer { + /// Create new Buffer Instance pub fn new(reclaim_threshold: usize) -> Self { Buffer { data_buffer: Vec::with_capacity(1024), @@ -116,11 +117,12 @@ impl Buffer { self.data_offset = MSG_TOTAL_HEADER_SIZE as usize; } - pub fn size_buffer(&mut self) -> Result<()> { + pub(crate) fn size_buffer(&mut self) -> Result<()> { let offset = self.data_offset; self.resize_buffer(offset) } + /// Size the Buffer Byte Vec to the given length pub fn resize_buffer(&mut self, size: usize) -> Result<()> { // Corrupted data streams can result in a huge length. // Do a sanity check here. @@ -137,12 +139,13 @@ impl Buffer { Ok(()) } + /// Reset the Buffer Offset to 0 pub fn reset_offset(&mut self) { // reset data offset self.data_offset = 0; } - pub fn end(&mut self) { + pub(crate) fn end(&mut self) { let size = ((self.data_offset - 8) as i64) | ((i64::from(CL_MSG_VERSION) << 56) as i64) | (i64::from(AS_MSG_TYPE) << 48); @@ -153,12 +156,12 @@ impl Buffer { } // Writes the command for write operations - pub fn set_write<'b, A: AsRef>>( + pub(crate) fn set_write( &mut self, policy: &WritePolicy, op_type: OperationType, key: &Key, - bins: &[A], + bins: &T, ) -> Result<()> { self.begin(); let mut field_count = self.estimate_key_size(key, policy.send_key); @@ -167,9 +170,10 @@ impl Buffer { field_count += 1; } - for bin in bins { - self.estimate_operation_size_for_bin(bin.as_ref()); - } + self.data_offset += bins.writable_bins_size(); + //for bin in bins { + // self.estimate_operation_size_for_bin(bin.as_ref()); + //} self.size_buffer()?; self.write_header_with_policy( @@ -177,23 +181,24 @@ impl Buffer { 0, INFO2_WRITE, field_count as u16, - bins.len() as u16, + bins.writable_bins_count() as u16, ); self.write_key(key, policy.send_key); if let Some(filter) = policy.filter_expression() { self.write_filter_expression(filter, filter_size); } - for bin in bins { + bins.write_as_bins(self, op_type as u8)?; + /*for bin in bins { self.write_operation_for_bin(bin.as_ref(), op_type); - } + }*/ self.end(); Ok(()) } // Writes the command for write operations - pub fn set_delete(&mut self, policy: &WritePolicy, key: &Key) -> Result<()> { + pub(crate) fn set_delete(&mut self, policy: &WritePolicy, key: &Key) -> Result<()> { self.begin(); let mut field_count = self.estimate_key_size(key, false); let filter_size = self.estimate_filter_size(policy.filter_expression()); @@ -214,7 +219,7 @@ impl Buffer { } // Writes the command for touch operations - pub fn set_touch(&mut self, policy: &WritePolicy, key: &Key) -> Result<()> { + pub(crate) fn set_touch(&mut self, policy: &WritePolicy, key: &Key) -> Result<()> { self.begin(); let mut field_count = self.estimate_key_size(key, policy.send_key); let filter_size = self.estimate_filter_size(policy.filter_expression()); @@ -236,7 +241,7 @@ impl Buffer { } // Writes the command for exist operations - pub fn set_exists(&mut self, policy: &WritePolicy, key: &Key) -> Result<()> { + pub(crate) fn set_exists(&mut self, policy: &WritePolicy, key: &Key) -> Result<()> { self.begin(); let mut field_count = self.estimate_key_size(key, false); let filter_size = self.estimate_filter_size(policy.filter_expression()); @@ -263,7 +268,7 @@ impl Buffer { } // Writes the command for get operations - pub fn set_read(&mut self, policy: &BasePolicy, key: &Key, bins: &Bins) -> Result<()> { + pub(crate) fn set_read(&mut self, policy: &BasePolicy, key: &Key, bins: &Bins) -> Result<()> { match bins { Bins::None => self.set_read_header(policy, key), Bins::All => self.set_read_for_key_only(policy, key), @@ -297,7 +302,7 @@ impl Buffer { } // Writes the command for getting metadata operations - pub fn set_read_header(&mut self, policy: &BasePolicy, key: &Key) -> Result<()> { + pub(crate) fn set_read_header(&mut self, policy: &BasePolicy, key: &Key) -> Result<()> { self.begin(); let mut field_count = self.estimate_key_size(key, false); let filter_size = self.estimate_filter_size(policy.filter_expression()); @@ -319,7 +324,7 @@ impl Buffer { Ok(()) } - pub fn set_read_for_key_only(&mut self, policy: &BasePolicy, key: &Key) -> Result<()> { + pub(crate) fn set_read_for_key_only(&mut self, policy: &BasePolicy, key: &Key) -> Result<()> { self.begin(); let mut field_count = self.estimate_key_size(key, false); @@ -341,10 +346,10 @@ impl Buffer { } // Writes the command for batch read operations - pub fn set_batch_read( + pub(crate) fn set_batch_read ( &mut self, policy: &BatchPolicy, - batch_reads: &[(BatchRead, usize)], + batch_reads: &[BatchRead], ) -> Result<()> { let field_count_row = if policy.send_set_name { 2 } else { 1 }; @@ -357,8 +362,8 @@ impl Buffer { field_count += 1; } - let mut prev: Option<&BatchRead> = None; - for (batch_read, _) in batch_reads { + let mut prev: Option<&BatchRead> = None; + for batch_read in batch_reads { self.data_offset += batch_read.key.digest.len() + 4; match prev { Some(prev) if batch_read.match_header(prev, policy.send_set_name) => { @@ -404,7 +409,7 @@ impl Buffer { self.write_u8(if policy.allow_inline { 1 } else { 0 }); prev = None; - for (idx, (batch_read, _)) in batch_reads.iter().enumerate() { + for (idx, batch_read) in batch_reads.iter().enumerate() { let key = &batch_read.key; self.write_u32(idx as u32); self.write_bytes(&key.digest); @@ -462,7 +467,7 @@ impl Buffer { } // Writes the command for getting metadata operations - pub fn set_operate<'a>( + pub(crate) fn set_operate<'a>( &mut self, policy: &WritePolicy, key: &Key, @@ -549,7 +554,7 @@ impl Buffer { Ok(()) } - pub fn set_udf( + pub(crate) fn set_udf( &mut self, policy: &WritePolicy, key: &Key, @@ -581,7 +586,7 @@ impl Buffer { Ok(()) } - pub fn set_scan( + pub(crate) fn set_scan( &mut self, policy: &ScanPolicy, namespace: &str, @@ -696,7 +701,7 @@ impl Buffer { } #[allow(clippy::cognitive_complexity)] - pub fn set_query( + pub(crate) fn set_query( &mut self, policy: &QueryPolicy, statement: &Statement, @@ -956,11 +961,6 @@ impl Buffer { 3 } - fn estimate_operation_size_for_bin(&mut self, bin: &Bin) { - self.data_offset += bin.name.len() + OPERATION_HEADER_SIZE as usize; - self.data_offset += bin.value.estimate_size(); - } - fn estimate_operation_size_for_bin_name(&mut self, bin_name: &str) { self.data_offset += bin_name.len() + OPERATION_HEADER_SIZE as usize; } @@ -1124,19 +1124,6 @@ impl Buffer { } } - fn write_operation_for_bin(&mut self, bin: &Bin, op_type: OperationType) { - let name_length = bin.name.len(); - let value_length = bin.value.estimate_size(); - - self.write_i32((name_length + value_length + 4) as i32); - self.write_u8(op_type as u8); - self.write_u8(bin.value.particle_type() as u8); - self.write_u8(0); - self.write_u8(name_length as u8); - self.write_str(bin.name); - bin.value.write_to(self); - } - fn write_operation_for_bin_name(&mut self, name: &str, op_type: OperationType) { self.write_i32(name.len() as i32 + 4); self.write_u8(op_type as u8); @@ -1156,22 +1143,22 @@ impl Buffer { // Data buffer implementations + /// Get the current Data Offset pub const fn data_offset(&self) -> usize { self.data_offset } - pub fn skip_bytes(&mut self, count: usize) { - self.data_offset += count; - } - + /// Skips the amount of next bytes pub fn skip(&mut self, count: usize) { self.data_offset += count; } + /// Returns the current byte without moving the index pub fn peek(&self) -> u8 { self.data_buffer[self.data_offset] } + /// Reads a u8 Value from the Buffer #[allow(clippy::option_if_let_else)] pub fn read_u8(&mut self, pos: Option) -> u8 { if let Some(pos) = pos { @@ -1183,6 +1170,7 @@ impl Buffer { } } + /// Reads a i8 Value from the Buffer #[allow(clippy::option_if_let_else)] pub fn read_i8(&mut self, pos: Option) -> i8 { if let Some(pos) = pos { @@ -1194,6 +1182,7 @@ impl Buffer { } } + /// Reads a u16 Value from the Buffer #[allow(clippy::option_if_let_else)] pub fn read_u16(&mut self, pos: Option) -> u16 { let len = 2; @@ -1208,11 +1197,13 @@ impl Buffer { } } + /// Reads a i16 Value from the Buffer pub fn read_i16(&mut self, pos: Option) -> i16 { let val = self.read_u16(pos); val as i16 } + /// Reads a u32 Value from the Buffer #[allow(clippy::option_if_let_else)] pub fn read_u32(&mut self, pos: Option) -> u32 { let len = 4; @@ -1227,11 +1218,13 @@ impl Buffer { } } + /// Reads a i32 Value from the Buffer pub fn read_i32(&mut self, pos: Option) -> i32 { let val = self.read_u32(pos); val as i32 } + /// Reads a u64 Value from the Buffer #[allow(clippy::option_if_let_else)] pub fn read_u64(&mut self, pos: Option) -> u64 { let len = 8; @@ -1246,17 +1239,19 @@ impl Buffer { } } + /// Reads a i16 Value from the Buffer pub fn read_i64(&mut self, pos: Option) -> i64 { let val = self.read_u64(pos); val as i64 } - pub fn read_msg_size(&mut self, pos: Option) -> usize { + pub(crate) fn read_msg_size(&mut self, pos: Option) -> usize { let size = self.read_i64(pos); let size = size & 0xFFFF_FFFF_FFFF; size as usize } + /// Reads a f32 Value from the Buffer #[allow(clippy::option_if_let_else)] pub fn read_f32(&mut self, pos: Option) -> f32 { let len = 4; @@ -1271,6 +1266,7 @@ impl Buffer { } } + /// Reads a f64 Value from the Buffer #[allow(clippy::option_if_let_else)] pub fn read_f64(&mut self, pos: Option) -> f64 { let len = 8; @@ -1285,38 +1281,56 @@ impl Buffer { } } + /// Reads a String Value from the Buffer pub fn read_str(&mut self, len: usize) -> Result { let s = str::from_utf8(&self.data_buffer[self.data_offset..self.data_offset + len])?; self.data_offset += len; Ok(s.to_owned()) } + /// Reads a raw Bytes from the Buffer pub fn read_bytes(&mut self, pos: usize, count: usize) -> &[u8] { &self.data_buffer[pos..pos + count] } + /// Reads a byte Slice from the Buffer pub fn read_slice(&mut self, count: usize) -> &[u8] { &self.data_buffer[self.data_offset..self.data_offset + count] } + /// Reads a Blob Value from teh Buffer pub fn read_blob(&mut self, len: usize) -> Vec { let val = self.data_buffer[self.data_offset..self.data_offset + len].to_vec(); self.data_offset += len; val } + /// Reads a bool Value from the Buffer + pub fn read_bool(&mut self, len: usize) -> bool { + if len <= 0 { + false + } else { + let val = self.data_buffer[self.data_offset]; + self.data_offset += len; + val != 0 + } + } + + /// Writes a u8 Value to the Buffer pub fn write_u8(&mut self, val: u8) -> usize { self.data_buffer[self.data_offset] = val; self.data_offset += 1; 1 } + /// Writes a i8 Value to the Buffer pub fn write_i8(&mut self, val: i8) -> usize { self.data_buffer[self.data_offset] = val as u8; self.data_offset += 1; 1 } + /// Writes a u16 Value to the Buffer pub fn write_u16(&mut self, val: u16) -> usize { NetworkEndian::write_u16( &mut self.data_buffer[self.data_offset..self.data_offset + 2], @@ -1326,6 +1340,7 @@ impl Buffer { 2 } + /// Writes a u16 as Little Endian Value to the Buffer pub fn write_u16_little_endian(&mut self, val: u16) -> usize { LittleEndian::write_u16( &mut self.data_buffer[self.data_offset..self.data_offset + 2], @@ -1335,10 +1350,12 @@ impl Buffer { 2 } + /// Writes a i16 Value to the Buffer pub fn write_i16(&mut self, val: i16) -> usize { self.write_u16(val as u16) } + /// Writes a u32 Value to the Buffer pub fn write_u32(&mut self, val: u32) -> usize { NetworkEndian::write_u32( &mut self.data_buffer[self.data_offset..self.data_offset + 4], @@ -1348,10 +1365,12 @@ impl Buffer { 4 } + /// Writes a i32 Value to the Buffer pub fn write_i32(&mut self, val: i32) -> usize { self.write_u32(val as u32) } + /// Writes a u64 Value to the Buffer pub fn write_u64(&mut self, val: u64) -> usize { NetworkEndian::write_u64( &mut self.data_buffer[self.data_offset..self.data_offset + 8], @@ -1361,15 +1380,18 @@ impl Buffer { 8 } + /// Writes a i64 Value to the Buffer pub fn write_i64(&mut self, val: i64) -> usize { self.write_u64(val as u64) } + /// Writes a bool Value to the Buffer pub fn write_bool(&mut self, val: bool) -> usize { let val = if val { 1 } else { 0 }; - self.write_i64(val) + self.write_i8(val) } + /// Writes a f32 Value to the Buffer pub fn write_f32(&mut self, val: f32) -> usize { NetworkEndian::write_f32( &mut self.data_buffer[self.data_offset..self.data_offset + 4], @@ -1379,6 +1401,7 @@ impl Buffer { 4 } + /// Writes a f64 Value to the Buffer pub fn write_f64(&mut self, val: f64) -> usize { NetworkEndian::write_f64( &mut self.data_buffer[self.data_offset..self.data_offset + 8], @@ -1388,6 +1411,7 @@ impl Buffer { 8 } + /// Writes raw Bytes to the Buffer pub fn write_bytes(&mut self, bytes: &[u8]) -> usize { for b in bytes { self.write_u8(*b); @@ -1395,10 +1419,12 @@ impl Buffer { bytes.len() } + /// Writes a String Reference to the Buffer pub fn write_str(&mut self, val: &str) -> usize { self.write_bytes(val.as_bytes()) } + /// Writes a GeoJSON Value to the Buffer pub fn write_geo(&mut self, value: &str) -> usize { self.write_u8(0); self.write_u8(0); @@ -1407,14 +1433,10 @@ impl Buffer { 3 + value.len() } - pub fn write_timeout(&mut self, val: Option) { + pub(crate) fn write_timeout(&mut self, val: Option) { if let Some(val) = val { let millis: i32 = (val.as_secs() * 1_000) as i32 + val.subsec_millis() as i32; NetworkEndian::write_i32(&mut self.data_buffer[22..22 + 4], millis); } } - - pub fn dump_buffer(&self) { - println!(">>>>>>>>>>>>>>> {:?}", self.data_buffer.clone()); - } } diff --git a/aerospike-core/src/commands/delete_command.rs b/aerospike-core/src/commands/delete_command.rs index 3cea6aed..511ab86f 100644 --- a/aerospike-core/src/commands/delete_command.rs +++ b/aerospike-core/src/commands/delete_command.rs @@ -44,6 +44,7 @@ impl<'a> DeleteCommand<'a> { #[async_trait::async_trait] impl<'a> Command for DeleteCommand<'a> { + type Output = (); async fn write_timeout( &mut self, conn: &mut Connection, diff --git a/aerospike-core/src/commands/execute_udf_command.rs b/aerospike-core/src/commands/execute_udf_command.rs index 6a216683..417248cc 100644 --- a/aerospike-core/src/commands/execute_udf_command.rs +++ b/aerospike-core/src/commands/execute_udf_command.rs @@ -23,15 +23,15 @@ use crate::net::Connection; use crate::policy::WritePolicy; use crate::{Bins, Key, Value}; -pub struct ExecuteUDFCommand<'a> { - pub read_command: ReadCommand<'a>, +pub struct ExecuteUDFCommand<'a, T: serde::de::DeserializeOwned + Send> { + pub read_command: ReadCommand<'a, T>, policy: &'a WritePolicy, package_name: &'a str, function_name: &'a str, args: Option<&'a [Value]>, } -impl<'a> ExecuteUDFCommand<'a> { +impl<'a, T: serde::de::DeserializeOwned + Send> ExecuteUDFCommand<'a, T> { pub fn new( policy: &'a WritePolicy, cluster: Arc, @@ -49,13 +49,14 @@ impl<'a> ExecuteUDFCommand<'a> { } } - pub async fn execute(&mut self) -> Result<()> { + pub async fn execute(&mut self) -> Result<::Output> { SingleCommand::execute(self.policy, self).await } } #[async_trait::async_trait] -impl<'a> Command for ExecuteUDFCommand<'a> { +impl<'a, T: serde::de::DeserializeOwned + Send> Command for ExecuteUDFCommand<'a, T> { + type Output = crate::Record; async fn write_timeout( &mut self, conn: &mut Connection, @@ -65,10 +66,6 @@ impl<'a> Command for ExecuteUDFCommand<'a> { Ok(()) } - async fn write_buffer(&mut self, conn: &mut Connection) -> Result<()> { - conn.flush().await - } - fn prepare_buffer(&mut self, conn: &mut Connection) -> Result<()> { conn.buffer.set_udf( self.policy, @@ -83,7 +80,11 @@ impl<'a> Command for ExecuteUDFCommand<'a> { self.read_command.get_node() } - async fn parse_result(&mut self, conn: &mut Connection) -> Result<()> { + async fn parse_result(&mut self, conn: &mut Connection) -> Result { self.read_command.parse_result(conn).await } + + async fn write_buffer(&mut self, conn: &mut Connection) -> Result<()> { + conn.flush().await + } } diff --git a/aerospike-core/src/commands/exists_command.rs b/aerospike-core/src/commands/exists_command.rs index d1f7f651..2caf98e2 100644 --- a/aerospike-core/src/commands/exists_command.rs +++ b/aerospike-core/src/commands/exists_command.rs @@ -44,6 +44,7 @@ impl<'a> ExistsCommand<'a> { #[async_trait::async_trait] impl<'a> Command for ExistsCommand<'a> { + type Output = (); async fn write_timeout( &mut self, conn: &mut Connection, diff --git a/aerospike-core/src/commands/mod.rs b/aerospike-core/src/commands/mod.rs index aad829f2..e9e86ef7 100644 --- a/aerospike-core/src/commands/mod.rs +++ b/aerospike-core/src/commands/mod.rs @@ -62,9 +62,10 @@ pub trait Command { conn: &mut Connection, timeout: Option, ) -> Result<()>; + type Output; fn prepare_buffer(&mut self, conn: &mut Connection) -> Result<()>; fn get_node(&mut self) -> Result>; - async fn parse_result(&mut self, conn: &mut Connection) -> Result<()>; + async fn parse_result(&mut self, conn: &mut Connection) -> Result; async fn write_buffer(&mut self, conn: &mut Connection) -> Result<()>; } diff --git a/aerospike-core/src/commands/operate_command.rs b/aerospike-core/src/commands/operate_command.rs index c524f0f1..2c911ca8 100644 --- a/aerospike-core/src/commands/operate_command.rs +++ b/aerospike-core/src/commands/operate_command.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::marker::PhantomData; use std::sync::Arc; use std::time::Duration; @@ -21,14 +22,15 @@ use crate::errors::Result; use crate::net::Connection; use crate::operations::Operation; use crate::policy::WritePolicy; -use crate::value::bytes_to_particle; -use crate::{Key, ResultCode, Value}; +use crate::{Key, Record, Value}; -pub struct OperateCommand<'a> { +use super::read_command; + +pub struct OperateCommand<'a, T: serde::de::DeserializeOwned + Send> { pub single_command: SingleCommand<'a>, - pub record: OperateRecord, policy: &'a WritePolicy, operations: &'a [Operation<'a>], + phantom: PhantomData, } /// The return value from operate. Like a record, but retains the order and duplicate keys of bins. @@ -48,7 +50,7 @@ pub struct OperateRecord { expiration: u32, } -impl<'a> OperateCommand<'a> { +impl<'a, T: serde::de::DeserializeOwned + Send> OperateCommand<'a, T> { pub fn new( policy: &'a WritePolicy, cluster: Arc, @@ -57,50 +59,21 @@ impl<'a> OperateCommand<'a> { ) -> Self { OperateCommand { single_command: SingleCommand::new(cluster, key, crate::policy::Replica::Master), - record: OperateRecord::default(), policy, operations, + phantom: Default::default(), } } - pub async fn execute(&mut self) -> Result<()> { + pub async fn execute(&mut self) -> Result<::Output> { SingleCommand::execute(self.policy, self).await } - - fn parse_record( - &mut self, - conn: &mut Connection, - op_count: usize, - field_count: usize, - ) -> Result<()> { - // There can be fields in the response (setname etc). For now, ignore them. Expose them to - // the API if needed in the future. - for _ in 0..field_count { - let field_size = conn.buffer.read_u32(None) as usize; - conn.buffer.skip(4 + field_size); - } - - self.record.bins.reserve_exact(op_count); - for _ in 0..op_count { - let op_size = conn.buffer.read_u32(None) as usize; - conn.buffer.skip(1); - let particle_type = conn.buffer.read_u8(None); - conn.buffer.skip(1); - let name_size = conn.buffer.read_u8(None) as usize; - let name: String = conn.buffer.read_str(name_size)?; - - let particle_bytes_size = op_size - (4 + name_size); - let value = bytes_to_particle(particle_type, &mut conn.buffer, particle_bytes_size)?; - self.record.bins.push((name, value)); - } - - Ok(()) - } - } #[async_trait::async_trait] -impl<'a> Command for OperateCommand<'a> { +impl<'a, T: serde::de::DeserializeOwned + Send> Command for OperateCommand<'a, T> { + type Output = Record; + async fn write_timeout( &mut self, conn: &mut Connection, @@ -110,10 +83,6 @@ impl<'a> Command for OperateCommand<'a> { Ok(()) } - async fn write_buffer(&mut self, conn: &mut Connection) -> Result<()> { - conn.flush().await - } - fn prepare_buffer(&mut self, conn: &mut Connection) -> Result<()> { conn.buffer.set_operate( self.policy, @@ -126,46 +95,11 @@ impl<'a> Command for OperateCommand<'a> { self.single_command.get_node() } - async fn parse_result(&mut self, conn: &mut Connection) -> Result<()> { - if let Err(err) = conn - .read_buffer(super::buffer::MSG_TOTAL_HEADER_SIZE as usize) - .await - { - warn!("Parse result error: {}", err); - bail!(err); - } - - conn.buffer.reset_offset(); - let sz = conn.buffer.read_u64(Some(0)); - let header_length = conn.buffer.read_u8(Some(8)); - let result_code = conn.buffer.read_u8(Some(13)); - self.record.generation = conn.buffer.read_u32(Some(14)); - self.record.expiration = conn.buffer.read_u32(Some(18)); - let field_count = conn.buffer.read_u16(Some(26)) as usize; // almost certainly 0 - let op_count = conn.buffer.read_u16(Some(28)) as usize; - let receive_size = ((sz & 0xFFFF_FFFF_FFFF) - u64::from(header_length)) as usize; - - // Read remaining message bytes - if receive_size > 0 { - if let Err(err) = conn.read_buffer(receive_size).await { - warn!("Parse result error: {}", err); - bail!(err); - } - } + async fn parse_result(&mut self, conn: &mut Connection) -> Result { + read_command::ReadCommand::parse_result_internal(conn, false).await + } - match ResultCode::from(result_code) { - ResultCode::Ok => { - self.parse_record(conn, op_count, field_count) - } - ResultCode::UdfBadResponse => { - // record bin "FAILURE" contains details about the UDF error - self.parse_record(conn, op_count, field_count)?; - let reason = self.record - .bins.iter().find(|(k, _)|k == "FAILURE").map(|(_, v)|v) - .map_or(String::from("UDF Error"), ToString::to_string); - Err(crate::ErrorKind::UdfBadResponse(reason).into()) - } - rc => Err(crate::ErrorKind::ServerError(rc).into()), - } + async fn write_buffer(&mut self, conn: &mut Connection) -> Result<()> { + conn.flush().await } } diff --git a/aerospike-core/src/commands/particle_type.rs b/aerospike-core/src/commands/particle_type.rs index c88260fc..215a064c 100644 --- a/aerospike-core/src/commands/particle_type.rs +++ b/aerospike-core/src/commands/particle_type.rs @@ -13,7 +13,10 @@ // License for the specific language governing permissions and limitations under // the License. -#[derive(Debug, Clone)] +use std::fmt; +use std::result::Result as StdResult; + +#[derive(Debug, Clone, PartialEq, Eq)] #[doc(hidden)] pub enum ParticleType { // Server particle types. Unsupported types are commented out. @@ -22,20 +25,8 @@ pub enum ParticleType { FLOAT = 2, STRING = 3, BLOB = 4, - // TIMESTAMP = 5, DIGEST = 6, - // JBLOB = 7, - // CSHARP_BLOB = 8, - // PYTHON_BLOB = 9, - // RUBY_BLOB = 10, - // PHP_BLOB = 11, - // ERLANG_BLOB = 12, - // SEGMENT_POINTER = 13, - // RTA_LIST = 14, - // RTA_DICT = 15, - // RTA_APPEND_DICT = 16, - // RTA_APPEND_LIST = 17, - // LUA_BLOB = 18, + BOOL = 17, HLL = 18, MAP = 19, LIST = 20, @@ -51,20 +42,8 @@ impl From for ParticleType { 2 => ParticleType::FLOAT, 3 => ParticleType::STRING, 4 => ParticleType::BLOB, - // 5 => ParticleType::TIMESTAMP , 6 => ParticleType::DIGEST, - // 7 => ParticleType::JBLOB , - // 8 => ParticleType::CSHARP_BLOB , - // 9 => ParticleType::PYTHON_BLOB , - // 10 => ParticleType::RUBY_BLOB , - // 11 => ParticleType::PHP_BLOB , - // 12 => ParticleType::ERLANG_BLOB , - // 13 => ParticleType::SEGMENT_POINTER, - // 14 => ParticleType::RTA_LIST , - // 15 => ParticleType::RTA_DICT , - // 16 => ParticleType::RTA_APPEND_DICT, - // 17 => ParticleType::RTA_APPEND_LIST, - // 18 => ParticleType::LUA_BLOB , + 17 => ParticleType::BOOL, 18 => ParticleType::HLL, 19 => ParticleType::MAP, 20 => ParticleType::LIST, @@ -74,3 +53,22 @@ impl From for ParticleType { } } } + +impl fmt::Display for ParticleType { + fn fmt(&self, f: &mut fmt::Formatter) -> StdResult<(), fmt::Error> { + match self { + ParticleType::NULL => write!(f, "NULL"), + ParticleType::INTEGER => write!(f, "INTEGER"), + ParticleType::FLOAT => write!(f, "FLOAT"), + ParticleType::STRING => write!(f, "STRING"), + ParticleType::BLOB => write!(f, "BLOB"), + ParticleType::DIGEST => write!(f, "DIGEST"), + ParticleType::BOOL => write!(f, "BOOL"), + ParticleType::HLL => write!(f, "HLL"), + ParticleType::MAP => write!(f, "MAP"), + ParticleType::LIST => write!(f, "LIST"), + ParticleType::LDT => write!(f, "LDT"), + ParticleType::GEOJSON => write!(f, "GEOJSON"), + } + } +} diff --git a/aerospike-core/src/commands/query_command.rs b/aerospike-core/src/commands/query_command.rs index 5205e7eb..3da5968f 100644 --- a/aerospike-core/src/commands/query_command.rs +++ b/aerospike-core/src/commands/query_command.rs @@ -22,19 +22,19 @@ use crate::net::Connection; use crate::policy::QueryPolicy; use crate::{Recordset, Statement}; -pub struct QueryCommand<'a> { - stream_command: StreamCommand, +pub struct QueryCommand<'a, T: serde::de::DeserializeOwned> { + stream_command: StreamCommand, policy: &'a QueryPolicy, statement: Arc, partitions: Vec, } -impl<'a> QueryCommand<'a> { +impl<'a, T: serde::de::DeserializeOwned + Send> QueryCommand<'a, T> { pub fn new( policy: &'a QueryPolicy, node: Arc, statement: Arc, - recordset: Arc, + recordset: Arc>, partitions: Vec, ) -> Self { QueryCommand { @@ -51,7 +51,8 @@ impl<'a> QueryCommand<'a> { } #[async_trait::async_trait] -impl<'a> Command for QueryCommand<'a> { +impl<'a, T: serde::de::DeserializeOwned + Send> Command for QueryCommand<'a, T> { + type Output = (); async fn write_timeout( &mut self, conn: &mut Connection, @@ -61,10 +62,6 @@ impl<'a> Command for QueryCommand<'a> { Ok(()) } - async fn write_buffer(&mut self, conn: &mut Connection) -> Result<()> { - conn.flush().await - } - fn prepare_buffer(&mut self, conn: &mut Connection) -> Result<()> { conn.buffer.set_query( self.policy, @@ -82,4 +79,8 @@ impl<'a> Command for QueryCommand<'a> { async fn parse_result(&mut self, conn: &mut Connection) -> Result<()> { StreamCommand::parse_result(&mut self.stream_command, conn).await } + + async fn write_buffer(&mut self, conn: &mut Connection) -> Result<()> { + conn.flush().await + } } diff --git a/aerospike-core/src/commands/read_command.rs b/aerospike-core/src/commands/read_command.rs index 9d157728..dee54117 100644 --- a/aerospike-core/src/commands/read_command.rs +++ b/aerospike-core/src/commands/read_command.rs @@ -12,28 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry::{Occupied, Vacant}; -use std::collections::HashMap; +use std::collections::VecDeque; use std::sync::Arc; use std::time::Duration; +use serde::Deserialize; + use crate::cluster::{Cluster, Node}; use crate::commands::buffer; use crate::commands::{Command, SingleCommand}; use crate::errors::{ErrorKind, Result}; use crate::net::Connection; use crate::policy::{BasePolicy, Replica}; -use crate::value::bytes_to_particle; -use crate::{Bins, Key, Record, ResultCode, Value}; +use crate::{derive, Bins, Key, Record, ResultCode}; -pub struct ReadCommand<'a> { +pub struct ReadCommand<'a, T: serde::de::DeserializeOwned + Send> { pub single_command: SingleCommand<'a>, - pub record: Option, + pub record: Option>, policy: &'a BasePolicy, bins: Bins, } -impl<'a> ReadCommand<'a> { +impl<'a, T: serde::de::DeserializeOwned + Send> ReadCommand<'a, T> { pub fn new(policy: &'a BasePolicy, cluster: Arc, key: &'a Key, bins: Bins, replica: Replica) -> Self { ReadCommand { single_command: SingleCommand::new(cluster, key, replica), @@ -43,83 +43,33 @@ impl<'a> ReadCommand<'a> { } } - pub async fn execute(&mut self) -> Result<()> { + pub async fn execute(&mut self) -> Result<::Output> { SingleCommand::execute(self.policy, self).await } - fn parse_record( - &mut self, + pub async fn parse_record( conn: &mut Connection, op_count: usize, field_count: usize, generation: u32, expiration: u32, - ) -> Result { - let mut bins: HashMap = HashMap::with_capacity(op_count); - + ) -> Result> { // There can be fields in the response (setname etc). For now, ignore them. Expose them to // the API if needed in the future. for _ in 0..field_count { + conn.read_buffer(4).await?; let field_size = conn.buffer.read_u32(None) as usize; - conn.buffer.skip(4 + field_size); + conn.read_buffer(field_size).await?; + conn.buffer.skip(field_size); } - for _ in 0..op_count { - let op_size = conn.buffer.read_u32(None) as usize; - conn.buffer.skip(1); - let particle_type = conn.buffer.read_u8(None); - conn.buffer.skip(1); - let name_size = conn.buffer.read_u8(None) as usize; - let name: String = conn.buffer.read_str(name_size)?; - - let particle_bytes_size = op_size - (4 + name_size); - let value = bytes_to_particle(particle_type, &mut conn.buffer, particle_bytes_size)?; - - if !value.is_nil() { - // list/map operations may return multiple values for the same bin. - match bins.entry(name) { - Vacant(entry) => { - entry.insert(value); - } - Occupied(entry) => match *entry.into_mut() { - Value::List(ref mut list) => list.push(value), - ref mut prev => { - *prev = as_list!(prev.clone(), value); - } - }, - } - } - } + let reader = crate::derive::readable::BinsDeserializer{ bins: conn.pre_parse_stream_bins(op_count).await?.into() }; + let bins = T::deserialize(reader)?; Ok(Record::new(None, bins, generation, expiration)) } -} -#[async_trait::async_trait] -impl<'a> Command for ReadCommand<'a> { - async fn write_timeout( - &mut self, - conn: &mut Connection, - timeout: Option, - ) -> Result<()> { - conn.buffer.write_timeout(timeout); - Ok(()) - } - - async fn write_buffer(&mut self, conn: &mut Connection) -> Result<()> { - conn.flush().await - } - - fn prepare_buffer(&mut self, conn: &mut Connection) -> Result<()> { - conn.buffer - .set_read(self.policy, self.single_command.key, &self.bins) - } - - fn get_node(&mut self) -> Result> { - self.single_command.get_node() - } - - async fn parse_result(&mut self, conn: &mut Connection) -> Result<()> { + pub async fn parse_result_internal(conn: &mut Connection, bins_none: bool) -> Result> { if let Err(err) = conn .read_buffer(buffer::MSG_TOTAL_HEADER_SIZE as usize) .await @@ -129,44 +79,87 @@ impl<'a> Command for ReadCommand<'a> { } conn.buffer.reset_offset(); - let sz = conn.buffer.read_u64(Some(0)); - let header_length = conn.buffer.read_u8(Some(8)); + conn.buffer.skip(9); let result_code = conn.buffer.read_u8(Some(13)); let generation = conn.buffer.read_u32(Some(14)); let expiration = conn.buffer.read_u32(Some(18)); let field_count = conn.buffer.read_u16(Some(26)) as usize; // almost certainly 0 let op_count = conn.buffer.read_u16(Some(28)) as usize; - let receive_size = ((sz & 0xFFFF_FFFF_FFFF) - u64::from(header_length)) as usize; - - // Read remaining message bytes - if receive_size > 0 { - if let Err(err) = conn.read_buffer(receive_size).await { - warn!("Parse result error: {}", err); - bail!(err); - } - } match ResultCode::from(result_code) { ResultCode::Ok => { - let record = if self.bins.is_none() { - Record::new(None, HashMap::new(), generation, expiration) + let record = if bins_none { + Record::new(None, T::deserialize(derive::readable::BinsDeserializer{bins: VecDeque::new()})?, generation, expiration) } else { - self.parse_record(conn, op_count, field_count, generation, expiration)? + Self::parse_record(conn, op_count, field_count, generation, expiration) + .await? }; - self.record = Some(record); - Ok(()) + Ok(record) } ResultCode::UdfBadResponse => { // record bin "FAILURE" contains details about the UDF error - let record = - self.parse_record(conn, op_count, field_count, generation, expiration)?; - let reason = record - .bins - .get("FAILURE") - .map_or(String::from("UDF Error"), ToString::to_string); + let reason = parse_udf_error(conn, op_count, field_count).await?; Err(ErrorKind::UdfBadResponse(reason).into()) } rc => Err(ErrorKind::ServerError(rc).into()), } } } + +pub(crate) async fn parse_udf_error( + conn: &mut Connection, + op_count: usize, + field_count: usize, +) -> Result { + // There can be fields in the response (setname etc). For now, ignore them. Expose them to + // the API if needed in the future. + for _ in 0..field_count { + conn.read_buffer(4).await?; + let field_size = conn.buffer.read_u32(None) as usize; + conn.read_buffer(field_size).await?; + conn.buffer.skip(field_size); + } + + let reader = crate::derive::readable::BinsDeserializer{ bins: conn.pre_parse_stream_bins(op_count).await?.into() }; + + #[derive(Deserialize)] + struct FailureReason { + #[serde(rename = "FAILURE")] + failure: Option, + } + let bins = FailureReason::deserialize(reader)?; + if let Some(fail) = bins.failure { + return Ok(fail); + } + Ok(String::from("UDF Error")) +} + +#[async_trait::async_trait] +impl<'a, T: serde::de::DeserializeOwned + Send> Command for ReadCommand<'a, T> { + type Output = Record; + async fn write_timeout( + &mut self, + conn: &mut Connection, + timeout: Option, + ) -> Result<()> { + conn.buffer.write_timeout(timeout); + Ok(()) + } + + fn prepare_buffer(&mut self, conn: &mut Connection) -> Result<()> { + conn.buffer + .set_read(self.policy, self.single_command.key, &self.bins) + } + + fn get_node(&mut self) -> Result> { + self.single_command.get_node() + } + + async fn parse_result(&mut self, conn: &mut Connection) -> Result> { + Self::parse_result_internal(conn, self.bins.is_none()).await + } + + async fn write_buffer(&mut self, conn: &mut Connection) -> Result<()> { + conn.flush().await + } +} diff --git a/aerospike-core/src/commands/scan_command.rs b/aerospike-core/src/commands/scan_command.rs index 42cff8f6..31ac0076 100644 --- a/aerospike-core/src/commands/scan_command.rs +++ b/aerospike-core/src/commands/scan_command.rs @@ -23,8 +23,8 @@ use crate::net::Connection; use crate::policy::ScanPolicy; use crate::{Bins, Recordset}; -pub struct ScanCommand<'a> { - stream_command: StreamCommand, +pub struct ScanCommand<'a, T: serde::de::DeserializeOwned> { + stream_command: StreamCommand, policy: &'a ScanPolicy, namespace: &'a str, set_name: &'a str, @@ -32,14 +32,14 @@ pub struct ScanCommand<'a> { partitions: Vec, } -impl<'a> ScanCommand<'a> { +impl<'a, T: serde::de::DeserializeOwned + Send> ScanCommand<'a, T> { pub fn new( policy: &'a ScanPolicy, node: Arc, namespace: &'a str, set_name: &'a str, bins: Bins, - recordset: Arc, + recordset: Arc>, partitions: Vec, ) -> Self { ScanCommand { @@ -58,7 +58,8 @@ impl<'a> ScanCommand<'a> { } #[async_trait::async_trait] -impl<'a> Command for ScanCommand<'a> { +impl<'a, T: serde::de::DeserializeOwned + Send> Command for ScanCommand<'a, T> { + type Output = (); async fn write_timeout( &mut self, conn: &mut Connection, @@ -68,10 +69,6 @@ impl<'a> Command for ScanCommand<'a> { Ok(()) } - async fn write_buffer(&mut self, conn: &mut Connection) -> Result<()> { - conn.flush().await - } - fn prepare_buffer(&mut self, conn: &mut Connection) -> Result<()> { conn.buffer.set_scan( self.policy, @@ -90,4 +87,8 @@ impl<'a> Command for ScanCommand<'a> { async fn parse_result(&mut self, conn: &mut Connection) -> Result<()> { StreamCommand::parse_result(&mut self.stream_command, conn).await } + + async fn write_buffer(&mut self, conn: &mut Connection) -> Result<()> { + conn.flush().await + } } diff --git a/aerospike-core/src/commands/single_command.rs b/aerospike-core/src/commands/single_command.rs index 76cd5382..50808921 100644 --- a/aerospike-core/src/commands/single_command.rs +++ b/aerospike-core/src/commands/single_command.rs @@ -81,10 +81,10 @@ impl<'a> SingleCommand<'a> { // EXECUTE // - pub async fn execute( + pub async fn execute( policy: &(dyn Policy + Send + Sync), - cmd: &'a mut (dyn commands::Command + Send), - ) -> Result<()> { + cmd: &'a mut C, + ) -> Result { let mut iterations = 0; // set timeout outside the loop @@ -149,18 +149,17 @@ impl<'a> SingleCommand<'a> { } // Parse results. - if let Err(err) = try_with_timeout(deadline, cmd.parse_result(&mut conn)).await { + let result = try_with_timeout(deadline, cmd.parse_result(&mut conn)).await; + if let Err(err) = result.as_ref() { // close the connection // cancelling/closing the batch/multi commands will return an error, which will // close the connection to throw away its data and signal the server about the // situation. We will not put back the connection in the buffer. - if !commands::keep_connection(&err) { + if !commands::keep_connection(err) { conn.invalidate(); } - return Err(err); - } else { - return Ok(()); } + return result; } bail!(ErrorKind::Connection("Timeout".to_string())) diff --git a/aerospike-core/src/commands/stream_command.rs b/aerospike-core/src/commands/stream_command.rs index 7c8f0e15..0f221bbe 100644 --- a/aerospike-core/src/commands/stream_command.rs +++ b/aerospike-core/src/commands/stream_command.rs @@ -12,39 +12,43 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; use std::sync::Arc; use std::thread; use std::time::Duration; +use serde::Deserialize; + use crate::cluster::Node; use crate::commands::buffer; use crate::commands::field_type::FieldType; use crate::commands::Command; +use crate::derive::readable::PreParsedValue; use crate::errors::{ErrorKind, Result}; use crate::net::Connection; use crate::query::Recordset; -use crate::value::bytes_to_particle; use crate::{Key, Record, ResultCode, Value}; -pub struct StreamCommand { +pub struct StreamCommand { node: Arc, - pub recordset: Arc, + pub recordset: Arc>, } -impl Drop for StreamCommand { +impl Drop for StreamCommand { fn drop(&mut self) { // signal_end self.recordset.signal_end(); } } -impl StreamCommand { - pub fn new(node: Arc, recordset: Arc) -> Self { +impl StreamCommand { + pub fn new(node: Arc, recordset: Arc>) -> Self { StreamCommand { node, recordset } } - async fn parse_record(conn: &mut Connection, size: usize) -> Result<(Option, bool)> { + async fn parse_record(conn: &mut Connection, size: usize) -> Result<(Option>, bool)> + where + T: serde::de::DeserializeOwned, + { let result_code = ResultCode::from(conn.buffer.read_u8(Some(5))); if result_code != ResultCode::Ok { if conn.bytes_read() < size { @@ -71,31 +75,16 @@ impl StreamCommand { let field_count = conn.buffer.read_u16(None) as usize; // almost certainly 0 let op_count = conn.buffer.read_u16(None) as usize; - let key = StreamCommand::parse_key(conn, field_count).await?; + let key = StreamCommand::::parse_key(conn, field_count).await?; // Partition is done, don't go further if info3 & buffer::_INFO3_PARTITION_DONE != 0 { return Ok((None, true)); } - let mut bins: HashMap = HashMap::with_capacity(op_count); + let reader = crate::derive::readable::BinsDeserializer{ bins: conn.pre_parse_stream_bins(op_count).await?.into() }; - for _ in 0..op_count { - conn.read_buffer(8).await?; - let op_size = conn.buffer.read_u32(None) as usize; - conn.buffer.skip(1); - let particle_type = conn.buffer.read_u8(None); - conn.buffer.skip(1); - let name_size = conn.buffer.read_u8(None) as usize; - conn.read_buffer(name_size).await?; - let name: String = conn.buffer.read_str(name_size)?; - - let particle_bytes_size = op_size - (4 + name_size); - conn.read_buffer(particle_bytes_size).await?; - let value = bytes_to_particle(particle_type, &mut conn.buffer, particle_bytes_size)?; - - bins.insert(name, value); - } + let bins = T::deserialize(reader)?; let record = Record::new(Some(key), bins, generation, expiration); Ok((Some(record), true)) @@ -124,7 +113,8 @@ impl StreamCommand { } } }, - Ok((None, cont)) => return Ok(cont), + Ok((None, false)) => return Ok(false), + Ok((None, true)) => continue, // handle partition done Err(err) => { self.recordset.push(Err(err)); return Ok(false); @@ -160,11 +150,8 @@ impl StreamCommand { x if x == FieldType::Key as u8 => { let particle_type = conn.buffer.read_u8(None); let particle_bytes_size = field_len - 2; - orig_key = Some(bytes_to_particle( - particle_type, - &mut conn.buffer, - particle_bytes_size, - )?); + let value = PreParsedValue{particle_type, name_len: 0, name: Default::default(), particle: conn.buffer.read_blob(particle_bytes_size)}; + orig_key = Some(Value::deserialize(value)?); } _ => unreachable!(), } @@ -180,7 +167,8 @@ impl StreamCommand { } #[async_trait::async_trait] -impl Command for StreamCommand { +impl Command for StreamCommand { + type Output = (); async fn write_timeout( &mut self, conn: &mut Connection, @@ -190,10 +178,6 @@ impl Command for StreamCommand { Ok(()) } - async fn write_buffer(&mut self, conn: &mut Connection) -> Result<()> { - conn.flush().await - } - #[allow(unused_variables)] fn prepare_buffer(&mut self, conn: &mut Connection) -> Result<()> { // should be implemented downstream @@ -220,4 +204,8 @@ impl Command for StreamCommand { Ok(()) } + + async fn write_buffer(&mut self, conn: &mut Connection) -> Result<()> { + conn.flush().await + } } diff --git a/aerospike-core/src/commands/touch_command.rs b/aerospike-core/src/commands/touch_command.rs index b4f822be..a7d28624 100644 --- a/aerospike-core/src/commands/touch_command.rs +++ b/aerospike-core/src/commands/touch_command.rs @@ -43,6 +43,7 @@ impl<'a> TouchCommand<'a> { #[async_trait::async_trait] impl<'a> Command for TouchCommand<'a> { + type Output = (); async fn write_timeout( &mut self, conn: &mut Connection, diff --git a/aerospike-core/src/commands/write_command.rs b/aerospike-core/src/commands/write_command.rs index 1adf39b9..a072da6d 100644 --- a/aerospike-core/src/commands/write_command.rs +++ b/aerospike-core/src/commands/write_command.rs @@ -18,25 +18,26 @@ use std::time::Duration; use crate::cluster::{Cluster, Node}; use crate::commands::buffer; use crate::commands::{Command, SingleCommand}; +use crate::derive::writable::WritableBins; use crate::errors::{ErrorKind, Result}; use crate::net::Connection; use crate::operations::OperationType; use crate::policy::WritePolicy; -use crate::{Bin, Key, ResultCode}; +use crate::{Key, ResultCode}; -pub struct WriteCommand<'a> { +pub struct WriteCommand<'a, T: WritableBins> { single_command: SingleCommand<'a>, policy: &'a WritePolicy, - bins: &'a [Bin<'a>], + bins: &'a T, operation: OperationType, } -impl<'a, 'b> WriteCommand<'a> { +impl<'a, T: WritableBins> WriteCommand<'a, T> { pub fn new( policy: &'a WritePolicy, cluster: Arc, key: &'a Key, - bins: &'a [Bin<'b>], + bins: &'a T, operation: OperationType, ) -> Self { WriteCommand { @@ -53,7 +54,8 @@ impl<'a, 'b> WriteCommand<'a> { } #[async_trait::async_trait] -impl<'a, 'b> Command for WriteCommand<'a> { +impl<'a, T: WritableBins> Command for WriteCommand<'a, T> { + type Output = (); async fn write_timeout( &mut self, conn: &mut Connection, diff --git a/aerospike-core/src/derive/mod.rs b/aerospike-core/src/derive/mod.rs new file mode 100644 index 00000000..024f86b3 --- /dev/null +++ b/aerospike-core/src/derive/mod.rs @@ -0,0 +1,19 @@ +// Copyright 2015-2020 Aerospike, Inc. +// +// Portions may be licensed to Aerospike, Inc. under one or more contributor +// license agreements. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +//! Traits and Implementations for reading and writing data from/into structs and variables + +pub mod readable; +pub mod writable; diff --git a/aerospike-core/src/derive/readable.rs b/aerospike-core/src/derive/readable.rs new file mode 100644 index 00000000..b1faddaf --- /dev/null +++ b/aerospike-core/src/derive/readable.rs @@ -0,0 +1,1657 @@ +// Copyright 2015-2020 Aerospike, Inc. +// +// Portions may be licensed to Aerospike, Inc. under one or more contributor +// license agreements. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +//! Traits and Implementations for reading data into structs and variables + +use serde::de::EnumAccess; +use serde::de::Error as _; +use serde::de::MapAccess; +use serde::de::SeqAccess; +use serde::de::VariantAccess; +use serde::de::Visitor; +use serde::Deserializer; + +use crate::errors::Result; +use crate::Error; +use crate::ParticleType; +use std::collections::VecDeque; +use std::convert::{TryInto, TryFrom}; + +// This serializer represents all the bins in a record. +pub(crate) struct BinsDeserializer { + pub bins: VecDeque, +} + +impl serde::de::Error for crate::errors::Error { + fn custom(msg:T) -> Self where T:std::fmt::Display { + crate::errors::Error::from_kind(crate::ErrorKind::Derive(msg.to_string())) + } +} + +impl<'de> serde::de::Deserializer<'de> for BinsDeserializer { + type Error = crate::errors::Error; + + fn deserialize_any(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_bool(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_i8(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_i16(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_i32(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_i64(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_u8(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_u16(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_u32(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_u64(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_f32(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_f64(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_char(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_str(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_string(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_bytes(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_byte_buf(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_option(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_some(self) + } + + fn deserialize_unit(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_unit_struct( + self, + _name: &'static str, + visitor: V, + ) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_newtype_struct( + self, + _name: &'static str, + visitor: V, + ) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_seq(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_tuple(self, len: usize, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + if self.bins.len() != len { + struct ExpectedLen(usize); + let len = ExpectedLen(len); + impl serde::de::Expected for ExpectedLen { + fn fmt(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "{}", self.0) + } + } + Err(::invalid_length(self.bins.len(), &len)) + } else { + visitor.visit_map(self) + } + } + + fn deserialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + visitor: V, + ) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_map(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_struct( + self, + __name: &'static str, + _fields: &'static [&'static str], + visitor: V, + ) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_enum( + self, + _name: &'static str, + _variants: &'static [&'static str], + visitor: V, + ) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_identifier(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } + + fn deserialize_ignored_any(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_map(self) + } +} + + +struct DeserializeStr<'a>(&'a str); +impl<'a, 'de> serde::de::Deserializer<'de> for DeserializeStr<'a> { + type Error = crate::errors::Error; + + fn deserialize_any(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_bool(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_i8(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_i16(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_i32(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_i64(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_u8(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_u16(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_u32(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_u64(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_f32(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_f64(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_char(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_str(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_string(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_bytes(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_byte_buf(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_option(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_unit(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_unit_struct( + self, + _name: &'static str, + visitor: V, + ) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_newtype_struct( + self, + _name: &'static str, + visitor: V, + ) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_seq(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_tuple(self, _len: usize, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + visitor: V, + ) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_map(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_struct( + self, + _name: &'static str, + _fields: &'static [&'static str], + visitor: V, + ) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_enum( + self, + _name: &'static str, + _variants: &'static [&'static str], + visitor: V, + ) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_identifier(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } + + fn deserialize_ignored_any(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_str(self.0) + } +} + +impl<'de> serde::de::MapAccess<'de> for BinsDeserializer { + type Error = crate::errors::Error; + + fn next_key_seed(&mut self, seed: K) -> std::result::Result, Self::Error> + where + K: serde::de::DeserializeSeed<'de> { + + if let Some(next_key) = self.bins.front() { + Some(seed.deserialize(DeserializeStr(next_key.name()?))).transpose() + } else { + Ok(None) + } + } + + fn next_value_seed(&mut self, seed: V) -> std::result::Result + where + V: serde::de::DeserializeSeed<'de> { + seed.deserialize(self.bins.pop_front().unwrap()) + } + + fn size_hint(&self) -> Option { + Some(self.bins.len()) + } +} + +impl<'de> serde::de::Deserializer<'de> for PreParsedValue { + type Error = crate::errors::Error; + + fn deserialize_any(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + match self.particle_type() { + ParticleType::NULL => { + visitor.visit_none() + } + ParticleType::INTEGER => { + visitor.visit_i64(self.as_int()?) + } + ParticleType::FLOAT => { + visitor.visit_f64(self.as_float()?) + } + ParticleType::STRING | ParticleType::GEOJSON => { + visitor.visit_string(self.into_string()?) + } + ParticleType::BLOB | ParticleType::HLL => { + visitor.visit_byte_buf(self.into_blob()) + } + ParticleType::BOOL => { + visitor.visit_bool(self.as_bool()?) + } + ParticleType::MAP | ParticleType::LIST => { + let mut read = 0; + let cdt_reader = CDTDecoder(self.particle(), &mut read); + cdt_reader.deserialize_any(visitor) + } + ParticleType::DIGEST => todo!(), + ParticleType::LDT => todo!(), + } + } + + fn deserialize_bool(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + self.deserialize_any(visitor) + } + + fn deserialize_i8(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + if self.particle_type() == ParticleType::INTEGER { + let integer = self.as_int()?; + if let Ok(as_int) = integer.try_into() { + return visitor.visit_i8(as_int); + } + } + self.deserialize_any(visitor) + } + + fn deserialize_i16(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + if self.particle_type() == ParticleType::INTEGER { + let integer = self.as_int()?; + if let Ok(as_int) = integer.try_into() { + return visitor.visit_i16(as_int); + } + } + self.deserialize_any(visitor) + } + + fn deserialize_i32(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + if self.particle_type() == ParticleType::INTEGER { + let integer = self.as_int()?; + return visitor.visit_i32(integer.try_into()?); + } else { + self.deserialize_any(visitor) + } + } + + fn deserialize_i64(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + if self.particle_type() == ParticleType::INTEGER { + let integer = self.as_int()?; + return visitor.visit_i64(integer); + } else { + self.deserialize_any(visitor) + } + } + + fn deserialize_u8(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + if self.particle_type() == ParticleType::INTEGER { + let integer = self.as_int()?; + visitor.visit_u8(integer.try_into()?) + } else { + self.deserialize_any(visitor) + } + } + + fn deserialize_u16(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + if self.particle_type() == ParticleType::INTEGER { + let integer = self.as_int()?; + if let Ok(as_int) = integer.try_into() { + return visitor.visit_u16(as_int); + } + } + self.deserialize_any(visitor) + } + + fn deserialize_u32(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + if self.particle_type() == ParticleType::INTEGER { + let integer = self.as_int()?; + visitor.visit_u32(integer.try_into()?) + } else { + self.deserialize_any(visitor) + } + } + + fn deserialize_u64(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + if self.particle_type() == ParticleType::INTEGER { + let integer = self.as_int()?; + return visitor.visit_u64(integer.try_into()?); + } else { + self.deserialize_any(visitor) + } + } + + fn deserialize_f32(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + if self.particle_type() == ParticleType::FLOAT { + let flt = self.as_float()?; + visitor.visit_f32(flt as f32) + } else { + self.deserialize_any(visitor) + } + } + + fn deserialize_f64(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + if self.particle_type() == ParticleType::FLOAT { + let flt = self.as_float()?; + visitor.visit_f64(flt) + } else { + self.deserialize_any(visitor) + } + } + + fn deserialize_char(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + self.deserialize_any(visitor) + } + + fn deserialize_str(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + self.deserialize_any(visitor) + } + + fn deserialize_string(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + self.deserialize_any(visitor) + } + + fn deserialize_bytes(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + self.deserialize_any(visitor) + } + + fn deserialize_byte_buf(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + match self.particle_type() { + ParticleType::NULL => { + visitor.visit_none() + } + ParticleType::INTEGER => { + visitor.visit_i64(self.as_int()?) + } + ParticleType::FLOAT => { + visitor.visit_f64(self.as_float()?) + } + ParticleType::STRING | ParticleType::GEOJSON => { + visitor.visit_string(self.into_string()?) + } + ParticleType::BLOB | ParticleType::HLL => { + visitor.visit_byte_buf(self.into_blob()) + } + ParticleType::BOOL => { + visitor.visit_bool(self.as_bool()?) + } + ParticleType::MAP | ParticleType::LIST => { + let mut read = 0; + let cdt_reader = CDTDecoder(self.particle(), &mut read); + cdt_reader.deserialize_any(visitor) + } + ParticleType::DIGEST => todo!(), + ParticleType::LDT => todo!(), + } + } + + fn deserialize_option(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + if self.particle_type() != ParticleType::NULL { + visitor.visit_some(self) + } else { + visitor.visit_none() + } + } + + fn deserialize_unit(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + self.deserialize_any(visitor) + } + + fn deserialize_unit_struct( + self, + _name: &'static str, + visitor: V, + ) -> std::result::Result + where + V: serde::de::Visitor<'de> { + self.deserialize_any(visitor) + } + + fn deserialize_newtype_struct( + self, + _name: &'static str, + visitor: V, + ) -> std::result::Result + where + V: serde::de::Visitor<'de> { + self.deserialize_any(visitor) + } + + fn deserialize_seq(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + self.deserialize_any(visitor) + } + + fn deserialize_tuple(self, _len: usize, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + self.deserialize_any(visitor) + } + + fn deserialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + visitor: V, + ) -> std::result::Result + where + V: serde::de::Visitor<'de> { + self.deserialize_any(visitor) + } + + fn deserialize_map(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + self.deserialize_any(visitor) + } + + fn deserialize_struct( + self, + _name: &'static str, + _fields: &'static [&'static str], + visitor: V, + ) -> std::result::Result + where + V: serde::de::Visitor<'de> { + self.deserialize_any(visitor) + } + + fn deserialize_enum( + self, + _name: &'static str, + _variants: &'static [&'static str], + visitor: V, + ) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_enum(EnumAdaptor{ particle_type: self.particle_type(), deserializer: self}) + } + + fn deserialize_identifier(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + self.deserialize_any(visitor) + } + + fn deserialize_ignored_any(self, visitor: V) -> std::result::Result + where + V: serde::de::Visitor<'de> { + visitor.visit_none() + } +} + +struct EnumAdaptor Deserializer<'a, Error = crate::Error>> { + particle_type: ParticleType, + deserializer: V, +} + +// This is specially designed for Value type, to retain current performance. +// There is a possibility that we can do this directly using a u8 enum tag. +impl<'de, Var: for<'a> Deserializer<'a, Error = crate::Error>> EnumAccess<'de> for EnumAdaptor { + type Error = crate::Error; + type Variant = Self; + + fn variant_seed(self, seed: V) -> std::prelude::v1::Result<(V::Value, Self::Variant), Self::Error> + where + V: serde::de::DeserializeSeed<'de> { + let name = match self.particle_type { + ParticleType::NULL => "Nil", + ParticleType::INTEGER => "Int", + ParticleType::FLOAT => "Float", + ParticleType::STRING => "String", + ParticleType::BLOB => "Blob", + ParticleType::DIGEST => todo!(), + ParticleType::BOOL => "Bool", + ParticleType::HLL => "HLL", + ParticleType::MAP => "HashMap", + ParticleType::LIST => "List", + ParticleType::LDT => todo!(), + ParticleType::GEOJSON => "GeoJSON", + }; + let val = seed.deserialize(DeserializeStr(name))?; + Ok((val, self)) + } +} + +impl<'de, Var: for<'a> Deserializer<'a, Error = crate::Error>> VariantAccess<'de> for EnumAdaptor { + type Error = crate::errors::Error; + + fn unit_variant(self) -> std::prelude::v1::Result<(), Self::Error> { + if self.particle_type == ParticleType::NULL { + Ok(()) + } else { + Err(serde::de::Error::invalid_type(serde::de::Unexpected::NewtypeVariant, &"unit variant")) + } + } + + fn newtype_variant_seed(self, seed: T) -> std::prelude::v1::Result + where + T: serde::de::DeserializeSeed<'de> { + seed.deserialize(self.deserializer) + } + + fn tuple_variant(self, _len: usize, _visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + Err(serde::de::Error::invalid_type(serde::de::Unexpected::NewtypeVariant, &"tuple variant")) + } + + fn struct_variant( + self, + _fields: &'static [&'static str], + _visitor: V, + ) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'de> { + Err(serde::de::Error::invalid_type(serde::de::Unexpected::NewtypeVariant, &"struct variant")) + } +} + +struct CDTDecoder<'m>(&'m [u8], &'m mut usize); + +struct CDTListOrMap<'m>(usize, &'m [u8], &'m mut usize); + +impl<'m> CDTDecoder<'m> { + fn as_unexpected(mut self, ptype: u8) -> std::result::Result, Error> { + Ok(match ptype { + 0x00..=0x7f => serde::de::Unexpected::Unsigned(ptype as u64), + 0x80..=0x8f => serde::de::Unexpected::Map, + 0x90..=0x9f => serde::de::Unexpected::Seq, + 0xa0..=0xbf => serde::de::Unexpected::Bytes(self.take_nbyte((ptype & 0x1f) as usize)?), + 0xc0 => serde::de::Unexpected::Unit, + 0xc1 => serde::de::Unexpected::Unit, // Don't actually support this type + 0xc2 => serde::de::Unexpected::Bool(false), + 0xc3 => serde::de::Unexpected::Bool(true), + 0xc4 | 0xd9 => { + let count = u8::from_be_bytes(self.take_bytes()?) as usize; + serde::de::Unexpected::Bytes(self.take_nbyte(count)?) + } + 0xc5 | 0xda => { + let count = u16::from_be_bytes(self.take_bytes()?) as usize; + serde::de::Unexpected::Bytes(self.take_nbyte(count)?) + } + 0xc6 | 0xdb => { + let count = u32::from_be_bytes(self.take_bytes()?) as usize; + serde::de::Unexpected::Bytes(self.take_nbyte(count)?) + } + 0xc7 | 0xc8 | 0xc9 => serde::de::Unexpected::Unit, // Don't actually support this type + 0xca => serde::de::Unexpected::Float(f32::from_be_bytes(self.take_bytes()?) as f64), + 0xcb => serde::de::Unexpected::Float(f64::from_be_bytes(self.take_bytes()?)), + 0xcc => serde::de::Unexpected::Unsigned(u8::from_be_bytes(self.take_bytes()?) as u64), + 0xcd => serde::de::Unexpected::Unsigned(u16::from_be_bytes(self.take_bytes()?) as u64), + 0xce => serde::de::Unexpected::Unsigned(u32::from_be_bytes(self.take_bytes()?) as u64), + 0xcf => serde::de::Unexpected::Unsigned(u64::from_be_bytes(self.take_bytes()?) as u64), + 0xd0 => serde::de::Unexpected::Signed(i8::from_be_bytes(self.take_bytes()?) as i64), + 0xd1 => serde::de::Unexpected::Signed(i16::from_be_bytes(self.take_bytes()?) as i64), + 0xd2 => serde::de::Unexpected::Signed(i32::from_be_bytes(self.take_bytes()?) as i64), + 0xd3 => serde::de::Unexpected::Signed(i64::from_be_bytes(self.take_bytes()?) as i64), + 0xd4..=0xd8 => serde::de::Unexpected::Unit, // Don't actually support this type + 0xdc => serde::de::Unexpected::Seq, + 0xdd => serde::de::Unexpected::Seq, + 0xde => serde::de::Unexpected::Map, + 0xdf => serde::de::Unexpected::Map, + 0xe0..=0xff => { + let value = i64::from(ptype) - 0xe0 - 32; + serde::de::Unexpected::Signed(value) + } + }) + } + + fn take_byte(&mut self) -> std::result::Result { + if *self.1 >= self.0.len() { + Err(Error::from_kind(crate::errors::ErrorKind::Derive("Ran out of data".to_string()))) + } else { + let out = self.0[*self.1]; + *self.1 += 1; + Ok(out) + } + } + + fn take_bytes(&mut self) -> std::result::Result<[u8; N], Error> { + if *self.1 + N >= self.0.len() { + Err(Error::from_kind(crate::errors::ErrorKind::Derive("Ran out of data".to_string()))) + } else { + let offset = *self.1 as isize; + *self.1 += N; + // SAFETY: ok because we just checked that the length fits + unsafe { + let ptr = self.0.as_ptr().offset(offset) as *const [u8; N]; + Ok(*ptr) + } + } + } + + + fn take_nbyte(&mut self, mid: usize) -> std::result::Result<&'m [u8], Error> { + if *self.1 + mid > self.0.len() { + Err(Error::from_kind(crate::errors::ErrorKind::Derive("Ran out of data".to_string()))) + } else { + let first = &self.0[*self.1..(*self.1 + mid)]; + *self.1 += mid; + Ok(first) + } + } +} + +impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { + type Error = crate::errors::Error; + + fn deserialize_any(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + fn deserialize_any_buffer<'l, 'm, V>(mut deserializer: CDTDecoder<'m>, visitor: V, count: usize) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = ParticleType::from(deserializer.take_byte()?); + let body = deserializer.take_nbyte(count - 1)?; + if matches!(ptype, ParticleType::STRING | ParticleType::GEOJSON) { + visitor.visit_str(std::str::from_utf8(body)?) + } else { + visitor.visit_bytes(body) + } + } + + let ptype = self.take_byte()?; + match ptype { + 0x00..=0x7f => visitor.visit_u8(ptype as u8), + 0x80..=0x8f => visitor.visit_map(CDTListOrMap((ptype & 0x0f) as usize, self.0, self.1)), + 0x90..=0x9f => visitor.visit_seq(CDTListOrMap((ptype & 0x0f) as usize, self.0, self.1)), + 0xa0..=0xbf => deserialize_any_buffer(self, visitor, (ptype & 0x1f) as usize), + 0xc0 => visitor.visit_none(), + 0xc2 => visitor.visit_bool(false), + 0xc3 => visitor.visit_bool(true), + 0xc4 | 0xd9 => { + let count = u8::from_be_bytes(self.take_bytes()?); + deserialize_any_buffer(self, visitor, count as usize) + } + 0xc5 | 0xda => { + let count = u16::from_be_bytes(self.take_bytes()?); + deserialize_any_buffer(self, visitor, count as usize) + } + 0xc6 | 0xdb => { + let count = u32::from_be_bytes(self.take_bytes()?); + deserialize_any_buffer(self, visitor, count as usize) + } + 0xca => visitor.visit_f32(f32::from_be_bytes(self.take_bytes()?)), + 0xcb => visitor.visit_f64(f64::from_be_bytes(self.take_bytes()?)), + 0xcc => visitor.visit_u8(u8::from_be_bytes(self.take_bytes()?)), + 0xcd => visitor.visit_u16(u16::from_be_bytes(self.take_bytes()?)), + 0xce => visitor.visit_u32(u32::from_be_bytes(self.take_bytes()?)), + 0xcf => visitor.visit_u64(u64::from_be_bytes(self.take_bytes()?)), + 0xd0 => visitor.visit_i8(i8::from_be_bytes(self.take_bytes()?)), + 0xd1 => visitor.visit_i16(i16::from_be_bytes(self.take_bytes()?)), + 0xd2 => visitor.visit_i32(i32::from_be_bytes(self.take_bytes()?)), + 0xd3 => visitor.visit_i64(i64::from_be_bytes(self.take_bytes()?)), + 0xdc => { + let count = u16::from_be_bytes(self.take_bytes()?) as usize; + visitor.visit_seq(CDTListOrMap(count, self.0, self.1)) + } + 0xdd => { + let count = u32::from_be_bytes(self.take_bytes()?) as usize; + visitor.visit_seq(CDTListOrMap(count, self.0, self.1)) + } + 0xde => { + let count = u16::from_be_bytes(self.take_bytes()?) as usize; + visitor.visit_map(CDTListOrMap(count, self.0, self.1)) + } + 0xdf => { + let count = u32::from_be_bytes(self.take_bytes()?) as usize; + visitor.visit_map(CDTListOrMap(count, self.0, self.1)) + } + 0xe0..=0xff => { + let value = (ptype - 0xe0) as i8 - 32; + visitor.visit_i8(value) + } + _ => todo!() + } + } + + fn deserialize_bool(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = self.take_byte()?; + match ptype { + 0xc2 => visitor.visit_bool(false), + 0xc3 => visitor.visit_bool(true), + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_i8(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = self.take_byte()?; + match ptype { + 0x00..=0x7f => visitor.visit_i8(ptype as i8), + 0xcc => visitor.visit_i8(i8::try_from(u8::from_be_bytes(self.take_bytes()?))?), + 0xd0 => visitor.visit_i8(i8::from_be_bytes(self.take_bytes()?)), + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_i16(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = self.take_byte()?; + match ptype { + 0x00..=0x7f => visitor.visit_i16(ptype as i16), + 0xcc => visitor.visit_i16(u8::from_be_bytes(self.take_bytes()?) as i16), + 0xd0 => visitor.visit_i16(i8::from_be_bytes(self.take_bytes()?) as i16), + 0xcd => visitor.visit_i16(u16::from_be_bytes(self.take_bytes()?).try_into()?), + 0xd1 => visitor.visit_i16(i16::from_be_bytes(self.take_bytes()?)), + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_i32(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = self.take_byte()?; + match ptype { + 0x00..=0x7f => visitor.visit_i32(ptype as i32), + 0xcc => visitor.visit_i32(u8::from_be_bytes(self.take_bytes()?) as i32), + 0xcd => visitor.visit_i32(u16::from_be_bytes(self.take_bytes()?) as i32), + 0xce => visitor.visit_i32(u32::from_be_bytes(self.take_bytes()?).try_into()?), + 0xd0 => visitor.visit_i32(i8::from_be_bytes(self.take_bytes()?) as i32), + 0xd1 => visitor.visit_i32(i16::from_be_bytes(self.take_bytes()?) as i32), + 0xd2 => visitor.visit_i32(i32::from_be_bytes(self.take_bytes()?)), + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_i64(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = self.take_byte()?; + match ptype { + 0x00..=0x7f => visitor.visit_i64(ptype as i64), + 0xcc => visitor.visit_i64(u8::from_be_bytes(self.take_bytes()?) as i64), + 0xcd => visitor.visit_i64(u16::from_be_bytes(self.take_bytes()?) as i64), + 0xce => visitor.visit_i64(u32::from_be_bytes(self.take_bytes()?) as i64), + 0xcf => visitor.visit_i64(u64::from_be_bytes(self.take_bytes()?).try_into()?), + 0xd0 => visitor.visit_i64(i8::from_be_bytes(self.take_bytes()?) as i64), + 0xd1 => visitor.visit_i64(i16::from_be_bytes(self.take_bytes()?) as i64), + 0xd2 => visitor.visit_i64(i32::from_be_bytes(self.take_bytes()?) as i64), + 0xd3 => visitor.visit_i64(i64::from_be_bytes(self.take_bytes()?)), + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_u8(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = self.take_byte()?; + match ptype { + 0x00..=0x7f => visitor.visit_u8(ptype), + 0xcc => visitor.visit_u8(u8::from_be_bytes(self.take_bytes()?)), + 0xd0 => visitor.visit_u8(i8::from_be_bytes(self.take_bytes()?).try_into()?), + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_u16(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = self.take_byte()?; + match ptype { + 0x00..=0x7f => visitor.visit_i16(ptype as i16), + 0xcc => visitor.visit_i16(u8::from_be_bytes(self.take_bytes()?) as i16), + 0xd0 => visitor.visit_i16(i8::from_be_bytes(self.take_bytes()?) as i16), + 0xcd => visitor.visit_i16(u16::from_be_bytes(self.take_bytes()?).try_into()?), + 0xd1 => visitor.visit_i16(i16::from_be_bytes(self.take_bytes()?)), + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_u32(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = self.take_byte()?; + match ptype { + 0x00..=0x7f => visitor.visit_u64(ptype as u64), + 0xcc => visitor.visit_u32(u8::from_be_bytes(self.take_bytes()?) as u32), + 0xcd => visitor.visit_u32(u16::from_be_bytes(self.take_bytes()?) as u32), + 0xce => visitor.visit_u32(u32::from_be_bytes(self.take_bytes()?)), + 0xd0 => visitor.visit_u32(i8::from_be_bytes(self.take_bytes()?) as u32), + 0xd1 => visitor.visit_u32(i16::from_be_bytes(self.take_bytes()?) as u32), + 0xd2 => visitor.visit_u32(i32::from_be_bytes(self.take_bytes()?).try_into()?), + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_u64(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = self.take_byte()?; + match ptype { + 0x00..=0x7f => visitor.visit_u64(ptype as u64), + 0xcc => visitor.visit_u64(u8::from_be_bytes(self.take_bytes()?) as u64), + 0xcd => visitor.visit_u64(u16::from_be_bytes(self.take_bytes()?) as u64), + 0xce => visitor.visit_u64(u32::from_be_bytes(self.take_bytes()?) as u64), + 0xcf => visitor.visit_u64(u64::from_be_bytes(self.take_bytes()?)), + 0xd0 => visitor.visit_u64(i8::from_be_bytes(self.take_bytes()?) as u64), + 0xd1 => visitor.visit_u64(i16::from_be_bytes(self.take_bytes()?) as u64), + 0xd2 => visitor.visit_u64(i32::from_be_bytes(self.take_bytes()?) as u64), + 0xd3 => visitor.visit_u64(i64::from_be_bytes(self.take_bytes()?).try_into()?), + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_f32(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = self.take_byte()?; + match ptype { + 0xca => visitor.visit_f32(f32::from_be_bytes(self.take_bytes()?)), + 0xcb => visitor.visit_f32(f64::from_be_bytes(self.take_bytes()?) as f32), + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_f64(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = self.take_byte()?; + match ptype { + 0xca => visitor.visit_f64(f32::from_be_bytes(self.take_bytes()?).into()), + 0xcb => visitor.visit_f64(f64::from_be_bytes(self.take_bytes()?)), + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_char(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + self.deserialize_any(visitor) + } + + fn deserialize_str(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + fn deserialize_any_buffer<'l, 'm, V>(mut deserializer: CDTDecoder<'m>, visitor: V, count: usize) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = ParticleType::from(deserializer.take_byte()?); + let body = deserializer.take_nbyte(count - 1)?; + if matches!(ptype, ParticleType::STRING | ParticleType::GEOJSON) { + visitor.visit_str(std::str::from_utf8(body)?) + } else { + Err(crate::errors::Error::invalid_type(serde::de::Unexpected::Bytes(body), &visitor)) + } + } + + let ptype = self.take_byte()?; + match ptype { + 0xa0..=0xbf => deserialize_any_buffer(self, visitor, (ptype & 0x1f) as usize), + 0xc4 | 0xd9 => { + let count = u8::from_be_bytes(self.take_bytes()?); + deserialize_any_buffer(self, visitor, count as usize) + } + 0xc5 | 0xda => { + let count = u16::from_be_bytes(self.take_bytes()?); + deserialize_any_buffer(self, visitor, count as usize) + } + 0xc6 | 0xdb => { + let count = u32::from_be_bytes(self.take_bytes()?); + deserialize_any_buffer(self, visitor, count as usize) + } + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_string(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + self.deserialize_str(visitor) + } + + fn deserialize_bytes(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + fn deserialize_any_buffer<'l, 'm, V>(mut deserializer: CDTDecoder<'m>, visitor: V, count: usize) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = ParticleType::from(deserializer.take_byte()?); + let body = deserializer.take_nbyte(count - 1)?; + if matches!(ptype, ParticleType::STRING | ParticleType::GEOJSON) { + Err(crate::errors::Error::invalid_type(serde::de::Unexpected::Str(std::str::from_utf8(body)?), &visitor)) + } else { + visitor.visit_bytes(body) + } + } + + let ptype = self.take_byte()?; + match ptype { + 0xa0..=0xbf => deserialize_any_buffer(self, visitor, (ptype & 0x1f) as usize), + 0xc4 | 0xd9 => { + let count = u8::from_be_bytes(self.take_bytes()?); + deserialize_any_buffer(self, visitor, count as usize) + } + 0xc5 | 0xda => { + let count = u16::from_be_bytes(self.take_bytes()?); + deserialize_any_buffer(self, visitor, count as usize) + } + 0xc6 | 0xdb => { + let count = u32::from_be_bytes(self.take_bytes()?); + deserialize_any_buffer(self, visitor, count as usize) + } + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_byte_buf(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + self.deserialize_bytes(visitor) + } + + fn deserialize_option(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + if self.0[*self.1] == 0xc0 { + *self.1 += 1; + visitor.visit_none() + } else { + visitor.visit_some(self) + } + } + + fn deserialize_unit(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + self.deserialize_any(visitor) + } + + fn deserialize_unit_struct( + self, + _name: &'static str, + visitor: V, + ) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + self.deserialize_any(visitor) + } + + fn deserialize_newtype_struct( + self, + _name: &'static str, + visitor: V, + ) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + self.deserialize_any(visitor) + } + + fn deserialize_seq(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = self.take_byte()?; + match ptype { + 0x90..=0x9f => visitor.visit_seq(CDTListOrMap((ptype & 0x0f) as usize, self.0, self.1)), + 0xdc => { + let count = u16::from_be_bytes(self.take_bytes()?) as usize; + visitor.visit_seq(CDTListOrMap(count, self.0, self.1)) + } + 0xdd => { + let count = u32::from_be_bytes(self.take_bytes()?) as usize; + visitor.visit_seq(CDTListOrMap(count, self.0, self.1)) + } + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_tuple(self, _len: usize, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + self.deserialize_any(visitor) + } + + fn deserialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + visitor: V, + ) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + self.deserialize_any(visitor) + } + + fn deserialize_map(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + let ptype = self.take_byte()?; + match ptype { + 0x80..=0x8f => visitor.visit_map(CDTListOrMap((ptype & 0x0f) as usize, self.0, self.1)), + 0xde => { + let count = u16::from_be_bytes(self.take_bytes()?) as usize; + visitor.visit_map(CDTListOrMap(count, self.0, self.1)) + } + 0xdf => { + let count = u32::from_be_bytes(self.take_bytes()?) as usize; + visitor.visit_map(CDTListOrMap(count, self.0, self.1)) + } + _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + } + } + + fn deserialize_struct( + self, + _name: &'static str, + _fields: &'static [&'static str], + visitor: V, + ) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + self.deserialize_map(visitor) + } + + fn deserialize_enum( + self, + _name: &'static str, + _variants: &'static [&'static str], + visitor: V, + ) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + + let ptype = self.0[*self.1]; + let particle_type = match ptype { + 0x00..=0x7f | 0xcc | 0xcd | 0xce | 0xcf | 0xd0 | 0xd1 | 0xd2 | 0xd3 | 0xe0..=0xff => ParticleType::INTEGER, + 0x80..=0x8f | 0xde | 0xdf => ParticleType::MAP, + 0x90..=0x9f | 0xdc | 0xdd => ParticleType::LIST, + 0xc0 => ParticleType::NULL, + 0xc2 | 0xc3 => ParticleType::BOOL, + // In blobs, the particle type is hiding just after the length + 0xa0..=0xbf => ParticleType::from(self.0[*self.1 + 1]), + 0xc4 | 0xd9 => ParticleType::from(self.0[*self.1 + 2]), + 0xc5 | 0xda => ParticleType::from(self.0[*self.1 + 3]), + 0xc6 | 0xdb => ParticleType::from(self.0[*self.1 + 5]), + 0xca | 0xcb => ParticleType::FLOAT, + _ => ParticleType::NULL + }; + visitor.visit_enum(EnumAdaptor{particle_type, deserializer: self}) + } + + fn deserialize_identifier(self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + self.deserialize_str(visitor) + } + + fn deserialize_ignored_any(mut self, visitor: V) -> std::prelude::v1::Result + where + V: serde::de::Visitor<'l> { + fn ignore_values<'l, 'm>(deserializer: CDTDecoder<'m>, entries: usize) { + struct IgnoreVisitor; + impl<'l> Visitor<'l> for IgnoreVisitor { + type Value = (); + + fn visit_none(self) -> std::prelude::v1::Result + where + E: serde::de::Error, { + Ok(()) + } + fn expecting(&self, _formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + Ok(()) + } + } + for _ in 0..entries { + let _ = CDTDecoder(deserializer.0, deserializer.1).deserialize_ignored_any(IgnoreVisitor); + } + } + + let ptype = self.take_byte()?; + match ptype { + 0x80..=0x8f => ignore_values(self, (ptype & 0x0f) as usize * 2), + 0x90..=0x9f => ignore_values(self, (ptype & 0x0f) as usize), + 0xa0..=0xbf => { *self.1 += (ptype & 0x1f) as usize; }, + 0xc4 | 0xd9 => { + let count = u8::from_be_bytes(self.take_bytes()?); + *self.1 += count as usize; + } + 0xc5 | 0xda => { + let count = u16::from_be_bytes(self.take_bytes()?); + *self.1 += count as usize; + } + 0xc6 | 0xdb => { + let count = u32::from_be_bytes(self.take_bytes()?); + *self.1 += count as usize; + } + 0xcc | 0xd0 => {self.take_byte()?;} + 0xcd | 0xd1 => {self.take_bytes::<2>()?;} + 0xca | 0xce | 0xd2 => {self.take_bytes::<4>()?;} + 0xcb | 0xcf | 0xd3 => {self.take_bytes::<8>()?;} + 0xdc => { + let count = u16::from_be_bytes(self.take_bytes()?) as usize; + ignore_values(self, count) + } + 0xdd => { + let count = u32::from_be_bytes(self.take_bytes()?) as usize; + ignore_values(self, count) + } + 0xde => { + let count = u16::from_be_bytes(self.take_bytes()?) as usize; + ignore_values(self, count * 2) + } + 0xdf => { + let count = u32::from_be_bytes(self.take_bytes()?) as usize; + ignore_values(self, count * 2) + } + _ => () + } + visitor.visit_none() + } +} + +impl<'l, 'm> MapAccess<'l> for CDTListOrMap<'m> { + type Error = crate::errors::Error; + + fn next_key_seed(&mut self, seed: K) -> std::prelude::v1::Result, Self::Error> + where + K: serde::de::DeserializeSeed<'l> { + if self.0 == 0 { + Ok(None) + } else { + self.0 -= 1; + seed.deserialize(CDTDecoder(self.1, self.2)).map(Some) + } + } + + fn next_value_seed(&mut self, seed: V) -> std::prelude::v1::Result + where + V: serde::de::DeserializeSeed<'l> { + seed.deserialize(CDTDecoder(self.1, self.2)) + } + + fn size_hint(&self) -> Option { + Some(self.0) + } +} + +impl<'l, 'm> SeqAccess<'l> for CDTListOrMap<'m> { + type Error = crate::errors::Error; + + fn next_element_seed(&mut self, seed: T) -> std::prelude::v1::Result, Self::Error> + where + T: serde::de::DeserializeSeed<'l> { + if self.0 == 0 { + Ok(None) + } else { + self.0 -= 1; + seed.deserialize(CDTDecoder(self.1, self.2)).map(Some) + } + } + + fn size_hint(&self) -> Option { + Some(self.0) + } +} + +/// Includes the data for the Value part of a Bin. +#[derive(Debug, Clone)] +pub(crate) struct PreParsedValue{ + pub particle_type: u8, + pub name_len: u8, + pub name: [u8; 15], + pub particle: Vec, +} + +impl PreParsedValue { + fn particle_type(&self) -> ParticleType { + ParticleType::from(self.particle_type) + } + + fn name(&self) -> Result<&str> { + let len = self.name_len as usize; + let s = std::str::from_utf8(&self.name[..len])?; + Ok(s) + } + + fn particle(&self) -> &[u8] { + &self.particle + } + + fn as_bool(&self) -> Result { + let [element]: [u8; 1] = self.particle().try_into()?; + Ok(element != 0) + } + + fn as_int(&self) -> Result { + Ok(i64::from_be_bytes(self.particle().try_into()?)) + } + + fn as_float(&self) -> Result { + Ok(f64::from_be_bytes(self.particle().try_into()?)) + } + + fn into_blob(self) -> Vec { + self.particle + } + + fn into_string(self) -> Result { + Ok(std::string::String::from_utf8(self.particle)?) + } +} + + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use serde::Deserialize; + + use crate::derive::readable::PreParsedValue; + + #[derive(Deserialize)] + struct SomeTupleThing(i32, String); + + #[derive(Deserialize, Clone, Copy)] + struct ANormalStruct { + one: u32, + two: i16, + } + + #[derive(Deserialize)] + struct MoreComplexStruct { + binname: SomeTupleThing, + another_binname: Option, + } + + fn new_preparsed(particle_type: u8, name: &str, particle: Vec) -> PreParsedValue { + let mut namebuf = [0_u8; 15]; + let name_len = name.as_bytes().len(); + namebuf[..name_len].copy_from_slice(name.as_bytes()); + PreParsedValue { + particle_type, + name_len: name_len as u8, + name: namebuf, + particle, + } + } + + #[test] + fn destream_structs() { + let mut buffer = crate::Buffer::new(1024); + let myval = crate::Value::List(vec![ + crate::Value::Int(2), + crate::Value::String("Hello world".to_string()), + ]); + + buffer.resize_buffer(myval.estimate_size()).unwrap(); + myval.write_to(&mut buffer); + + let as_bin = new_preparsed(20, "binname", buffer.data_buffer); + + let deserialized = SomeTupleThing::deserialize(as_bin.clone()).unwrap(); + assert_eq!(deserialized.0, 2); + assert_eq!(deserialized.1, "Hello world"); + + let deserialized = MoreComplexStruct::deserialize(crate::derive::readable::BinsDeserializer{bins: vec![as_bin.clone()].into()}).unwrap(); + assert_eq!(deserialized.binname.0, 2); + assert_eq!(deserialized.binname.1, "Hello world"); + assert!(deserialized.another_binname.is_none()); + + let mut buffer = crate::Buffer::new(1024); + let myval = crate::Value::HashMap(HashMap::from([ + (crate::Value::String("one".to_string()), crate::Value::Int(1)), + (crate::Value::String("two".to_string()), crate::Value::Int(2)), + ])); + + buffer.resize_buffer(myval.estimate_size()).unwrap(); + myval.write_to(&mut buffer); + + let another_bin = new_preparsed(20, "another_binname", buffer.data_buffer); + + let deserialized = ANormalStruct::deserialize(another_bin.clone()).unwrap(); + assert_eq!(deserialized.one, 1); + assert_eq!(deserialized.two, 2); + + let deserialized = MoreComplexStruct::deserialize(crate::derive::readable::BinsDeserializer{bins: vec![as_bin.clone(), another_bin.clone()].into()}).unwrap(); + assert_eq!(deserialized.binname.0, 2); + assert_eq!(deserialized.binname.1, "Hello world"); + assert_eq!(deserialized.another_binname.unwrap().one, 1); + assert_eq!(deserialized.another_binname.unwrap().two, 2); + } + + + #[test] + fn destream_value() { + let mut buffer = crate::Buffer::new(1024); + let myval = crate::Value::String("Hello world".to_string()); + + buffer.resize_buffer(myval.estimate_size()).unwrap(); + myval.write_to(&mut buffer); + + let as_bin = new_preparsed(myval.particle_type() as u8, "binname", buffer.data_buffer); + + let deserialized = crate::Value::deserialize(as_bin.clone()).unwrap(); + assert_eq!(deserialized, crate::Value::String("Hello world".to_string())); + + let mut buffer = crate::Buffer::new(1024); + let myval = crate::Value::List(vec![ + crate::Value::Int(2), + crate::Value::String("Hello world".to_string()), + ]); + + buffer.resize_buffer(myval.estimate_size()).unwrap(); + myval.write_to(&mut buffer); + + let as_bin = new_preparsed(20, "binname", buffer.data_buffer); + let deserialized = crate::Value::deserialize(as_bin.clone()).unwrap(); + assert_eq!(deserialized, crate::Value::List(vec![ + crate::Value::Int(2), + crate::Value::String("Hello world".to_string()), + ])); + } +} diff --git a/aerospike-core/src/derive/writable.rs b/aerospike-core/src/derive/writable.rs new file mode 100644 index 00000000..e5836193 --- /dev/null +++ b/aerospike-core/src/derive/writable.rs @@ -0,0 +1,291 @@ +// Copyright 2015-2020 Aerospike, Inc. +// +// Portions may be licensed to Aerospike, Inc. under one or more contributor +// license agreements. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +//! Traits and Implementations for writing data from structs and variables + +use std::collections::HashMap; +use crate::errors::Result; +use crate::{Bin, Buffer, ParticleType}; + +/// The WritableBins Trait is used to convert Objects to Aerospike Wire Data +pub trait WritableBins: Sync { + /// Writes the Object as Bins to the Wire + fn write_as_bins(&self, buffer: &mut Buffer, op_type: u8) -> Result<()>; + /// The encoded size to size the buffer and set the offsets accordingly. + /// Calculated by bin_name_bytes + value_bytes + 8 + /// Defaults to 0 + fn writable_bins_size(&self) -> usize { + 0 + } + /// The amount of bins that will be processed. This is usually just the amount of struct members or list entries. + /// Defaults to 0 + fn writable_bins_count(&self) -> usize { + 0 + } +} + +/// The WritableValue Trait is used to convert Object Values to Aerospike Wire Data +pub trait WritableValue: Sync { + /// Write the Object as Value of a Bin + /// Requires `writable_value_size` and `writable_value_particle_type` to be overwritten to return the correct values + /// Needs to return the byte size of the value + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize; + /// Writes the Object as Value of a CDT + /// Most CDT Objects (content of maps/lists etc.) are encoded differently from the normal Values + fn write_as_cdt_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + self.write_as_value(buffer) + } + /// The particle Type of the value to write. + /// This sets the Value Type for the Aerospike Server + fn writable_value_particle_type(&self) -> ParticleType; + /// Defines if the Object can be encoded + /// For example empty Lists or Options should return false if no data is inside + /// Defaults to true + fn writable_value_encodable(&self) -> bool { + true + } +} + +macro_rules! impl_writable_value_for_num { + ($ty:ident) => { + impl WritableValue for $ty { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + if let Some(ref mut buf) = *buffer { + buf.write_i64(*self as i64); + } + 8 + } + fn write_as_cdt_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + crate::msgpack::encoder::pack_integer(buffer, *self as i64) + } + + fn writable_value_particle_type(&self) -> ParticleType { + ParticleType::INTEGER + } + } + }; +} + +macro_rules! impl_writable_value_for_float { + ($ty:ident) => { + impl WritableValue for $ty { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + if let Some(ref mut buf) = *buffer { + buf.write_f64(f64::from(*self)); + } + 8 + } + fn write_as_cdt_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + crate::msgpack::encoder::pack_f64(buffer, f64::from(*self)) + } + + fn writable_value_particle_type(&self) -> ParticleType { + ParticleType::FLOAT + } + } + }; +} + +impl_writable_value_for_num!(u8); +impl_writable_value_for_num!(i8); +impl_writable_value_for_num!(u16); +impl_writable_value_for_num!(i16); +impl_writable_value_for_num!(u32); +impl_writable_value_for_num!(i32); +impl_writable_value_for_num!(u64); +impl_writable_value_for_num!(i64); +impl_writable_value_for_num!(usize); +impl_writable_value_for_num!(isize); +impl_writable_value_for_float!(f64); +impl_writable_value_for_float!(f32); + +impl WritableValue for Option { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + if let Some(v) = self { + return v.write_as_value(buffer); + } + 0 + } + + fn write_as_cdt_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + if let Some(v) = self { + return v.write_as_cdt_value(buffer); + } + 0 + } + + fn writable_value_particle_type(&self) -> ParticleType { + if let Some(v) = self { + return v.writable_value_particle_type(); + } + ParticleType::NULL + } + + fn writable_value_encodable(&self) -> bool { + self.is_some() + } +} + +impl WritableValue for Vec { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + let mut size = 0; + size += crate::msgpack::encoder::pack_array_begin(buffer, self.len()); + for v in self { + size += v.write_as_cdt_value(buffer) + } + size + } + + fn writable_value_particle_type(&self) -> ParticleType { + ParticleType::LIST + } + + fn writable_value_encodable(&self) -> bool { + !self.is_empty() + } +} + +impl WritableValue for HashMap { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + let mut size = 0; + size += crate::msgpack::encoder::pack_map_begin(buffer, self.len()); + for (k, v) in self { + size += k.write_as_cdt_value(buffer); + size += v.write_as_cdt_value(buffer); + } + size + } + fn writable_value_particle_type(&self) -> ParticleType { + ParticleType::MAP + } + fn writable_value_encodable(&self) -> bool { + !self.is_empty() + } +} + +impl WritableValue for String { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + if let Some(ref mut buf) = *buffer { + buf.write_str(&self); + } + self.len() + } + + fn write_as_cdt_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + crate::msgpack::encoder::pack_string(buffer, &self) + } + + fn writable_value_particle_type(&self) -> ParticleType { + ParticleType::STRING + } + + fn writable_value_encodable(&self) -> bool { + !self.is_empty() + } +} + +impl WritableValue for bool { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + if let Some(ref mut buf) = *buffer { + buf.write_bool(*self); + } + 1 + } + + fn write_as_cdt_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + crate::msgpack::encoder::pack_bool(buffer, *self) + } + + fn writable_value_particle_type(&self) -> ParticleType { + ParticleType::BOOL + } +} + +impl WritableValue for &str { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + if let Some(ref mut buf) = *buffer { + buf.write_str(self); + } + self.len() + } + + fn write_as_cdt_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + crate::msgpack::encoder::pack_string(buffer, self) + } + + fn writable_value_particle_type(&self) -> ParticleType { + ParticleType::STRING + } + + fn writable_value_encodable(&self) -> bool { + !self.is_empty() + } +} +impl WritableBins for [Bin; COUNT] { + fn write_as_bins(&self, buffer: &mut Buffer, op_type: u8) -> Result<()> { + legacy_bins_slice_write_wire(self.as_slice(), buffer, op_type) + } + fn writable_bins_size(&self) -> usize { + legacy_bins_slice_writable_size(self.as_slice()) + } + fn writable_bins_count(&self) -> usize { + self.len() + } +} + +impl WritableBins for &[Bin] { + fn write_as_bins(&self, buffer: &mut Buffer, op_type: u8) -> Result<()> { + legacy_bins_slice_write_wire(&self, buffer, op_type) + } + fn writable_bins_size(&self) -> usize { + legacy_bins_slice_writable_size(&self) + } + fn writable_bins_count(&self) -> usize { + self.len() + } +} + +impl WritableBins for Vec { + fn write_as_bins(&self, buffer: &mut Buffer, op_type: u8) -> Result<()> { + legacy_bins_slice_write_wire(self.as_slice(), buffer, op_type) + } + fn writable_bins_size(&self) -> usize { + legacy_bins_slice_writable_size(self.as_slice()) + } + fn writable_bins_count(&self) -> usize { + self.len() + } +} + +fn legacy_bins_slice_write_wire(bins: &[Bin], buffer: &mut Buffer, op_type: u8) -> Result<()> { + bins.iter().for_each(|b| { + buffer.write_i32((b.name.len() + b.value.estimate_size() + 4) as i32); + buffer.write_u8(op_type); + buffer.write_u8(b.value.particle_type() as u8); + buffer.write_u8(0); + buffer.write_u8(b.name.len() as u8); + buffer.write_str(&b.name); + b.value.write_to(buffer); + }); + Ok(()) +} + +fn legacy_bins_slice_writable_size(bins: &[Bin]) -> usize { + let mut size: usize = 0; + bins.iter().for_each(|b| { + size += b.name.len() + b.value.estimate_size() + 8; + }); + size +} diff --git a/aerospike-core/src/errors.rs b/aerospike-core/src/errors.rs index f4344920..c0df2422 100644 --- a/aerospike-core/src/errors.rs +++ b/aerospike-core/src/errors.rs @@ -73,6 +73,12 @@ error_chain! { #[doc = "Error parsing an integer"]; PwHash(::pwhash::error::Error) #[doc = "Error returned while hashing a password for user authentication"]; + WrongParticleLength(std::array::TryFromSliceError) + #[doc = "Wrong particle length in data"]; + BadUTFData(std::string::FromUtf8Error) + #[doc = "Bad string"]; + TryFromInt(std::num::TryFromIntError) + #[doc = "Bad Int conversion"]; } // Additional `ErrorKind` variants. @@ -121,11 +127,21 @@ error_chain! { display("UDF Bad Response: {}", details) } -/// Error returned when a tasked timeed out before it could be completed. +/// Error returned when a tasked timed out before it could be completed. Timeout(details: String) { description("Timeout") display("Timeout: {}", details) } +/// Error returned when a derive operation fails to encode/decode data. + Derive(details: String) { + description("Derive") + display("Derive error: {}", details) + } + /// Error returned when a derive operation fails to encode/decode data from bins. + WrongTypeForBins { + description("WrongTypeForBins") + display("Records should destream into a map or struct type") + } } } diff --git a/aerospike-core/src/expressions/mod.rs b/aerospike-core/src/expressions/mod.rs index 057d10c0..046a28c6 100644 --- a/aerospike-core/src/expressions/mod.rs +++ b/aerospike-core/src/expressions/mod.rs @@ -54,7 +54,7 @@ pub enum ExpType { #[derive(Debug, Clone, Copy)] #[doc(hidden)] -pub enum ExpOp { +pub(crate) enum ExpOp { Unknown = 0, EQ = 1, NE = 2, @@ -759,6 +759,24 @@ pub const fn or(exps: Vec) -> FilterExpression { } } +/// Create "xor" (^) operator that applies to a variable number of expressions. +/// ``` +/// // a == 0 ^ b == 0 +/// use aerospike::expressions::{xor, eq, int_bin, int_val}; +/// xor(vec![eq(int_bin("a".to_string()), int_val(0)), eq(int_bin("b".to_string()), int_val(0))]); +/// ``` +pub const fn xor(exps: Vec) -> FilterExpression { + FilterExpression { + cmd: Some(ExpOp::Xor), + val: None, + bin: None, + flags: None, + module: None, + exps: Some(exps), + arguments: None, + } +} + /// Create equal (==) expression. /// ``` /// // a == 11 @@ -1130,6 +1148,26 @@ pub const fn int_and(exps: Vec) -> FilterExpression { } } +/// Create integer "or" (|) operator that is applied to two or more integers. +/// All arguments must resolve to integers. +/// Requires server version 5.6.0+. +/// ``` +/// // a | 0xff == 0xff +/// use aerospike::expressions::{eq, int_val, int_or, int_bin}; +/// eq(int_or(vec![int_bin("a".to_string()), int_val(0xFF)]), int_val(0xFF)); +/// ``` +pub const fn int_or(exps: Vec) -> FilterExpression { + FilterExpression { + cmd: Some(ExpOp::IntOr), + val: None, + bin: None, + flags: None, + module: None, + exps: Some(exps), + arguments: None, + } +} + /// Create integer "xor" (^) operator that is applied to two or more integers. /// All arguments must resolve to integers. /// Requires server version 5.6.0+. diff --git a/aerospike-core/src/lib.rs b/aerospike-core/src/lib.rs index f13dec57..abdfdd64 100644 --- a/aerospike-core/src/lib.rs +++ b/aerospike-core/src/lib.rs @@ -183,8 +183,11 @@ mod batch; mod client; mod cluster; mod commands; + +pub use commands::buffer::Buffer; + pub mod expressions; -mod msgpack; +pub mod msgpack; mod net; pub mod operations; pub mod policy; @@ -196,3 +199,5 @@ mod user; #[cfg(test)] extern crate hex; + +pub mod derive; diff --git a/aerospike-core/src/msgpack/decoder.rs b/aerospike-core/src/msgpack/decoder.rs deleted file mode 100644 index 2a8c0012..00000000 --- a/aerospike-core/src/msgpack/decoder.rs +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2015-2018 Aerospike, Inc. -// -// Portions may be licensed to Aerospike, Inc. under one or more contributor -// license agreements. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -use std::collections::HashMap; -use std::vec::Vec; - -use crate::commands::buffer::Buffer; -use crate::commands::ParticleType; -use crate::errors::{ErrorKind, Result}; -use crate::value::Value; - -pub fn unpack_value_list(buf: &mut Buffer) -> Result { - if buf.data_buffer.is_empty() { - return Ok(Value::List(vec![])); - } - - let ltype: u8 = buf.read_u8(None); - - let count: usize = match ltype { - 0x90..=0x9f => (ltype & 0x0f) as usize, - 0xdc => buf.read_u16(None) as usize, - 0xdd => buf.read_u32(None) as usize, - _ => unreachable!(), - }; - - unpack_list(buf, count) -} - -pub fn unpack_value_map(buf: &mut Buffer) -> Result { - if buf.data_buffer.is_empty() { - return Ok(Value::from(HashMap::with_capacity(0))); - } - - let ltype: u8 = buf.read_u8(None); - - let count: usize = match ltype { - 0x80..=0x8f => (ltype & 0x0f) as usize, - 0xde => buf.read_u16(None) as usize, - 0xdf => buf.read_u32(None) as usize, - _ => unreachable!(), - }; - - unpack_map(buf, count) -} - -fn unpack_list(buf: &mut Buffer, mut count: usize) -> Result { - if count > 0 && is_ext(buf.peek()) { - let _uv = unpack_value(buf); - count -= 1; - } - - let mut list: Vec = Vec::with_capacity(count); - for _ in 0..count { - let val = unpack_value(buf)?; - list.push(val); - } - - Ok(Value::from(list)) -} - -fn unpack_map(buf: &mut Buffer, mut count: usize) -> Result { - if count > 0 && is_ext(buf.peek()) { - let _uv = unpack_value(buf); - let _uv = unpack_value(buf); - count -= 1; - } - - let mut map: HashMap = HashMap::with_capacity(count); - for _ in 0..count { - let key = unpack_value(buf)?; - let val = unpack_value(buf)?; - map.insert(key, val); - } - - Ok(Value::from(map)) -} - -fn unpack_blob(buf: &mut Buffer, count: usize) -> Result { - let vtype = buf.read_u8(None); - let count = count - 1; - - match ParticleType::from(vtype) { - ParticleType::STRING => { - let val = buf.read_str(count)?; - Ok(Value::String(val)) - } - - ParticleType::BLOB => Ok(Value::Blob(buf.read_blob(count))), - - ParticleType::GEOJSON => { - let val = buf.read_str(count)?; - Ok(Value::GeoJSON(val)) - } - - _ => bail!( - "Error while unpacking BLOB. Type-header with code `{}` not recognized.", - vtype - ), - } -} - -fn unpack_value(buf: &mut Buffer) -> Result { - let obj_type = buf.read_u8(None); - - match obj_type { - 0x00..=0x7f => Ok(Value::from(obj_type)), - 0x80..=0x8f => unpack_map(buf, (obj_type & 0x0f) as usize), - 0x90..=0x9f => unpack_list(buf, (obj_type & 0x0f) as usize), - 0xa0..=0xbf => unpack_blob(buf, (obj_type & 0x1f) as usize), - 0xc0 => Ok(Value::Nil), - 0xc2 => Ok(Value::from(false)), - 0xc3 => Ok(Value::from(true)), - 0xc4 | 0xd9 => { - let count = buf.read_u8(None); - Ok(unpack_blob(buf, count as usize)?) - } - 0xc5 | 0xda => { - let count = buf.read_u16(None); - Ok(unpack_blob(buf, count as usize)?) - } - 0xc6 | 0xdb => { - let count = buf.read_u32(None); - Ok(unpack_blob(buf, count as usize)?) - } - 0xc7 => { - let count = 1 + buf.read_u8(None); - buf.skip_bytes(count as usize); - Ok(Value::Nil) - } - 0xc8 => { - let count = 1 + buf.read_u16(None); - buf.skip_bytes(count as usize); - Ok(Value::Nil) - } - 0xc9 => { - let count = 1 + buf.read_u32(None); - buf.skip_bytes(count as usize); - Ok(Value::Nil) - } - 0xca => Ok(Value::from(buf.read_f32(None))), - 0xcb => Ok(Value::from(buf.read_f64(None))), - 0xcc => Ok(Value::from(buf.read_u8(None))), - 0xcd => Ok(Value::from(buf.read_u16(None))), - 0xce => Ok(Value::from(buf.read_u32(None))), - 0xcf => Ok(Value::from(buf.read_u64(None))), - 0xd0 => Ok(Value::from(buf.read_i8(None))), - 0xd1 => Ok(Value::from(buf.read_i16(None))), - 0xd2 => Ok(Value::from(buf.read_i32(None))), - 0xd3 => Ok(Value::from(buf.read_i64(None))), - 0xd4 => { - let count = (1 + 1) as usize; - buf.skip_bytes(count); - Ok(Value::Nil) - } - 0xd5 => { - let count = (1 + 2) as usize; - buf.skip_bytes(count); - Ok(Value::Nil) - } - 0xd6 => { - let count = (1 + 4) as usize; - buf.skip_bytes(count); - Ok(Value::Nil) - } - 0xd7 => { - let count = (1 + 8) as usize; - buf.skip_bytes(count); - Ok(Value::Nil) - } - 0xd8 => { - let count = (1 + 16) as usize; - buf.skip_bytes(count); - Ok(Value::Nil) - } - 0xdc => { - let count = buf.read_u16(None); - unpack_list(buf, count as usize) - } - 0xdd => { - let count = buf.read_u32(None); - unpack_list(buf, count as usize) - } - 0xde => { - let count = buf.read_u16(None); - unpack_map(buf, count as usize) - } - 0xdf => { - let count = buf.read_u32(None); - unpack_map(buf, count as usize) - } - 0xe0..=0xff => { - let value = i16::from(obj_type) - 0xe0 - 32; - Ok(Value::from(value)) - } - _ => Err( - ErrorKind::BadResponse(format!("Error unpacking value of type '{:x}'", obj_type)) - .into(), - ), - } -} - -const fn is_ext(byte: u8) -> bool { - matches!(byte, 0xc7 | 0xc8 | 0xc9 | 0xd4 | 0xd5 | 0xd6 | 0xd7 | 0xd8) -} diff --git a/aerospike-core/src/msgpack/encoder.rs b/aerospike-core/src/msgpack/encoder.rs index f6c42f15..9e4b646c 100644 --- a/aerospike-core/src/msgpack/encoder.rs +++ b/aerospike-core/src/msgpack/encoder.rs @@ -13,6 +13,8 @@ // License for the specific language governing permissions and limitations under // the License. +//! General Functions to Encode Aerospike Wire Data + use std::collections::HashMap; use std::num::Wrapping; use std::{i16, i32, i64, i8}; @@ -24,7 +26,7 @@ use crate::operations::cdt_context::CdtContext; use crate::value::{FloatValue, Value}; #[doc(hidden)] -pub fn pack_value(buf: &mut Option<&mut Buffer>, val: &Value) -> usize { +pub(crate) fn pack_value(buf: &mut Option<&mut Buffer>, val: &Value) -> usize { match *val { Value::Nil => pack_nil(buf), Value::Int(ref val) => pack_integer(buf, *val), @@ -44,7 +46,7 @@ pub fn pack_value(buf: &mut Option<&mut Buffer>, val: &Value) -> usize { } #[doc(hidden)] -pub fn pack_empty_args_array(buf: &mut Option<&mut Buffer>) -> usize { +pub(crate) fn pack_empty_args_array(buf: &mut Option<&mut Buffer>) -> usize { let mut size = 0; size += pack_array_begin(buf, 0); @@ -52,7 +54,7 @@ pub fn pack_empty_args_array(buf: &mut Option<&mut Buffer>) -> usize { } #[doc(hidden)] -pub fn pack_cdt_op( +pub(crate) fn pack_cdt_op( buf: &mut Option<&mut Buffer>, cdt_op: &CdtOperation, ctx: &[CdtContext], @@ -98,7 +100,7 @@ pub fn pack_cdt_op( } #[doc(hidden)] -pub fn pack_hll_op( +pub(crate) fn pack_hll_op( buf: &mut Option<&mut Buffer>, hll_op: &CdtOperation, _ctx: &[CdtContext], @@ -122,7 +124,7 @@ pub fn pack_hll_op( } #[doc(hidden)] -pub fn pack_cdt_bit_op( +pub(crate) fn pack_cdt_bit_op( buf: &mut Option<&mut Buffer>, cdt_op: &CdtOperation, ctx: &[CdtContext], @@ -162,7 +164,7 @@ pub fn pack_cdt_bit_op( } #[doc(hidden)] -pub fn pack_array(buf: &mut Option<&mut Buffer>, values: &[Value]) -> usize { +pub(crate) fn pack_array(buf: &mut Option<&mut Buffer>, values: &[Value]) -> usize { let mut size = 0; size += pack_array_begin(buf, values.len()); @@ -174,7 +176,7 @@ pub fn pack_array(buf: &mut Option<&mut Buffer>, values: &[Value]) -> usize { } #[doc(hidden)] -pub fn pack_map(buf: &mut Option<&mut Buffer>, map: &HashMap) -> usize { +pub(crate) fn pack_map(buf: &mut Option<&mut Buffer>, map: &HashMap) -> usize { let mut size = 0; size += pack_map_begin(buf, map.len()); @@ -205,13 +207,15 @@ const MSGPACK_MARKER_I64: u8 = 0xd3; // This method is not compatible with MsgPack specs and is only used by aerospike client<->server // for wire transfer only #[doc(hidden)] -pub fn pack_raw_u16(buf: &mut Option<&mut Buffer>, value: u16) -> usize { +pub(crate) fn pack_raw_u16(buf: &mut Option<&mut Buffer>, value: u16) -> usize { if let Some(ref mut buf) = *buf { buf.write_u16(value); } 2 } +/// Packs a byte without Marker. Writes it to the Buffer if given. +/// Returns the size of the packaged Data #[doc(hidden)] pub fn pack_half_byte(buf: &mut Option<&mut Buffer>, value: u8) -> usize { if let Some(ref mut buf) = *buf { @@ -220,14 +224,18 @@ pub fn pack_half_byte(buf: &mut Option<&mut Buffer>, value: u8) -> usize { 1 } +/// Packs a byte with Marker. Writes it to the Buffer if given. +/// Returns the size of the packaged Data #[doc(hidden)] -pub fn pack_nil(buf: &mut Option<&mut Buffer>) -> usize { +pub(crate) fn pack_nil(buf: &mut Option<&mut Buffer>) -> usize { if let Some(ref mut buf) = *buf { buf.write_u8(MSGPACK_MARKER_NIL); } 1 } +/// Packs a bool. Writes it to the Buffer if given. +/// Returns the size of the packaged Data #[doc(hidden)] pub fn pack_bool(buf: &mut Option<&mut Buffer>, value: bool) -> usize { if let Some(ref mut buf) = *buf { @@ -240,8 +248,10 @@ pub fn pack_bool(buf: &mut Option<&mut Buffer>, value: bool) -> usize { 1 } +/// Packs a Map Marker including the length. Writes it to the Buffer if given. +/// Returns the size of the packaged Data #[doc(hidden)] -fn pack_map_begin(buf: &mut Option<&mut Buffer>, length: usize) -> usize { +pub fn pack_map_begin(buf: &mut Option<&mut Buffer>, length: usize) -> usize { if length < 16 { pack_half_byte(buf, 0x80 | (length as u8)) } else if length < 1 << 16 { @@ -251,6 +261,8 @@ fn pack_map_begin(buf: &mut Option<&mut Buffer>, length: usize) -> usize { } } +/// Packs a List Marker including the length. Writes it to the Buffer if given. +/// Returns the size of the packaged Data #[doc(hidden)] pub fn pack_array_begin(buf: &mut Option<&mut Buffer>, length: usize) -> usize { if length < 16 { @@ -262,6 +274,8 @@ pub fn pack_array_begin(buf: &mut Option<&mut Buffer>, length: usize) -> usize { } } +/// Packs a Byte Array Marker. Writes it to the Buffer if given. +/// Returns the size of the packaged Data #[doc(hidden)] pub fn pack_string_begin(buf: &mut Option<&mut Buffer>, length: usize) -> usize { if length < 32 { @@ -273,6 +287,8 @@ pub fn pack_string_begin(buf: &mut Option<&mut Buffer>, length: usize) -> usize } } +/// Packs a blob. Writes it to the Buffer if given. +/// Returns the size of the packaged Data #[doc(hidden)] pub fn pack_blob(buf: &mut Option<&mut Buffer>, value: &[u8]) -> usize { let mut size = value.len() + 1; @@ -286,6 +302,8 @@ pub fn pack_blob(buf: &mut Option<&mut Buffer>, value: &[u8]) -> usize { size } +/// Packs a string including Marker. Writes it to the Buffer if given. +/// Returns the size of the packaged Data #[doc(hidden)] pub fn pack_string(buf: &mut Option<&mut Buffer>, value: &str) -> usize { let mut size = value.len() + 1; @@ -299,6 +317,8 @@ pub fn pack_string(buf: &mut Option<&mut Buffer>, value: &str) -> usize { size } +/// Packs a String without Marker. Writes it to the Buffer if given. +/// Returns the size of the packaged Data #[doc(hidden)] pub fn pack_raw_string(buf: &mut Option<&mut Buffer>, value: &str) -> usize { let mut size = value.len(); @@ -311,6 +331,8 @@ pub fn pack_raw_string(buf: &mut Option<&mut Buffer>, value: &str) -> usize { size } +/// Packs a GeoJSON Object. Writes it to the Buffer if given. +/// Returns the size of the packaged Data #[doc(hidden)] fn pack_geo_json(buf: &mut Option<&mut Buffer>, value: &str) -> usize { let mut size = value.len() + 1; @@ -324,6 +346,9 @@ fn pack_geo_json(buf: &mut Option<&mut Buffer>, value: &str) -> usize { size } +/// Packs a Integer. Writes it to the Buffer if given. +/// Returns the size of the packaged Data +/// The exact Integer size is handled by this function. #[doc(hidden)] pub fn pack_integer(buf: &mut Option<&mut Buffer>, value: i64) -> usize { if value >= 0 { diff --git a/aerospike-core/src/msgpack/mod.rs b/aerospike-core/src/msgpack/mod.rs index 1ee543a4..84197168 100644 --- a/aerospike-core/src/msgpack/mod.rs +++ b/aerospike-core/src/msgpack/mod.rs @@ -13,5 +13,6 @@ // License for the specific language governing permissions and limitations under // the License. -pub mod decoder; +//! General Functions for Aerospike Wire encoding and decoding + pub mod encoder; diff --git a/aerospike-core/src/net/connection.rs b/aerospike-core/src/net/connection.rs index fd5d8ce9..b981cdd8 100644 --- a/aerospike-core/src/net/connection.rs +++ b/aerospike-core/src/net/connection.rs @@ -15,6 +15,7 @@ use crate::commands::admin_command::AdminCommand; use crate::commands::buffer::Buffer; +use crate::derive::readable::PreParsedValue; use crate::errors::{ErrorKind, Result}; use crate::policy::ClientPolicy; #[cfg(all(any(feature = "rt-async-std"), not(feature = "rt-tokio")))] @@ -25,6 +26,7 @@ use aerospike_rt::net::TcpStream; use aerospike_rt::time::{Duration, Instant}; #[cfg(all(any(feature = "rt-async-std"), not(feature = "rt-tokio")))] use futures::{AsyncReadExt, AsyncWriteExt}; +use std::convert::TryInto; use std::ops::Add; #[derive(Debug)] @@ -129,4 +131,30 @@ impl Connection { pub const fn bytes_read(&self) -> usize { self.bytes_read } + + pub(crate) async fn pre_parse_stream_bins( + &mut self, + op_count: usize, + ) -> Result> { + let mut data_points = Vec::new(); + data_points.reserve_exact(op_count); + + for _ in 0..op_count { + let mut head = [0; 8]; + self.conn.read_exact(&mut head).await?; + let next_len = u32::from_be_bytes(head[..4].try_into().unwrap()); + let particle_type = head[5]; + let name_len = head[7] as usize; + let mut name = [0; 15]; + self.conn.read_exact(&mut name[..name_len]).await?; + + let mut particle = Vec::new(); + particle.resize(next_len as usize - 4 - name_len, 0); + self.conn.read_exact(&mut particle).await?; + + data_points.push(PreParsedValue{particle_type, name, name_len: head[7], particle}); + } + + Ok(data_points) + } } diff --git a/aerospike-core/src/operations/scalar.rs b/aerospike-core/src/operations/scalar.rs index a23ba348..0f4f393c 100644 --- a/aerospike-core/src/operations/scalar.rs +++ b/aerospike-core/src/operations/scalar.rs @@ -40,7 +40,7 @@ pub const fn get_header<'a>() -> Operation<'a> { } /// Create read bin database operation. -pub const fn get_bin(bin_name: &str) -> Operation { +pub fn get_bin(bin_name: &str) -> Operation { Operation { op: OperationType::Read, ctx: DEFAULT_CTX, @@ -50,41 +50,41 @@ pub const fn get_bin(bin_name: &str) -> Operation { } /// Create set database operation. -pub const fn put<'a>(bin: &'a Bin) -> Operation<'a> { +pub fn put<'a>(bin: &'a Bin) -> Operation<'a> { Operation { op: OperationType::Write, ctx: DEFAULT_CTX, - bin: OperationBin::Name(bin.name), + bin: OperationBin::Name(&bin.name), data: OperationData::Value(&bin.value), } } /// Create string append database operation. -pub const fn append<'a>(bin: &'a Bin) -> Operation<'a> { +pub fn append<'a>(bin: &'a Bin) -> Operation<'a> { Operation { op: OperationType::Append, ctx: DEFAULT_CTX, - bin: OperationBin::Name(bin.name), + bin: OperationBin::Name(&bin.name), data: OperationData::Value(&bin.value), } } /// Create string prepend database operation. -pub const fn prepend<'a>(bin: &'a Bin) -> Operation<'a> { +pub fn prepend<'a>(bin: &'a Bin) -> Operation<'a> { Operation { op: OperationType::Prepend, ctx: DEFAULT_CTX, - bin: OperationBin::Name(bin.name), + bin: OperationBin::Name(&bin.name), data: OperationData::Value(&bin.value), } } /// Create integer add database operation. -pub const fn add<'a>(bin: &'a Bin) -> Operation<'a> { +pub fn add<'a>(bin: &'a Bin) -> Operation<'a> { Operation { op: OperationType::Incr, ctx: DEFAULT_CTX, - bin: OperationBin::Name(bin.name), + bin: OperationBin::Name(&bin.name), data: OperationData::Value(&bin.value), } } diff --git a/aerospike-core/src/query/recordset.rs b/aerospike-core/src/query/recordset.rs index ce2ceecc..975af29f 100644 --- a/aerospike-core/src/query/recordset.rs +++ b/aerospike-core/src/query/recordset.rs @@ -28,16 +28,19 @@ use crate::Record; /// multiple threads will retrieve records from the server nodes and put these records on an /// internal queue managed by the recordset. The single user thread consumes these records from the /// queue. -pub struct Recordset { +pub struct Recordset +where + T: serde::de::DeserializeOwned, +{ instances: AtomicUsize, record_queue_count: AtomicUsize, record_queue_size: AtomicUsize, - record_queue: SegQueue>, + record_queue: SegQueue>>, active: AtomicBool, task_id: AtomicUsize, } -impl Recordset { +impl Recordset { #[doc(hidden)] pub fn new(rec_queue_size: usize, nodes: usize) -> Self { let mut rng = rand::thread_rng(); @@ -64,7 +67,7 @@ impl Recordset { } #[doc(hidden)] - pub fn push(&self, record: Result) -> Option> { + pub fn push(&self, record: Result>) -> Option>> { if self.record_queue_count.fetch_add(1, Ordering::Relaxed) < self.record_queue_size.load(Ordering::Relaxed) { @@ -88,10 +91,10 @@ impl Recordset { } } -impl<'a> Iterator for &'a Recordset { - type Item = Result; +impl<'a, T: serde::de::DeserializeOwned> Iterator for &'a Recordset { + type Item = Result>; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option>> { loop { if self.is_active() || !self.record_queue.is_empty() { let result = self.record_queue.pop(); diff --git a/aerospike-core/src/record.rs b/aerospike-core/src/record.rs index 8804de53..0c789b84 100644 --- a/aerospike-core/src/record.rs +++ b/aerospike-core/src/record.rs @@ -16,12 +16,9 @@ #[cfg(feature = "serialization")] use serde::Serialize; -use std::collections::HashMap; -use std::fmt; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::{collections::HashMap, time::{Duration, SystemTime, UNIX_EPOCH}}; use crate::Key; -use crate::Value; lazy_static! { // Fri Jan 1 00:00:00 UTC 2010 @@ -31,13 +28,13 @@ lazy_static! { /// Container object for a database record. #[derive(Debug, Clone)] #[cfg_attr(feature = "serialization", derive(Serialize))] -pub struct Record { +pub struct Record> { /// Record key. When reading a record from the database, the key is not set in the returned /// Record struct. pub key: Option, /// Map of named record bins. - pub bins: HashMap, + pub bins: T, /// Record modification count. pub generation: u32, @@ -46,15 +43,10 @@ pub struct Record { expiration: u32, } -impl Record { +impl Record { /// Construct a new Record. For internal use only. #[doc(hidden)] - pub const fn new( - key: Option, - bins: HashMap, - generation: u32, - expiration: u32, - ) -> Self { + pub const fn new(key: Option, bins: T, generation: u32, expiration: u32) -> Self { Record { key, bins, @@ -81,25 +73,6 @@ impl Record { } } -impl fmt::Display for Record { - fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { - write!(f, "key: {:?}", self.key)?; - write!(f, ", bins: {{")?; - for (i, (k, v)) in self.bins.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{}: {}", k, v)?; - } - write!(f, "}}, generation: {}", self.generation)?; - write!(f, ", ttl: ")?; - match self.time_to_live() { - None => "none".fmt(f), - Some(duration) => duration.as_secs().fmt(f), - } - } -} - #[cfg(test)] mod tests { use super::{Record, CITRUSLEAF_EPOCH}; @@ -113,7 +86,7 @@ mod tests { .duration_since(*CITRUSLEAF_EPOCH) .unwrap() .as_secs(); - let record = Record::new(None, HashMap::new(), 0, secs_since_epoch as u32); + let record = Record::>::new(None, HashMap::new(), 0, secs_since_epoch as u32); let ttl = record.time_to_live(); assert!(ttl.is_some()); assert!(1000 - ttl.unwrap().as_secs() <= 1); @@ -121,13 +94,13 @@ mod tests { #[test] fn ttl_expiration_past() { - let record = Record::new(None, HashMap::new(), 0, 0x0d00_d21c); + let record = Record::>::new(None, HashMap::new(), 0, 0x0d00_d21c); assert_eq!(record.time_to_live(), Some(Duration::new(1u64, 0))); } #[test] fn ttl_never_expires() { - let record = Record::new(None, HashMap::new(), 0, 0); + let record = Record::>::new(None, HashMap::new(), 0, 0); assert_eq!(record.time_to_live(), None); } } diff --git a/aerospike-core/src/traits.rs b/aerospike-core/src/traits.rs new file mode 100644 index 00000000..56f2b9b8 --- /dev/null +++ b/aerospike-core/src/traits.rs @@ -0,0 +1,586 @@ +// Copyright 2015-2018 Aerospike, Inc. +// +// Commonly used Traits for Data input and output handling + +use crate::commands::buffer::Buffer; +use crate::errors::Result; +use crate::{Bin, ParticleType, Value}; +use std::collections::HashMap; + +use crate::value::bytes_to_particle; +pub use aerospike_macro::{ReadableBins, WritableBins, WritableValue}; + +/// The WritableBins Trait is used to convert Objects to Aerospike Wire Data +pub trait WritableBins: Sync { + /// Writes the Object as Bins to the Wire + fn write_as_bins(&self, buffer: &mut Buffer, op_type: u8) -> Result<()>; + /// The encoded size to size the buffer and set the offsets accordingly. + /// Calculated by bin_name_bytes + value_bytes + 8 + /// Defaults to 0 + fn writable_bins_size(&self) -> usize { + 0 + } + /// The amount of bins that will be processed. This is usually just the amount of struct members or list entries. + /// Defaults to 0 + fn writable_bins_count(&self) -> usize { + 0 + } +} + +/// The ReadableBins Trait is used to convert Aerospike Wire Data to Objects +pub trait ReadableBins: Sync + Sized + Send + Clone { + /// Convert the pre-parsed Bins to a compatible Object + /// The String in `data_points` is the field name returned by the Server. + /// This can vary from the actual name in the Object if the rename attribute is used. + fn read_bins_from_bytes(data_points: &mut HashMap) -> Result; + /// Default Fallback for Empty Bins + /// Should be implemented for Types like Options and Lists. + /// Defaults to throwing an Error + fn new_empty() -> Result { + bail!("No empty implementation found") + } +} + +/// The ReadableValue Trait is used to convert Aerospike Wire Data into the Value of Objects +pub trait ReadableValue: Sync + Sized + Send + Clone { + /// Read the data from the Wire Buffer. + /// This method is primarily used for pre-parsing checks + fn read_value_from_bytes(data_point: &mut PreParsedBin) -> Result; + /// Actual conversion of the bytes to the value + fn parse_value(data_point: &mut PreParsedValue) -> Result; + /// CDT values are parsed differently from normal Values. This buffer is not a copy, so modifications can cause wrong behavior. + fn parse_cdt_value(buff: &mut Buffer) -> Result; +} + +/// Before giving data to the Readable Traits, the client pre-parses the wire data into this format#[derive(Debug)] +#[derive(Debug)] +pub struct PreParsedBin { + /// Value is always given for any datatype. + pub value: PreParsedValue, + /// Lists and Maps can have other sub-values for entries. In this case, they need to be appended to the parsed value + pub sub_values: Vec, +} + +/// Includes the data for the Value part of a Bin. +#[derive(Debug)] +pub struct PreParsedValue { + /// The Particle Type the Sever stored the Value as. + pub particle_type: u8, + /// Part of the wire Buffer with only the relevant Value Data inside. Value starts at offset 0 without meta around it. + pub buffer: Buffer, + /// Amount of bytes that should be parsed as Value in the buffer. Should be used instead of buffer length for safety reasons. + pub byte_length: usize, +} + +/// The WritableValue Trait is used to convert Object Values to Aerospike Wire Data +pub trait WritableValue: Sync { + /// Write the Object as Value of a Bin + /// Requires `writable_value_size` and `writable_value_particle_type` to be overwritten to return the correct values + /// Needs to return the byte size of the value + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize; + /// Writes the Object as Value of a CDT + /// Most CDT Objects (content of maps/lists etc.) are encoded differently from the normal Values + fn write_as_cdt_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + self.write_as_value(buffer) + } + /// The particle Type of the value to write. + /// This sets the Value Type for the Aerospike Server + fn writable_value_particle_type(&self) -> ParticleType; + /// Defines if the Object can be encoded + /// For example empty Lists or Options should return false if no data is inside + /// Defaults to true + fn writable_value_encodable(&self) -> bool { + true + } +} + +macro_rules! impl_writable_value_for_num { + ($ty:ident) => { + impl WritableValue for $ty { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + if let Some(ref mut buf) = *buffer { + buf.write_i64(*self as i64); + } + 8 + } + fn write_as_cdt_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + crate::msgpack::encoder::pack_integer(buffer, *self as i64) + } + + fn writable_value_particle_type(&self) -> ParticleType { + ParticleType::INTEGER + } + } + }; +} + +macro_rules! impl_writable_value_for_float { + ($ty:ident) => { + impl WritableValue for $ty { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + if let Some(ref mut buf) = *buffer { + buf.write_f64(f64::from(*self)); + } + 8 + } + fn write_as_cdt_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + crate::msgpack::encoder::pack_f64(buffer, f64::from(*self)) + } + + fn writable_value_particle_type(&self) -> ParticleType { + ParticleType::FLOAT + } + } + }; +} + +impl_writable_value_for_num!(u8); +impl_writable_value_for_num!(i8); +impl_writable_value_for_num!(u16); +impl_writable_value_for_num!(i16); +impl_writable_value_for_num!(u32); +impl_writable_value_for_num!(i32); +impl_writable_value_for_num!(u64); +impl_writable_value_for_num!(i64); +impl_writable_value_for_num!(usize); +impl_writable_value_for_num!(isize); +impl_writable_value_for_float!(f64); +impl_writable_value_for_float!(f32); + +impl WritableValue for Option { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + if let Some(v) = self { + return v.write_as_value(buffer); + } + 0 + } + + fn write_as_cdt_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + if let Some(v) = self { + return v.write_as_cdt_value(buffer); + } + 0 + } + + fn writable_value_particle_type(&self) -> ParticleType { + if let Some(v) = self { + return v.writable_value_particle_type(); + } + ParticleType::NULL + } + + fn writable_value_encodable(&self) -> bool { + self.is_some() + } +} + +impl WritableValue for Vec { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + let mut size = 0; + size += crate::msgpack::encoder::pack_array_begin(buffer, self.len()); + for v in self { + size += v.write_as_cdt_value(buffer) + } + size + } + + fn writable_value_particle_type(&self) -> ParticleType { + ParticleType::LIST + } + + fn writable_value_encodable(&self) -> bool { + !self.is_empty() + } +} + +impl WritableValue for String { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + if let Some(ref mut buf) = *buffer { + buf.write_str(&self); + } + self.len() + } + + fn write_as_cdt_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + crate::msgpack::encoder::pack_string(buffer, &self) + } + + fn writable_value_particle_type(&self) -> ParticleType { + ParticleType::STRING + } + + fn writable_value_encodable(&self) -> bool { + !self.is_empty() + } +} + +impl WritableValue for bool { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + if let Some(ref mut buf) = *buffer { + buf.write_bool(*self); + } + 1 + } + + fn write_as_cdt_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + crate::msgpack::encoder::pack_bool(buffer, *self) + } + + fn writable_value_particle_type(&self) -> ParticleType { + ParticleType::BOOL + } +} + +impl WritableValue for &str { + fn write_as_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + if let Some(ref mut buf) = *buffer { + buf.write_str(self); + } + self.len() + } + + fn write_as_cdt_value(&self, buffer: &mut Option<&mut Buffer>) -> usize { + crate::msgpack::encoder::pack_string(buffer, self) + } + + fn writable_value_particle_type(&self) -> ParticleType { + ParticleType::STRING + } + + fn writable_value_encodable(&self) -> bool { + !self.is_empty() + } +} +impl WritableBins for [Bin; COUNT] { + fn write_as_bins(&self, buffer: &mut Buffer, op_type: u8) -> Result<()> { + legacy_bins_slice_write_wire(self.as_slice(), buffer, op_type) + } + fn writable_bins_size(&self) -> usize { + legacy_bins_slice_writable_size(self.as_slice()) + } + fn writable_bins_count(&self) -> usize { + self.len() + } +} + +impl WritableBins for &[Bin] { + fn write_as_bins(&self, buffer: &mut Buffer, op_type: u8) -> Result<()> { + legacy_bins_slice_write_wire(&self, buffer, op_type) + } + fn writable_bins_size(&self) -> usize { + legacy_bins_slice_writable_size(&self) + } + fn writable_bins_count(&self) -> usize { + self.len() + } +} + +impl WritableBins for Vec { + fn write_as_bins(&self, buffer: &mut Buffer, op_type: u8) -> Result<()> { + legacy_bins_slice_write_wire(self.as_slice(), buffer, op_type) + } + fn writable_bins_size(&self) -> usize { + legacy_bins_slice_writable_size(self.as_slice()) + } + fn writable_bins_count(&self) -> usize { + self.len() + } +} + +fn legacy_bins_slice_write_wire(bins: &[Bin], buffer: &mut Buffer, op_type: u8) -> Result<()> { + bins.iter().for_each(|b| { + buffer.write_i32((b.name.len() + b.value.estimate_size() + 4) as i32); + buffer.write_u8(op_type); + buffer.write_u8(b.value.particle_type() as u8); + buffer.write_u8(0); + buffer.write_u8(b.name.len() as u8); + buffer.write_str(&b.name); + b.value.write_to(buffer); + }); + Ok(()) +} + +fn legacy_bins_slice_writable_size(bins: &[Bin]) -> usize { + let mut size: usize = 0; + bins.iter().for_each(|b| { + size += b.name.len() + b.value.estimate_size() + 8; + }); + size +} + +impl ReadableValue for Value { + fn read_value_from_bytes(data_point: &mut PreParsedBin) -> Result { + let mut val = Value::parse_value(&mut data_point.value)?; + + for sv in &mut data_point.sub_values { + let sval = Value::parse_value(sv)?; + match val { + Value::List(ref mut list) => list.push(sval), + ref mut prev => { + *prev = as_list!(prev.clone(), sval); + } + } + } + return Ok(val); + } + + fn parse_value(data_point: &mut PreParsedValue) -> Result { + bytes_to_particle( + data_point.particle_type, + &mut data_point.buffer, + data_point.byte_length, + ) + } + + fn parse_cdt_value(buff: &mut Buffer) -> Result { + crate::msgpack::decoder::unpack_value(buff) + } +} + +impl ReadableBins for HashMap { + fn read_bins_from_bytes(data_points: &mut HashMap) -> Result { + let mut hm = HashMap::new(); + for (k, d) in data_points { + let x = Value::read_value_from_bytes(d)?; + hm.insert(k.to_string(), x); + } + + Ok(hm) + } + + fn new_empty() -> Result { + Ok(HashMap::new()) + } +} + +impl ReadableValue for i64 { + fn read_value_from_bytes(data_point: &mut PreParsedBin) -> Result { + if data_point.value.particle_type == 0 { + bail!("No Value received for Integer") + } + if !data_point.sub_values.is_empty() { + bail!("Multiple Values received for Integer") + } + return Self::parse_value(&mut data_point.value); + } + + fn parse_value(data_point: &mut PreParsedValue) -> Result { + return Ok(data_point.buffer.read_i64(None)); + } + + fn parse_cdt_value(buff: &mut Buffer) -> Result { + let ptype = buff.read_u8(None); + match ptype { + 0x00..=0x7f => Ok(i64::from(ptype) as i64), + 0xcc => Ok(buff.read_u8(None) as i64), + 0xcd => Ok(buff.read_u16(None) as i64), + 0xce => Ok(buff.read_u32(None) as i64), + 0xcf => Ok(buff.read_u64(None) as i64), + 0xd0 => Ok(buff.read_i8(None) as i64), + 0xd1 => Ok(buff.read_i16(None) as i64), + 0xd2 => Ok(buff.read_i32(None) as i64), + 0xd3 => Ok(buff.read_i64(None) as i64), + _ => bail!("Invalid Data Type for derive i64 CDT Value"), + } + } +} + +impl ReadableValue for usize { + fn read_value_from_bytes(data_point: &mut PreParsedBin) -> Result { + if data_point.value.particle_type == 0 { + bail!("No Value received for Integer") + } + if !data_point.sub_values.is_empty() { + bail!("Multiple Values received for Integer") + } + return Self::parse_value(&mut data_point.value); + } + + fn parse_value(data_point: &mut PreParsedValue) -> Result { + return Ok(data_point.buffer.read_i64(None) as usize); + } + + fn parse_cdt_value(buff: &mut Buffer) -> Result { + let ptype = buff.read_u8(None); + match ptype { + 0x00..=0x7f => Ok(i64::from(ptype) as usize), + 0xcc => Ok(buff.read_u8(None) as usize), + 0xcd => Ok(buff.read_u16(None) as usize), + 0xce => Ok(buff.read_u32(None) as usize), + 0xcf => Ok(buff.read_u64(None) as usize), + 0xd0 => Ok(buff.read_i8(None) as usize), + 0xd1 => Ok(buff.read_i16(None) as usize), + 0xd2 => Ok(buff.read_i32(None) as usize), + 0xd3 => Ok(buff.read_i64(None) as usize), + _ => bail!("Invalid Data Type for derive usize CDT Value"), + } + } +} + +impl ReadableValue for isize { + fn read_value_from_bytes(data_point: &mut PreParsedBin) -> Result { + if data_point.value.particle_type == 0 { + bail!("No Value received for Integer") + } + if !data_point.sub_values.is_empty() { + bail!("Multiple Values received for Integer") + } + return Self::parse_value(&mut data_point.value); + } + + fn parse_value(data_point: &mut PreParsedValue) -> Result { + return Ok(data_point.buffer.read_i64(None) as isize); + } + + fn parse_cdt_value(buff: &mut Buffer) -> Result { + let ptype = buff.read_u8(None); + match ptype { + 0x00..=0x7f => Ok(i64::from(ptype) as isize), + 0xcc => Ok(buff.read_u8(None) as isize), + 0xcd => Ok(buff.read_u16(None) as isize), + 0xce => Ok(buff.read_u32(None) as isize), + 0xcf => Ok(buff.read_u64(None) as isize), + 0xd0 => Ok(buff.read_i8(None) as isize), + 0xd1 => Ok(buff.read_i16(None) as isize), + 0xd2 => Ok(buff.read_i32(None) as isize), + 0xd3 => Ok(buff.read_i64(None) as isize), + _ => bail!("Invalid Data Type for derive isize CDT Value"), + } + } +} + +impl ReadableValue for u64 { + fn read_value_from_bytes(data_point: &mut PreParsedBin) -> Result { + if data_point.value.particle_type == 0 { + bail!("No Value received for Integer") + } + if !data_point.sub_values.is_empty() { + bail!("Multiple Values received for Integer") + } + return Self::parse_value(&mut data_point.value); + } + + fn parse_value(data_point: &mut PreParsedValue) -> Result { + return Ok(data_point.buffer.read_i64(None) as u64); + } + + fn parse_cdt_value(buff: &mut Buffer) -> Result { + let ptype = buff.read_u8(None); + match ptype { + 0x00..=0x7f => Ok(i64::from(ptype) as u64), + 0xcc => Ok(buff.read_u8(None) as u64), + 0xcd => Ok(buff.read_u16(None) as u64), + 0xce => Ok(buff.read_u32(None) as u64), + 0xcf => Ok(buff.read_u64(None)), + 0xd0 => Ok(buff.read_i8(None) as u64), + 0xd1 => Ok(buff.read_i16(None) as u64), + 0xd2 => Ok(buff.read_i32(None) as u64), + 0xd3 => Ok(buff.read_i64(None) as u64), + _ => bail!("Invalid Data Type for derive u64 CDT Value"), + } + } +} + +impl ReadableValue for f64 { + fn read_value_from_bytes(data_point: &mut PreParsedBin) -> Result { + if data_point.value.particle_type == 0 { + bail!("No Value received for Float") + } + if !data_point.sub_values.is_empty() { + bail!("Multiple Values received for Float") + } + return Self::parse_value(&mut data_point.value); + } + + fn parse_value(data_point: &mut PreParsedValue) -> Result { + return Ok(data_point.buffer.read_f64(None)); + } + + fn parse_cdt_value(buff: &mut Buffer) -> Result { + let ptype = buff.read_u8(None); + match ptype { + 0xca => Ok(buff.read_f32(None) as f64), + 0xcb => Ok(buff.read_f64(None)), + _ => bail!("Invalid Data Type for derive float CDT Value"), + } + } +} + +impl ReadableValue for String { + fn read_value_from_bytes(data_point: &mut PreParsedBin) -> Result { + if data_point.value.particle_type == 0 { + bail!("No Value received for String") + } + if !data_point.sub_values.is_empty() { + bail!("Multiple Values received for string") + } + return Self::parse_value(&mut data_point.value); + } + + fn parse_value(data_point: &mut PreParsedValue) -> Result { + return data_point.buffer.read_str(data_point.byte_length); + } + + fn parse_cdt_value(buff: &mut Buffer) -> Result { + let len = (buff.read_u8(None) & 0x1f) as usize; + let ptype = buff.read_u8(None); + if ptype != ParticleType::STRING as u8 { + bail!("Invalid Data Type for derive string CDT Value") + } + return buff.read_str(len - 1); + } +} + +impl ReadableValue for Option { + fn read_value_from_bytes(data_point: &mut PreParsedBin) -> Result { + if data_point.value.particle_type == 0 { + return Ok(None); + } + Ok(Some(T::read_value_from_bytes(data_point)?)) + } + + fn parse_value(data_point: &mut PreParsedValue) -> Result { + Ok(Some(T::parse_value(data_point)?)) + } + + fn parse_cdt_value(buff: &mut Buffer) -> Result { + Ok(Some(T::parse_cdt_value(buff)?)) + } +} + +impl ReadableValue for Vec { + fn read_value_from_bytes(data_point: &mut PreParsedBin) -> Result { + let mut v: Vec = Vec::with_capacity(data_point.sub_values.len() + 1); + v.extend(Self::parse_value(&mut data_point.value)?); + for sv in &mut data_point.sub_values { + v.extend(Self::parse_value(sv)?) + } + return Ok(v); + } + + fn parse_value(data_point: &mut PreParsedValue) -> Result { + if data_point.particle_type == ParticleType::LIST as u8 { + Self::parse_cdt_value(&mut data_point.buffer) + } else { + bail!("Invalid Data Type for derive List CDT Type") + } + } + + fn parse_cdt_value(buff: &mut Buffer) -> Result { + let ltype = buff.read_u8(None); + let count: usize = match ltype { + 0x90..=0x9f => (ltype & 0x0f) as usize, + 0xdc => buff.read_u16(None) as usize, + 0xdd => buff.read_u32(None) as usize, + _ => { + bail!("Invalid Data Type for derive List CDT Type") + } + }; + + let mut list = Vec::with_capacity(count); + for _ in 0..count { + list.push(T::parse_cdt_value(buff)?); + } + return Ok(list); + } +} diff --git a/aerospike-core/src/value.rs b/aerospike-core/src/value.rs index 164a1984..9f976368 100644 --- a/aerospike-core/src/value.rs +++ b/aerospike-core/src/value.rs @@ -14,6 +14,7 @@ // the License. use std::collections::HashMap; +use std::convert::TryFrom; use std::fmt; use std::hash::{Hash, Hasher}; use std::result::Result as StdResult; @@ -28,7 +29,7 @@ use std::vec::Vec; use crate::commands::buffer::Buffer; use crate::commands::ParticleType; use crate::errors::Result; -use crate::msgpack::{decoder, encoder}; +use crate::msgpack::encoder; #[cfg(feature = "serialization")] use serde::ser::{SerializeMap, SerializeSeq}; @@ -44,6 +45,60 @@ pub enum FloatValue { F64(u64), } +impl<'l> serde::de::Deserialize<'l> for FloatValue { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'l> { + struct MyVistor; + + impl<'l> serde::de::Visitor<'l> for MyVistor { + type Value = FloatValue; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "f32, f64 or byte buffer") + } + + fn visit_f64(self, v: f64) -> StdResult + where + E: serde::de::Error, { + Ok(FloatValue::F64(v.to_bits())) + } + + fn visit_f32(self, v: f32) -> StdResult + where + E: serde::de::Error, { + Ok(FloatValue::F32(v.to_bits())) + } + + fn visit_u64(self, v: u64) -> StdResult + where + E: serde::de::Error, { + Ok(FloatValue::F64(v)) + } + + fn visit_u32(self, v: u32) -> StdResult + where + E: serde::de::Error, { + Ok(FloatValue::F32(v)) + } + + fn visit_bytes(self, v: &[u8]) -> StdResult + where + E: serde::de::Error, { + if let Ok(array) = <&[u8; 8]>::try_from(v) { + Ok(FloatValue::F64(u64::from_le_bytes(*array))) + } else if let Ok(array) = <&[u8; 8]>::try_from(v) { + Ok(FloatValue::F64(u64::from_le_bytes(*array))) + } else { + Err(E::custom("Floats may be 8 or 4 bytes")) + } + } + } + + deserializer.deserialize_bytes(MyVistor) + } +} + impl From for f64 { fn from(val: FloatValue) -> f64 { match val { @@ -142,7 +197,7 @@ impl fmt::Display for FloatValue { } /// Container for bin values stored in the Aerospike database. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] pub enum Value { /// Empty value. Nil, @@ -227,7 +282,7 @@ impl Value { pub fn particle_type(&self) -> ParticleType { match *self { Value::Nil => ParticleType::NULL, - Value::Int(_) | Value::Bool(_) => ParticleType::INTEGER, + Value::Int(_) => ParticleType::INTEGER, Value::UInt(_) => panic!( "Aerospike does not support u64 natively on server-side. Use casting to \ store and retrieve u64 values." @@ -235,6 +290,7 @@ impl Value { Value::Float(_) => ParticleType::FLOAT, Value::String(_) => ParticleType::STRING, Value::Blob(_) => ParticleType::BLOB, + Value::Bool(_) => ParticleType::BOOL, Value::List(_) => ParticleType::LIST, Value::HashMap(_) => ParticleType::MAP, Value::OrderedMap(_) => panic!("The library never passes ordered maps to the server."), @@ -265,13 +321,14 @@ impl Value { pub fn estimate_size(&self) -> usize { match *self { Value::Nil => 0, - Value::Int(_) | Value::Bool(_) | Value::Float(_) => 8, + Value::Int(_) | Value::Float(_) => 8, Value::UInt(_) => panic!( "Aerospike does not support u64 natively on server-side. Use casting to \ store and retrieve u64 values." ), Value::String(ref s) => s.len(), Value::Blob(ref b) => b.len(), + Value::Bool(_) => 1, Value::List(_) | Value::HashMap(_) => encoder::pack_value(&mut None, self), Value::OrderedMap(_) => panic!("The library never passes ordered maps to the server."), Value::GeoJSON(ref s) => 1 + 2 + s.len(), // flags + ncells + jsonstr @@ -554,46 +611,103 @@ impl<'a> From<&'a Value> for i64 { } } -#[doc(hidden)] -pub fn bytes_to_particle(ptype: u8, buf: &mut Buffer, len: usize) -> Result { - match ParticleType::from(ptype) { - ParticleType::NULL => Ok(Value::Nil), - ParticleType::INTEGER => { - let val = buf.read_i64(None); - Ok(Value::Int(val)) +impl TryFrom for String { + type Error = String; + fn try_from(val: Value) -> std::result::Result { + match val { + Value::String(v) => Ok(v), + Value::GeoJSON(v) => Ok(v), + _ => bail!(format!( + "Invalid type conversion from Value::{} to {}", + val.particle_type(), + std::any::type_name::() + )), } - ParticleType::FLOAT => { - let val = buf.read_f64(None); - Ok(Value::Float(FloatValue::from(val))) + } +} + +impl TryFrom for Vec { + type Error = String; + fn try_from(val: Value) -> std::result::Result { + match val { + Value::Blob(v) => Ok(v), + Value::HLL(v) => Ok(v), + _ => bail!(format!( + "Invalid type conversion from Value::{} to {}", + val.particle_type(), + std::any::type_name::() + )), } - ParticleType::STRING => { - let val = buf.read_str(len)?; - Ok(Value::String(val)) + } +} + +impl TryFrom for Vec { + type Error = String; + fn try_from(val: Value) -> std::result::Result { + match val { + Value::List(v) => Ok(v), + _ => bail!(format!( + "Invalid type conversion from Value::{} to {}", + val.particle_type(), + std::any::type_name::() + )), + } + } +} + +impl TryFrom for HashMap { + type Error = String; + fn try_from(val: Value) -> std::result::Result { + match val { + Value::HashMap(v) => Ok(v), + _ => bail!(format!( + "Invalid type conversion from Value::{} to {}", + val.particle_type(), + std::any::type_name::() + )), } - ParticleType::GEOJSON => { - buf.skip(1); - let ncells = buf.read_i16(None) as usize; - let header_size: usize = ncells * 8; - - buf.skip(header_size); - let val = buf.read_str(len - header_size - 3)?; - Ok(Value::GeoJSON(val)) + } +} + +impl TryFrom for Vec<(Value, Value)> { + type Error = String; + fn try_from(val: Value) -> std::result::Result { + match val { + Value::OrderedMap(v) => Ok(v), + _ => bail!(format!( + "Invalid type conversion from Value::{} to {}", + val.particle_type(), + std::any::type_name::() + )), } - ParticleType::BLOB => Ok(Value::Blob(buf.read_blob(len))), - ParticleType::LIST => { - let val = decoder::unpack_value_list(buf)?; - Ok(val) + } +} + +impl TryFrom for f64 { + type Error = String; + fn try_from(val: Value) -> std::result::Result { + match val { + Value::Float(v) => Ok(f64::from(v)), + _ => bail!(format!( + "Invalid type conversion from Value::{} to {}", + val.particle_type(), + std::any::type_name::() + )), } - ParticleType::MAP => { - let val = decoder::unpack_value_map(buf)?; - Ok(val) + } +} + +impl TryFrom for bool { + type Error = String; + fn try_from(val: Value) -> std::result::Result { + match val { + Value::Bool(v) => Ok(v), + _ => bail!("Invalid type bool"), } - ParticleType::DIGEST => Ok(Value::from("A DIGEST, NOT IMPLEMENTED YET!")), - ParticleType::LDT => Ok(Value::from("A LDT, NOT IMPLEMENTED YET!")), - ParticleType::HLL => Ok(Value::HLL(buf.read_blob(len))), } } + /// Constructs a new Value from one of the supported native data types. #[macro_export] macro_rules! as_val { @@ -760,6 +874,23 @@ impl Serialize for Value { #[cfg(test)] mod tests { use super::Value; + use std::collections::HashMap; + use std::convert::TryInto; + + #[test] + fn try_into() { + let _: i64 = Value::Int(42).try_into().unwrap(); + let _: f64 = Value::from(42.1).try_into().unwrap(); + let _: String = Value::String("hello".into()).try_into().unwrap(); + let _: String = Value::GeoJSON(r#"{"type":"Point"}"#.into()) + .try_into() + .unwrap(); + let _: Vec = Value::Blob("hello!".into()).try_into().unwrap(); + let _: Vec = Value::HLL("hello!".into()).try_into().unwrap(); + let _: bool = Value::Bool(false).try_into().unwrap(); + let _: HashMap = Value::HashMap(HashMap::new()).try_into().unwrap(); + let _: Vec<(Value, Value)> = Value::OrderedMap(Vec::new()).try_into().unwrap(); + } #[test] fn as_string() { @@ -787,11 +918,23 @@ mod tests { #[test] #[cfg(feature = "serialization")] fn serializer() { - let val: Value = as_list!("0", 9, 8, 7, 1, 2.1f64, -1, as_list!(5, 6, 7, 8, "asd")); + let val: Value = as_list!( + Value::Nil, + "0", + 9, + 8, + 7, + 1, + 2.1f64, + -1, + as_list!(5, 6, 7, 8, "asd"), + true, + false + ); let json = serde_json::to_string(&val); assert_eq!( json.unwrap(), - "[\"0\",9,8,7,1,2.1,-1,[5,6,7,8,\"asd\"]]", + "[null,\"0\",9,8,7,1,4611911198408756429,-1,[5,6,7,8,\"asd\"],true,false]", "List Serialization failed" ); diff --git a/aerospike-macro/src/lib.rs b/aerospike-macro/src/lib.rs index 63abec43..f9083301 100644 --- a/aerospike-macro/src/lib.rs +++ b/aerospike-macro/src/lib.rs @@ -1,10 +1,16 @@ -extern crate proc_macro; -use proc_macro::TokenStream; +use crate::traits::readable::{convert_readable_bins, convert_readable_value_source}; +use crate::traits::writable::{build_writable, convert_writable_value_source}; use quote::quote; +use syn::{parse_macro_input, parse_quote, DeriveInput, GenericParam}; + +mod traits; #[doc(hidden)] #[proc_macro_attribute] -pub fn test(_attr: TokenStream, input: TokenStream) -> TokenStream { +pub fn test( + _attr: proc_macro::TokenStream, + input: proc_macro::TokenStream, +) -> proc_macro::TokenStream { let input = syn::parse_macro_input!(input as syn::ItemFn); let ret = &input.sig.output; @@ -34,3 +40,104 @@ pub fn test(_attr: TokenStream, input: TokenStream) -> TokenStream { result.into() } + +#[proc_macro_derive(WritableBins, attributes(aerospike))] +pub fn writable_bins(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let mut input = parse_macro_input!(input as DeriveInput); + let name = input.ident; + for param in &mut input.generics.params { + if let GenericParam::Type(ref mut type_param) = *param { + type_param + .bounds + .push(parse_quote!(aerospike::WritableValue)); + } + } + let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + let ops = build_writable(&input.data); + + let expanded = quote! { + impl #impl_generics WritableBins for #name #ty_generics #where_clause { + #ops + } + }; + proc_macro::TokenStream::from(expanded) +} + +#[proc_macro_derive(WritableValue, attributes(aerospike))] +pub fn writable_value(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let mut input = parse_macro_input!(input as DeriveInput); + let name = input.ident; + for param in &mut input.generics.params { + if let GenericParam::Type(ref mut type_param) = *param { + type_param + .bounds + .push(parse_quote!(aerospike::derive::writable::WritableValue)); + } + } + let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + let vals = convert_writable_value_source(&input.data); + + let expanded = quote! { + impl #impl_generics WritableValue for #name #ty_generics #where_clause { + fn write_as_value(&self, buffer: &mut Option<&mut aerospike::Buffer>) -> usize{ + // Vec for bin Values + // Bins token Stream + let mut size: usize = 0; + #vals + size + } + + fn writable_value_particle_type(&self) -> aerospike::ParticleType { + aerospike::ParticleType::MAP + } + } + }; + proc_macro::TokenStream::from(expanded) +} + +#[proc_macro_derive(ReadableBins, attributes(aerospike))] +pub fn readable_bins(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let mut input = parse_macro_input!(input as DeriveInput); + let name = input.ident; + for param in &mut input.generics.params { + if let GenericParam::Type(ref mut type_param) = *param { + type_param + .bounds + .push(parse_quote!(aerospike::derive::readable::ReadableValue)); + } + } + let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + let vals = convert_readable_bins(&input.data, &name); + + let expanded = quote! { + impl #impl_generics ReadableBins for #name #ty_generics #where_clause { + fn read_bins_from_bytes(data_points: &mut HashMap) -> aerospike::errors::Result{ + #vals + } + + } + }; + proc_macro::TokenStream::from(expanded) +} + +#[proc_macro_derive(ReadableValue, attributes(aerospike))] +pub fn readable_value(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let mut input = parse_macro_input!(input as DeriveInput); + let name = input.ident; + for param in &mut input.generics.params { + if let GenericParam::Type(ref mut type_param) = *param { + type_param + .bounds + .push(parse_quote!(aerospike::derive::readable::ReadableValue)); + } + } + let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + let vals = convert_readable_value_source(&input.data, &name); + + let expanded = quote! { + impl #impl_generics ReadableValue for #name #ty_generics #where_clause { + #vals + } + }; + proc_macro::TokenStream::from(expanded) +} diff --git a/aerospike-macro/src/traits/mod.rs b/aerospike-macro/src/traits/mod.rs new file mode 100644 index 00000000..5393aef6 --- /dev/null +++ b/aerospike-macro/src/traits/mod.rs @@ -0,0 +1,2 @@ +pub mod readable; +pub mod writable; diff --git a/aerospike-macro/src/traits/readable.rs b/aerospike-macro/src/traits/readable.rs new file mode 100644 index 00000000..eb51e427 --- /dev/null +++ b/aerospike-macro/src/traits/readable.rs @@ -0,0 +1,184 @@ +use quote::{quote, quote_spanned}; +use syn::spanned::Spanned; +use syn::Expr::{Assign, Lit, Path}; +use syn::{Data, Expr, Field, Fields, Ident, Type}; + +pub struct ReadableFieldAttributes<'a> { + field: &'a Field, + ident: &'a Option, + name: String, + ty: Type, +} + +fn readable_field_arguments(field: &Field) -> ReadableFieldAttributes { + // Collect initial Information + let mut attributes = ReadableFieldAttributes { + field, + ident: &field.ident, + name: field.ident.clone().unwrap().to_string(), + ty: field.ty.clone(), + }; + + for a in &field.attrs { + // Filter for aerospike() attributes + if !a.path.is_ident("aerospike") { + continue; + } + + // Parse field attributes to Expression + let expr: Expr = a.parse_args().unwrap(); + + match expr { + // Assign based Attributes like rename + Assign(assign) => { + match assign.left.as_ref() { + Path(path) => { + // Rename Attribute extraction + if path.path.is_ident("rename") { + if let Lit(lit) = *assign.right { + // Currently only accepts Strings as Field Name + if let syn::Lit::Str(ls) = lit.lit { + attributes.name = ls.value(); + } else { + panic!("Invalid Aerospike Rename Value") + } + } else { + panic!("Invalid Aerospike Rename Value") + } + } + } + _ => { + panic!("Invalid Aerospike Derive Attribute") + } + } + } + // Path based Attributes that just serve as markers + /*Path(path) => { + if let Some(ident) = path.path.get_ident() { + match ident.to_string().as_ref() { + _ => { panic!("Invalid Aerospike Derive Attribute") } + } + } + }*/ + _ => { + panic!("Invalid Aerospike Derive Attribute") + } + } + } + if attributes.name.len() > 15 { + panic!("Aerospike Derive Bin Names can not be longer than 15 bytes!") + } + attributes +} + +pub(crate) fn convert_readable_bins(data: &Data, name: &Ident) -> proc_macro2::TokenStream { + match *data { + Data::Struct(ref data) => { + match data.fields { + Fields::Named(ref fields) => { + let field_args = fields + .named + .iter() + .map(|f| readable_field_arguments(&f)) + .collect::>(); + + let field_recurse = field_args.iter().map(|f| { + let field_name = &f.name; + let name = f.ident; + quote_spanned! {f.field.span()=> + #name: aerospike::derive::readable::ReadableValue::read_value_from_bytes(data_points.get_mut(#field_name).unwrap_or(&mut aerospike::derive::readable::PreParsedBin {sub_values: vec![], value: aerospike::derive::readable::PreParsedValue { particle_type: 0, buffer: Default::default(), byte_length: 0 }}))?, + } + }); + quote! { + // Build the final struct + Ok(#name { + #(#field_recurse)* + }) + } + } + _ => unimplemented!(), + } + } + Data::Enum(_) | Data::Union(_) => unimplemented!(), + } +} + +pub(crate) fn convert_readable_value_source(data: &Data, name: &Ident) -> proc_macro2::TokenStream { + match *data { + Data::Struct(ref data) => { + match data.fields { + Fields::Named(ref fields) => { + let field_args = fields + .named + .iter() + .map(|f| readable_field_arguments(&f)) + .collect::>(); + + let field_recurse = field_args.iter().map(|f| { + let field_name = &f.name; + let name = f.ident; + let ty = &f.ty; + quote_spanned! {f.field.span()=> + #name: aerospike::derive::readable::read_map_value_bytes(buff, keys.get(#field_name))?, + } + }); + quote! { + fn read_value_from_bytes(data_point: &mut aerospike::derive::readable::PreParsedBin) -> aerospike::errors::Result { + let mut nbuf = aerospike::Buffer::new(1); + let mut op_count: u64 = 0; + let mut ndbuf = vec![]; + let mut ndval = vec![&mut data_point.value]; + ndval.extend(&mut data_point.sub_values); + for sv in &mut ndval { + let ltype = sv.buffer.peek(); + let count: usize = match ltype { + 0x80..=0x8f => (ltype & 0x0f) as usize, + 0xde => sv.buffer.read_u16(None) as usize, + 0xdf => sv.buffer.read_u32(None) as usize, + _ => { + return Err(aerospike::ErrorKind::Derive("Invalid Data Type for derive Map (Struct) CDT Type".to_string()).into()) + } + }; + op_count += count as u64; + ndbuf.extend(sv.buffer.data_buffer[1..].iter()); + } + nbuf.resize_buffer(8)?; + nbuf.reset_offset(); + nbuf.write_u64(op_count); + nbuf.data_buffer.extend(ndbuf.iter()); + Self::parse_cdt_value(&mut nbuf) + } + + fn parse_value(data_point: &mut aerospike::derive::readable::PreParsedValue) -> aerospike::errors::Result { + unreachable!() + } + + fn parse_cdt_value(buff: &mut aerospike::Buffer) -> aerospike::errors::Result { + buff.reset_offset(); + let count = buff.read_u64(None); + + let mut keys: std::collections::HashMap = std::collections::HashMap::new(); + for _ in 0..count { + let len = buff.read_u8(None) as usize & 0x1f; + let vtype = buff.read_u8(None); + if(vtype != 3) { + return Err(aerospike::ErrorKind::Derive("Only map keys of type string are allowed for Struct reading".to_string()).into()) + } + let name = buff.read_str(len-1)?; + let ofs = buff.data_offset(); + aerospike::derive::readable::skip_map_value_bytes(buff)?; + keys.insert(name.clone(), ofs); + } + + Ok(#name { + #(#field_recurse)* + }) + } + } + } + _ => unimplemented!(), + } + } + Data::Enum(_) | Data::Union(_) => unimplemented!(), + } +} diff --git a/aerospike-macro/src/traits/writable.rs b/aerospike-macro/src/traits/writable.rs new file mode 100644 index 00000000..e9f6b20f --- /dev/null +++ b/aerospike-macro/src/traits/writable.rs @@ -0,0 +1,400 @@ +use proc_macro2::TokenStream; +use quote::{quote, quote_spanned}; +use syn::spanned::Spanned; +use syn::Expr::{Assign, Lit, Path}; +use syn::{Data, Expr, Field, Fields, Ident}; + +pub struct WritableFieldAttributes<'a> { + field: &'a Field, + ident: &'a Option, + name: String, + default: Option, + skip: bool, +} + +impl<'a> WritableFieldAttributes<'a> { + pub fn default_write_value_token_stream(&self) -> (TokenStream, usize, u8) { + // Unwarp is fine since this function can only get called if default is Some. + let default = self.default.clone().unwrap(); + match default { + syn::Lit::Str(s) => { + let val = &s.value(); + return ( + quote! { + buffer.write_str(#val); + }, + val.len(), + 3, + ); + } + syn::Lit::Int(i) => { + if let Ok(val) = i.base10_parse::() { + return ( + quote! { + buffer.write_i64(#val); + }, + 8, + 1, + ); + } else { + panic!("Aerospike Default value could not be parsed as i64") + } + } + syn::Lit::Float(f) => { + if let Ok(val) = f.base10_parse::() { + return ( + quote! { + buffer.write_f64(#val); + }, + 8, + 2, + ); + } else { + panic!("Aerospike Default value could not be parsed as f64") + } + } + syn::Lit::Bool(b) => { + let val = b.value(); + return ( + quote! { + buffer.write_bool(#val); + }, + 1, + 17, + ); + } + _ => { + panic!( + "Aerospike Default value is not supported for the value on {}", + &self.name + ) + } + } + } + + pub fn default_write_value_cdt_token_stream(&self) -> TokenStream { + // Unwarp is fine since this function can only get called if default is Some. + let default = self.default.clone().unwrap(); + match default { + syn::Lit::Str(s) => { + let val = &s.value(); + return quote! { + size += aerospike::msgpack::encoder::pack_string(buffer, #val); + }; + } + syn::Lit::Int(i) => { + if let Ok(val) = i.base10_parse::() { + return quote! { + size += aerospike::msgpack::encoder::pack_integer(buffer, #val); + }; + } else { + panic!("Aerospike Default value could not be parsed as i64") + } + } + syn::Lit::Float(f) => { + if let Ok(val) = f.base10_parse::() { + // Default Values are always encoded as f64 + return quote! { + size += aerospike::msgpack::encoder::pack_f64(buffer, #val); + }; + } else { + panic!("Aerospike Default value could not be parsed as f64") + } + } + syn::Lit::Bool(b) => { + let val = b.value(); + return quote! { + size += aerospike::msgpack::encoder::pack_bool(buffer, #val); + }; + } + _ => { + panic!( + "Aerospike Default value is not supported for the value on {}", + &self.name + ) + } + } + } +} + +fn writable_field_arguments(field: &Field) -> WritableFieldAttributes { + // Collect initial Information + let mut attributes = WritableFieldAttributes { + field, + ident: &field.ident, + name: field.ident.clone().unwrap().to_string(), + default: None, + skip: false, + }; + + for a in &field.attrs { + // Filter for aerospike() attributes + if !a.path.is_ident("aerospike") { + continue; + } + + // Parse field attributes to Expression + let expr: Expr = a.parse_args().unwrap(); + + match expr { + // Assign based Attributes like rename + Assign(assign) => { + match assign.left.as_ref() { + Path(path) => { + // Rename Attribute extraction + if path.path.is_ident("rename") { + if let Lit(lit) = *assign.right { + // Currently only accepts Strings as Field Name + if let syn::Lit::Str(ls) = lit.lit { + attributes.name = ls.value(); + } else { + panic!("Invalid Aerospike Rename Value") + } + } else { + panic!("Invalid Aerospike Rename Value") + } + } else if path.path.is_ident("default") { + if let Lit(lit) = *assign.right { + attributes.default = Some(lit.lit); + } + } + } + _ => { + panic!("Invalid Aerospike Derive Attribute") + } + } + } + // Path based Attributes that just serve as markers + Path(path) => { + if let Some(ident) = path.path.get_ident() { + match ident.to_string().as_ref() { + // Ignore Attribute with skip as alias + "ignore" | "skip" => attributes.skip = true, + _ => { + panic!("Invalid Aerospike Derive Attribute") + } + } + } + } + _ => { + panic!("Invalid Aerospike Derive Attribute") + } + } + } + if attributes.name.len() > 15 { + panic!("Aerospike Derive Bin Names can not be longer than 15 bytes!") + } + attributes +} + +pub(crate) fn build_writable(data: &Data) -> TokenStream { + match *data { + Data::Struct(ref data) => { + match data.fields { + Fields::Named(ref fields) => { + // Collect all the Field Info + let field_args = fields + .named + .iter() + .map(|f| writable_field_arguments(&f)) + .collect::>(); + + // Build the `write_as_bins` function + let writer_recurse = field_args.iter().map(|f| { + let name = f.ident; + let skip = f.skip; + let name_str = &f.name; + + let has_default = f.default.is_some(); + // Build the bin Token Stream. + if has_default { + let default = f.default_write_value_token_stream(); + let default_writer = default.0; + let default_length = default.1; + let default_type = default.2; + + quote_spanned! {f.field.span()=> + if !#skip { + { + let encodable = aerospike::derive::writable::WritableValue::writable_value_encodable(&self.#name); + if encodable { + buffer.write_i32((#name_str.len() + aerospike::derive::writable::WritableValue::write_as_value(&self.#name, &mut None) + 4) as i32); + } else { + buffer.write_i32((#name_str.len() + #default_length + 4) as i32); + } + buffer.write_u8(op_type); + + if encodable { + buffer.write_u8(aerospike::derive::writable::WritableValue::writable_value_particle_type(&self.#name) as u8); + } else { + buffer.write_u8(#default_type); + } + buffer.write_u8(0); + buffer.write_u8(#name_str.len() as u8); + buffer.write_str(#name_str); + if encodable { + aerospike::derive::writable::WritableValue::write_as_value(&self.#name, &mut Some(buffer)); + } else { + #default_writer + } + } + } + } + + } else { + quote_spanned! {f.field.span()=> + if !#skip && aerospike::derive::writable::WritableValue::writable_value_encodable(&self.#name) { + buffer.write_i32((#name_str.len() + aerospike::derive::writable::WritableValue::write_as_value(&self.#name, &mut None) + 4) as i32); + buffer.write_u8(op_type); + buffer.write_u8(aerospike::derive::writable::WritableValue::writable_value_particle_type(&self.#name) as u8); + buffer.write_u8(0); + buffer.write_u8(#name_str.len() as u8); + buffer.write_str(#name_str); + aerospike::derive::writable::WritableValue::write_as_value(&self.#name, &mut Some(buffer)); + } + } + } + + }); + + // Build the `writable_bins_size` function + let length_recurse = field_args.iter().map(|f| { + let name = f.ident; + let name_len = f.name.len(); + let skip = f.skip; + let has_default = f.default.is_some(); + // Build the bin Token Stream. + if has_default { + let default = f.default_write_value_token_stream(); + let default_length = default.1; + + quote_spanned! {f.field.span()=> + if !#skip { + if aerospike::derive::writable::WritableValue::writable_value_encodable(&self.#name) { + size += #name_len + aerospike::derive::writable::WritableValue::write_as_value(&self.#name, &mut None) + 8; + } else { + size += #name_len + #default_length + 8; + } + } + } + } else { + quote_spanned! {f.field.span()=> + if !#skip && aerospike::derive::writable::WritableValue::writable_value_encodable(&self.#name) { + size += #name_len + aerospike::derive::writable::WritableValue::write_as_value(&self.#name, &mut None) + 8; + } + } + } + }); + + // Build the `writable_bins_count` function + let op_count_recurse = field_args.iter().map(|f| { + let name = f.ident; + let skip = f.skip; + let has_default = f.default.is_some(); + if has_default { + quote_spanned! {f.field.span()=> + if !#skip { + count += 1; + } + } + } else { + quote_spanned! {f.field.span()=> + if !#skip && aerospike::derive::writable::WritableValue::writable_value_encodable(&self.#name) { + count += 1; + } + } + } + + }); + + // Build the final functions for the Trait impl + quote! { + fn write_as_bins(&self, buffer: &mut aerospike::Buffer, op_type: u8) -> aerospike::errors::Result<()>{ + #(#writer_recurse)* + Ok(()) + } + + fn writable_bins_size(&self) -> usize { + let mut size: usize = 0; + #(#length_recurse)* + size + } + + fn writable_bins_count(&self) -> usize { + let mut count: usize = 0; + #(#op_count_recurse)* + count + } + + } + } + _ => panic!("Aerospike Bin Derive is not supported for unnamed Structs"), + } + } + Data::Enum(_) | Data::Union(_) => { + panic!("Aerospike Bin Derive is only supported for Enum and Union") + } + } +} + +// WritableValue +pub(crate) fn convert_writable_value_source(data: &Data) -> proc_macro2::TokenStream { + match *data { + Data::Struct(ref data) => match data.fields { + Fields::Named(ref fields) => { + let field_args = fields + .named + .iter() + .map(|f| writable_field_arguments(&f)) + .collect::>(); + + let recurse = field_args.iter().map(|f| { + let name = f.ident; + let name_str = &f.name; + let skip = &f.skip; + let has_default = f.default.is_some(); + + if has_default { + let default_writer = f.default_write_value_cdt_token_stream(); + quote_spanned! {f.field.span()=> + if !#skip { + size += aerospike::msgpack::encoder::pack_string(buffer, #name_str); + if aerospike::derive::writable::WritableValue::writable_value_encodable(&self.#name) { + size += aerospike::derive::writable::WritableValue::write_as_cdt_value(&self.#name, buffer); + } else { + #default_writer + } + } + } + } else { + quote_spanned! {f.field.span()=> + if !#skip && aerospike::derive::writable::WritableValue::writable_value_encodable(&self.#name) { + size += aerospike::msgpack::encoder::pack_string(buffer, #name_str); + size += aerospike::derive::writable::WritableValue::write_as_cdt_value(&self.#name, buffer); + } + } + } + }); + let len_recurse = field_args.iter().map(|f| { + let skip = f.skip; + let name = f.ident; + let has_default = f.default.is_some(); + quote_spanned! {f.field.span()=> + if !#skip && (#has_default || aerospike::derive::writable::WritableValue::writable_value_encodable(&self.#name)) { + len += 1; + } + } + }); + quote! { + let mut len = 0; + #(#len_recurse)* + + size += aerospike::msgpack::encoder::pack_map_begin(buffer, len); + #(#recurse)* + } + } + _ => unimplemented!(), + }, + Data::Enum(_) | Data::Union(_) => unimplemented!(), + } +} diff --git a/aerospike-rt/src/lib.rs b/aerospike-rt/src/lib.rs index 83919230..5e1c68a3 100644 --- a/aerospike-rt/src/lib.rs +++ b/aerospike-rt/src/lib.rs @@ -1,4 +1,5 @@ #[cfg(not(any(feature = "rt-tokio", feature = "rt-async-std")))] + compile_error!("Please select a runtime from ['rt-tokio', 'rt-async-std']"); #[cfg(any(all(feature = "rt-async-std", feature = "rt-tokio")))] diff --git a/benches/client_server.rs b/benches/client_server.rs index e12eeece..22ef76df 100644 --- a/benches/client_server.rs +++ b/benches/client_server.rs @@ -13,16 +13,16 @@ // License for the specific language governing permissions and limitations under // the License. -#[macro_use] -extern crate bencher; #[macro_use] extern crate lazy_static; extern crate rand; -use aerospike::{Bins, ReadPolicy, WritePolicy}; - use aerospike::{as_bin, as_key}; -use bencher::Bencher; +use aerospike::{Bins, Client, Key, ReadPolicy, Value, WritePolicy}; +use criterion::{criterion_group, criterion_main, Criterion}; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; #[path = "../tests/common/mod.rs"] mod common; @@ -33,34 +33,59 @@ lazy_static! { static ref TEST_SET: String = common::rand_str(10); } -fn single_key_read(bench: &mut Bencher) { - let client = block_on(common::client()); - let namespace = common::namespace(); - let key = as_key!(namespace, &TEST_SET, common::rand_str(10)); - let wbin = as_bin!("i", 1); - let bins = vec![wbin]; +async fn single_key_read(client: Arc, key: &Key) { let rpolicy = ReadPolicy::default(); - let wpolicy = WritePolicy::default(); - block_on(client.put(&wpolicy, &key, &bins)).unwrap(); - - bench.iter(|| block_on(client.get(&rpolicy, &key, Bins::All)).unwrap()); + client + .get::, Bins>(&rpolicy, &key, Bins::All) + .await + .unwrap(); } -fn single_key_read_header(bench: &mut Bencher) { - let client = block_on(common::client()); +fn run_single_key_read(bench: &mut Criterion) { + let rt = tokio::runtime::Runtime::new().unwrap(); + let client = Arc::new(rt.block_on(common::client())); let namespace = common::namespace(); let key = as_key!(namespace, &TEST_SET, common::rand_str(10)); let wbin = as_bin!("i", 1); let bins = vec![wbin]; - let rpolicy = ReadPolicy::default(); + let wpolicy = WritePolicy::default(); - block_on(client.put(&wpolicy, &key, &bins)).unwrap(); + rt.block_on(client.put(&wpolicy, &key, &bins)).unwrap(); + + let key2 = as_key!(namespace, &TEST_SET, common::rand_str(10)); + rt.block_on(client.put(&wpolicy, &key2, &bins)).unwrap(); - bench.iter(|| block_on(client.get(&rpolicy, &key, Bins::None)).unwrap()); + let mut group = bench.benchmark_group("single operations"); + group + .sample_size(1000) + .measurement_time(Duration::from_secs(40)); + + group.bench_function("single_key_read", |b| { + b.to_async(&rt) + .iter(|| single_key_read(client.clone(), &key)) + }); + + group.bench_function("single_key_read_header", |b| { + b.to_async(&rt) + .iter(|| single_key_read_header(client.clone(), &key2)) + }); + + group.bench_function("single_key_write", |b| { + b.to_async(&rt).iter(|| single_key_write(client.clone())) + }); + + group.finish() } -fn single_key_write(bench: &mut Bencher) { - let client = block_on(common::client()); +async fn single_key_read_header(client: Arc, key: &Key) { + let rpolicy = ReadPolicy::default(); + client + .get::, Bins>(&rpolicy, &key, Bins::None) + .await + .unwrap(); +} + +async fn single_key_write(client: Arc) { let namespace = common::namespace(); let key = as_key!(namespace, &TEST_SET, common::rand_str(10)); let wpolicy = WritePolicy::default(); @@ -71,15 +96,13 @@ fn single_key_write(bench: &mut Bencher) { let bin4 = as_bin!("str1", common::rand_str(256)); let bins = [bin1, bin2, bin3, bin4]; - bench.iter(|| { - block_on(client.put(&wpolicy, &key, &bins)).unwrap(); - }); + client.put(&wpolicy, &key, &bins).await.unwrap(); } -benchmark_group!( +criterion_group!( benches, - single_key_read, - single_key_read_header, - single_key_write, + run_single_key_read, + //single_key_read_header, + //single_key_write, ); -benchmark_main!(benches); +criterion_main!(benches); diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 771706a6..d7ff54b0 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -60,5 +60,5 @@ pub async fn client() -> Client { pub fn rand_str(sz: usize) -> String { let rng = rand::thread_rng(); - String::from_utf8(rng.sample_iter(&Alphanumeric).take(sz).collect()).unwrap() + String::from_utf8(rng.sample_iter(&Alphanumeric).take(sz).collect::>()).unwrap() } diff --git a/tests/src/batch.rs b/tests/src/batch.rs index b11d7041..3361404f 100644 --- a/tests/src/batch.rs +++ b/tests/src/batch.rs @@ -15,7 +15,8 @@ use aerospike::BatchRead; use aerospike::Bins; -use aerospike::{as_bin, as_key, BatchPolicy, Concurrency, WritePolicy}; +use aerospike::{as_bin, as_key, BatchPolicy, Concurrency, Value, WritePolicy}; +use std::collections::HashMap; @@ -58,7 +59,8 @@ async fn batch_get() { BatchRead::new(key3.clone(), none.clone()), BatchRead::new(key4.clone(), none), ]; - let mut results = client.batch_get(&bpolicy, batch).await.unwrap(); + let mut results: Vec>> = + client.batch_get(&bpolicy, batch).await.unwrap(); let result = results.remove(0); assert_eq!(result.key, key1); diff --git a/tests/src/cdt_bitwise.rs b/tests/src/cdt_bitwise.rs index dcc880f9..3617c162 100644 --- a/tests/src/cdt_bitwise.rs +++ b/tests/src/cdt_bitwise.rs @@ -14,7 +14,8 @@ // the License. use crate::common; - +use env_logger; +use std::collections::HashMap; use aerospike::operations::bitwise; use aerospike::operations::bitwise::{BitPolicy, BitwiseOverflowActions}; @@ -42,12 +43,18 @@ async fn cdt_bitwise() { bitwise::insert("bin", 0, &val, &bpolicy), bitwise::get("bin", 9, 5), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::Blob(vec![0b10000000])); // Verify the Count command let ops = &vec![bitwise::count("bin", 20, 4)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::Int(2)); // Verify the set command @@ -56,7 +63,10 @@ async fn cdt_bitwise() { bitwise::set("bin", 13, 3, &val, &bpolicy), bitwise::get("bin", 0, 40), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), Value::Blob(vec![ @@ -69,7 +79,10 @@ async fn cdt_bitwise() { bitwise::remove("bin", 0, 1, &bpolicy), bitwise::get("bin", 0, 8), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::Blob(vec![0b01000111])); // Verify OR command @@ -78,7 +91,10 @@ async fn cdt_bitwise() { bitwise::or("bin", 0, 8, &val, &bpolicy), bitwise::get("bin", 0, 8), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::Blob(vec![0b11101111])); // Verify XOR command @@ -87,7 +103,10 @@ async fn cdt_bitwise() { bitwise::xor("bin", 0, 8, &val, &bpolicy), bitwise::get("bin", 0, 8), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::Blob(vec![0b01000011])); // Verify AND command @@ -96,7 +115,10 @@ async fn cdt_bitwise() { bitwise::and("bin", 0, 8, &val, &bpolicy), bitwise::get("bin", 0, 8), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::Blob(vec![0b01000010])); // Verify NOT command @@ -104,7 +126,10 @@ async fn cdt_bitwise() { bitwise::not("bin", 0, 8, &bpolicy), bitwise::get("bin", 0, 8), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::Blob(vec![0b10111101])); // Verify LSHIFT command @@ -112,7 +137,10 @@ async fn cdt_bitwise() { bitwise::lshift("bin", 24, 8, 3, &bpolicy), bitwise::get("bin", 24, 8), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::Blob(vec![0b00101000])); // Verify RSHIFT command @@ -120,7 +148,10 @@ async fn cdt_bitwise() { bitwise::rshift("bin", 0, 9, 1, &bpolicy), bitwise::get("bin", 0, 16), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), Value::Blob(vec![0b01011110, 0b10000011]) @@ -139,7 +170,10 @@ async fn cdt_bitwise() { ), bitwise::get("bin", 0, 32), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), Value::Blob(vec![0b11011110, 0b10000011, 0b00000100, 0b00101000]) @@ -158,7 +192,10 @@ async fn cdt_bitwise() { ), bitwise::get("bin", 0, 32), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), Value::Blob(vec![0b01011110, 0b10000011, 0b00000100, 0b00101000]) @@ -169,7 +206,10 @@ async fn cdt_bitwise() { bitwise::set_int("bin", 8, 8, 255, &bpolicy), bitwise::get("bin", 0, 32), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), Value::Blob(vec![0b01011110, 0b11111111, 0b00000100, 0b00101000]) @@ -177,17 +217,26 @@ async fn cdt_bitwise() { // Verify the get int command let ops = &vec![bitwise::get_int("bin", 8, 8, false)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::Int(255)); // Verify the LSCAN command let ops = &vec![bitwise::lscan("bin", 19, 8, true)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::Int(2)); // Verify the RSCAN command let ops = &vec![bitwise::rscan("bin", 19, 8, true)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::Int(7)); client.close().await.unwrap(); } diff --git a/tests/src/cdt_list.rs b/tests/src/cdt_list.rs index 5dc19a29..052dbfbb 100644 --- a/tests/src/cdt_list.rs +++ b/tests/src/cdt_list.rs @@ -14,12 +14,14 @@ // the License. use crate::common; - +use env_logger; +use std::collections::HashMap; use aerospike::operations; use aerospike::operations::lists; use aerospike::operations::lists::{ListPolicy, ListReturnType, ListSortFlags}; use aerospike::{as_bin, as_key, as_list, as_val, as_values, Bins, ReadPolicy, Value, WritePolicy}; +use aerospike_core::Record; #[aerospike_macro::test] fn cdt_list() { @@ -41,11 +43,14 @@ fn cdt_list() { client.delete(&wpolicy, &key).await.unwrap(); client.put(&wpolicy, &key, &bins).await.unwrap(); - let rec = client.get(&policy, &key, Bins::All).await.unwrap(); + let rec: Record> = client.get(&policy, &key, Bins::All).await.unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), val); let ops = &vec![lists::size("bin")]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::from(3)); let values = vec![as_val!(9), as_val!(8), as_val!(7)]; @@ -53,28 +58,40 @@ fn cdt_list() { lists::insert_items(&lpolicy, "bin", 1, &values), operations::get_bin("bin"), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(6, as_list!("0", 9, 8, 7, 1, 2.1f64)) ); let ops = &vec![lists::pop("bin", 0), operations::get_bin("bin")]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!("0", as_list!(9, 8, 7, 1, 2.1f64)) ); let ops = &vec![lists::pop_range("bin", 0, 2), operations::get_bin("bin")]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(9, 8, as_list!(7, 1, 2.1f64)) ); let ops = &vec![lists::pop_range_from("bin", 1), operations::get_bin("bin")]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(1, 2.1f64, as_list!(7)) @@ -86,25 +103,37 @@ fn cdt_list() { lists::append_items(&lpolicy, "bin", &values), operations::get_bin("bin"), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(6, as_list!("0", 9, 8, 7, 1, 2.1f64)) ); let ops = &vec![lists::increment(&lpolicy, "bin", 1, 4)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::from(13)); let ops = &vec![lists::remove("bin", 1), operations::get_bin("bin")]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(1, as_list!("0", 8, 7, 1, 2.1f64)) ); let ops = &vec![lists::remove_range("bin", 1, 2), operations::get_bin("bin")]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(2, as_list!("0", 1, 2.1f64)) @@ -114,12 +143,18 @@ fn cdt_list() { lists::remove_range_from("bin", -1), operations::get_bin("bin"), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!(1, as_list!("0", 1))); let v = as_val!(2); let ops = &vec![lists::set("bin", -1, &v), operations::get_bin("bin")]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!("0", 2)); let values = as_values!["0", 9, 8, 7, 1, 2.1f64, -1]; @@ -128,14 +163,20 @@ fn cdt_list() { lists::append_items(&lpolicy, "bin", &values), operations::get_bin("bin"), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(7, as_list!("0", 9, 8, 7, 1, 2.1f64, -1)) ); let ops = &vec![lists::trim("bin", 1, 1), operations::get_bin("bin")]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!(6, as_list!(9))); let values = as_values!["0", 9, 8, 7, 1, 2.1f64, -1]; @@ -144,30 +185,45 @@ fn cdt_list() { lists::append_items(&lpolicy, "bin", &values), operations::get_bin("bin"), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(7, as_list!("0", 9, 8, 7, 1, 2.1f64, -1)) ); let ops = &vec![lists::get("bin", 1)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_val!(9)); let ops = &vec![lists::get_range("bin", 1, -1)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(9, 8, 7, 1, 2.1f64, -1) ); let ops = &vec![lists::get_range_from("bin", 2)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!(8, 7, 1, 2.1f64, -1)); let rval = Value::from(9); let ops = &vec![lists::remove_by_value("bin", &rval, ListReturnType::Count)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::from(1)); let rval = vec![Value::from(8), Value::from(7)]; @@ -176,7 +232,10 @@ fn cdt_list() { &rval, ListReturnType::Count, )]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::from(2)); let values = as_values!["0", 9, 8, 7, 1, 2.1f64, -1]; @@ -185,7 +244,10 @@ fn cdt_list() { lists::append_items(&lpolicy, "bin", &values), operations::get_bin("bin"), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(7, as_list!("0", 9, 8, 7, 1, 2.1f64, -1)) @@ -199,7 +261,10 @@ fn cdt_list() { &beg, &end, )]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::from(2)); let values = as_values!["0", 9, 8, 7, 1, 2.1f64, -1]; @@ -208,24 +273,36 @@ fn cdt_list() { lists::append_items(&lpolicy, "bin", &values), operations::get_bin("bin"), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(7, as_list!("0", 9, 8, 7, 1, 2.1f64, -1)) ); let ops = &vec![lists::sort("bin", ListSortFlags::Default)]; - client.operate(&wpolicy, &key, ops).await.unwrap(); + client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); let ops = &vec![operations::get_bin("bin")]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(-1, 1, 7, 8, 9, "0", 2.1f64) ); let ops = &vec![lists::remove_by_index("bin", 1, ListReturnType::Values)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::from(1)); let ops = &vec![lists::remove_by_index_range( @@ -233,7 +310,10 @@ fn cdt_list() { 4, ListReturnType::Values, )]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!("0", 2.1f64)); let values = as_values!["0", 9, 8, 7, 1, 2.1f64, -1]; @@ -242,7 +322,10 @@ fn cdt_list() { lists::append_items(&lpolicy, "bin", &values), operations::get_bin("bin"), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(7, as_list!("0", 9, 8, 7, 1, 2.1f64, -1)) @@ -254,11 +337,17 @@ fn cdt_list() { 2, ListReturnType::Values, )]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!("0", 9)); let ops = &vec![lists::remove_by_rank("bin", 2, ListReturnType::Values)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::from(7)); let ops = &vec![lists::remove_by_rank_range( @@ -266,7 +355,10 @@ fn cdt_list() { 2, ListReturnType::Values, )]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!(8, 2.1f64)); let values = as_values!["0", 9, 8, 7, 1, 2.1f64, -1]; @@ -275,7 +367,10 @@ fn cdt_list() { lists::append_items(&lpolicy, "bin", &values), operations::get_bin("bin"), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(7, as_list!("0", 9, 8, 7, 1, 2.1f64, -1)) @@ -287,7 +382,10 @@ fn cdt_list() { 2, ListReturnType::Values, )]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!(8, 7)); let values = as_values!["0", 9, 8, 7, 1, 2.1f64, -1]; @@ -296,7 +394,10 @@ fn cdt_list() { lists::append_items(&lpolicy, "bin", &values), operations::get_bin("bin"), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(7, as_list!("0", 9, 8, 7, 1, 2.1f64, -1)) @@ -309,7 +410,10 @@ fn cdt_list() { &val, 1, )]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(7, 8, 9, "0", 2.1f64) @@ -321,7 +425,10 @@ fn cdt_list() { lists::append_items(&lpolicy, "bin", &values), operations::get_bin("bin"), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(7, as_list!("0", 9, 8, 7, 1, 2.1f64, -1)) @@ -335,7 +442,10 @@ fn cdt_list() { 1, 2, )]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!(8, 7)); let values = as_values!["0", 9, 8, 7, 1, 2.1f64, -1]; @@ -344,7 +454,10 @@ fn cdt_list() { lists::append_items(&lpolicy, "bin", &values), operations::get_bin("bin"), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(7, as_list!("0", 9, 8, 7, 1, 2.1f64, -1)) @@ -358,17 +471,26 @@ fn cdt_list() { 2, ListReturnType::Values, )]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!(8, 9)); let val = Value::from(1); let ops = &vec![lists::get_by_value("bin", &val, ListReturnType::Count)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::from(1)); let val = vec![Value::from(1), Value::from("0")]; let ops = &vec![lists::get_by_value_list("bin", &val, ListReturnType::Count)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::from(2)); let beg = Value::from(1); @@ -379,15 +501,24 @@ fn cdt_list() { &end, ListReturnType::Count, )]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::from(3)); let ops = &vec![lists::get_by_index("bin", 3, ListReturnType::Values)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::from(7)); let ops = &vec![lists::get_by_index_range("bin", 3, ListReturnType::Values)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!(7, 1, 2.1f64, -1)); let ops = &vec![lists::get_by_index_range_count( @@ -396,7 +527,10 @@ fn cdt_list() { 2, ListReturnType::Values, )]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!("0", 9)); let values = as_values!["0", 9, 8, 7, 1, 2.1f64, -1]; @@ -405,18 +539,27 @@ fn cdt_list() { lists::append_items(&lpolicy, "bin", &values), operations::get_bin("bin"), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(7, as_list!("0", 9, 8, 7, 1, 2.1f64, -1)) ); let ops = &vec![lists::get_by_rank("bin", 2, ListReturnType::Values)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), Value::from(7)); let ops = &vec![lists::get_by_rank_range("bin", 4, ListReturnType::Values)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!(9, "0", 2.1f64)); let ops = &vec![lists::get_by_rank_range_count( @@ -425,7 +568,10 @@ fn cdt_list() { 2, ListReturnType::Values, )]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!(8, 7)); let val = Value::from(1); @@ -435,7 +581,10 @@ fn cdt_list() { 2, ListReturnType::Values, )]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!(8, 9, "0", 2.1f64)); let val = Value::from(1); @@ -446,7 +595,10 @@ fn cdt_list() { 2, ListReturnType::Values, )]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!(*rec.bins.get("bin").unwrap(), as_list!(8, 9)); client.close().await.unwrap(); } diff --git a/tests/src/cdt_map.rs b/tests/src/cdt_map.rs index 6e66fe98..8789e960 100644 --- a/tests/src/cdt_map.rs +++ b/tests/src/cdt_map.rs @@ -21,8 +21,8 @@ use crate::common; use aerospike::operations::cdt_context::{ctx_map_key, ctx_map_key_create}; use aerospike::operations::{maps, MapOrder}; use aerospike::{ - as_bin, as_key, as_list, as_map, as_val, Bins, MapPolicy, MapReturnType, ReadPolicy, - WritePolicy, + as_bin, as_key, as_list, as_map, as_val, Bins, MapPolicy, MapReturnType, ReadPolicy, Record, + Value, WritePolicy, }; #[aerospike_macro::test] @@ -51,16 +51,16 @@ async fn map_operations() { let (k, v) = (as_val!("c"), as_val!(3)); let op = maps::put(&mpolicy, bin_name, &k, &v); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); // returns size of map after put assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(3)); let op = maps::size(bin_name); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); // returns size of map assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(3)); - let rec = client.get(&rpolicy, &key, Bins::All).await.unwrap(); + let rec: Record> = client.get(&rpolicy, &key, Bins::All).await.unwrap(); assert_eq!( *rec.bins.get(bin_name).unwrap(), as_map!("a" => 1, "b" => 2, "c" => 3) @@ -70,24 +70,24 @@ async fn map_operations() { items.insert(as_val!("d"), as_val!(4)); items.insert(as_val!("e"), as_val!(5)); let op = maps::put_items(&mpolicy, bin_name, &items); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); // returns size of map after put assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(5)); let k = as_val!("e"); let op = maps::remove_by_key(bin_name, &k, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(5)); let (k, i) = (as_val!("a"), as_val!(19)); let op = maps::increment_value(&mpolicy, bin_name, &k, &i); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); // returns value of the key after increment assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(20)); let (k, i) = (as_val!("a"), as_val!(10)); let op = maps::decrement_value(&mpolicy, bin_name, &k, &i); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); // returns value of the key after decrement assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(10)); @@ -95,12 +95,13 @@ async fn map_operations() { let dec = maps::decrement_value(&mpolicy, bin_name, &k, &i); let (k, i) = (as_val!("a"), as_val!(7)); let inc = maps::increment_value(&mpolicy, bin_name, &k, &i); - let rec = client.operate(&wpolicy, &key, &[dec, inc]).await.unwrap(); + let rec: Record> = + client.operate(&wpolicy, &key, &[dec, inc]).await.unwrap(); // returns values from multiple ops returned as list assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(5, 12)); let op = maps::clear(bin_name); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); // map_clear returns no result assert!(rec.bins.get(bin_name).is_none()); @@ -111,78 +112,78 @@ async fn map_operations() { let bin = as_bin!(bin_name, val); let bins = vec![bin]; - client.put(&wpolicy, &key, bins.as_slice()).await.unwrap(); + client.put(&wpolicy, &key, &bins).await.unwrap(); let op = maps::get_by_index(bin_name, 0, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(1)); let op = maps::get_by_index_range(bin_name, 1, 2, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(2, 3)); let op = maps::get_by_index_range_from(bin_name, 3, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(4, 5)); let val = as_val!(5); let op = maps::get_by_value(bin_name, &val, MapReturnType::Index); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(4)); let beg = as_val!(3); let end = as_val!(5); let op = maps::get_by_value_range(bin_name, &beg, &end, MapReturnType::Count); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(2)); let op = maps::get_by_rank(bin_name, 2, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(3)); let op = maps::get_by_rank_range(bin_name, 2, 3, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(3, 4, 5)); let op = maps::get_by_rank_range_from(bin_name, 2, MapReturnType::Count); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(3)); let mkey = as_val!("b"); let op = maps::get_by_key(bin_name, &mkey, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(2)); let mkey = as_val!("b"); let mkey2 = as_val!("d"); let op = maps::get_by_key_range(bin_name, &mkey, &mkey2, MapReturnType::Count); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(2)); let mkey = vec![as_val!("b"), as_val!("d")]; let op = maps::get_by_key_list(bin_name, &mkey, MapReturnType::Count); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(2)); let mkey = vec![as_val!(2), as_val!(3)]; let op = maps::get_by_value_list(bin_name, &mkey, MapReturnType::Count); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(2)); let mkey = vec![as_val!("b"), as_val!("d")]; let op = maps::remove_by_key_list(bin_name, &mkey, MapReturnType::Count); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(2)); let mkey = as_val!("a"); let mkey2 = as_val!("c"); let op = maps::remove_by_key_range(bin_name, &mkey, &mkey2, MapReturnType::Count); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(1)); let mkey = as_val!(5); let op = maps::remove_by_value(bin_name, &mkey, MapReturnType::Count); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(1)); client.delete(&wpolicy, &key).await.unwrap(); @@ -190,46 +191,46 @@ async fn map_operations() { let mkey = vec![as_val!(4), as_val!(5)]; let op = maps::remove_by_value_list(bin_name, &mkey, MapReturnType::Count); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(2)); let mkey = as_val!(1); let mkey2 = as_val!(3); let op = maps::remove_by_value_range(bin_name, &mkey, &mkey2, MapReturnType::Count); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(2)); client.delete(&wpolicy, &key).await.unwrap(); client.put(&wpolicy, &key, &bins).await.unwrap(); let op = maps::remove_by_index(bin_name, 1, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(2)); let op = maps::remove_by_index_range(bin_name, 1, 2, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(3, 4)); let op = maps::remove_by_index_range_from(bin_name, 1, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(5)); client.delete(&wpolicy, &key).await.unwrap(); client.put(&wpolicy, &key, &bins).await.unwrap(); let op = maps::remove_by_rank(bin_name, 1, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(2)); let op = maps::remove_by_rank_range(bin_name, 1, 2, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(3, 4)); client.delete(&wpolicy, &key).await.unwrap(); client.put(&wpolicy, &key, &bins).await.unwrap(); let op = maps::remove_by_rank_range_from(bin_name, 3, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(4, 5)); client.delete(&wpolicy, &key).await.unwrap(); @@ -237,13 +238,13 @@ async fn map_operations() { let mkey = as_val!("b"); let op = maps::remove_by_key_relative_index_range(bin_name, &mkey, 2, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(4, 5)); let mkey = as_val!("c"); let op = maps::remove_by_key_relative_index_range_count(bin_name, &mkey, 0, 2, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(3)); client.delete(&wpolicy, &key).await.unwrap(); @@ -257,12 +258,12 @@ async fn map_operations() { 2, MapReturnType::Value, ); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(5)); let mkey = as_val!(2); let op = maps::remove_by_value_relative_rank_range(bin_name, &mkey, 1, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(3, 4)); client.delete(&wpolicy, &key).await.unwrap(); @@ -270,35 +271,38 @@ async fn map_operations() { let mkey = as_val!("a"); let op = maps::get_by_key_relative_index_range(bin_name, &mkey, 1, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(2, 3, 4, 5)); let mkey = as_val!("a"); let op = maps::get_by_key_relative_index_range_count(bin_name, &mkey, 1, 2, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(2, 3)); let mkey = as_val!(2); let op = maps::get_by_value_relative_rank_range(bin_name, &mkey, 1, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(3, 4, 5)); let mkey = as_val!(2); let op = maps::get_by_value_relative_rank_range_count(bin_name, &mkey, 1, 1, MapReturnType::Value); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_list!(3)); let mkey = as_val!("ctxtest"); let mval = as_map!("x" => 7, "y" => 8, "z" => 9); let op = maps::put(&mpolicy, bin_name, &mkey, &mval); - client.operate(&wpolicy, &key, &[op]).await.unwrap(); + client + .operate::>(&wpolicy, &key, &[op]) + .await + .unwrap(); let ctx = &vec![ctx_map_key(mkey)]; let xkey = as_val!("y"); let op = maps::get_by_key(bin_name, &xkey, MapReturnType::Value).set_context(ctx); - let rec = client.operate(&wpolicy, &key, &[op]).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &[op]).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(8)); let mkey = as_val!("ctxtest2"); @@ -306,9 +310,12 @@ async fn map_operations() { let xkey = as_val!("y"); let xval = as_val!(8); let op = [maps::put(&mpolicy, bin_name, &xkey, &xval).set_context(ctx)]; - client.operate(&wpolicy, &key, &op).await.unwrap(); + client + .operate::>(&wpolicy, &key, &op) + .await + .unwrap(); let op = [maps::get_by_key(bin_name, &xkey, MapReturnType::Value).set_context(ctx)]; - let rec = client.operate(&wpolicy, &key, &op).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &op).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(8)); let mkey2 = as_val!("ctxtest3"); @@ -319,9 +326,12 @@ async fn map_operations() { let xkey = as_val!("c"); let xval = as_val!(9); let op = [maps::put(&mpolicy, bin_name, &xkey, &xval).set_context(ctx)]; - client.operate(&wpolicy, &key, &op).await.unwrap(); + client + .operate::>(&wpolicy, &key, &op) + .await + .unwrap(); let op = [maps::get_by_key(bin_name, &xkey, MapReturnType::Value).set_context(ctx)]; - let rec = client.operate(&wpolicy, &key, &op).await.unwrap(); + let rec: Record> = client.operate(&wpolicy, &key, &op).await.unwrap(); assert_eq!(*rec.bins.get(bin_name).unwrap(), as_val!(9)); client.close().await.unwrap(); diff --git a/tests/src/derive.rs b/tests/src/derive.rs new file mode 100644 index 00000000..2d3239dd --- /dev/null +++ b/tests/src/derive.rs @@ -0,0 +1,278 @@ +use crate::common; +use aerospike::as_key; +use aerospike::as_val; +use aerospike::derive::writable::{WritableBins, WritableValue}; +use aerospike_macro::{WritableBins, WritableValue}; +use aerospike::WritePolicy; +use aerospike::{Bins, ReadPolicy, Record, Value}; +use std::collections::HashMap; + +#[aerospike_macro::test] +async fn derive_writable() { + let client = common::client().await; + let namespace: &str = common::namespace(); + let set_name = &common::rand_str(10); + let key = as_key!(namespace, set_name, "derive_struct"); + + #[derive(WritableValue)] + struct TestValue { + string: String, + no_val_cdt: Option, + has_val_cdt: Option, + #[aerospike(default = 1.231f64)] + default_val_cdt: Option, + #[aerospike(rename = "int32")] + int: i32, + #[aerospike(skip)] + skipped_bool: bool, + } + + #[derive(WritableBins)] + struct TestData<'a> { + #[aerospike(rename = "renamed_int32")] + int32: i32, + string: String, + refstr: &'a str, + uint16: u16, + test: TestValue, + #[aerospike(skip)] + no_write: i32, + #[aerospike(default = "test")] + default_value: Option, + no_value: Option, + has_value: Option, + list: Vec, + } + + let testv = TestValue { + string: "asd".to_string(), + no_val_cdt: None, + has_val_cdt: Some(123456), + default_val_cdt: None, + int: 1234, + skipped_bool: true, + }; + + let test = TestData { + int32: 65521, + string: "string".to_string(), + refstr: "str", + uint16: 7, + test: testv, + no_write: 123, + default_value: None, + no_value: None, + has_value: Some(12345), + list: Vec::from(["test1".to_string(), "test2".to_string()]), + }; + + let res = client.put(&WritePolicy::default(), &key, &test).await; + assert_eq!(res.is_ok(), true, "Derive writer failed"); + let res: Record> = client + .get(&ReadPolicy::default(), &key, Bins::All) + .await + .unwrap(); + + let bins = res.bins; + + assert_eq!(bins.get("int32"), None, "Derive Bin renaming failed"); + assert_eq!( + bins.get("renamed_int32"), + Some(&as_val!(65521)), + "Derive Bin renaming failed" + ); + + assert_eq!(bins.get("no_value"), None, "Derive Bin empty Option failed"); + assert_eq!( + bins.get("has_value"), + Some(&as_val!(12345)), + "Derive Bin filled Option failed" + ); + + assert_eq!( + bins.get("default_value"), + Some(&as_val!("test")), + "Derive Bin default value failed" + ); + + assert_eq!(bins.get("no_write"), None, "Derive Bin skipping failed"); + + assert_eq!( + bins.get("uint16"), + Some(&as_val!(7)), + "Derive Bin encoding failed for uint16" + ); + assert_eq!( + bins.get("string"), + Some(&as_val!("string")), + "Derive Bin encoding failed for string" + ); + assert_eq!( + bins.get("refstr"), + Some(&as_val!("str")), + "Derive Bin encoding failed for refstr" + ); + + assert_eq!( + bins.get("test").is_some(), + true, + "Derive Bin encoding failed for cdt map" + ); + + assert_eq!( + bins.get("list"), + Some(&Value::List(Vec::from([ + as_val!("test1"), + as_val!("test2") + ]))), + "Derive Bin encoding for list failed" + ); + + if let Some(bin) = bins.get("test") { + match bin { + Value::HashMap(m) => { + assert_eq!( + m.get(&as_val!("string")), + Some(&as_val!("asd")), + "Derive Value encoding failed for string" + ); + assert_eq!( + m.get(&as_val!("no_val_cdt")), + None, + "Derive Value encoding failed for no_val_cdt" + ); + assert_eq!( + m.get(&as_val!("default_val_cdt")), + Some(&as_val!(1.231f64)), + "Derive Value encoding failed for default_val_cdt" + ); + assert_eq!( + m.get(&as_val!("int32")), + Some(&as_val!(1234)), + "Derive Value encoding failed for renamed int" + ); + assert_eq!( + m.get(&as_val!("has_val_cdt")), + Some(&as_val!(123456)), + "Derive Value encoding failed for has_val_cdt" + ); + assert_eq!( + m.get(&as_val!("skipped_bool")), + None, + "Derive Value encoding failed for skipped_bool" + ); + } + _ => panic!("Derive Bin encoding for map returned wrong type"), + } + } else { + panic!("Derive Bin encoding for map undefined") + } +} + +#[aerospike_macro::test] +async fn derive_readable() { + let client = common::client().await; + let namespace: &str = common::namespace(); + let set_name = &common::rand_str(10); + let key = as_key!(namespace, set_name, "derive_struct"); + + #[derive(serde::Deserialize, WritableValue, Clone, Debug)] + struct TestValue { + string: String, + int: i64, + float: f64, + option: Option, + list: Vec, + list_i: Vec, + no_val: Option, + map: HashMap, + nested_list: Vec>, + } + + #[derive(serde::Deserialize, WritableBins, Clone, Debug)] + struct TestData { + string: String, + int: i64, + float: f64, + option: Option, + list: Vec, + list_i: Vec, + no_val: Option, + nested_list: Vec>, + test: TestValue, + map: HashMap + } + + let tv = TestValue { + string: "eeeeeeee".to_string(), + int: 56789, + float: 567.765, + option: Some("ffffff".to_string()), + list_i: vec![5, 2, 1, 11, 15], + list: vec!["eee".to_string(), "fff".to_string(), "ggg".to_string()], + no_val: None, + map: HashMap::from([("asdasd".to_string(), 1), ("sdfsdf".to_string(), 1234567890)]), + nested_list: vec![vec![3, 2, 1], vec![6, 5, 4]], + }; + + let write_data = TestData { + string: "asdfsd".to_string(), + int: 1234, + float: 123.456, + option: Some("asd".to_string()), + list_i: vec![1, 5, 8, 9, 15], + list: vec!["asd".to_string(), "ase".to_string(), "asf".to_string()], + no_val: None, + nested_list: vec![vec![1, 2, 3], vec![4, 5, 6]], + test: tv.clone(), + map: HashMap::from([("testasd".to_string(), 1), ("testfgh".to_string(), 1234567890)]) + }; + + let res = client.put(&WritePolicy::default(), &key, &write_data).await; + let res: aerospike::errors::Result> = + client.get(&ReadPolicy::default(), &key, Bins::All).await; + println!("{:?}", res); + assert_eq!(res.is_ok(), true, "Aerospike derive reader failed"); + let res = res.unwrap().bins; + assert_eq!( + res.string, "asdfsd", + "Aerospike derive reader failed for String" + ); + assert_eq!(res.int, 1234, "Aerospike derive reader failed for Int"); + assert_eq!( + res.float, 123.456, + "Aerospike derive reader failed for Float" + ); + assert_eq!( + res.option, + Some("asd".to_string()), + "Aerospike derive reader failed for Option Some" + ); + assert_eq!( + res.no_val, None, + "Aerospike derive reader failed for Option None" + ); + assert_eq!( + res.list_i, + vec![1, 5, 8, 9, 15], + "Aerospike derive reader failed for Int List" + ); + assert_eq!( + res.nested_list, + vec![vec![1, 2, 3], vec![4, 5, 6]], + "Aerospike derive reader failed for Nested List" + ); + assert_eq!( + res.list, + vec!["asd".to_string(), "ase".to_string(), "asf".to_string()], + "Aerospike derive reader failed for String List" + ); + assert_eq!( + res.test.string, tv.string, + "Aerospike derive reader failed for ReadableValue string" + ); + assert_eq!( + res.test.int, tv.int, + "Aerospike derive reader failed for ReadableValue int" + ); +} diff --git a/tests/src/exp.rs b/tests/src/exp.rs index fff8cd03..d8a39d1d 100644 --- a/tests/src/exp.rs +++ b/tests/src/exp.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; // Copyright 2015-2020 Aerospike, Inc. // // Portions may be licensed to Aerospike, Inc. under one or more contributor @@ -16,8 +17,9 @@ use crate::common; use aerospike::expressions::*; -use aerospike::ParticleType; use aerospike::*; +use aerospike::{ParticleType, Record}; +use aerospike_core::Recordset; use std::sync::Arc; const EXPECTED: usize = 100; @@ -26,7 +28,7 @@ async fn create_test_set(client: &Client, no_records: usize) -> String { let namespace = common::namespace(); let set_name = common::rand_str(10); - let wpolicy = WritePolicy::default(); + let wpolicy = WritePolicy::new(0, Expiration::Seconds(60)); for i in 0..no_records as i64 { let key = as_key!(namespace, &set_name, i); let ibin = as_bin!("bin", i); @@ -541,8 +543,8 @@ fn expression_rec_ops() { let count = count_results(rs); assert_eq!(count, 100, "SINCE UPDATE Test Failed"); - // Records dont expire - let rs = test_filter(&client, ge(void_time(), int_val(0)), &set_name).await; + // Records don't expire + let rs = test_filter(&client, ge(void_time(), int_val(50)), &set_name).await; let count = count_results(rs); assert_eq!(count, 100, "VOID TIME Test Failed"); @@ -646,12 +648,12 @@ async fn expression_commands() { // GET let key = as_key!(namespace, &set_name, 35); rpolicy.base_policy.filter_expression = Some(eq(int_bin("bin".to_string()), int_val(15))); - let test = client.get(&rpolicy, &key, Bins::All).await; - assert!(test.is_err(), "GET Err Test Failed"); + let test: Result>> = client.get(&rpolicy, &key, Bins::All).await; + assert_eq!(test.is_err(), true, "GET Err Test Failed"); rpolicy.base_policy.filter_expression = Some(eq(int_bin("bin".to_string()), int_val(35))); - let test = client.get(&rpolicy, &key, Bins::All).await; - assert!(test.is_ok(), "GET Ok Test Failed"); + let test: Result>> = client.get(&rpolicy, &key, Bins::All).await; + assert_eq!(test.is_ok(), true, "GET Ok Test Failed"); // EXISTS let key = as_key!(namespace, &set_name, 45); @@ -703,7 +705,10 @@ async fn expression_commands() { // SCAN spolicy.filter_expression = Some(eq(int_bin("bin".to_string()), int_val(75))); - match client.scan(&spolicy, namespace, &set_name, Bins::All).await { + + let res: aerospike::Result>>> = + client.scan(&spolicy, namespace, &set_name, Bins::All).await; + match res { Ok(records) => { let mut count = 0; for record in &*records { @@ -723,20 +728,24 @@ async fn expression_commands() { let key = as_key!(namespace, &set_name, 85); wpolicy.filter_expression = Some(eq(int_bin("bin".to_string()), int_val(15))); - let op = client.operate(&wpolicy, &key, &ops).await; - assert!(op.is_err(), "OPERATE Err Test Failed"); + let op = client + .operate::>(&wpolicy, &key, &ops) + .await; + assert_eq!(op.is_err(), true, "OPERATE Err Test Failed"); let key = as_key!(namespace, &set_name, 85); wpolicy.filter_expression = Some(eq(int_bin("bin".to_string()), int_val(85))); - let op = client.operate(&wpolicy, &key, &ops).await; - assert!(op.is_ok(), "OPERATE Ok Test Failed"); + let op = client + .operate::>(&wpolicy, &key, &ops) + .await; + assert_eq!(op.is_ok(), true, "OPERATE Ok Test Failed"); // BATCH GET let mut batch_reads = vec![]; let b = Bins::from(["bin"]); for i in 85..90 { let key = as_key!(namespace, &set_name, i); - batch_reads.push(BatchRead::new(key, b.clone())); + batch_reads.push(BatchRead::>::new(key, b.clone())); } bpolicy.filter_expression = Some(gt(int_bin("bin".to_string()), int_val(84))); match client.batch_get(&bpolicy, batch_reads).await { @@ -756,7 +765,11 @@ async fn expression_commands() { client.close().await.unwrap(); } -async fn test_filter(client: &Client, filter: FilterExpression, set_name: &str) -> Arc { +async fn test_filter( + client: &Client, + filter: FilterExpression, + set_name: &str, +) -> Arc>> { let namespace = common::namespace(); let mut qpolicy = QueryPolicy::default(); @@ -766,7 +779,7 @@ async fn test_filter(client: &Client, filter: FilterExpression, set_name: &str) client.query(&qpolicy, statement).await.unwrap() } -fn count_results(rs: Arc) -> usize { +fn count_results(rs: Arc>>) -> usize { let mut count = 0; for res in &*rs { diff --git a/tests/src/exp_bitwise.rs b/tests/src/exp_bitwise.rs index da773ef3..601e96da 100644 --- a/tests/src/exp_bitwise.rs +++ b/tests/src/exp_bitwise.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; // Copyright 2015-2020 Aerospike, Inc. // // Portions may be licensed to Aerospike, Inc. under one or more contributor @@ -46,7 +47,7 @@ async fn expression_bitwise() { let set_name = create_test_set(&client, EXPECTED).await; // EQ - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( count(int_val(0), int_val(16), blob_bin("bin".to_string())), @@ -58,7 +59,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "COUNT Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( count( @@ -79,7 +80,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "RESIZE Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( count( @@ -100,7 +101,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "INSERT Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( count( @@ -121,7 +122,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "REMOVE Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( count( @@ -143,7 +144,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "SET Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( count( @@ -165,13 +166,13 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "OR Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( count( int_val(0), int_val(8), - xor( + bitwise::xor( &BitPolicy::default(), int_val(0), int_val(8), @@ -187,7 +188,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "XOR Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( count( @@ -209,7 +210,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "AND Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( count( @@ -230,7 +231,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "NOT Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( count( @@ -252,7 +253,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "LSHIFT Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( count( @@ -274,7 +275,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "RSHIFT Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( count( @@ -298,7 +299,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "ADD Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( count( @@ -322,7 +323,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "SUBTRACT Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( count( @@ -344,7 +345,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "SET INT Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( get(int_val(0), int_val(8), blob_bin("bin".to_string())), @@ -356,7 +357,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "GET Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( lscan( @@ -373,7 +374,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "LSCAN Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( rscan( @@ -390,7 +391,7 @@ async fn expression_bitwise() { let item_count = count_results(rs); assert_eq!(item_count, 100, "RSCAN Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( get_int(int_val(0), int_val(8), false, blob_bin("bin".to_string())), @@ -405,7 +406,11 @@ async fn expression_bitwise() { client.close().await.unwrap(); } -async fn test_filter(client: &Client, filter: FilterExpression, set_name: &str) -> Arc { +async fn test_filter( + client: &Client, + filter: FilterExpression, + set_name: &str, +) -> Arc>> { let namespace = common::namespace(); let mut qpolicy = QueryPolicy::default(); @@ -415,7 +420,7 @@ async fn test_filter(client: &Client, filter: FilterExpression, set_name: &str) client.query(&qpolicy, statement).await.unwrap() } -fn count_results(rs: Arc) -> usize { +fn count_results(rs: Arc>>) -> usize { let mut count = 0; for res in &*rs { diff --git a/tests/src/exp_hll.rs b/tests/src/exp_hll.rs index c158dbd4..a226850e 100644 --- a/tests/src/exp_hll.rs +++ b/tests/src/exp_hll.rs @@ -15,13 +15,13 @@ use crate::common; - use aerospike::expressions::hll::*; use aerospike::expressions::lists::*; use aerospike::expressions::*; use aerospike::operations::hll::HLLPolicy; use aerospike::operations::lists::ListReturnType; use aerospike::*; +use std::collections::HashMap; use std::sync::Arc; const EXPECTED: usize = 100; @@ -57,7 +57,7 @@ async fn create_test_set(client: &Client, no_records: usize) -> String { ), ]; client - .operate(&WritePolicy::default(), &key, &ops) + .operate::>(&WritePolicy::default(), &key, &ops) .await .unwrap(); } @@ -70,7 +70,7 @@ async fn expression_hll() { let client = common::client().await; let set_name = create_test_set(&client, EXPECTED).await; - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( get_count(add_with_index_and_min_hash( @@ -88,7 +88,7 @@ async fn expression_hll() { let count = count_results(rs); assert_eq!(count, 99, "HLL INIT Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( may_contain( @@ -103,7 +103,7 @@ async fn expression_hll() { let count = count_results(rs); assert_eq!(count, 1, "HLL MAY CONTAIN Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, lt( get_by_index( @@ -121,7 +121,7 @@ async fn expression_hll() { let count = count_results(rs); assert_eq!(count, 100, "HLL DESCRIBE Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( get_count(get_union( @@ -136,7 +136,7 @@ async fn expression_hll() { let count = count_results(rs); assert_eq!(count, 98, "HLL GET UNION Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( get_union_count( @@ -151,7 +151,7 @@ async fn expression_hll() { let count = count_results(rs); assert_eq!(count, 98, "HLL GET UNION COUNT Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, eq( get_intersect_count( @@ -166,7 +166,7 @@ async fn expression_hll() { let count = count_results(rs); assert_eq!(count, 99, "HLL GET INTERSECT COUNT Test Failed"); - let rs = test_filter( + let rs: Arc>> = test_filter( &client, gt( get_similarity( @@ -184,7 +184,11 @@ async fn expression_hll() { client.close().await.unwrap(); } -async fn test_filter(client: &Client, filter: FilterExpression, set_name: &str) -> Arc { +async fn test_filter( + client: &Client, + filter: FilterExpression, + set_name: &str, +) -> Arc>> { let namespace = common::namespace(); let mut qpolicy = QueryPolicy::default(); @@ -194,7 +198,7 @@ async fn test_filter(client: &Client, filter: FilterExpression, set_name: &str) client.query(&qpolicy, statement).await.unwrap() } -fn count_results(rs: Arc) -> usize { +fn count_results(rs: Arc>>) -> usize { let mut count = 0; for res in &*rs { diff --git a/tests/src/exp_list.rs b/tests/src/exp_list.rs index 2694f7b4..d0116956 100644 --- a/tests/src/exp_list.rs +++ b/tests/src/exp_list.rs @@ -1,10 +1,10 @@ use crate::common; - use aerospike::expressions::lists::*; use aerospike::expressions::*; use aerospike::operations::lists::{ListPolicy, ListReturnType}; use aerospike::*; +use std::collections::HashMap; use std::sync::Arc; const EXPECTED: usize = 100; @@ -558,7 +558,11 @@ async fn expression_list() { client.close().await.unwrap(); } -async fn test_filter(client: &Client, filter: FilterExpression, set_name: &str) -> Arc { +async fn test_filter( + client: &Client, + filter: FilterExpression, + set_name: &str, +) -> Arc>> { let namespace = common::namespace(); let mut qpolicy = QueryPolicy::default(); @@ -568,7 +572,7 @@ async fn test_filter(client: &Client, filter: FilterExpression, set_name: &str) client.query(&qpolicy, statement).await.unwrap() } -fn count_results(rs: Arc) -> usize { +fn count_results(rs: Arc>>) -> usize { let mut count = 0; for res in &*rs { diff --git a/tests/src/exp_map.rs b/tests/src/exp_map.rs index 75e2578f..5871e300 100644 --- a/tests/src/exp_map.rs +++ b/tests/src/exp_map.rs @@ -658,7 +658,11 @@ fn expression_map() { client.close().await.unwrap(); } -async fn test_filter(client: &Client, filter: FilterExpression, set_name: &str) -> Arc { +async fn test_filter( + client: &Client, + filter: FilterExpression, + set_name: &str, +) -> Arc>> { let namespace = common::namespace(); let mut qpolicy = QueryPolicy::default(); @@ -668,7 +672,7 @@ async fn test_filter(client: &Client, filter: FilterExpression, set_name: &str) client.query(&qpolicy, statement).await.unwrap() } -fn count_results(rs: Arc) -> usize { +fn count_results(rs: Arc>>) -> usize { let mut count = 0; for res in &*rs { diff --git a/tests/src/exp_op.rs b/tests/src/exp_op.rs index 918c4196..4f01d98c 100644 --- a/tests/src/exp_op.rs +++ b/tests/src/exp_op.rs @@ -1,7 +1,8 @@ use crate::common; use aerospike::expressions::{int_bin, int_val, num_add}; use aerospike::operations::exp::{read_exp, write_exp, ExpReadFlags, ExpWriteFlags}; -use aerospike::{as_bin, as_key, as_val, Bins, ReadPolicy, WritePolicy}; +use aerospike::{as_bin, as_key, as_val, Bins, ReadPolicy, Record, Value, WritePolicy}; +use std::collections::HashMap; #[aerospike_macro::test] async fn exp_ops() { @@ -21,7 +22,7 @@ async fn exp_ops() { client.delete(&wpolicy, &key).await.unwrap(); client.put(&wpolicy, &key, &bins).await.unwrap(); - let rec = client.get(&policy, &key, Bins::All).await.unwrap(); + let rec: Record> = client.get(&policy, &key, Bins::All).await.unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_val!(25), @@ -29,7 +30,9 @@ async fn exp_ops() { ); let flt = num_add(vec![int_bin("bin".to_string()), int_val(4)]); let ops = &vec![read_exp("example", &flt, ExpReadFlags::Default)]; - let rec = client.operate(&wpolicy, &key, ops).await; + let rec = client + .operate::>(&wpolicy, &key, ops) + .await; let rec = rec.unwrap(); assert_eq!( @@ -44,7 +47,9 @@ async fn exp_ops() { read_exp("example", &flt2, ExpReadFlags::Default), ]; - let rec = client.operate(&wpolicy, &key, ops).await; + let rec = client + .operate::>(&wpolicy, &key, ops) + .await; let rec = rec.unwrap(); assert_eq!( diff --git a/tests/src/hll.rs b/tests/src/hll.rs index 7c5a2c31..21e099ac 100644 --- a/tests/src/hll.rs +++ b/tests/src/hll.rs @@ -1,9 +1,11 @@ use crate::common; - +use env_logger; +use std::collections::HashMap; use aerospike::operations::hll; use aerospike::operations::hll::HLLPolicy; use aerospike::{as_key, as_list, as_val, Bins, FloatValue, ReadPolicy, Value, WritePolicy}; +use aerospike_core::Record; #[aerospike_macro::test] async fn hll() { @@ -20,11 +22,17 @@ async fn hll() { let rpolicy = ReadPolicy::default(); let ops = &vec![hll::init(&hpolicy, "bin", 4)]; - client.operate(&wpolicy, &key, ops).await.unwrap(); + client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); let v = vec![Value::from("asd123")]; let ops = &vec![hll::add(&hpolicy, "bin", &v)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), Value::Int(1), @@ -32,7 +40,10 @@ async fn hll() { ); let ops = &vec![hll::get_count("bin")]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), Value::Int(1), @@ -40,14 +51,23 @@ async fn hll() { ); let ops = &vec![hll::init_with_min_hash(&hpolicy, "bin2", 8, 0)]; - client.operate(&wpolicy, &key, ops).await.unwrap(); + client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); let ops = &vec![hll::fold("bin2", 6)]; - client.operate(&wpolicy, &key, ops).await.unwrap(); + client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); let v2 = vec![Value::from("123asd")]; let ops = &vec![hll::add(&hpolicy, "bin2", &v2)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin2").unwrap(), Value::Int(1), @@ -55,21 +75,27 @@ async fn hll() { ); let ops = &vec![hll::describe("bin")]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), as_list!(4, 0), "Index bits did not match" ); - let rec = client + let rec: Record> = client .get(&rpolicy, &key, Bins::from(["bin2"])) .await .unwrap(); let bin2val = vec![rec.bins.get("bin2").unwrap().clone()]; let ops = &vec![hll::get_intersect_count("bin", &bin2val)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), Value::from(0), @@ -77,7 +103,10 @@ async fn hll() { ); let ops = &vec![hll::get_union_count("bin", &bin2val)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), Value::from(2), @@ -85,14 +114,20 @@ async fn hll() { ); let ops = &vec![hll::get_union("bin", &bin2val)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); let val = Value::HLL(vec![ 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, ]); assert_eq!(*rec.bins.get("bin").unwrap(), val, "Union does not match"); let ops = &vec![hll::refresh_count("bin")]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), Value::Int(1), @@ -103,7 +138,10 @@ async fn hll() { hll::set_union(&hpolicy, "bin", &bin2val), hll::get_count("bin"), ]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), Value::from(2), @@ -111,7 +149,10 @@ async fn hll() { ); let ops = &vec![hll::get_similarity("bin", &bin2val)]; - let rec = client.operate(&wpolicy, &key, ops).await.unwrap(); + let rec = client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); assert_eq!( *rec.bins.get("bin").unwrap(), Value::Float(FloatValue::F64(4602678819172646912)), diff --git a/tests/src/index.rs b/tests/src/index.rs index ccb6b536..c9aaaa75 100644 --- a/tests/src/index.rs +++ b/tests/src/index.rs @@ -41,8 +41,7 @@ async fn create_test_set(client: &Client, no_records: usize) -> String { } #[aerospike_macro::test] -#[should_panic(expected = "IndexFound")] -async fn recreate_index() { +async fn create_index() { let _ = env_logger::try_init(); let client = common::client().await; @@ -54,12 +53,6 @@ async fn recreate_index() { let _ = client.drop_index(ns, &set, &index).await; thread::sleep(Duration::from_millis(1000)); - let task = client - .create_index(ns, &set, bin, &index, IndexType::Numeric) - .await - .expect("Failed to create index"); - task.wait_till_complete(None).await.unwrap(); - let task = client .create_index(ns, &set, bin, &index, IndexType::Numeric) .await diff --git a/tests/src/kv.rs b/tests/src/kv.rs index cb135f27..ad3a52c7 100644 --- a/tests/src/kv.rs +++ b/tests/src/kv.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; // Copyright 2015-2018 Aerospike, Inc. // // Portions may be licensed to Aerospike, Inc. under one or more contributor @@ -12,12 +13,11 @@ // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the // License for the specific language governing permissions and limitations under // the License. -use aerospike::operations; +use aerospike::{operations, Record}; use aerospike::{ as_bin, as_blob, as_geo, as_key, as_list, as_map, as_val, Bins, ReadPolicy, Value, WritePolicy, }; - use crate::common; #[aerospike_macro::test] @@ -35,9 +35,10 @@ async fn connect() { let bins = [ as_bin!("bin999", "test string"), + as_bin!("bin bool", true), as_bin!("bin vec![int]", as_list![1u32, 2u32, 3u32]), as_bin!("bin vec![u8]", as_blob!(vec![1u8, 2u8, 3u8])), - as_bin!("bin map", as_map!(1 => 1, 2 => 2, 3 => "hi!")), + as_bin!("bin map", as_map!(1 => 1, 2 => 2, 3 => "hi!", 4 => false)), as_bin!("bin f64", 1.64f64), as_bin!("bin Nil", None), // Writing None erases the bin! as_bin!( @@ -51,9 +52,11 @@ async fn connect() { ]; client.put(&wpolicy, &key, &bins).await.unwrap(); - let record = client.get(&policy, &key, Bins::All).await.unwrap(); + let record: Record> = + client.get(&policy, &key, Bins::All).await.unwrap(); let bins = record.bins; - assert_eq!(bins.len(), 7); + assert_eq!(bins.len(), 8); + assert_eq!(bins.get("bin bool"), Some(&Value::Bool(true))); assert_eq!(bins.get("bin999"), Some(&Value::from("test string"))); assert_eq!(bins.get("bin vec![int]"), Some(&as_list![1u32, 2u32, 3u32])); assert_eq!( @@ -62,7 +65,7 @@ async fn connect() { ); assert_eq!( bins.get("bin map"), - Some(&as_map!(1 => 1, 2 => 2, 3 => "hi!")) + Some(&as_map!(1 => 1, 2 => 2, 3 => "hi!", 4 => false)) ); assert_eq!(bins.get("bin f64"), Some(&Value::from(1.64f64))); assert_eq!( @@ -79,10 +82,11 @@ async fn connect() { client.touch(&wpolicy, &key).await.unwrap(); let bins = Bins::from(["bin999", "bin f64"]); - let record = client.get(&policy, &key, bins).await.unwrap(); + let record: Record> = client.get(&policy, &key, bins).await.unwrap(); assert_eq!(record.bins.len(), 2); - let record = client.get(&policy, &key, Bins::None).await.unwrap(); + let record: Record> = + client.get(&policy, &key, Bins::None).await.unwrap(); assert_eq!(record.bins.len(), 0); let exists = client.exists(&wpolicy, &key).await.unwrap(); @@ -90,7 +94,10 @@ async fn connect() { let bin = as_bin!("bin999", "test string"); let ops = &vec![operations::put(&bin), operations::get()]; - client.operate(&wpolicy, &key, ops).await.unwrap(); + client + .operate::>(&wpolicy, &key, ops) + .await + .unwrap(); let existed = client.delete(&wpolicy, &key).await.unwrap(); assert!(existed); diff --git a/tests/src/mod.rs b/tests/src/mod.rs index 1d3765d9..2a68dea7 100644 --- a/tests/src/mod.rs +++ b/tests/src/mod.rs @@ -17,6 +17,7 @@ mod batch; mod cdt_bitwise; mod cdt_list; mod cdt_map; +mod derive; mod exp; mod exp_bitwise; mod exp_hll; diff --git a/tests/src/query.rs b/tests/src/query.rs index 9bd7f57f..0f030968 100644 --- a/tests/src/query.rs +++ b/tests/src/query.rs @@ -13,6 +13,7 @@ // License for the specific language governing permissions and limitations under // the License. +use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::thread; @@ -65,7 +66,8 @@ async fn query_single_consumer() { // Filter Query let mut statement = Statement::new(namespace, &set_name, Bins::All); statement.add_filter(as_eq!("bin", 1)); - let rs = client.query(&qpolicy, statement).await.unwrap(); + let rs: Arc>> = + client.query(&qpolicy, statement).await.unwrap(); let mut count = 0; for res in &*rs { match res { @@ -81,7 +83,8 @@ async fn query_single_consumer() { // Range Query let mut statement = Statement::new(namespace, &set_name, Bins::All); statement.add_filter(as_range!("bin", 0, 9)); - let rs = client.query(&qpolicy, statement).await.unwrap(); + let rs: Arc>> = + client.query(&qpolicy, statement).await.unwrap(); let mut count = 0; for res in &*rs { match res { @@ -110,7 +113,8 @@ async fn query_nobins() { let mut statement = Statement::new(namespace, &set_name, Bins::None); statement.add_filter(as_range!("bin", 0, 9)); - let rs = client.query(&qpolicy, statement).await.unwrap(); + let rs: Arc>> = + client.query(&qpolicy, statement).await.unwrap(); let mut count = 0; for res in &*rs { match res { @@ -140,7 +144,8 @@ async fn query_multi_consumer() { let mut statement = Statement::new(namespace, &set_name, Bins::All); let f = as_range!("bin", 0, 9); statement.add_filter(f); - let rs = client.query(&qpolicy, statement).await.unwrap(); + let rs: Arc>> = + client.query(&qpolicy, statement).await.unwrap(); let count = Arc::new(AtomicUsize::new(0)); let mut threads = vec![]; @@ -191,7 +196,8 @@ async fn query_node() { let qpolicy = QueryPolicy::default(); let mut statement = Statement::new(namespace, &set_name, Bins::All); statement.add_filter(as_range!("bin", 0, 99)); - let rs = client.query_node(&qpolicy, node, statement).await.unwrap(); + let rs: Arc>> = + client.query_node(&qpolicy, node, statement).await.unwrap(); let ok = (&*rs).filter(Result::is_ok).count(); count.fetch_add(ok, Ordering::Relaxed); })); @@ -205,3 +211,39 @@ async fn query_node() { client.close().await.unwrap(); } + +// https://github.com/aerospike/aerospike-client-rust/issues/115 +#[aerospike_macro::test] +async fn query_large_i64() { + const SET: &str = "large_i64"; + const BIN: &str = "val"; + + let client = Arc::new(common::client().await); + let value = Value::from(i64::max_value()); + let key = Key::new(common::namespace(), SET, value.clone()).unwrap(); + let wpolicy = WritePolicy::default(); + + let res = client + .put(&wpolicy, &key, &[aerospike::Bin::new(BIN.into(), value)]) + .await; + + assert!(res.is_ok()); + + let mut qpolicy = aerospike::QueryPolicy::new(); + let bin_name = aerospike::expressions::int_bin(BIN.into()); + let bin_val = aerospike::expressions::int_val(i64::max_value()); + qpolicy + .filter_expression + .replace(aerospike::expressions::eq(bin_name, bin_val)); + let stmt = aerospike::Statement::new(common::namespace(), SET, aerospike::Bins::All); + let recordset: Arc>> = + client.query(&qpolicy, stmt).await.unwrap(); + + for r in &*recordset { + assert!(r.is_ok()); + let int = r.unwrap().bins.remove(BIN).unwrap(); + assert_eq!(int, Value::Int(i64::max_value())); + } + + let _ = client.truncate(common::namespace(), SET, 0).await; +} diff --git a/tests/src/scan.rs b/tests/src/scan.rs index 6f096ff3..2ab7d55d 100644 --- a/tests/src/scan.rs +++ b/tests/src/scan.rs @@ -13,6 +13,7 @@ // License for the specific language governing permissions and limitations under // the License. +use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::thread; @@ -49,7 +50,7 @@ async fn scan_single_consumer() { let set_name = create_test_set(&client, EXPECTED).await; let spolicy = ScanPolicy::default(); - let rs = client + let rs: Arc>> = client .scan(&spolicy, namespace, &set_name, Bins::All) .await .unwrap(); @@ -70,7 +71,7 @@ async fn scan_multi_consumer() { let mut spolicy = ScanPolicy::default(); spolicy.record_queue_size = 4096; - let rs = client + let rs: Arc>> = client .scan(&spolicy, namespace, &set_name, Bins::All) .await .unwrap(); @@ -113,7 +114,7 @@ async fn scan_node() { let set_name = set_name.clone(); threads.push(aerospike_rt::spawn(async move { let spolicy = ScanPolicy::default(); - let rs = client + let rs: Arc>> = client .scan_node(&spolicy, node, namespace, &set_name, Bins::All) .await .unwrap(); diff --git a/tests/src/serialization.rs b/tests/src/serialization.rs index 620be5c3..bdd5bb44 100644 --- a/tests/src/serialization.rs +++ b/tests/src/serialization.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; // Copyright 2015-2018 Aerospike, Inc. // // Portions may be licensed to Aerospike, Inc. under one or more contributor @@ -13,7 +14,8 @@ // License for the specific language governing permissions and limitations under // the License. use aerospike::{ - as_bin, as_blob, as_geo, as_key, as_list, as_map, as_val, Bins, ReadPolicy, WritePolicy, + as_bin, as_blob, as_geo, as_key, as_list, as_map, as_val, Bins, ReadPolicy, Record, Value, + WritePolicy, }; @@ -50,7 +52,8 @@ async fn serialize() { ]; client.put(&wpolicy, &key, &bins).await.unwrap(); - let record = client.get(&policy, &key, Bins::All).await.unwrap(); + let record: Record> = + client.get(&policy, &key, Bins::All).await.unwrap(); let json = serde_json::to_string(&record); if json.is_err() { From e8806759691891f7c7df9e5295618725b36c550d Mon Sep 17 00:00:00 2001 From: Caleb Moore Date: Wed, 24 Apr 2024 13:45:22 +1000 Subject: [PATCH 02/12] Fixed destreaming floats --- aerospike-core/src/client.rs | 7 +- aerospike-core/src/commands/read_command.rs | 5 +- aerospike-core/src/derive/readable.rs | 108 ++++++++++++++------ aerospike-core/src/value.rs | 6 +- 4 files changed, 82 insertions(+), 44 deletions(-) diff --git a/aerospike-core/src/client.rs b/aerospike-core/src/client.rs index 70bf75d3..b7b0ba10 100644 --- a/aerospike-core/src/client.rs +++ b/aerospike-core/src/client.rs @@ -191,8 +191,7 @@ impl Client { { let bins = bins.into(); let mut command = ReadCommand::new(&policy.base_policy, self.cluster.clone(), key, bins, policy.replica); - command.execute().await?; - Ok(command.record.unwrap()) + command.execute().await } /// Read multiple record for specified batch keys in one batch call. This method allows @@ -590,9 +589,7 @@ impl Client { args, ); - command.execute().await?; - - let record = command.read_command.record.unwrap(); + let record = command.execute().await?; // User defined functions don't have to return a value. if record.bins.is_empty() { diff --git a/aerospike-core/src/commands/read_command.rs b/aerospike-core/src/commands/read_command.rs index dee54117..3489ab3a 100644 --- a/aerospike-core/src/commands/read_command.rs +++ b/aerospike-core/src/commands/read_command.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::VecDeque; +use std::marker::PhantomData; use std::sync::Arc; use std::time::Duration; @@ -28,7 +29,7 @@ use crate::{derive, Bins, Key, Record, ResultCode}; pub struct ReadCommand<'a, T: serde::de::DeserializeOwned + Send> { pub single_command: SingleCommand<'a>, - pub record: Option>, + record: PhantomData, policy: &'a BasePolicy, bins: Bins, } @@ -39,7 +40,7 @@ impl<'a, T: serde::de::DeserializeOwned + Send> ReadCommand<'a, T> { single_command: SingleCommand::new(cluster, key, replica), bins, policy, - record: None, + record: Default::default(), } } diff --git a/aerospike-core/src/derive/readable.rs b/aerospike-core/src/derive/readable.rs index b1faddaf..fd654bc2 100644 --- a/aerospike-core/src/derive/readable.rs +++ b/aerospike-core/src/derive/readable.rs @@ -21,6 +21,7 @@ use serde::de::MapAccess; use serde::de::SeqAccess; use serde::de::VariantAccess; use serde::de::Visitor; +use serde::Deserialize; use serde::Deserializer; use crate::errors::Result; @@ -655,7 +656,12 @@ impl<'de> serde::de::Deserializer<'de> for PreParsedValue { fn deserialize_bytes(self, visitor: V) -> std::result::Result where V: serde::de::Visitor<'de> { - self.deserialize_any(visitor) + match self.particle_type() { + ParticleType::NULL => { + visitor.visit_none() + } + _ => visitor.visit_bytes(self.particle()) + } } fn deserialize_byte_buf(self, visitor: V) -> std::result::Result @@ -665,28 +671,7 @@ impl<'de> serde::de::Deserializer<'de> for PreParsedValue { ParticleType::NULL => { visitor.visit_none() } - ParticleType::INTEGER => { - visitor.visit_i64(self.as_int()?) - } - ParticleType::FLOAT => { - visitor.visit_f64(self.as_float()?) - } - ParticleType::STRING | ParticleType::GEOJSON => { - visitor.visit_string(self.into_string()?) - } - ParticleType::BLOB | ParticleType::HLL => { - visitor.visit_byte_buf(self.into_blob()) - } - ParticleType::BOOL => { - visitor.visit_bool(self.as_bool()?) - } - ParticleType::MAP | ParticleType::LIST => { - let mut read = 0; - let cdt_reader = CDTDecoder(self.particle(), &mut read); - cdt_reader.deserialize_any(visitor) - } - ParticleType::DIGEST => todo!(), - ParticleType::LDT => todo!(), + _ => visitor.visit_byte_buf(self.into_blob()) } } @@ -918,7 +903,7 @@ impl<'m> CDTDecoder<'m> { } fn take_bytes(&mut self) -> std::result::Result<[u8; N], Error> { - if *self.1 + N >= self.0.len() { + if *self.1 + N > self.0.len() { Err(Error::from_kind(crate::errors::ErrorKind::Derive("Ran out of data".to_string()))) } else { let offset = *self.1 as isize; @@ -1215,37 +1200,49 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { self.deserialize_str(visitor) } + // this is a very permissive handler that allows fn deserialize_bytes(mut self, visitor: V) -> std::prelude::v1::Result where V: serde::de::Visitor<'l> { fn deserialize_any_buffer<'l, 'm, V>(mut deserializer: CDTDecoder<'m>, visitor: V, count: usize) -> std::prelude::v1::Result where V: serde::de::Visitor<'l> { - let ptype = ParticleType::from(deserializer.take_byte()?); - let body = deserializer.take_nbyte(count - 1)?; - if matches!(ptype, ParticleType::STRING | ParticleType::GEOJSON) { - Err(crate::errors::Error::invalid_type(serde::de::Unexpected::Str(std::str::from_utf8(body)?), &visitor)) - } else { - visitor.visit_bytes(body) - } + let body = deserializer.take_nbyte(count)?; + // Allows string or geojson to be gotten as bytes + visitor.visit_bytes(&body[1..]) } - let ptype = self.take_byte()?; + // Since we allow the permissive parsing below, do not tamper with what we have here + let ptype = self.0[*self.1]; match ptype { - 0xa0..=0xbf => deserialize_any_buffer(self, visitor, (ptype & 0x1f) as usize), + 0xa0..=0xbf => { + let _ = self.take_byte(); + deserialize_any_buffer(self, visitor, (ptype & 0x1f) as usize) + } 0xc4 | 0xd9 => { + let _ = self.take_byte(); let count = u8::from_be_bytes(self.take_bytes()?); deserialize_any_buffer(self, visitor, count as usize) } 0xc5 | 0xda => { + let _ = self.take_byte(); let count = u16::from_be_bytes(self.take_bytes()?); deserialize_any_buffer(self, visitor, count as usize) } 0xc6 | 0xdb => { + let _ = self.take_byte(); let count = u32::from_be_bytes(self.take_bytes()?); deserialize_any_buffer(self, visitor, count as usize) } - _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) + _ => { + // Permissivly parse _anything_ into a byte array for pass-through. + // The byte array is always immediately after this particle. + let start_at = *self.1 + 1; + // Deserialize whatever we have here to see how long it is. + serde::de::IgnoredAny::deserialize(CDTDecoder(self.0, self.1))?; + // The end of whatever is here must be where the upto pointer is now at. + visitor.visit_bytes(&self.0[start_at..*self.1]) + } } } @@ -1654,4 +1651,47 @@ mod tests { crate::Value::String("Hello world".to_string()), ])); } + + #[test] + fn destream_f64_value() { + let mut buffer = crate::Buffer::new(1024); + let myval = crate::Value::from(0.0023_f64); + + buffer.resize_buffer(myval.estimate_size()).unwrap(); + myval.write_to(&mut buffer); + + let as_bin = new_preparsed(myval.particle_type() as u8, "binname", buffer.data_buffer); + + let deserialized = crate::Value::deserialize(as_bin.clone()).unwrap(); + assert_eq!(deserialized, myval); + + let mut buffer = crate::Buffer::new(1024); + let myval = crate::Value::List(vec![ + myval + ]); + + buffer.resize_buffer(myval.estimate_size()).unwrap(); + myval.write_to(&mut buffer); + + let as_bin = new_preparsed(20, "binname", buffer.data_buffer); + let deserialized = crate::Value::deserialize(as_bin.clone()).unwrap(); + assert_eq!(deserialized, myval); + } + + #[test] + fn destream_f32_value() { + let myval = crate::Value::from(0.0023_f32); + + let mut buffer = crate::Buffer::new(1024); + let myval = crate::Value::List(vec![ + myval + ]); + + buffer.resize_buffer(myval.estimate_size()).unwrap(); + myval.write_to(&mut buffer); + + let as_bin = new_preparsed(20, "binname", buffer.data_buffer); + let deserialized = crate::Value::deserialize(as_bin.clone()).unwrap(); + assert_eq!(deserialized, myval); + } } diff --git a/aerospike-core/src/value.rs b/aerospike-core/src/value.rs index 9f976368..431ad6c7 100644 --- a/aerospike-core/src/value.rs +++ b/aerospike-core/src/value.rs @@ -85,10 +85,10 @@ impl<'l> serde::de::Deserialize<'l> for FloatValue { fn visit_bytes(self, v: &[u8]) -> StdResult where E: serde::de::Error, { - if let Ok(array) = <&[u8; 8]>::try_from(v) { - Ok(FloatValue::F64(u64::from_le_bytes(*array))) + if let Ok(array) = <&[u8; 4]>::try_from(v) { + Ok(FloatValue::F32(u32::from_be_bytes(*array))) } else if let Ok(array) = <&[u8; 8]>::try_from(v) { - Ok(FloatValue::F64(u64::from_le_bytes(*array))) + Ok(FloatValue::F64(u64::from_be_bytes(*array))) } else { Err(E::custom("Floats may be 8 or 4 bytes")) } From 7557518e6a6019544ac13f5955a1b4e8def27037 Mon Sep 17 00:00:00 2001 From: Caleb Moore Date: Wed, 24 Apr 2024 15:20:18 +1000 Subject: [PATCH 03/12] Added assert --- aerospike-core/src/derive/readable.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aerospike-core/src/derive/readable.rs b/aerospike-core/src/derive/readable.rs index fd654bc2..25f7027e 100644 --- a/aerospike-core/src/derive/readable.rs +++ b/aerospike-core/src/derive/readable.rs @@ -509,7 +509,9 @@ impl<'de> serde::de::Deserializer<'de> for PreParsedValue { ParticleType::MAP | ParticleType::LIST => { let mut read = 0; let cdt_reader = CDTDecoder(self.particle(), &mut read); - cdt_reader.deserialize_any(visitor) + let out = cdt_reader.deserialize_any(visitor)?; + assert_eq!(read, self.particle().len()); + Ok(out) } ParticleType::DIGEST => todo!(), ParticleType::LDT => todo!(), From fba9e209322315aac58438230503b0baa2a53a2d Mon Sep 17 00:00:00 2001 From: Caleb Moore Date: Wed, 24 Apr 2024 18:01:45 +1000 Subject: [PATCH 04/12] Added more debuggin g --- .../src/commands/batch_read_command.rs | 10 +++++++--- .../src/commands/operate_command.rs | 19 +------------------ 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/aerospike-core/src/commands/batch_read_command.rs b/aerospike-core/src/commands/batch_read_command.rs index 6c4a7e49..8f0b1434 100644 --- a/aerospike-core/src/commands/batch_read_command.rs +++ b/aerospike-core/src/commands/batch_read_command.rs @@ -148,18 +148,22 @@ impl BatchReadCommand { } async fn parse_group(batch_reads: &mut [BatchRead], conn: &mut Connection, size: usize) -> Result { + let mut count = 0; while conn.bytes_read() < size { + let bytes_read = conn.bytes_read(); conn.read_buffer(commands::buffer::MSG_REMAINING_HEADER_SIZE as usize) .await?; match Self::parse_record(conn).await? { None => return Ok(false), Some(batch_record) => { - let batch_read = batch_reads - .get_mut(batch_record.batch_index) - .expect("Invalid batch index"); + let Some(batch_read) = batch_reads + .get_mut(batch_record.batch_index) else { + panic!("Invalid batch index {} of {}, bytes: {} of {}", count, batch_reads.len(), bytes_read, size); + }; batch_read.record = batch_record.record; } } + count += 1; } Ok(true) } diff --git a/aerospike-core/src/commands/operate_command.rs b/aerospike-core/src/commands/operate_command.rs index 2c911ca8..d4eeeb96 100644 --- a/aerospike-core/src/commands/operate_command.rs +++ b/aerospike-core/src/commands/operate_command.rs @@ -22,7 +22,7 @@ use crate::errors::Result; use crate::net::Connection; use crate::operations::Operation; use crate::policy::WritePolicy; -use crate::{Key, Record, Value}; +use crate::{Key, Record}; use super::read_command; @@ -33,23 +33,6 @@ pub struct OperateCommand<'a, T: serde::de::DeserializeOwned + Send> { phantom: PhantomData, } -/// The return value from operate. Like a record, but retains the order and duplicate keys of bins. -#[derive(Default)] -pub struct OperateRecord { - /// Record key. When reading a record from the database, the key is not set in the returned - /// Record struct. - pub key: Option, - - /// Map of named record bins. - pub bins: Vec<(String, Value)>, - - /// Record modification count. - pub generation: u32, - - /// Date record will expire, in seconds from Jan 01 2010, 00:00:00 UTC. - expiration: u32, -} - impl<'a, T: serde::de::DeserializeOwned + Send> OperateCommand<'a, T> { pub fn new( policy: &'a WritePolicy, From c7cc8080f7e4ed1b2a1dd6a4f54ea9d6718e666c Mon Sep 17 00:00:00 2001 From: Caleb Moore Date: Wed, 24 Apr 2024 18:03:22 +1000 Subject: [PATCH 05/12] Removed invalid use --- aerospike-core/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/aerospike-core/src/lib.rs b/aerospike-core/src/lib.rs index abdfdd64..c2bfecee 100644 --- a/aerospike-core/src/lib.rs +++ b/aerospike-core/src/lib.rs @@ -169,7 +169,6 @@ pub use result_code::ResultCode; pub use task::{IndexTask, RegisterTask, Task}; pub use user::User; pub use value::{FloatValue, Value}; -pub use commands::operate_command::OperateRecord; #[macro_use] pub mod errors; From 9d0fb5ad5dbd6ea69bb7858a7a4b62a25fadf59f Mon Sep 17 00:00:00 2001 From: Caleb Moore Date: Wed, 24 Apr 2024 18:09:43 +1000 Subject: [PATCH 06/12] Fixed bytes read number. --- aerospike-core/src/net/connection.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aerospike-core/src/net/connection.rs b/aerospike-core/src/net/connection.rs index b981cdd8..c7c40407 100644 --- a/aerospike-core/src/net/connection.rs +++ b/aerospike-core/src/net/connection.rs @@ -142,6 +142,7 @@ impl Connection { for _ in 0..op_count { let mut head = [0; 8]; self.conn.read_exact(&mut head).await?; + self.bytes_read += 8; let next_len = u32::from_be_bytes(head[..4].try_into().unwrap()); let particle_type = head[5]; let name_len = head[7] as usize; @@ -151,6 +152,7 @@ impl Connection { let mut particle = Vec::new(); particle.resize(next_len as usize - 4 - name_len, 0); self.conn.read_exact(&mut particle).await?; + self.bytes_read += particle.len(); data_points.push(PreParsedValue{particle_type, name, name_len: head[7], particle}); } From f11b710c4ca2a683e857d678a7a95daf3c120390 Mon Sep 17 00:00:00 2001 From: Caleb Moore Date: Wed, 24 Apr 2024 18:14:56 +1000 Subject: [PATCH 07/12] Fixed bytes read number... again. --- aerospike-core/src/net/connection.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/aerospike-core/src/net/connection.rs b/aerospike-core/src/net/connection.rs index c7c40407..2f83f02f 100644 --- a/aerospike-core/src/net/connection.rs +++ b/aerospike-core/src/net/connection.rs @@ -148,6 +148,7 @@ impl Connection { let name_len = head[7] as usize; let mut name = [0; 15]; self.conn.read_exact(&mut name[..name_len]).await?; + self.bytes_read += name_len; let mut particle = Vec::new(); particle.resize(next_len as usize - 4 - name_len, 0); From c4872bfee06e00be5be39fc07c995878f75a4fc6 Mon Sep 17 00:00:00 2001 From: Caleb Moore Date: Wed, 24 Apr 2024 22:53:07 +1000 Subject: [PATCH 08/12] Fixed bytes destream --- aerospike-core/Cargo.toml | 5 +++-- aerospike-core/src/derive/readable.rs | 17 +++++++++++++++++ aerospike-core/src/net/connection.rs | 15 +++++++++------ aerospike-core/src/value.rs | 10 ++++++++-- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/aerospike-core/Cargo.toml b/aerospike-core/Cargo.toml index 56248c56..e3a469ff 100644 --- a/aerospike-core/Cargo.toml +++ b/aerospike-core/Cargo.toml @@ -15,14 +15,15 @@ rand = "0.8" lazy_static = "1.4" error-chain = { version = "0.12.4", default-features = false } pwhash = "1.0" -serde = { version = "1.0", features = ["derive"], optional = true } +serde = { version = "1.0", features = ["derive"] } +serde_bytes = "0.11" aerospike-rt = {path = "../aerospike-rt"} futures = {version = "0.3.16" } async-trait = "0.1.51" num = "0.4.0" [features] -serialization = ["serde"] +serialization = [] rt-tokio = ["aerospike-rt/rt-tokio"] rt-async-std = ["aerospike-rt/rt-async-std"] diff --git a/aerospike-core/src/derive/readable.rs b/aerospike-core/src/derive/readable.rs index 25f7027e..42dbce92 100644 --- a/aerospike-core/src/derive/readable.rs +++ b/aerospike-core/src/derive/readable.rs @@ -1696,4 +1696,21 @@ mod tests { let deserialized = crate::Value::deserialize(as_bin.clone()).unwrap(); assert_eq!(deserialized, myval); } + + #[test] + fn destream_bytes() { + let myval = crate::Value::from(Vec::from([0_u8, 1_u8, 2_u8, 3_u8, 4_u8, 5_u8])); + + let mut buffer = crate::Buffer::new(1024); + let myval = crate::Value::List(vec![ + myval + ]); + + buffer.resize_buffer(myval.estimate_size()).unwrap(); + myval.write_to(&mut buffer); + + let as_bin = new_preparsed(20, "binname", buffer.data_buffer); + let deserialized = crate::Value::deserialize(as_bin.clone()).unwrap(); + assert_eq!(deserialized, myval); + } } diff --git a/aerospike-core/src/net/connection.rs b/aerospike-core/src/net/connection.rs index 2f83f02f..08e297d5 100644 --- a/aerospike-core/src/net/connection.rs +++ b/aerospike-core/src/net/connection.rs @@ -132,6 +132,12 @@ impl Connection { self.bytes_read } + async fn read_exact_direct<'a>(&'a mut self, buf: &'a mut [u8]) -> std::io::Result { + let result = self.conn.read_exact(buf).await?; + self.bytes_read += buf.len(); + Ok(result) + } + pub(crate) async fn pre_parse_stream_bins( &mut self, op_count: usize, @@ -141,19 +147,16 @@ impl Connection { for _ in 0..op_count { let mut head = [0; 8]; - self.conn.read_exact(&mut head).await?; - self.bytes_read += 8; + self.read_exact_direct(&mut head).await?; let next_len = u32::from_be_bytes(head[..4].try_into().unwrap()); let particle_type = head[5]; let name_len = head[7] as usize; let mut name = [0; 15]; - self.conn.read_exact(&mut name[..name_len]).await?; - self.bytes_read += name_len; + self.read_exact_direct(&mut name[..name_len]).await?; let mut particle = Vec::new(); particle.resize(next_len as usize - 4 - name_len, 0); - self.conn.read_exact(&mut particle).await?; - self.bytes_read += particle.len(); + self.read_exact_direct(&mut particle).await?; data_points.push(PreParsedValue{particle_type, name, name_len: head[7], particle}); } diff --git a/aerospike-core/src/value.rs b/aerospike-core/src/value.rs index 431ad6c7..f3cc3988 100644 --- a/aerospike-core/src/value.rs +++ b/aerospike-core/src/value.rs @@ -226,7 +226,10 @@ pub enum Value { String(String), /// Byte array value. - Blob(Vec), + Blob( + #[serde(with = "serde_bytes")] + Vec + ), /// List data type is an ordered collection of values. Lists can contain values of any /// supported data type. List data order is maintained on writes and reads. @@ -246,7 +249,10 @@ pub enum Value { GeoJSON(String), /// HLL value - HLL(Vec), + HLL( + #[serde(with = "serde_bytes")] + Vec + ), } #[allow(clippy::derive_hash_xor_eq)] From 4dce08794b557fd73b1414625595cf7a0a701f77 Mon Sep 17 00:00:00 2001 From: Caleb Moore Date: Wed, 24 Apr 2024 23:40:23 +1000 Subject: [PATCH 09/12] Skip extentions --- aerospike-core/src/derive/readable.rs | 86 ++++++++++++++++++++------- 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/aerospike-core/src/derive/readable.rs b/aerospike-core/src/derive/readable.rs index 42dbce92..59fd69c0 100644 --- a/aerospike-core/src/derive/readable.rs +++ b/aerospike-core/src/derive/readable.rs @@ -846,7 +846,33 @@ impl<'de, Var: for<'a> Deserializer<'a, Error = crate::Error>> VariantAccess<'de struct CDTDecoder<'m>(&'m [u8], &'m mut usize); -struct CDTListOrMap<'m>(usize, &'m [u8], &'m mut usize); +struct CDTListOrMap<'m> { + remaining: usize, + buffer: &'m [u8], + upto: &'m mut usize, +} + +impl<'m> CDTListOrMap<'m> { + fn new( + remaining: usize, + buffer: &'m [u8], + upto: &'m mut usize, + is_map: bool, + ) -> Self { + // Skip the extensions + if remaining > 0 && is_ext(buffer[*upto]) { + serde::de::IgnoredAny::deserialize(CDTDecoder(buffer, upto)).unwrap(); + if is_map { + serde::de::IgnoredAny::deserialize(CDTDecoder(buffer, upto)).unwrap(); + } + } + CDTListOrMap { + remaining, + buffer, + upto, + } + } +} impl<'m> CDTDecoder<'m> { fn as_unexpected(mut self, ptype: u8) -> std::result::Result, Error> { @@ -951,8 +977,8 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { let ptype = self.take_byte()?; match ptype { 0x00..=0x7f => visitor.visit_u8(ptype as u8), - 0x80..=0x8f => visitor.visit_map(CDTListOrMap((ptype & 0x0f) as usize, self.0, self.1)), - 0x90..=0x9f => visitor.visit_seq(CDTListOrMap((ptype & 0x0f) as usize, self.0, self.1)), + 0x80..=0x8f => visitor.visit_map(CDTListOrMap::new((ptype & 0x0f) as usize, self.0, self.1, true)), + 0x90..=0x9f => visitor.visit_seq(CDTListOrMap::new((ptype & 0x0f) as usize, self.0, self.1, false)), 0xa0..=0xbf => deserialize_any_buffer(self, visitor, (ptype & 0x1f) as usize), 0xc0 => visitor.visit_none(), 0xc2 => visitor.visit_bool(false), @@ -981,19 +1007,19 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { 0xd3 => visitor.visit_i64(i64::from_be_bytes(self.take_bytes()?)), 0xdc => { let count = u16::from_be_bytes(self.take_bytes()?) as usize; - visitor.visit_seq(CDTListOrMap(count, self.0, self.1)) + visitor.visit_seq(CDTListOrMap::new(count, self.0, self.1, false)) } 0xdd => { let count = u32::from_be_bytes(self.take_bytes()?) as usize; - visitor.visit_seq(CDTListOrMap(count, self.0, self.1)) + visitor.visit_seq(CDTListOrMap::new(count, self.0, self.1, false)) } 0xde => { let count = u16::from_be_bytes(self.take_bytes()?) as usize; - visitor.visit_map(CDTListOrMap(count, self.0, self.1)) + visitor.visit_map(CDTListOrMap::new(count, self.0, self.1, true)) } 0xdf => { let count = u32::from_be_bytes(self.take_bytes()?) as usize; - visitor.visit_map(CDTListOrMap(count, self.0, self.1)) + visitor.visit_map(CDTListOrMap::new(count, self.0, self.1, true)) } 0xe0..=0xff => { let value = (ptype - 0xe0) as i8 - 32; @@ -1296,14 +1322,14 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { V: serde::de::Visitor<'l> { let ptype = self.take_byte()?; match ptype { - 0x90..=0x9f => visitor.visit_seq(CDTListOrMap((ptype & 0x0f) as usize, self.0, self.1)), + 0x90..=0x9f => visitor.visit_seq(CDTListOrMap::new((ptype & 0x0f) as usize, self.0, self.1, false)), 0xdc => { let count = u16::from_be_bytes(self.take_bytes()?) as usize; - visitor.visit_seq(CDTListOrMap(count, self.0, self.1)) + visitor.visit_seq(CDTListOrMap::new(count, self.0, self.1, false)) } 0xdd => { let count = u32::from_be_bytes(self.take_bytes()?) as usize; - visitor.visit_seq(CDTListOrMap(count, self.0, self.1)) + visitor.visit_seq(CDTListOrMap::new(count, self.0, self.1, false)) } _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) } @@ -1331,14 +1357,14 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { V: serde::de::Visitor<'l> { let ptype = self.take_byte()?; match ptype { - 0x80..=0x8f => visitor.visit_map(CDTListOrMap((ptype & 0x0f) as usize, self.0, self.1)), + 0x80..=0x8f => visitor.visit_map(CDTListOrMap::new((ptype & 0x0f) as usize, self.0, self.1, true)), 0xde => { let count = u16::from_be_bytes(self.take_bytes()?) as usize; - visitor.visit_map(CDTListOrMap(count, self.0, self.1)) + visitor.visit_map(CDTListOrMap::new(count, self.0, self.1, true)) } 0xdf => { let count = u32::from_be_bytes(self.take_bytes()?) as usize; - visitor.visit_map(CDTListOrMap(count, self.0, self.1)) + visitor.visit_map(CDTListOrMap::new(count, self.0, self.1, true)) } _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) } @@ -1427,6 +1453,18 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { let count = u32::from_be_bytes(self.take_bytes()?); *self.1 += count as usize; } + 0xc7 => { + let count = 1 + u8::from_be_bytes(self.take_bytes()?); + *self.1 += count as usize; + } + 0xc8 => { + let count = 1 + u16::from_be_bytes(self.take_bytes()?); + *self.1 += count as usize; + } + 0xc9 => { + let count = 1 + u32::from_be_bytes(self.take_bytes()?); + *self.1 += count as usize; + } 0xcc | 0xd0 => {self.take_byte()?;} 0xcd | 0xd1 => {self.take_bytes::<2>()?;} 0xca | 0xce | 0xd2 => {self.take_bytes::<4>()?;} @@ -1459,22 +1497,22 @@ impl<'l, 'm> MapAccess<'l> for CDTListOrMap<'m> { fn next_key_seed(&mut self, seed: K) -> std::prelude::v1::Result, Self::Error> where K: serde::de::DeserializeSeed<'l> { - if self.0 == 0 { + if self.remaining == 0 { Ok(None) } else { - self.0 -= 1; - seed.deserialize(CDTDecoder(self.1, self.2)).map(Some) + self.remaining -= 1; + seed.deserialize(CDTDecoder(self.buffer, self.upto)).map(Some) } } fn next_value_seed(&mut self, seed: V) -> std::prelude::v1::Result where V: serde::de::DeserializeSeed<'l> { - seed.deserialize(CDTDecoder(self.1, self.2)) + seed.deserialize(CDTDecoder(self.buffer, self.upto)) } fn size_hint(&self) -> Option { - Some(self.0) + Some(self.remaining) } } @@ -1484,19 +1522,23 @@ impl<'l, 'm> SeqAccess<'l> for CDTListOrMap<'m> { fn next_element_seed(&mut self, seed: T) -> std::prelude::v1::Result, Self::Error> where T: serde::de::DeserializeSeed<'l> { - if self.0 == 0 { + if self.remaining == 0 { Ok(None) } else { - self.0 -= 1; - seed.deserialize(CDTDecoder(self.1, self.2)).map(Some) + self.remaining -= 1; + seed.deserialize(CDTDecoder(self.buffer, self.upto)).map(Some) } } fn size_hint(&self) -> Option { - Some(self.0) + Some(self.remaining) } } +const fn is_ext(byte: u8) -> bool { + matches!(byte, 0xc7 | 0xc8 | 0xc9 | 0xd4 | 0xd5 | 0xd6 | 0xd7 | 0xd8) +} + /// Includes the data for the Value part of a Bin. #[derive(Debug, Clone)] pub(crate) struct PreParsedValue{ From 6a0d15739601f615dc05405c0c122024685e8d59 Mon Sep 17 00:00:00 2001 From: Caleb Moore Date: Wed, 24 Apr 2024 23:44:10 +1000 Subject: [PATCH 10/12] Decrement count --- aerospike-core/src/derive/readable.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aerospike-core/src/derive/readable.rs b/aerospike-core/src/derive/readable.rs index 59fd69c0..bba0c971 100644 --- a/aerospike-core/src/derive/readable.rs +++ b/aerospike-core/src/derive/readable.rs @@ -854,7 +854,7 @@ struct CDTListOrMap<'m> { impl<'m> CDTListOrMap<'m> { fn new( - remaining: usize, + mut remaining: usize, buffer: &'m [u8], upto: &'m mut usize, is_map: bool, @@ -865,6 +865,7 @@ impl<'m> CDTListOrMap<'m> { if is_map { serde::de::IgnoredAny::deserialize(CDTDecoder(buffer, upto)).unwrap(); } + remaining -= 1; } CDTListOrMap { remaining, From cde75966d7914bd9ab0f20163ac31c3254195e86 Mon Sep 17 00:00:00 2001 From: Caleb Moore Date: Fri, 26 Apr 2024 13:14:15 +1000 Subject: [PATCH 11/12] Fixed destreaming small negative numbers with hint --- aerospike-core/src/derive/readable.rs | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/aerospike-core/src/derive/readable.rs b/aerospike-core/src/derive/readable.rs index bba0c971..2af7ef7e 100644 --- a/aerospike-core/src/derive/readable.rs +++ b/aerospike-core/src/derive/readable.rs @@ -1049,6 +1049,10 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { 0x00..=0x7f => visitor.visit_i8(ptype as i8), 0xcc => visitor.visit_i8(i8::try_from(u8::from_be_bytes(self.take_bytes()?))?), 0xd0 => visitor.visit_i8(i8::from_be_bytes(self.take_bytes()?)), + 0xe0..=0xff => { + let value = (ptype - 0xe0) as i8 - 32; + visitor.visit_i8(value) + } _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) } } @@ -1063,6 +1067,10 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { 0xd0 => visitor.visit_i16(i8::from_be_bytes(self.take_bytes()?) as i16), 0xcd => visitor.visit_i16(u16::from_be_bytes(self.take_bytes()?).try_into()?), 0xd1 => visitor.visit_i16(i16::from_be_bytes(self.take_bytes()?)), + 0xe0..=0xff => { + let value = (ptype - 0xe0) as i8 - 32; + visitor.visit_i16(value as i16) + } _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) } } @@ -1079,6 +1087,10 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { 0xd0 => visitor.visit_i32(i8::from_be_bytes(self.take_bytes()?) as i32), 0xd1 => visitor.visit_i32(i16::from_be_bytes(self.take_bytes()?) as i32), 0xd2 => visitor.visit_i32(i32::from_be_bytes(self.take_bytes()?)), + 0xe0..=0xff => { + let value = (ptype - 0xe0) as i8 - 32; + visitor.visit_i32(value as i32) + } _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) } } @@ -1097,6 +1109,10 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { 0xd1 => visitor.visit_i64(i16::from_be_bytes(self.take_bytes()?) as i64), 0xd2 => visitor.visit_i64(i32::from_be_bytes(self.take_bytes()?) as i64), 0xd3 => visitor.visit_i64(i64::from_be_bytes(self.take_bytes()?)), + 0xe0..=0xff => { + let value = (ptype - 0xe0) as i8 - 32; + visitor.visit_i64(value as i64) + } _ => Err(Self::Error::invalid_type(self.as_unexpected(ptype)?, &visitor)) } } @@ -1740,6 +1756,23 @@ mod tests { assert_eq!(deserialized, myval); } + #[test] + fn destream_i8_value() { + let myval = crate::Value::from(-2); + + let mut buffer = crate::Buffer::new(1024); + let myval = crate::Value::List(vec![ + myval + ]); + + buffer.resize_buffer(myval.estimate_size()).unwrap(); + myval.write_to(&mut buffer); + + let as_bin = new_preparsed(20, "binname", buffer.data_buffer); + let deserialized = crate::Value::deserialize(as_bin.clone()).unwrap(); + assert_eq!(deserialized, myval); + } + #[test] fn destream_bytes() { let myval = crate::Value::from(Vec::from([0_u8, 1_u8, 2_u8, 3_u8, 4_u8, 5_u8])); From 40276d72f0f76b8dff32aa9e312633903e873cb9 Mon Sep 17 00:00:00 2001 From: Caleb Moore Date: Thu, 2 May 2024 18:18:16 +1000 Subject: [PATCH 12/12] Skip enum layer when building values from CDT --- aerospike-core/src/derive/readable.rs | 546 +++++++------------------- aerospike-core/src/value.rs | 124 +++++- 2 files changed, 258 insertions(+), 412 deletions(-) diff --git a/aerospike-core/src/derive/readable.rs b/aerospike-core/src/derive/readable.rs index 2af7ef7e..65aadaaa 100644 --- a/aerospike-core/src/derive/readable.rs +++ b/aerospike-core/src/derive/readable.rs @@ -21,6 +21,7 @@ use serde::de::MapAccess; use serde::de::SeqAccess; use serde::de::VariantAccess; use serde::de::Visitor; +use serde::forward_to_deserialize_any; use serde::Deserialize; use serde::Deserializer; @@ -254,207 +255,6 @@ impl<'de> serde::de::Deserializer<'de> for BinsDeserializer { } -struct DeserializeStr<'a>(&'a str); -impl<'a, 'de> serde::de::Deserializer<'de> for DeserializeStr<'a> { - type Error = crate::errors::Error; - - fn deserialize_any(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_bool(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_i8(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_i16(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_i32(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_i64(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_u8(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_u16(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_u32(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_u64(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_f32(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_f64(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_char(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_str(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_string(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_bytes(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_byte_buf(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_option(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_unit(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_unit_struct( - self, - _name: &'static str, - visitor: V, - ) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_newtype_struct( - self, - _name: &'static str, - visitor: V, - ) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_seq(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_tuple(self, _len: usize, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_tuple_struct( - self, - _name: &'static str, - _len: usize, - visitor: V, - ) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_map(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_struct( - self, - _name: &'static str, - _fields: &'static [&'static str], - visitor: V, - ) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_enum( - self, - _name: &'static str, - _variants: &'static [&'static str], - visitor: V, - ) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_identifier(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } - - fn deserialize_ignored_any(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'de> { - visitor.visit_str(self.0) - } -} impl<'de> serde::de::MapAccess<'de> for BinsDeserializer { type Error = crate::errors::Error; @@ -464,7 +264,7 @@ impl<'de> serde::de::MapAccess<'de> for BinsDeserializer { K: serde::de::DeserializeSeed<'de> { if let Some(next_key) = self.bins.front() { - Some(seed.deserialize(DeserializeStr(next_key.name()?))).transpose() + Some(seed.deserialize(serde::de::value::StrDeserializer::::new(next_key.name()?))).transpose() } else { Ok(None) } @@ -518,12 +318,6 @@ impl<'de> serde::de::Deserializer<'de> for PreParsedValue { } } - fn deserialize_bool(self, visitor: V) -> std::result::Result - where - V: serde::de::Visitor<'de> { - self.deserialize_any(visitor) - } - fn deserialize_i8(self, visitor: V) -> std::result::Result where V: serde::de::Visitor<'de> { @@ -637,24 +431,6 @@ impl<'de> serde::de::Deserializer<'de> for PreParsedValue { } } - fn deserialize_char(self, visitor: V) -> std::result::Result - where - V: serde::de::Visitor<'de> { - self.deserialize_any(visitor) - } - - fn deserialize_str(self, visitor: V) -> std::result::Result - where - V: serde::de::Visitor<'de> { - self.deserialize_any(visitor) - } - - fn deserialize_string(self, visitor: V) -> std::result::Result - where - V: serde::de::Visitor<'de> { - self.deserialize_any(visitor) - } - fn deserialize_bytes(self, visitor: V) -> std::result::Result where V: serde::de::Visitor<'de> { @@ -687,72 +463,6 @@ impl<'de> serde::de::Deserializer<'de> for PreParsedValue { } } - fn deserialize_unit(self, visitor: V) -> std::result::Result - where - V: serde::de::Visitor<'de> { - self.deserialize_any(visitor) - } - - fn deserialize_unit_struct( - self, - _name: &'static str, - visitor: V, - ) -> std::result::Result - where - V: serde::de::Visitor<'de> { - self.deserialize_any(visitor) - } - - fn deserialize_newtype_struct( - self, - _name: &'static str, - visitor: V, - ) -> std::result::Result - where - V: serde::de::Visitor<'de> { - self.deserialize_any(visitor) - } - - fn deserialize_seq(self, visitor: V) -> std::result::Result - where - V: serde::de::Visitor<'de> { - self.deserialize_any(visitor) - } - - fn deserialize_tuple(self, _len: usize, visitor: V) -> std::result::Result - where - V: serde::de::Visitor<'de> { - self.deserialize_any(visitor) - } - - fn deserialize_tuple_struct( - self, - _name: &'static str, - _len: usize, - visitor: V, - ) -> std::result::Result - where - V: serde::de::Visitor<'de> { - self.deserialize_any(visitor) - } - - fn deserialize_map(self, visitor: V) -> std::result::Result - where - V: serde::de::Visitor<'de> { - self.deserialize_any(visitor) - } - - fn deserialize_struct( - self, - _name: &'static str, - _fields: &'static [&'static str], - visitor: V, - ) -> std::result::Result - where - V: serde::de::Visitor<'de> { - self.deserialize_any(visitor) - } - fn deserialize_enum( self, _name: &'static str, @@ -761,13 +471,7 @@ impl<'de> serde::de::Deserializer<'de> for PreParsedValue { ) -> std::result::Result where V: serde::de::Visitor<'de> { - visitor.visit_enum(EnumAdaptor{ particle_type: self.particle_type(), deserializer: self}) - } - - fn deserialize_identifier(self, visitor: V) -> std::result::Result - where - V: serde::de::Visitor<'de> { - self.deserialize_any(visitor) + visitor.visit_enum(EnumAdaptor{ particle_type: self.particle_type, deserializer: self}) } fn deserialize_ignored_any(self, visitor: V) -> std::result::Result @@ -775,10 +479,16 @@ impl<'de> serde::de::Deserializer<'de> for PreParsedValue { V: serde::de::Visitor<'de> { visitor.visit_none() } + + forward_to_deserialize_any! { + bool i128 u128 char str string + unit unit_struct newtype_struct seq tuple + tuple_struct map struct identifier + } } struct EnumAdaptor Deserializer<'a, Error = crate::Error>> { - particle_type: ParticleType, + particle_type: u8, deserializer: V, } @@ -791,21 +501,7 @@ impl<'de, Var: for<'a> Deserializer<'a, Error = crate::Error>> EnumAccess<'de> f fn variant_seed(self, seed: V) -> std::prelude::v1::Result<(V::Value, Self::Variant), Self::Error> where V: serde::de::DeserializeSeed<'de> { - let name = match self.particle_type { - ParticleType::NULL => "Nil", - ParticleType::INTEGER => "Int", - ParticleType::FLOAT => "Float", - ParticleType::STRING => "String", - ParticleType::BLOB => "Blob", - ParticleType::DIGEST => todo!(), - ParticleType::BOOL => "Bool", - ParticleType::HLL => "HLL", - ParticleType::MAP => "HashMap", - ParticleType::LIST => "List", - ParticleType::LDT => todo!(), - ParticleType::GEOJSON => "GeoJSON", - }; - let val = seed.deserialize(DeserializeStr(name))?; + let val = seed.deserialize(serde::de::value::U8Deserializer::::new(self.particle_type))?; Ok((val, self)) } } @@ -814,7 +510,7 @@ impl<'de, Var: for<'a> Deserializer<'a, Error = crate::Error>> VariantAccess<'de type Error = crate::errors::Error; fn unit_variant(self) -> std::prelude::v1::Result<(), Self::Error> { - if self.particle_type == ParticleType::NULL { + if self.particle_type == ParticleType::NULL as u8 { Ok(()) } else { Err(serde::de::Error::invalid_type(serde::de::Unexpected::NewtypeVariant, &"unit variant")) @@ -957,12 +653,12 @@ impl<'m> CDTDecoder<'m> { } } -impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { +impl<'de, 'm> Deserializer<'de> for CDTDecoder<'m> { type Error = crate::errors::Error; fn deserialize_any(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { fn deserialize_any_buffer<'l, 'm, V>(mut deserializer: CDTDecoder<'m>, visitor: V, count: usize) -> std::prelude::v1::Result where V: serde::de::Visitor<'l> { @@ -1032,7 +728,7 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { fn deserialize_bool(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { let ptype = self.take_byte()?; match ptype { 0xc2 => visitor.visit_bool(false), @@ -1043,7 +739,7 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { fn deserialize_i8(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { let ptype = self.take_byte()?; match ptype { 0x00..=0x7f => visitor.visit_i8(ptype as i8), @@ -1059,7 +755,7 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { fn deserialize_i16(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { let ptype = self.take_byte()?; match ptype { 0x00..=0x7f => visitor.visit_i16(ptype as i16), @@ -1077,7 +773,7 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { fn deserialize_i32(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { let ptype = self.take_byte()?; match ptype { 0x00..=0x7f => visitor.visit_i32(ptype as i32), @@ -1097,7 +793,7 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { fn deserialize_i64(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { let ptype = self.take_byte()?; match ptype { 0x00..=0x7f => visitor.visit_i64(ptype as i64), @@ -1119,7 +815,7 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { fn deserialize_u8(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { let ptype = self.take_byte()?; match ptype { 0x00..=0x7f => visitor.visit_u8(ptype), @@ -1131,7 +827,7 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { fn deserialize_u16(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { let ptype = self.take_byte()?; match ptype { 0x00..=0x7f => visitor.visit_i16(ptype as i16), @@ -1145,7 +841,7 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { fn deserialize_u32(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { let ptype = self.take_byte()?; match ptype { 0x00..=0x7f => visitor.visit_u64(ptype as u64), @@ -1161,7 +857,7 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { fn deserialize_u64(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { let ptype = self.take_byte()?; match ptype { 0x00..=0x7f => visitor.visit_u64(ptype as u64), @@ -1179,7 +875,7 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { fn deserialize_f32(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { let ptype = self.take_byte()?; match ptype { 0xca => visitor.visit_f32(f32::from_be_bytes(self.take_bytes()?)), @@ -1190,7 +886,7 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { fn deserialize_f64(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { let ptype = self.take_byte()?; match ptype { 0xca => visitor.visit_f64(f32::from_be_bytes(self.take_bytes()?).into()), @@ -1201,13 +897,13 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { fn deserialize_char(self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { self.deserialize_any(visitor) } fn deserialize_str(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { fn deserialize_any_buffer<'l, 'm, V>(mut deserializer: CDTDecoder<'m>, visitor: V, count: usize) -> std::prelude::v1::Result where V: serde::de::Visitor<'l> { @@ -1241,14 +937,14 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { fn deserialize_string(self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { self.deserialize_str(visitor) } // this is a very permissive handler that allows fn deserialize_bytes(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { fn deserialize_any_buffer<'l, 'm, V>(mut deserializer: CDTDecoder<'m>, visitor: V, count: usize) -> std::prelude::v1::Result where V: serde::de::Visitor<'l> { @@ -1293,13 +989,13 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { fn deserialize_byte_buf(self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { self.deserialize_bytes(visitor) } fn deserialize_option(self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { if self.0[*self.1] == 0xc0 { *self.1 += 1; visitor.visit_none() @@ -1308,35 +1004,9 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { } } - fn deserialize_unit(self, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'l> { - self.deserialize_any(visitor) - } - - fn deserialize_unit_struct( - self, - _name: &'static str, - visitor: V, - ) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'l> { - self.deserialize_any(visitor) - } - - fn deserialize_newtype_struct( - self, - _name: &'static str, - visitor: V, - ) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'l> { - self.deserialize_any(visitor) - } - fn deserialize_seq(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { let ptype = self.take_byte()?; match ptype { 0x90..=0x9f => visitor.visit_seq(CDTListOrMap::new((ptype & 0x0f) as usize, self.0, self.1, false)), @@ -1352,26 +1022,9 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { } } - fn deserialize_tuple(self, _len: usize, visitor: V) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'l> { - self.deserialize_any(visitor) - } - - fn deserialize_tuple_struct( - self, - _name: &'static str, - _len: usize, - visitor: V, - ) -> std::prelude::v1::Result - where - V: serde::de::Visitor<'l> { - self.deserialize_any(visitor) - } - fn deserialize_map(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { let ptype = self.take_byte()?; match ptype { 0x80..=0x8f => visitor.visit_map(CDTListOrMap::new((ptype & 0x0f) as usize, self.0, self.1, true)), @@ -1394,46 +1047,95 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { visitor: V, ) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { self.deserialize_map(visitor) } + // Specifically designed for parsing Value. Will take a shortcut to some internal types fn deserialize_enum( - self, + mut self, _name: &'static str, _variants: &'static [&'static str], visitor: V, ) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { + fn deserialize_any_buffer<'l, 'm, V: serde::de::Visitor<'l>>(mut deserializer: CDTDecoder<'m>, visitor: V, count: usize) -> std::prelude::v1::Result { + let particle_type = deserializer.take_byte()?; + let body = deserializer.take_nbyte(count - 1)?; + visitor.visit_enum(EnumAdaptor{ particle_type, deserializer: BytesOrStrDeserializer(body) }) + } - let ptype = self.0[*self.1]; - let particle_type = match ptype { - 0x00..=0x7f | 0xcc | 0xcd | 0xce | 0xcf | 0xd0 | 0xd1 | 0xd2 | 0xd3 | 0xe0..=0xff => ParticleType::INTEGER, - 0x80..=0x8f | 0xde | 0xdf => ParticleType::MAP, - 0x90..=0x9f | 0xdc | 0xdd => ParticleType::LIST, - 0xc0 => ParticleType::NULL, - 0xc2 | 0xc3 => ParticleType::BOOL, - // In blobs, the particle type is hiding just after the length - 0xa0..=0xbf => ParticleType::from(self.0[*self.1 + 1]), - 0xc4 | 0xd9 => ParticleType::from(self.0[*self.1 + 2]), - 0xc5 | 0xda => ParticleType::from(self.0[*self.1 + 3]), - 0xc6 | 0xdb => ParticleType::from(self.0[*self.1 + 5]), - 0xca | 0xcb => ParticleType::FLOAT, - _ => ParticleType::NULL - }; - visitor.visit_enum(EnumAdaptor{particle_type, deserializer: self}) + let ptype = self.take_byte()?; + match ptype { + 0x00..=0x7f => visitor.visit_u8(ptype as u8), + 0x80..=0x8f => visitor.visit_map(CDTListOrMap::new((ptype & 0x0f) as usize, self.0, self.1, true)), + 0x90..=0x9f => visitor.visit_seq(CDTListOrMap::new((ptype & 0x0f) as usize, self.0, self.1, false)), + 0xa0..=0xbf => deserialize_any_buffer(self, visitor, (ptype & 0x1f) as usize), + 0xc0 => visitor.visit_none(), + 0xc2 => visitor.visit_bool(false), + 0xc3 => visitor.visit_bool(true), + 0xc4 | 0xd9 => { + let count = u8::from_be_bytes(self.take_bytes()?); + deserialize_any_buffer(self, visitor, count as usize) + } + 0xc5 | 0xda => { + let count = u16::from_be_bytes(self.take_bytes()?); + deserialize_any_buffer(self, visitor, count as usize) + } + 0xc6 | 0xdb => { + let count = u32::from_be_bytes(self.take_bytes()?); + deserialize_any_buffer(self, visitor, count as usize) + } + 0xca => { + let body = self.take_bytes::<4>()?; + visitor.visit_enum(EnumAdaptor{ particle_type: ParticleType::FLOAT as u8, deserializer: BytesOrStrDeserializer(&body) }) + } + 0xcb => { + let body = self.take_bytes::<8>()?; + visitor.visit_enum(EnumAdaptor{ particle_type: ParticleType::FLOAT as u8, deserializer: BytesOrStrDeserializer(&body) }) + } + 0xcc => visitor.visit_u8(u8::from_be_bytes(self.take_bytes()?)), + 0xcd => visitor.visit_u16(u16::from_be_bytes(self.take_bytes()?)), + 0xce => visitor.visit_u32(u32::from_be_bytes(self.take_bytes()?)), + 0xcf => visitor.visit_u64(u64::from_be_bytes(self.take_bytes()?)), + 0xd0 => visitor.visit_i8(i8::from_be_bytes(self.take_bytes()?)), + 0xd1 => visitor.visit_i16(i16::from_be_bytes(self.take_bytes()?)), + 0xd2 => visitor.visit_i32(i32::from_be_bytes(self.take_bytes()?)), + 0xd3 => visitor.visit_i64(i64::from_be_bytes(self.take_bytes()?)), + 0xdc => { + let count = u16::from_be_bytes(self.take_bytes()?) as usize; + visitor.visit_seq(CDTListOrMap::new(count, self.0, self.1, false)) + } + 0xdd => { + let count = u32::from_be_bytes(self.take_bytes()?) as usize; + visitor.visit_seq(CDTListOrMap::new(count, self.0, self.1, false)) + } + 0xde => { + let count = u16::from_be_bytes(self.take_bytes()?) as usize; + visitor.visit_map(CDTListOrMap::new(count, self.0, self.1, true)) + } + 0xdf => { + let count = u32::from_be_bytes(self.take_bytes()?) as usize; + visitor.visit_map(CDTListOrMap::new(count, self.0, self.1, true)) + } + 0xe0..=0xff => { + let value = (ptype - 0xe0) as i8 - 32; + visitor.visit_i8(value) + } + _ => todo!() + } } fn deserialize_identifier(self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { self.deserialize_str(visitor) } fn deserialize_ignored_any(mut self, visitor: V) -> std::prelude::v1::Result where - V: serde::de::Visitor<'l> { + V: serde::de::Visitor<'de> { fn ignore_values<'l, 'm>(deserializer: CDTDecoder<'m>, entries: usize) { struct IgnoreVisitor; impl<'l> Visitor<'l> for IgnoreVisitor { @@ -1506,6 +1208,10 @@ impl<'l, 'm> Deserializer<'l> for CDTDecoder<'m> { } visitor.visit_none() } + + forward_to_deserialize_any! { + unit unit_struct newtype_struct tuple tuple_struct + } } impl<'l, 'm> MapAccess<'l> for CDTListOrMap<'m> { @@ -1602,6 +1308,40 @@ impl PreParsedValue { } } +/// A deserializer holding a `&[u8]`. +pub struct BytesOrStrDeserializer<'a>(&'a [u8]); + +impl<'de, 'a> Deserializer<'de> for BytesOrStrDeserializer<'a> { + type Error = Error; + + fn deserialize_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_bytes(self.0) + } + + fn deserialize_str(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_str(std::str::from_utf8(self.0)?) + } + + fn deserialize_string(self, visitor: V) -> std::prelude::v1::Result + where + V: Visitor<'de> { + self.deserialize_str(visitor) + } + + + forward_to_deserialize_any! { + bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char + bytes byte_buf option unit unit_struct newtype_struct seq tuple + tuple_struct map struct enum identifier ignored_any + } +} + #[cfg(test)] mod tests { @@ -1698,7 +1438,7 @@ mod tests { let mut buffer = crate::Buffer::new(1024); let myval = crate::Value::List(vec![ - crate::Value::Int(2), + crate::Value::UInt(2), crate::Value::String("Hello world".to_string()), ]); @@ -1708,7 +1448,7 @@ mod tests { let as_bin = new_preparsed(20, "binname", buffer.data_buffer); let deserialized = crate::Value::deserialize(as_bin.clone()).unwrap(); assert_eq!(deserialized, crate::Value::List(vec![ - crate::Value::Int(2), + crate::Value::UInt(2), crate::Value::String("Hello world".to_string()), ])); } diff --git a/aerospike-core/src/value.rs b/aerospike-core/src/value.rs index f3cc3988..d8d9601b 100644 --- a/aerospike-core/src/value.rs +++ b/aerospike-core/src/value.rs @@ -23,6 +23,7 @@ use byteorder::{ByteOrder, NetworkEndian}; use ripemd::digest::Digest; use ripemd::Ripemd160; +use serde::de::Error; use std::vec::Vec; @@ -197,7 +198,7 @@ impl fmt::Display for FloatValue { } /// Container for bin values stored in the Aerospike database. -#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum Value { /// Empty value. Nil, @@ -226,10 +227,7 @@ pub enum Value { String(String), /// Byte array value. - Blob( - #[serde(with = "serde_bytes")] - Vec - ), + Blob(Vec), /// List data type is an ordered collection of values. Lists can contain values of any /// supported data type. List data order is maintained on writes and reads. @@ -249,10 +247,118 @@ pub enum Value { GeoJSON(String), /// HLL value - HLL( - #[serde(with = "serde_bytes")] - Vec - ), + HLL(Vec), +} + +impl<'l> serde::de::Deserialize<'l> for Value { + fn deserialize(deserializer: D) -> StdResult + where + D: serde::Deserializer<'l> { + struct Visitor; + impl<'d> serde::de::Visitor<'d> for Visitor { + type Value = Value; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("enum") + } + + fn visit_none(self) -> StdResult { + Ok(Value::Nil) + } + + fn visit_string(self, v: String) -> StdResult { + Ok(Value::String(v)) + } + + fn visit_str(self, v: &str) -> StdResult { + Ok(Value::String(v.to_string())) + } + + fn visit_byte_buf(self, v: Vec) -> StdResult { + Ok(Value::Blob(v)) + } + + fn visit_bytes(self, v: &[u8]) -> StdResult { + Ok(Value::Blob(v.to_vec())) + } + + fn visit_bool(self, v: bool) -> StdResult { + Ok(Value::Bool(v)) + } + + fn visit_i64(self, v: i64) -> StdResult { + Ok(Value::Int(v)) + } + + fn visit_u64(self, v: u64) -> StdResult { + Ok(Value::UInt(v)) + } + + fn visit_seq>(self, mut seq: A) -> StdResult { + let mut values = Vec::with_capacity(seq.size_hint().unwrap_or_default()); + + while let Some(value) = seq.next_element()? { + values.push(value); + } + + Ok(Value::List(values)) + } + + fn visit_map>(self, mut map: A) -> StdResult { + let mut values = HashMap::with_capacity_and_hasher(map.size_hint().unwrap_or_default(), Default::default()); + + while let Some((key, value)) = map.next_entry()? { + values.insert(key, value); + } + + Ok(Value::HashMap(values)) + } + + + fn visit_enum>(self, data: A) -> StdResult { + use serde::de::VariantAccess; + let (key, variant_access) = data.variant::()?; + match ParticleType::from(key) { + ParticleType::NULL => { + variant_access.unit_variant()?; + Ok(Value::Nil) + }, + ParticleType::INTEGER => { + Ok(Value::Int(variant_access.newtype_variant()?)) + }, + ParticleType::FLOAT => { + Ok(Value::Float(variant_access.newtype_variant()?)) + }, + ParticleType::STRING => { + Ok(Value::String(variant_access.newtype_variant()?)) + }, + ParticleType::BLOB => { + Ok(Value::Blob(variant_access.newtype_variant::()?.into_vec())) + }, + ParticleType::BOOL => { + Ok(Value::Bool(variant_access.newtype_variant()?)) + }, + ParticleType::HLL => { + Ok(Value::HLL(variant_access.newtype_variant::()?.into_vec())) + }, + ParticleType::MAP => { + Ok(Value::HashMap(variant_access.newtype_variant()?)) + }, + ParticleType::LIST => { + Ok(Value::List(variant_access.newtype_variant()?)) + }, + ParticleType::LDT | ParticleType::DIGEST => { + Err(Error::custom(format_args!("unknown varient: {}", key))) + }, + ParticleType::GEOJSON => { + Ok(Value::GeoJSON(variant_access.newtype_variant()?)) + }, + } + } + } + + deserializer.deserialize_enum("Value", &[], Visitor) + } } #[allow(clippy::derive_hash_xor_eq)]