//! # Benchmarks inspired by `dmonad/crdt-benchmarks` //! //! Rust port of the benchmarks at [`dmonad/crdt-benchmarks`]. The results will //! not be directly comparable for a number of reasons, but at least we're //! measuring the same operations which should give an idea about overall //! performance characteristics. //! //! ## Currently implemented benchmarks //! //! - B1.1 (`append_chars`) //! - B1.2 (`insert_string`) //! - B1.3 (`prepend_chars`) //! - B1.4 (`insert_chars_at_random_positions`) //! - B1.5 (`insert_words_at_random_positions`) //! - B1.6 (`insert_and_delete_string`) //! - B1.7 (`insert_and_delete_strings_at_random_positions`) //! //! ## TODO //! //! - Measure memory usage //! - Add missing benchmarks //! //! [`dmonad/crdt-benchmarks`]: https://github.com/dmonad/crdt-benchmarks use chronofold::{Chronofold, LogIndex, Session}; use criterion::{criterion_group, criterion_main, Criterion}; use rand::{rngs::ThreadRng, seq::IteratorRandom, Rng}; use std::ops::RangeInclusive; use std::time::{Duration, Instant}; type CFold = Chronofold; type Sess<'a> = Session<'a, u8, char>; const N: usize = 6000; fn append_chars(c: &mut Criterion) { let mut rng = rand::thread_rng(); let input = random_string(&mut rng, N..=N); bench( c, &format!("[B1.1] Append {} characters", N), input.chars().collect(), |sess, c| { sess.push_back(c); }, assert_docs_equal(&input), ); } fn insert_string(c: &mut Criterion) { let mut rng = rand::thread_rng(); let input = random_string(&mut rng, N..=N); bench( c, &format!("[B1.2] Insert string of length {}", N), vec![input.clone()], |sess, s| { sess.extend(s.chars()); }, assert_docs_equal(&input), ); } fn prepend_chars(c: &mut Criterion) { let mut rng = rand::thread_rng(); let input = random_string(&mut rng, N..=N); bench( c, &format!("[B1.3] Prepend {} characters", N), input.chars().collect(), |sess, c| { sess.push_front(c); }, assert_docs_equal(&input.chars().rev().collect::()), ); } fn insert_chars_at_random_positions(c: &mut Criterion) { let mut rng = rand::thread_rng(); let mut expected = Vec::::new(); let positions_and_chars: Vec<_> = (0..N) .map(|i| { let pos = (0..=i).choose(&mut rng).unwrap(); let c = random_char(&mut rng); expected.insert(pos, c); (pos, c) }) .collect(); bench( c, &format!("[B1.4] Insert {} characters at random positions", N), positions_and_chars, |sess, (pos, c)| { let idx = if pos == 0 { LogIndex(0) // insert as first element } else { // This is expected to be really slow, as accessing a specific // position (as opposed to a log index) requires walking the // linked list up to that position. sess.as_ref().iter().nth(pos - 1).unwrap().1 }; sess.insert_after(idx, c); }, assert_docs_equal(&expected.into_iter().collect::()), ); } fn insert_words_at_random_positions(c: &mut Criterion) { let mut rng = rand::thread_rng(); let mut expected = Vec::::new(); let positions_and_words: Vec<_> = (0..N) .map(|_i| { let pos = (0..=expected.len()).choose(&mut rng).unwrap(); let s = random_string(&mut rng, 2..=10); expected.splice(pos..pos, s.chars()); (pos, s) }) .collect(); bench( c, &format!("[B1.5] Insert {} words at random positions", N), positions_and_words, |sess, (pos, s)| { // This is expected to be really slow, as accessing a specific // position (as opposed to a log index) requires walking the // linked list up to that position. let e = sess.as_ref().iter().nth(pos); match e { Some((_, idx)) => sess.splice(idx..idx, s.chars()), None => sess.extend(s.chars()), }; }, assert_docs_equal(&expected.into_iter().collect::()), ); } fn insert_and_delete_string(c: &mut Criterion) { let mut rng = rand::thread_rng(); let s = random_string(&mut rng, N..=N); bench( c, &format!("[B1.6] Insert string of length {}, then delete it", N), vec![s], |sess, s| { let last_idx = sess.extend(s.chars()).unwrap(); sess.splice(LogIndex(0)..=last_idx, "".chars()); }, assert_docs_equal(""), ); } fn insert_and_delete_strings_at_random_positions(c: &mut Criterion) { let mut rng = rand::thread_rng(); let mut expected = Vec::::new(); let input: Vec<_> = (0..N) .map(|_i| { let pos = (0..=expected.len()).choose(&mut rng).unwrap(); if expected.len() == pos || rng.gen() { // Insert let s = random_string(&mut rng, 2..=10); expected.splice(pos..pos, s.chars()); (pos, pos, s) } else { // Delete let del_count = (1..=usize::min(9, expected.len() - pos)) .choose(&mut rng) .unwrap(); expected.splice(pos..pos + del_count, "".chars()); (pos, pos + del_count, "".to_owned()) } }) .collect(); bench( c, &format!("[B1.7] Insert/Delete {} strings at random positions", N), input, |sess, (start, end, s)| { let a = sess.as_ref().iter().nth(start).map(|(_, idx)| idx); let b = sess.as_ref().iter().nth(end).map(|(_, idx)| idx); match (a, b) { (None, _) => sess.extend(s.chars()), (Some(start_idx), None) => sess.splice(start_idx.., s.chars()), (Some(start_idx), Some(end_idx)) => sess.splice(start_idx..end_idx, s.chars()), }; }, assert_docs_equal(&expected.iter().collect::()), ); } fn bench, T) + Copy, G: Fn(&CFold, &CFold) + Copy>( c: &mut Criterion, name: &str, input: Vec, apply_change: F, check: G, ) { c.bench_function(&format!("{} (time)", name), |b| { b.iter_custom(|iters| { let mut elapsed = Duration::new(0, 0); for _i in 0..iters { elapsed += measure_time(input.clone(), apply_change, check); } elapsed }); }); measure_space(name, input, apply_change); } fn measure_time, T), G: Fn(&CFold, &CFold)>( input: Vec, apply_change: F, check: G, ) -> Duration { let mut doc1 = CFold::default(); let mut doc2 = CFold::default(); let start = Instant::now(); for d in input.into_iter() { let mut session = doc1.session(1); apply_change(&mut session, d); for op in session.iter_ops() { let _ = doc2.apply(op.cloned()); } } let elapsed = start.elapsed(); check(&doc1, &doc2); elapsed } fn measure_space, T)>(name: &str, input: Vec, apply_change: F) { let mut cfold = CFold::default(); let mut total_ops_bytes = 0; let n = input.len(); for d in input.into_iter() { let mut session = cfold.session(1); apply_change(&mut session, d); for op in session.iter_ops::<&char>() { total_ops_bytes += serde_json::to_vec(&op).unwrap().len(); } } let avg_update_size = total_ops_bytes / n; let doc_size = serde_json::to_vec(&cfold).unwrap().len(); println!("{} (avgUpdateSize): {} bytes", name, avg_update_size); println!("{} (docSize): {} bytes", name, doc_size); println!(); } fn assert_docs_equal(expected: &str) -> impl Fn(&CFold, &CFold) + Copy + '_ { move |doc1, doc2| { let text1 = format!("{}", doc1); let text2 = format!("{}", doc2); assert_eq!(text1, text2); assert_eq!(expected, text1); } } fn random_char(rng: &mut ThreadRng) -> char { "abcdefghijklmnopqrstuvwxyz".chars().choose(rng).unwrap() } fn random_string(rng: &mut ThreadRng, len_range: RangeInclusive) -> String { let len = len_range.choose(rng).unwrap(); (0..len).map(|_| random_char(rng)).collect() } criterion_group!( name = benches; config = Criterion::default(); targets = append_chars, insert_string, prepend_chars, insert_chars_at_random_positions, insert_words_at_random_positions, insert_and_delete_string, insert_and_delete_strings_at_random_positions, ); criterion_main!(benches);