use std::io::Cursor; use bytesize::ByteSize; use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput}; use deepsize::DeepSizeOf; use lil::{compress_bytes, decompress_bytes}; use litl::MeasureValue; pub fn convert_json(c: &mut Criterion) { let tests = [ ("gists", "/Users/anselm/Downloads/gists.json", None), ( "citm_catalog", "/Users/anselm/Downloads/citm_catalog.json", None, ), ("canada", "/Users/anselm/Downloads/canada.json", None), ("twitter", "/Users/anselm/Downloads/twitter.json", None), // ( // "citylots", // "/Users/anselm/Downloads/citylots.json", // Some(10), // ), ]; for (test_name, test_file, samples) in tests { let mut group = c.benchmark_group(test_name); if let Some(samples) = samples { group.sample_size(samples); } let json_str = std::fs::read_to_string(test_file).unwrap(); let value: serde_json::Value = serde_json::from_str(&json_str).unwrap(); let val: litl::Val = litl::val_from_str(&json_str).unwrap(); let value_bytes = litl::to_vec(&value).unwrap(); let val_bytes = litl::to_vec(&val).unwrap(); let compressed = compress_bytes(&value_bytes); let decompressed = decompress_bytes(&compressed); let mut json_zstd_compressed = Cursor::new(vec![]); zstd::stream::copy_encode( Cursor::new(json_str.as_bytes()), &mut json_zstd_compressed, 3, ) .unwrap(); let mut stream_zstd_compressed = Cursor::new(vec![]); zstd::stream::copy_encode(Cursor::new(&value_bytes), &mut stream_zstd_compressed, 3) .unwrap(); let stream_zstd_compressed = stream_zstd_compressed.into_inner(); assert_eq!(value_bytes, decompressed); group.throughput(Throughput::Bytes(json_str.len() as u64)); group.bench_function("parse_to_value", |b| { b.iter(|| black_box(serde_json::from_str::(&json_str).unwrap())); }); group.bench_function("parse_to_val", |b| { b.iter(|| black_box(litl::val_from_str(&json_str).unwrap())); }); group.bench_function("value_to_stream", |b| { b.iter(|| black_box(litl::to_vec(&value))); }); group.bench_function("val_to_stream", |b| { b.iter(|| black_box(litl::to_vec(&val))); }); group.bench_function("to_compressed_stream", |b| { b.iter(|| black_box(compress_bytes(&value_bytes))); }); // group.bench_function("zstd compress json", |b| { // b.iter(|| { // let mut zstd_compressed = Cursor::new(vec![]); // zstd::stream::copy_encode( // Cursor::new(json_str.as_bytes()), // &mut zstd_compressed, // 3, // ) // .unwrap(); // black_box(zstd_compressed) // }); // }); group.bench_function("zstd compress stream", |b| { b.iter(|| { let mut zstd_compressed = Cursor::new(vec![]); zstd::stream::copy_encode(Cursor::new(&value_bytes), &mut zstd_compressed, 3) .unwrap(); black_box(zstd_compressed) }); }); // group.bench_function("write_litl", |b| { // b.iter(|| black_box(Litl::write_all(vec![&litl_value]))); // }); // let litl_bytes = Litl::write_all(vec![&litl_value]); group.bench_function("to_decompressed_stream", |b| { b.iter(|| black_box(decompress_bytes(&compressed))); }); group.bench_function("zstd decompress stream", |b| { b.iter(|| { let mut zstd_decompressed = Cursor::new(vec![]); zstd::stream::copy_decode( Cursor::new(&stream_zstd_compressed), &mut zstd_decompressed, ) .unwrap(); black_box(zstd_decompressed) }) }); // group.bench_function("read_litl", |b| { // b.iter(|| black_box(Litl::read_all(&litl_stream))); // }); // assert_eq!(Litl::read_one(&litl_bytes), litl_value); // println!( // "JSON ZSTD: {}: {} vs {}, ratio: {}", // test_name, // ByteSize(json_str.len() as u64), // ByteSize(json_zstd_compressed.position() as u64), // json_zstd_compressed.position() as f64 / json_str.len() as f64 // ); println!( "Mem Value: {}", ByteSize(DeepSizeOf::deep_size_of(&MeasureValue(value.clone())) as u64) ); println!( "Mem Val: {}", ByteSize(DeepSizeOf::deep_size_of(&val) as u64) ); println!( "Stream ZSTD: {}: {} vs {}, ratio: {}", test_name, ByteSize(value_bytes.len() as u64), ByteSize(stream_zstd_compressed.len() as u64), stream_zstd_compressed.len() as f64 / value_bytes.len() as f64 ); println!( "Compressed stream: {}: {} vs {}, ratio: {}", test_name, ByteSize(value_bytes.len() as u64), ByteSize(compressed.len() as u64), compressed.len() as f64 / value_bytes.len() as f64 ); } } criterion_group!(benches, convert_json); criterion_main!(benches);