Advanced performance regression detection with statistical analysis and trend identification.

use benchkit::prelude::*;
use std::collections::HashMap;
use std::time::{ Duration, SystemTime };

fn regression_analysis_example() -> Result< (), Box< dyn std::error::Error > > 
{
    // Current benchmark results
    let mut current_results = HashMap::new();
    let current_times = vec![ Duration::from_micros( 85 ), Duration::from_micros( 88 ), Duration::from_micros( 82 ) ];
    current_results.insert( "fast_sort".to_string(), BenchmarkResult::new( "fast_sort", current_times ) );
    
    // Historical baseline data
    let mut baseline_data = HashMap::new();
    let baseline_times = vec![ Duration::from_micros( 110 ), Duration::from_micros( 115 ), Duration::from_micros( 108 ) ];
    baseline_data.insert( "fast_sort".to_string(), BenchmarkResult::new( "fast_sort", baseline_times ) );
    
    let historical = HistoricalResults::new().with_baseline( baseline_data );
    
    // Configure regression analyzer
    let analyzer = RegressionAnalyzer::new()
        .with_baseline_strategy( BaselineStrategy::FixedBaseline )
        .with_significance_threshold( 0.05 )  // 5% significance level
        .with_trend_window( 5 );
    
    // Perform regression analysis
    let regression_report = analyzer.analyze( &current_results, &historical );
    
    // Check results
    if regression_report.has_significant_changes() {
        println!( "📊 Significant performance changes detected!" );
        
        if let Some( trend ) = regression_report.get_trend_for( "fast_sort" ) {
            match trend {
                PerformanceTrend::Improving => println!( "🟢 Performance improved!" ),
                PerformanceTrend::Degrading => println!( "🔴 Performance regression detected!" ),
                PerformanceTrend::Stable => println!( "🟡 Performance remains stable" ),
            }
        }
        
        // Generate professional markdown report
        let markdown_report = regression_report.format_markdown();
        println!( "{}", markdown_report );
    }
    
    Ok(())
}

Key Features:

• Three Baseline Strategies: Fixed baseline, rolling average, and previous run comparison
• Statistical Significance: Configurable thresholds with proper statistical testing
• Trend Detection: Automatic identification of improving, degrading, or stable performance
• Professional Reports: Publication-quality markdown with statistical analysis
• CI/CD Integration: Automated regression detection for deployment pipelines
• Historical Data Management: Long-term performance tracking with quality validation

Use Cases:

• Automated performance regression detection in CI/CD pipelines
• Long-term performance monitoring and trend analysis
• Code optimization validation with statistical confidence
• Production deployment gates with zero-regression tolerance
• Performance documentation with automated updates

Coordinate multiple markdown section updates atomically - either all succeed or none are modified.

use benchkit::prelude::*;

fn update_markdown_atomically() -> Result< (), Box< dyn std::error::Error > > 
{
    let performance_markdown = "## Performance Results\n\nFast!";
    let memory_markdown = "## Memory Usage\n\nLow!";
    let cpu_markdown = "## CPU Usage\n\nOptimal!";
    
    // Update multiple sections atomically
    let chain = MarkdownUpdateChain::new("readme.md")?
        .add_section("Performance Benchmarks", performance_markdown)
        .add_section("Memory Analysis", memory_markdown)
        .add_section("CPU Profiling", cpu_markdown);

    // Validate all sections before any updates
    let conflicts = chain.check_all_conflicts()?;
    if !conflicts.is_empty() {
        return Err(format!("Section conflicts detected: {:?}", conflicts).into());
    }

    // Atomic update - either all succeed or all fail
    chain.execute()?;
    Ok(())
}

Key Features:

• Atomic Operations: Either all sections update successfully or none are modified
• Conflict Detection: Validates all sections exist and are unambiguous before any changes
• Automatic Rollback: Failed operations restore original file state
• Reduced I/O: Single read and write operation instead of multiple file accesses
• Error Recovery: Comprehensive error handling with detailed diagnostics

Use Cases:

• Multi-section benchmark reports that must stay synchronized
• CI/CD pipelines requiring consistent documentation updates
• Coordinated updates across large documentation projects
• Production deployments where partial updates would be problematic

Advanced Example:

use benchkit::prelude::*;

fn complex_update_example() -> Result< (), Box< dyn std::error::Error > > 
{
    let performance_report = "Performance analysis results";
    let memory_report = "Memory usage analysis";
    let comparison_report = "Algorithm comparison data";
    let validation_report = "Quality assessment report";
    
    // Complex coordinated update across multiple report types
    let chain = MarkdownUpdateChain::new("PROJECT_BENCHMARKS.md")?
        .add_section("Performance Analysis", performance_report)
        .add_section("Memory Usage Analysis", memory_report)
        .add_section("Algorithm Comparison", comparison_report)
        .add_section("Quality Assessment", validation_report);

    // Validate everything before committing any changes
    match chain.check_all_conflicts() {
        Ok(conflicts) if conflicts.is_empty() => {
            println!("✅ All {} sections validated", chain.len());
            chain.execute()?;
        },
        Ok(conflicts) => {
            eprintln!("⚠️ Conflicts: {:?}", conflicts);
            // Handle conflicts or use more specific section names
        },
        Err(e) => eprintln!("❌ Validation failed: {}", e),
    }
    Ok(())
}

Generate standardized, publication-quality reports with full statistical analysis and customizable sections.

use benchkit::prelude::*;
use std::collections::HashMap;

fn generate_reports() -> Result< (), Box< dyn std::error::Error > > 
{
    let results = HashMap::new();
    let comparison_results = HashMap::new();
    
    // Comprehensive performance analysis
    let performance_template = PerformanceReport::new()
        .title("Algorithm Performance Analysis")
        .add_context("Comparing sequential vs parallel processing approaches")
        .include_statistical_analysis(true)
        .include_regression_analysis(true)
        .add_custom_section(CustomSection::new(
            "Implementation Notes", 
            "Detailed implementation considerations and optimizations applied"
        ));

    let performance_report = performance_template.generate(&results)?;

    // A/B testing comparison with statistical significance
    let comparison_template = ComparisonReport::new()
        .title("Sequential vs Parallel Processing Comparison")
        .baseline("Sequential Processing")
        .candidate("Parallel Processing") 
        .significance_threshold(0.01)     // 1% statistical significance
        .practical_significance_threshold(0.05);  // 5% practical significance

    let comparison_report = comparison_template.generate(&comparison_results)?;
    Ok(())
}

Performance Report Features:

• Executive Summary: Key metrics and performance indicators
• Statistical Analysis: Confidence intervals, coefficient of variation, reliability assessment
• Performance Tables: Sorted results with throughput, latency, and quality indicators
• Custom Sections: Domain-specific analysis and recommendations
• Professional Formatting: Publication-ready markdown with proper statistical notation

Comparison Report Features:

• Significance Testing: Both statistical and practical significance analysis
• Confidence Intervals: 95% CI analysis with overlap detection
• Performance Ratios: Clear improvement/regression percentages
• Reliability Assessment: Quality validation for both baseline and candidate
• Decision Support: Clear recommendations based on statistical analysis

Advanced Template Composition:

use benchkit::prelude::*;

fn create_enterprise_template() -> PerformanceReport 
{
    // Create domain-specific template with multiple custom sections
    let enterprise_template = PerformanceReport::new()
        .title("Enterprise Algorithm Performance Audit")
        .add_context("Monthly performance review for production trading systems")
        .include_statistical_analysis(true)
        .add_custom_section(CustomSection::new(
            "Risk Assessment",
            r#"### Performance Risk Analysis
            
    | Algorithm | Latency Risk | Throughput Risk | Stability | Overall |
    |-----------|-------------|-----------------|-----------|----------|
    | Current   | 🟢 Low     | 🟡 Medium      | 🟢 Low   | 🟡 Medium |
    | Proposed  | 🟢 Low     | 🟢 Low        | 🟢 Low   | 🟢 Low    |"#
        ))
        .add_custom_section(CustomSection::new(
            "Business Impact",
            r#"### Projected Business Impact

    - **Latency Improvement**: 15% faster response times
    - **Throughput Increase**: +2,000 req/sec capacity
    - **Cost Reduction**: -$50K/month in infrastructure
    - **SLA Compliance**: 99.9% → 99.99% uptime"#
        ));
    enterprise_template
}

Comprehensive quality assessment system with configurable criteria and automatic reliability analysis.

use benchkit::prelude::*;
use std::collections::HashMap;

fn validate_benchmark_results() 
{
    let results = HashMap::new();
    
    // Configure validator for your specific requirements
    let validator = BenchmarkValidator::new()
        .min_samples(20)                           // Require 20+ measurements
        .max_coefficient_variation(0.10)           // 10% maximum variability
        .require_warmup(true)                      // Detect warm-up periods
        .max_time_ratio(3.0)                       // 3x max/min ratio
        .min_measurement_time(Duration::from_micros(50)); // 50μs minimum duration

    // Validate all results with detailed analysis
    let validated_results = ValidatedResults::new(results, validator);

    println!("Reliability: {:.1}%", validated_results.reliability_rate());

    // Get detailed quality warnings
    if let Some(warnings) = validated_results.reliability_warnings() {
        println!("⚠️ Quality Issues Detected:");
        for warning in warnings {
            println!("  - {}", warning);
        }
    }

    // Work with only statistically reliable results
    let reliable_only = validated_results.reliable_results();
    println!("Using {}/{} reliable benchmarks for analysis", 
             reliable_only.len(), validated_results.results.len());
}

Validation Criteria:

• Sample Size: Ensure sufficient measurements for statistical power
• Variability: Detect high coefficient of variation indicating noise
• Measurement Duration: Flag measurements that may be timing-resolution limited
• Performance Range: Identify outliers and wide performance distributions
• Warm-up Detection: Verify proper system warm-up for consistent results

Warning Types:

• InsufficientSamples: Too few measurements for reliable statistics
• HighVariability: Coefficient of variation exceeds threshold
• ShortMeasurementTime: Measurements may be affected by timer resolution
• WidePerformanceRange: Large ratio between fastest/slowest measurements
• NoWarmup: Missing warm-up period may indicate measurement issues

Domain-Specific Validation:

use benchkit::prelude::*;
use std::collections::HashMap;

fn domain_specific_validation() 
{
    let results = HashMap::new();
    
    // Real-time systems validation (very strict)
    let realtime_validator = BenchmarkValidator::new()
        .min_samples(50)
        .max_coefficient_variation(0.02)  // 2% maximum
        .max_time_ratio(1.5);             // Very tight timing

    // Interactive systems validation (balanced)  
    let interactive_validator = BenchmarkValidator::new()
        .min_samples(15)
        .max_coefficient_variation(0.15)  // 15% acceptable
        .require_warmup(false);           // Interactive may not show warmup

    // Batch processing validation (lenient)
    let batch_validator = BenchmarkValidator::new()
        .min_samples(10)
        .max_coefficient_variation(0.25)  // 25% acceptable  
        .max_time_ratio(5.0);             // Allow more variation

    // Apply appropriate validator for your domain
    let domain_results = ValidatedResults::new(results, realtime_validator);
}

Quality Reporting:

use benchkit::prelude::*;
use std::collections::HashMap;

fn generate_validation_report() 
{
    let results = HashMap::new();
    let validator = BenchmarkValidator::new();
    
    // Generate comprehensive validation report
    let validation_report = validator.generate_validation_report(&results);

    // Validation report includes:
    // - Summary statistics and reliability rates
    // - Detailed warnings with improvement recommendations  
    // - Validation criteria documentation
    // - Quality assessment for each benchmark
    // - Actionable steps to improve measurement quality

    println!("{}", validation_report);
}

Comprehensive examples demonstrating real-world usage patterns and advanced integration scenarios.

Development Workflow Integration:

use benchkit::prelude::*;

// Complete development cycle: benchmark → validate → document → commit
fn development_workflow() -> Result< (), Box< dyn std::error::Error > > 
{
    // Mock implementations for doc test
    fn quicksort_implementation() {}
    fn mergesort_implementation() {}
    
    // 1. Run benchmarks
    let mut suite = BenchmarkSuite::new("Algorithm Performance");
    suite.benchmark("quicksort", || quicksort_implementation());
    suite.benchmark("mergesort", || mergesort_implementation());
    let results = suite.run_all();
    
    // 2. Validate quality
    let validator = BenchmarkValidator::new()
        .min_samples(15)
        .max_coefficient_variation(0.15);
    let validated_results = ValidatedResults::new(results.results, validator);
    
    if validated_results.reliability_rate() < 80.0 {
        return Err("Benchmark quality insufficient for analysis".into());
    }
    
    // 3. Generate professional report
    let template = PerformanceReport::new()
        .title("Algorithm Performance Analysis")
        .include_statistical_analysis(true)
        .add_custom_section(CustomSection::new(
            "Development Notes",
            "Analysis conducted during algorithm optimization phase"
        ));
    
    let report = template.generate(&validated_results.results)?;
    
    // 4. Update documentation atomically
    let chain = MarkdownUpdateChain::new("README.md")?
        .add_section("Performance Analysis", report)
        .add_section("Quality Assessment", validated_results.validation_report());
    
    chain.execute()?;
    println!("✅ Development documentation updated successfully");
    
    Ok(())
}

CI/CD Pipeline Integration:

use benchkit::prelude::*;
use std::collections::HashMap;

// Automated performance regression detection
fn cicd_performance_check(baseline_results: HashMap<String, BenchmarkResult>, 
                          pr_results: HashMap<String, BenchmarkResult>) -> Result< bool, Box< dyn std::error::Error > > {
    // Validate both result sets
    let validator = BenchmarkValidator::new().require_warmup(false);
    let baseline_validated = ValidatedResults::new(baseline_results.clone(), validator.clone());
    let pr_validated = ValidatedResults::new(pr_results.clone(), validator);
    
    // Require high quality for regression analysis
    if baseline_validated.reliability_rate() < 90.0 || pr_validated.reliability_rate() < 90.0 {
        println!("❌ BLOCK: Insufficient benchmark quality for regression analysis");
        return Ok(false);
    }
    
    // Compare performance for regression detection
    let comparison = ComparisonReport::new()
        .title("Performance Regression Analysis")
        .baseline("baseline_version")
        .candidate("pr_version")
        .practical_significance_threshold(0.05);  // 5% regression threshold
    
    // Create combined results for comparison
    let mut combined = HashMap::new();
    combined.insert("baseline_version".to_string(), 
                   baseline_results.values().next().unwrap().clone());
    combined.insert("pr_version".to_string(), 
                   pr_results.values().next().unwrap().clone());
    
    let regression_report = comparison.generate(&combined)?;
    
    // Check for regressions
    let has_regression = regression_report.contains("slower");
    
    if has_regression {
        println!("❌ BLOCK: Performance regression detected");
        // Save detailed report for review
        std::fs::write("regression_analysis.md", regression_report)?;
        Ok(false)
    } else {
        println!("✅ ALLOW: No performance regressions detected");
        Ok(true)
    }
}

Multi-Project Coordination:

use benchkit::prelude::*;
use std::collections::HashMap;

// Coordinate benchmark updates across multiple related projects
fn coordinate_multi_project_benchmarks() -> Result< (), Box< dyn std::error::Error > > 
{
    let projects = vec!["web-api", "batch-processor", "realtime-analyzer"];
    let mut all_results = HashMap::new();
    
    // Collect results from all projects  
    for project in &projects {
        let project_results = run_project_benchmarks(project)?;
        all_results.extend(project_results);
    }
    
    // Cross-project validation with lenient criteria
    let validator = BenchmarkValidator::new()
        .max_coefficient_variation(0.25)  // Different environments have more noise
        .require_warmup(false);
    
    let cross_project_validated = ValidatedResults::new(all_results.clone(), validator);
    
    // Generate consolidated impact analysis
    let impact_template = PerformanceReport::new()
        .title("Cross-Project Performance Impact Analysis")
        .add_context("Shared library upgrade impact across all dependent projects")
        .include_statistical_analysis(true)
        .add_custom_section(CustomSection::new(
            "Project Impact Summary",
            format_project_impact_analysis(&projects, &all_results)
        ));
    
    let impact_report = impact_template.generate(&all_results)?;
    
    // Update shared documentation
    let shared_chain = MarkdownUpdateChain::new("SHARED_LIBRARY_IMPACT.md")?
        .add_section("Current Impact Analysis", &impact_report)
        .add_section("Quality Assessment", &cross_project_validated.validation_report());
    
    shared_chain.execute()?;
    
    // Notify project maintainers
    notify_project_teams(&projects, &impact_report)?;
    
    Ok(())
}

// Helper functions for the example
fn run_project_benchmarks(_project: &str) -> Result< HashMap< String, BenchmarkResult >, Box< dyn std::error::Error > > 
{
    // Mock implementation for doc test
    Ok(HashMap::new())
}

fn format_project_impact_analysis(_projects: &[&str], _results: &HashMap< String, BenchmarkResult >) -> String 
{
    // Mock implementation for doc test  
    "Impact analysis summary".to_string()
}

fn notify_project_teams(_projects: &[&str], _report: &str) -> Result< (), Box< dyn std::error::Error > > 
{
    // Mock implementation for doc test
    Ok(())
}

At its heart, benchkit provides simple and accurate measurement primitives.

use benchkit::prelude::*;

// A robust measurement with multiple iterations and statistical cleanup.
let result = bench_function
(
  "summation_1000",
  ||
  {
    ( 0..1000 ).fold( 0, | acc, x | acc + x )
  }
);
println!( "Avg time: {:.2?}", result.mean_time() );
println!( "Throughput: {:.0} ops/sec", result.operations_per_second() );

// Track memory usage patterns alongside timing.
let memory_benchmark = MemoryBenchmark::new( "allocation_test" );
let ( timing, memory_stats ) = memory_benchmark.run_with_tracking
(
  10,
  ||
  {
    let data = vec![ 0u8; 1024 ];
    memory_benchmark.tracker.record_allocation( 1024 );
    std::hint::black_box( data );
  }
);
println!( "Peak memory usage: {} bytes", memory_stats.peak_usage );

Turn raw numbers into actionable insights.

use benchkit::prelude::*;

// Compare multiple implementations to find the best one.
let report = ComparativeAnalysis::new( "Hashing" )
.algorithm( "fnv", || { /* ... */ } )
.algorithm( "siphash", || { /* ... */ } )
.run();

if let Some( ( fastest_name, _ ) ) = report.fastest()
{
  println!( "Fastest algorithm: {}", fastest_name );
}

// Example benchmark results
let result_a = bench_function( "test_a", || { /* ... */ } );
let result_b = bench_function( "test_b", || { /* ... */ } );

// Compare two benchmark results
let comparison = result_a.compare( &result_b );
if comparison.is_improvement()
{
  println!( "Performance improved!" );
}

Stop writing boilerplate to create test data. benchkit provides generators for common scenarios.

use benchkit::prelude::*;

// Generate a comma-separated list of 100 items.
let list_data = generate_list_data( DataSize::Medium );

// Generate realistic unilang command strings for parser benchmarking.
let command_generator = DataGenerator::new()
.complexity( DataComplexity::Complex );
let commands = command_generator.generate_unilang_commands( 10 );

// Create reproducible data with a specific seed.
let mut seeded_gen = SeededGenerator::new( 42 );
let random_data = seeded_gen.random_string( 1024 );

The "documentation-first" philosophy is enabled by powerful report generation and file updating tools.

use benchkit::prelude::*;

fn main() -> Result< (), Box< dyn std::error::Error > >
{
  let mut suite = BenchmarkSuite::new( "api_performance" );
  suite.benchmark( "get_user", || { /* ... */ } );
  suite.benchmark( "create_user", || { /* ... */ } );
  let results = suite.run_analysis();

  // Generate a markdown report from the results.
  let markdown_report = results.generate_markdown_report().generate();

  // Automatically update the "## Performance" section of a file.
  let updater = MarkdownUpdater::new( "readme.md", "Performance" )?;
  updater.update_section( &markdown_report )?;
  
  Ok( () )
}