// Copyright ยฉ 2024 StaticDataGen. All rights reserved. // SPDX-License-Identifier: Apache-2.0 OR MIT //! # StaticDataGen Robots.txt Examples //! //! This program demonstrates the usage of robots.txt file generation //! in the StaticDataGen library, showing various ways to create and //! manage website crawling directives and sitemap references. use staticdatagen::models::data::TxtData; use staticdatagen::modules::robots::{ create_txt_data, generate_txt_content, }; use std::collections::HashMap; /// Entry point for the StaticDataGen Robots.txt Examples program. /// /// Demonstrates various robots.txt file generation scenarios and shows /// different ways to control search engine crawling behavior. /// /// # Errors /// /// Returns a `Result` containing a `Box` if any error /// occurs during the execution of the examples. pub fn main() -> Result<(), Box> { println!("\n๐Ÿงช StaticDataGen Robots.txt Examples\n"); basic_robots_example()?; custom_directives_example()?; sitemap_reference_example()?; subdirectory_rules_example()?; multi_sitemap_example()?; crawl_delay_example()?; user_agent_example()?; validation_example()?; println!("\n๐ŸŽ‰ All robots.txt examples completed successfully!"); Ok(()) } /// Demonstrates basic robots.txt file creation. fn basic_robots_example() -> Result<(), Box> { println!("๐Ÿฆ€ Basic Robots.txt Example"); println!("---------------------------------------------"); let txt_data = TxtData::new("https://example.com".to_string()); match txt_data.validate() { Ok(_) => { let content = generate_txt_content(&txt_data); println!(" โœ… Generated robots.txt content:"); println!("{}", content); } Err(e) => println!(" โŒ Validation error: {:?}", e), } Ok(()) } /// Demonstrates custom crawling directives. fn custom_directives_example() -> Result<(), Box> { println!("\n๐Ÿฆ€ Custom Directives Example"); println!("---------------------------------------------"); let mut metadata = HashMap::new(); let _ = metadata.insert( "permalink".to_string(), "https://example.com".to_string(), ); let _ = metadata.insert("allow".to_string(), "/blog/".to_string()); let _ = metadata.insert("disallow".to_string(), "/admin/".to_string()); let txt_data = create_txt_data(&metadata); match txt_data.validate() { Ok(_) => { println!(" โœ… Custom directives added:"); println!(" ๐ŸŒ Site: {}", txt_data.permalink); let content = generate_txt_content(&txt_data); println!(" ๐Ÿ“ Content:"); println!("{}", content); } Err(e) => println!(" โŒ Validation error: {:?}", e), } Ok(()) } /// Demonstrates sitemap reference addition. fn sitemap_reference_example() -> Result<(), Box> { println!("\n๐Ÿฆ€ Sitemap Reference Example"); println!("---------------------------------------------"); let mut metadata = HashMap::new(); let _ = metadata.insert( "permalink".to_string(), "https://example.com".to_string(), ); let _ = metadata.insert( "sitemap".to_string(), "https://example.com/sitemap.xml".to_string(), ); let txt_data = create_txt_data(&metadata); match txt_data.validate() { Ok(_) => { let content = generate_txt_content(&txt_data); println!(" โœ… Generated content with sitemap:"); println!("{}", content); } Err(e) => println!(" โŒ Validation error: {:?}", e), } Ok(()) } /// Demonstrates subdirectory crawling rules. fn subdirectory_rules_example() -> Result<(), Box> { println!("\n๐Ÿฆ€ Subdirectory Rules Example"); println!("---------------------------------------------"); let mut metadata = HashMap::new(); let _ = metadata.insert( "permalink".to_string(), "https://example.com".to_string(), ); let _ = metadata.insert( "disallow_paths".to_string(), "/private/*,/tmp/*,/admin/*".to_string(), ); let txt_data = create_txt_data(&metadata); match txt_data.validate() { Ok(_) => { let content = generate_txt_content(&txt_data); println!(" โœ… Generated content with path rules:"); println!("{}", content); } Err(e) => println!(" โŒ Validation error: {:?}", e), } Ok(()) } /// Demonstrates multiple sitemap configurations. fn multi_sitemap_example() -> Result<(), Box> { println!("\n๐Ÿฆ€ Multiple Sitemaps Example"); println!("---------------------------------------------"); let mut metadata = HashMap::new(); let _ = metadata.insert( "permalink".to_string(), "https://example.com".to_string(), ); let _ = metadata.insert( "sitemaps".to_string(), "sitemap.xml,news-sitemap.xml,images-sitemap.xml".to_string(), ); let txt_data = create_txt_data(&metadata); match txt_data.validate() { Ok(_) => { let content = generate_txt_content(&txt_data); println!( " โœ… Generated content with multiple sitemaps:" ); println!("{}", content); } Err(e) => println!(" โŒ Validation error: {:?}", e), } Ok(()) } /// Demonstrates crawl delay settings. fn crawl_delay_example() -> Result<(), Box> { println!("\n๐Ÿฆ€ Crawl Delay Example"); println!("---------------------------------------------"); let mut metadata = HashMap::new(); let _ = metadata.insert( "permalink".to_string(), "https://example.com".to_string(), ); let _ = metadata.insert("crawl_delay".to_string(), "10".to_string()); let txt_data = create_txt_data(&metadata); match txt_data.validate() { Ok(_) => { let content = generate_txt_content(&txt_data); println!(" โœ… Generated content with crawl delay:"); println!("{}", content); } Err(e) => println!(" โŒ Validation error: {:?}", e), } Ok(()) } /// Demonstrates user agent specific rules. fn user_agent_example() -> Result<(), Box> { println!("\n๐Ÿฆ€ User Agent Example"); println!("---------------------------------------------"); let agents = vec![ ("*", "Allow: /"), ("Googlebot", "Allow: /blog/\nDisallow: /private/"), ("Bingbot", "Allow: /public/\nDisallow: /internal/"), ]; for (agent, rules) in agents { let mut metadata = HashMap::new(); let _ = metadata.insert( "permalink".to_string(), "https://example.com".to_string(), ); let _ = metadata .insert("user_agent".to_string(), agent.to_string()); let _ = metadata.insert("rules".to_string(), rules.to_string()); let txt_data = create_txt_data(&metadata); match txt_data.validate() { Ok(_) => { println!(" โœ… Rules for {}: {}", agent, rules); } Err(e) => println!(" โŒ Error for {}: {:?}", agent, e), } } Ok(()) } /// Demonstrates validation of robots.txt data. fn validation_example() -> Result<(), Box> { println!("\n๐Ÿฆ€ Validation Example"); println!("---------------------------------------------"); let test_cases = vec![ (TxtData::new("".to_string()), false, "Empty permalink"), ( TxtData::new("https://example.com".to_string()), true, "Valid permalink", ), ( TxtData::new("invalid-url".to_string()), false, "Invalid URL format", ), ( TxtData::new("http://example.com/invalid/path".to_string()), false, "URL with path", ), ]; for (data, should_be_valid, case) in test_cases { match data.validate() { Ok(_) => { if should_be_valid { println!(" โœ… Valid case: {}", case); } else { println!( " โŒ Unexpected validation success: {}", case ); } } Err(e) => { if !should_be_valid { println!( " โœ… Expected validation failure: {}", case ); } else { println!( " โŒ Unexpected validation error for {}: {:?}", case, e ); } } } } Ok(()) }