pub mod common; use std::{ env::temp_dir, fs::{self, read_to_string, File}, path::Path, }; use ::serde::{ser::Error, Deserialize, Serialize}; use rand::distributions::{Alphanumeric, DistString}; use serde::Serializer; use spring_batch_rs::{ core::{ item::{ItemProcessor, ItemProcessorResult}, job::{Job, JobBuilder}, step::{Step, StepBuilder, StepInstance, StepStatus}, }, item::csv::csv_reader::CsvItemReaderBuilder, item::csv::csv_writer::CsvItemWriterBuilder, item::json::json_reader::JsonItemReaderBuilder, item::json::json_writer::JsonItemWriterBuilder, }; use time::{format_description, Date, Month}; fn date_serializer(date: &Date, serializer: S) -> Result where S: Serializer, { let result = format_description::parse("[year]-[month]-[day]"); match result { Ok(format) => { let s = date.format(&format).unwrap(); serializer.serialize_str(&s) } Err(error) => Err(Error::custom(error.to_string())), } } #[derive(Serialize, Deserialize, Clone)] pub struct Person { first_name: String, last_name: String, title: String, email: String, #[serde(serialize_with = "date_serializer")] birth_date: Date, } #[derive(Default)] struct UpperCaseProcessor; impl ItemProcessor for UpperCaseProcessor { fn process(&self, item: &Person) -> ItemProcessorResult { let person = Person { first_name: item.first_name.to_uppercase(), last_name: item.last_name.to_uppercase(), title: item.title.to_uppercase(), email: item.email.to_uppercase(), birth_date: Date::from_calendar_date(2019, Month::January, 1).unwrap(), }; Ok(person) } } #[test] fn transform_from_json_file_to_csv_file_without_error() { let path = Path::new("examples/data/persons.json"); let file = File::open(path).expect("Unable to open file"); let reader = JsonItemReaderBuilder::new().from_reader(file); let processor = UpperCaseProcessor::default(); let writer = CsvItemWriterBuilder::new() .has_headers(true) .from_path(temp_dir().join("persons.csv")); let step: StepInstance = StepBuilder::new() .reader(&reader) .processor(&processor) .writer(&writer) .chunk(3) .build(); let job = JobBuilder::new().start(&step).build(); let result = job.run(); assert!(result.is_ok()); assert!(step.get_status() == StepStatus::Success); assert!(step.get_read_count() == 4); assert!(step.get_write_count() == 4); assert!(step.get_read_error_count() == 0); assert!(step.get_write_error_count() == 0); let file_content = read_to_string(temp_dir().join("persons.csv")) .expect("Should have been able to read the file"); assert_eq!( file_content, "first_name,last_name,title,email,birth_date OCÉANE,DUPOND,MR.,LEOPOLD_ENIM@ORANGE.FR,2019-01-01 AMANDINE,ÉVRAT,MRS.,AMANDINE_IURE@OUTLOOK.FR,2019-01-01 UGO,NIELS,SIR.,XAVIER_VOLUPTATEM@SFR.FR,2019-01-01 LÉO,ZOLA,DR.,UGO_PRAESENTIUM@ORANGE.FR,2019-01-01 " ); } #[derive(Deserialize, Serialize, Debug, Clone)] struct Car { year: u16, make: String, model: String, description: String, } #[test] fn convert_csv_file_to_json_file_without_error() { let path = Path::new("examples/data/cars_with_headers.csv"); let file = File::open(path).expect("Unable to open file"); let reader = CsvItemReaderBuilder::new() .has_headers(true) .from_reader(file); let writer = JsonItemWriterBuilder::new().from_path(temp_dir().join("cars.json")); let step: StepInstance = StepBuilder::new() .reader(&reader) .writer(&writer) .chunk(3) .build(); let job = JobBuilder::new().start(&step).build(); let result = job.run(); assert!(result.is_ok()); assert!(step.get_status() == StepStatus::Success); assert!(step.get_read_count() == 7); assert!(step.get_write_count() == 7); assert!(step.get_read_error_count() == 0); assert!(step.get_write_error_count() == 0); let file_content = fs::read_to_string(temp_dir().join("cars.json")) .expect("Should have been able to read the file"); assert_eq!( file_content, r#"[{"year":1948,"make":"Porsche","model":"356","description":"Luxury sports car 1"},{"year":1949,"make":"Porsche","model":"357","description":"Luxury sports car 2"},{"year":1950,"make":"Porsche","model":"358","description":"Luxury sports car 3"},{"year":1951,"make":"Porsche","model":"359","description":"Luxury sports car 4"},{"year":1952,"make":"Porsche","model":"360","description":"Luxury sports car 5"},{"year":1967,"make":"Ford","model":"Mustang fastback 1967","description":"American car"},{"year":1967,"make":"Ford","model":"Mustang fastback 1967","description":"American car"}] "# ); } #[test] fn transform_csv_stream_to_writer_with_one_error_should_succeded() { let csv = "year,make,model,description 1948,Porsche,356,Luxury sports car 2011d,Peugeot,206+,City car 2012,Citroën,C4 Picasso,SUV 2021,Mazda,CX-30,SUV Compact 1967,Ford,Mustang fastback 1967,American car"; let reader = CsvItemReaderBuilder::new() .has_headers(true) .from_reader(csv.as_bytes()); let file_name = Alphanumeric.sample_string(&mut rand::thread_rng(), 16); let file_writer = File::create(temp_dir().join(file_name.clone())).expect("Unable to open file"); let writer = JsonItemWriterBuilder::new().from_writer(file_writer); let step: StepInstance = StepBuilder::new() .reader(&reader) .writer(&writer) .chunk(3) .skip_limit(1) .build(); let job = JobBuilder::new().start(&step).build(); let result = job.run(); assert!(result.is_ok()); assert!(step.get_status() == StepStatus::Success); assert!(step.get_read_count() == 4); assert!(step.get_write_count() == 4); assert!(step.get_read_error_count() == 1); assert!(step.get_write_error_count() == 0); let file_content = fs::read_to_string(temp_dir().join(file_name)) .expect("Should have been able to read the file"); assert_eq!( file_content, r#"[{"year":1948,"make":"Porsche","model":"356","description":"Luxury sports car"},{"year":2012,"make":"Citroën","model":"C4 Picasso","description":"SUV"},{"year":2021,"make":"Mazda","model":"CX-30","description":"SUV Compact"},{"year":1967,"make":"Ford","model":"Mustang fastback 1967","description":"American car"}] "# ); } #[test] fn transform_csv_stream_to_writer_with_3_errors_should_failed() { let csv = "year,make,model,description 1948,Porsche,356,Luxury sports car 2011d,Peugeot,206+,City car 2012,Citroën,C4 Picasso,SUV 1995,Peugeot,205,City car 2021d,Mazda,CX-30,SUV Compact 1967d,Ford,Mustang fastback 1967,American car"; let reader = CsvItemReaderBuilder::new() .has_headers(true) .from_reader(csv.as_bytes()); let file_name = Alphanumeric.sample_string(&mut rand::thread_rng(), 16); let file_writer = File::create(temp_dir().join(file_name.clone())).expect("Unable to open file"); let writer = JsonItemWriterBuilder::new().from_writer(file_writer); let step: StepInstance = StepBuilder::new() .reader(&reader) .writer(&writer) .chunk(3) .skip_limit(2) .build(); let job = JobBuilder::new().start(&step).build(); let result = job.run(); assert!(result.is_err()); assert!(step.get_status() == StepStatus::ReadError); assert!(step.get_read_count() == 3); assert!(step.get_write_count() == 3); assert!(step.get_read_error_count() == 3); assert!(step.get_write_error_count() == 0); let file_content = fs::read_to_string(temp_dir().join(file_name)) .expect("Should have been able to read the file"); assert_eq!( file_content, r#"[{"year":1948,"make":"Porsche","model":"356","description":"Luxury sports car"},{"year":2012,"make":"Citroën","model":"C4 Picasso","description":"SUV"},{"year":1995,"make":"Peugeot","model":"205","description":"City car"}] "# ); }