[package]
name = "sws-crawler"
version = "0.1.0"
edition = "2021"
license = "MIT OR Apache-2.0"
description = "Web crawler with plugable scraping logic"
repository = "https://github.com/lerouxrgd/sws"

[dependencies]
anyhow = "1"
clap = { version = "4", optional = true }
crossbeam-channel = "0.5"
flate2 = "1"
futures = "0.3"
lazy_static = "1"
log = "0.4"
num_cpus = "1"
pin-project-lite = "0.2"
reqwest = { version = "0.11", features = ["gzip", "deflate"] }
serde = { version = "1", features = ["derive"] }
sxd-document = "0.3"
sxd-xpath = "0.4"
texting_robots = "0.2"
tokio = { version = "1", features = ["signal", "sync"] }
tokio-stream = "0.1"

[features]
clap = ["dep:clap"]

[dev-dependencies]
tokio = { version = "1", features = ["macros"] }