use libipld::Cid; use std::convert::TryFrom; use std::fmt; use std::io::{Error as IoError, Read, Write}; use std::path::PathBuf; use unixfs_v1::file::{visit::IdleFileVisit, FileReadFailed}; fn main() { let cid = match std::env::args().nth(1).map(Cid::try_from) { Some(Ok(cid)) => cid, Some(Err(e)) => { eprintln!("Invalid cid given as argument: {}", e); std::process::exit(1); } None => { eprintln!("USAGE: {} CID\n", std::env::args().next().unwrap()); eprintln!( "Will walk the unixfs file pointed out by the CID from default go-ipfs 0.5 \ configuration flatfs blockstore and write all content to stdout." ); std::process::exit(0); } }; let ipfs_path = match std::env::var("IPFS_PATH") { Ok(s) => s, Err(e) => { eprintln!("IPFS_PATH is not set or could not be read: {}", e); std::process::exit(1); } }; let mut blocks = PathBuf::from(ipfs_path); blocks.push("blocks"); let blockstore = ShardedBlockStore { root: blocks }; match walk(blockstore, &cid) { Ok((read, content)) => { eprintln!("Content bytes: {}", content); eprintln!("Total bytes: {}", read); } Err(Error::OpeningFailed(e)) => { eprintln!("{}\n", e); eprintln!("This is likely caused by either:"); eprintln!(" - ipfs does not have the block"); eprintln!(" - ipfs is configured to use non-flatfs storage"); eprintln!(" - ipfs is configured to use flatfs with different sharding"); std::process::exit(1); } Err(e) => { eprintln!("Failed to walk the merkle tree: {}", e); std::process::exit(1); } } } fn walk(blocks: ShardedBlockStore, start: &Cid) -> Result<(u64, u64), Error> { let stdout = std::io::stdout(); let mut stdout = stdout.lock(); let mut read_bytes = 0; let mut content_bytes = 0; // The blockstore specific way of reading the block. Here we assume go-ipfs 0.5 default flatfs // configuration, which puts the files at sharded directories and names the blocks as base32 // upper and a suffix of "data". // // For the ipfs-unixfs it is important that the raw block data lives long enough that the // possible content gets to be processed, at minimum one step of the walk as shown in this // example. let mut buf = Vec::new(); read_bytes += blocks.as_file(&start.to_bytes())?.read_to_end(&mut buf)? as u64; // First step of the walk can give content or continued visitation but not both. let (content, _, _metadata, mut step) = IdleFileVisit::default().start(&buf)?; stdout.write_all(content)?; content_bytes += content.len() as u64; // Following steps repeat the same pattern: while let Some(visit) = step { // Read the next link. The `pending_links()` gives the next link and an iterator over the // following links. The iterator lists the known links in the order of traversal, with the // exception of possible new links appearing before the older. let (first, _) = visit.pending_links(); buf.clear(); read_bytes += blocks.as_file(&first.to_bytes())?.read_to_end(&mut buf)? as u64; // Similar to first step, except we no longer get the file metadata. It is still accessible // from the `visit` via `AsRef` but likely only needed in // the first step. let (content, next_step) = visit.continue_walk(&buf, &mut None)?; stdout.write_all(content)?; content_bytes += content.len() as u64; // Using a while loop combined with `let Some(visit) = step` allows for easy walking. step = next_step; } stdout.flush()?; Ok((read_bytes, content_bytes)) } enum Error { OpeningFailed(IoError), Other(IoError), Traversal(unixfs_v1::file::FileReadFailed), } impl From for Error { fn from(e: IoError) -> Error { Error::Other(e) } } impl From for Error { fn from(e: FileReadFailed) -> Error { Error::Traversal(e) } } impl fmt::Display for Error { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { use Error::*; match self { OpeningFailed(e) => write!(fmt, "File opening failed: {}", e), Other(e) => write!(fmt, "Other file related io error: {}", e), Traversal(e) => write!(fmt, "Traversal failed, please report this as a bug: {}", e), } } } struct ShardedBlockStore { root: PathBuf, } impl ShardedBlockStore { fn as_path(&self, key: &[u8]) -> PathBuf { // assume that we have a block store with second-to-last/2 sharding // files in Base32Upper let encoded = libipld::multibase::Base::Base32Upper.encode(key); let len = encoded.len(); // this is safe because base32 is ascii let dir = &encoded[(len - 3)..(len - 1)]; assert_eq!(dir.len(), 2); let mut path = self.root.clone(); path.push(dir); path.push(encoded); path.set_extension("data"); path } fn as_file(&self, key: &[u8]) -> Result { let path = self.as_path(key); std::fs::OpenOptions::new() .read(true) .open(path) .map_err(Error::OpeningFailed) } }