-
// [dependencies] // reqwest = { version = "0.10", features = ["json"] } // tokio = { version = "0.2", features = ["full"] } // # so modular that we even have competing async executors // serde_json = "1.0.52" // # Why? because json parser in the stdlib of a modern language is BLOATTTTT // serde = { version = "1.0", features = ["derive"] } // # the fact that I depend on serde_json is pretty vague // # so I need to mention that I actually am using serde // futures = "0.3.4" // # Because join_all in the stdlib future module is CRAZY TALK! struct Link { board: String, api: String, } // Extract board and thread from the url fn extract_board_and_thread(uri: &str) -> (Option<String>, Option<String>) { let mut board: Option<String> = None; let mut thread: Option<String> = None; for (i, token) in uri.split(|c| c == '/' || c == '#').enumerate() { if i == 3 { board = Some(token.into()); } else if i == 5 { thread = Some(token.into()); break; } } (board, thread) } impl Link { pub fn new(raw_link: &str) -> Option<Self> { match extract_board_and_thread(raw_link) { (Some(board), Some(thread)) => Some(Self { board: board.clone(), api: format!("https://a.4cdn.org/{}/thread/{}.json", board, thread), }), _ => None, } } } #[derive(serde::Deserialize)] struct Post { // i32 won't work and say vague messages like it needs an "integer" tim: Option<i64>, ext: Option<String>, } #[derive(serde::Deserialize)] struct RestResp { posts: Vec<Post>, } // TODO: Parallelize this async fn download(uri: String, name: String, dir: std::path::PathBuf) { let mut file_path = dir; file_path.push(name); println!("Downloading {} as {}...", uri, file_path.display()); let resp_bytes = reqwest::get(&uri) .await .expect("Download error") .bytes() .await .unwrap(); let mut resp = resp_bytes.as_ref(); let mut file = std::fs::File::create(&file_path).expect("Cannot access file"); std::io::copy(&mut resp, &mut file).expect("Write error"); println!( "[SUCCESS] Downloading {} as {}...", uri, file_path.display() ); } #[tokio::main] async fn main() { let dir = std::path::PathBuf::from( std::env::args() .nth(2) .expect("Argument not found: destination folder"), ); if !dir.exists() { std::fs::create_dir(dir.clone()).expect("Error creating directory"); } let raw_link = std::env::args().nth(1).expect("Argument not found: link"); let link = Link::new(&raw_link).expect("Could not recognize URI"); let j = reqwest::get(&link.api) .await .expect("HTTP GET error") .json::<RestResp>() .await .expect("Deserialization error"); let mut download_futures = Vec::new(); for post in j.posts { match (post.tim, post.ext) { (Some(tim), Some(ext)) => { let file_name = format!("{}{}", tim, ext); let uri = format!("https://i.4cdn.org/{}/{}", link.board, file_name); download_futures.push(download(uri, file_name, dir.clone())); } _ => {} } } futures::future::join_all(download_futures).await; }
-
private void assertEq(T)(inout auto ref T lhs, inout auto ref T rhs, int line = __LINE__) { debug { if (lhs != rhs) { import std.stdio : writefln, stderr; stderr.writefln("Assertion failure in line %s:\n\tlhs: %s\n\trhs: %s", line, lhs, rhs); } } assert(lhs == rhs); } private struct Link { string board; string thread; this(string uri) { int dashCount; foreach (c; uri) { if (c == '/') ++dashCount; if (dashCount == 3 && c != '/') this.board ~= c; else if (dashCount == 5 && c >= '0' && c <= '9') this.thread ~= c; if (c == '#') break; } } string getJSONAPI() inout { import std.string : format; return format!"https://a.4cdn.org/%s/thread/%s.json"(board, thread); } } unittest { auto link = Link(`https://boards.4channel.org/g/thread/76298934#p76298934`); assertEq(link.board, "g"); assertEq(link.thread, "76298934"); assertEq(link.getJSONAPI(), `https://a.4cdn.org/g/thread/76298934.json`); } string getMediaLink(T)(inout auto ref T board, string filename, T ext) { // https://is2.4chan.org/g/1591830015154.png import std.string : format; return format!"https://i.4cdn.org/%s/%s%s"(board, filename, ext); } unittest { assertEq(getMediaLink("g", 1591830015154, ".png"), `https://is2.4chan.org/g/1591830015154.png`); } class Listener { import core.sync.mutex : Mutex; Mutex mtx; size_t downloaded, toDownload; this(size_t toDownload) shared { mtx = new shared Mutex(); this.toDownload = toDownload; } void addOne() shared { mtx.lock(); ++cast()downloaded; debug { import std.stdio : writeln; writeln("Downloaded ", cast() downloaded, " of ", cast() toDownload); } mtx.unlock(); } } void download(shared Listener listener, string uri, string path, string name, string ext) { import std.net.curl : curlDownload = download; curlDownload(uri, path ~ "/" ~ name ~ ext); listener.addOne(); } void main(in string[] args) { if (args.length < 3) { import std.stdio : stderr, writeln; stderr.writeln("Arguments missing. Usage: [program] [thread] [location]"); import core.stdc.stdlib : exit, EXIT_FAILURE; exit(EXIT_FAILURE); } const string downloadFolder = (string dir) { import std.file : exists, mkdir, getcwd; if (!dir.exists()) mkdir(getcwd() ~ "/" ~ dir ~ "/"); return dir; }(args[2]); import std.net.curl : get; import std.json : parseJSON, JSONValue, JSONType; auto l = Link(args[1]); const JSONValue j = l.getJSONAPI().get().parseJSON(); const JSONValue[] downloadLinks = (JSONValue jv) { import std.algorithm : splitter, filter; import std.range : array; return jv["posts"].array().filter!(node => "tim" in node).array(); }(j); auto listener = new shared Listener(downloadLinks.length); import std.parallelism : parallel; foreach (post; downloadLinks.parallel()) { import std.conv : to; const string name = post["tim"].integer.to!string, ext = post["ext"].str; listener.download(getMediaLink(l.board, name, ext), downloadFolder, name, ext); } }
Please register or sign in to comment