Skip to content
  • // [dependencies]
    // reqwest = { version = "0.10", features = ["json"] }
    // tokio = { version = "0.2", features = ["full"] }
    // # so modular that we even have competing async executors
    // serde_json = "1.0.52" 
    // # Why? because json parser in the stdlib of a modern language is BLOATTTTT 
    // serde = { version = "1.0", features = ["derive"] }
    // # the fact that I depend on serde_json is pretty vague 
    // # so I need to mention that I actually am using serde
    // futures = "0.3.4" 
    // # Because join_all in the stdlib future module is CRAZY TALK!
    
    struct Link {
        board: String,
        api: String,
    }
    
    // Extract board and thread from the url
    fn extract_board_and_thread(uri: &str) -> (Option<String>, Option<String>) {
        let mut board: Option<String> = None;
        let mut thread: Option<String> = None;
        for (i, token) in uri.split(|c| c == '/' || c == '#').enumerate() {
            if i == 3 {
                board = Some(token.into());
            } else if i == 5 {
                thread = Some(token.into());
                break;
            }
        }
        (board, thread)
    }
    
    impl Link {
        pub fn new(raw_link: &str) -> Option<Self> {
            match extract_board_and_thread(raw_link) {
                (Some(board), Some(thread)) => Some(Self {
                    board: board.clone(),
                    api: format!("https://a.4cdn.org/{}/thread/{}.json", board, thread),
                }),
                _ => None,
            }
        }
    }
    
    #[derive(serde::Deserialize)]
    struct Post {
        // i32 won't work and say vague messages like it needs an "integer"
        tim: Option<i64>,
        ext: Option<String>,
    }
    
    #[derive(serde::Deserialize)]
    struct RestResp {
        posts: Vec<Post>,
    }
    
    // TODO: Parallelize this
    async fn download(uri: String, name: String, dir: std::path::PathBuf) {
        let mut file_path = dir;
        file_path.push(name);
        println!("Downloading {} as {}...", uri, file_path.display());
    
        let resp_bytes = reqwest::get(&uri)
            .await
            .expect("Download error")
            .bytes()
            .await
            .unwrap();
    
        let mut resp = resp_bytes.as_ref();
        let mut file = std::fs::File::create(&file_path).expect("Cannot access file");
    
        std::io::copy(&mut resp, &mut file).expect("Write error");
    
        println!(
            "[SUCCESS] Downloading {} as {}...",
            uri,
            file_path.display()
        );
    }
    
    #[tokio::main]
    async fn main() {
        let dir = std::path::PathBuf::from(
            std::env::args()
                .nth(2)
                .expect("Argument not found: destination folder"),
        );
        if !dir.exists() {
            std::fs::create_dir(dir.clone()).expect("Error creating directory");
        }
    
        let raw_link = std::env::args().nth(1).expect("Argument not found: link");
        let link = Link::new(&raw_link).expect("Could not recognize URI");
    
        let j = reqwest::get(&link.api)
            .await
            .expect("HTTP GET error")
            .json::<RestResp>()
            .await
            .expect("Deserialization error");
    
        let mut download_futures = Vec::new();
    
        for post in j.posts {
            match (post.tim, post.ext) {
                (Some(tim), Some(ext)) => {
                    let file_name = format!("{}{}", tim, ext);
                    let uri = format!("https://i.4cdn.org/{}/{}", link.board, file_name);
                    download_futures.push(download(uri, file_name, dir.clone()));
                }
                _ => {}
            }
        }
        futures::future::join_all(download_futures).await;
    }
  • private void assertEq(T)(inout auto ref T lhs, inout auto ref T rhs, int line = __LINE__)
    {
    	debug
    	{
    		if (lhs != rhs)
    		{
    			import std.stdio : writefln, stderr;
    
    			stderr.writefln("Assertion failure in line %s:\n\tlhs: %s\n\trhs: %s", line, lhs, rhs);
    		}
    	}
    	assert(lhs == rhs);
    }
    
    private struct Link
    {
    	string board;
    	string thread;
    
    	this(string uri)
    	{
    		int dashCount;
    		foreach (c; uri)
    		{
    			if (c == '/')
    				++dashCount;
    			if (dashCount == 3 && c != '/')
    				this.board ~= c;
    			else if (dashCount == 5 && c >= '0' && c <= '9')
    				this.thread ~= c;
    			if (c == '#')
    				break;
    		}
    	}
    
    	string getJSONAPI() inout
    	{
    		import std.string : format;
    
    		return format!"https://a.4cdn.org/%s/thread/%s.json"(board, thread);
    	}
    }
    
    unittest
    {
    	auto link = Link(`https://boards.4channel.org/g/thread/76298934#p76298934`);
    	assertEq(link.board, "g");
    	assertEq(link.thread, "76298934");
    	assertEq(link.getJSONAPI(), `https://a.4cdn.org/g/thread/76298934.json`);
    }
    
    string getMediaLink(T)(inout auto ref T board, string filename, T ext)
    {
    	// https://is2.4chan.org/g/1591830015154.png
    	import std.string : format;
    
    	return format!"https://i.4cdn.org/%s/%s%s"(board, filename, ext);
    }
    
    unittest
    {
    	assertEq(getMediaLink("g", 1591830015154, ".png"), `https://is2.4chan.org/g/1591830015154.png`);
    }
    
    class Listener
    {
    	import core.sync.mutex : Mutex;
    
    	Mutex mtx;
    	size_t downloaded, toDownload;
    
    	this(size_t toDownload) shared
    	{
    		mtx = new shared Mutex();
    		this.toDownload = toDownload;
    	}
    
    	void addOne() shared
    	{
    		mtx.lock();
    		++cast()downloaded;
    		debug
    		{
    			import std.stdio : writeln;
    
    			writeln("Downloaded ", cast() downloaded, " of ", cast() toDownload);
    		}
    		mtx.unlock();
    	}
    
    }
    
    void download(shared Listener listener, string uri, string path, string name, string ext)
    {
    	import std.net.curl : curlDownload = download;
    
    	curlDownload(uri, path ~ "/" ~ name ~ ext);
    	listener.addOne();
    }
    
    void main(in string[] args)
    {
    	if (args.length < 3)
    	{
    		import std.stdio : stderr, writeln;
    
    		stderr.writeln("Arguments missing. Usage: [program] [thread] [location]");
    		import core.stdc.stdlib : exit, EXIT_FAILURE;
    
    		exit(EXIT_FAILURE);
    	}
    
    	const string downloadFolder = (string dir) {
    		import std.file : exists, mkdir, getcwd;
    
    		if (!dir.exists())
    			mkdir(getcwd() ~ "/" ~ dir ~ "/");
    		return dir;
    	}(args[2]);
    
    	import std.net.curl : get;
    	import std.json : parseJSON, JSONValue, JSONType;
    
    	auto l = Link(args[1]);
    	const JSONValue j = l.getJSONAPI().get().parseJSON();
    	const JSONValue[] downloadLinks = (JSONValue jv) {
    		import std.algorithm : splitter, filter;
    		import std.range : array;
    
    		return jv["posts"].array().filter!(node => "tim" in node).array();
    	}(j);
    
    	auto listener = new shared Listener(downloadLinks.length);
    
    	import std.parallelism : parallel;
    
    	foreach (post; downloadLinks.parallel())
    	{
    		import std.conv : to;
    
    		const string name = post["tim"].integer.to!string, ext = post["ext"].str;
    		listener.download(getMediaLink(l.board, name, ext), downloadFolder, name, ext);
    	}
    }
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment