.cargo/config

+[build]
+rustflags = ["-C", "target-cpu=native"]
\ No newline at end of file

.gitignore

+/target
+**/*.rs.bk
+.vscode/**
+*.csv
+*.router
+dumps/*.json
+plot.py
+*.tmp
+*.idx

.gitlab-ci.yml

+# This file is a template, and might need editing before it works on your project.
+# Official language image. Look for the different tagged releases at:
+# https://hub.docker.com/r/library/rust/tags/
+image: "rust:latest"
+
+# Optional: Pick zero or more services to be used on all builds.
+# Only needed when using a docker container to run your tests in.
+# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+# services:
+#   - mysql:latest
+#   - redis:latest
+#   - postgres:latest
+
+# Optional: Install a C compiler, cmake and git into the container.
+# You will often need this when you (or any of your dependencies) depends on C code.
+before_script:
+  - apt-get update -yqq
+  - apt-get install -yqq --no-install-recommends build-essential
+  - rustup update nightly
+  - rustup default nightly
+
+# Use cargo to test the project
+test:cargo:
+  script:
+    - rustc --version && cargo --version  # Print version info for debugging
+    - cargo build --release

Cargo.toml

+[package]
+name = "ed_lrr"
+version = "0.1.0"
+authors = ["Daniel Seiller <earthnuker@gmail.com>"]
+edition = "2018"
+repository = "https://gitlab.com/Earthnuker/ed_lrr.git"
+license = "WTFPL"
+
+
+[profile.release]
+# debug=true
+
+[dependencies]
+csv = "1.1.0"
+serde = "1.0.94"
+serde_derive = "1.0.94"
+rstar = {version="0.4.0",features=["serde"]}
+humantime = "1.2.0"
+structopt = "0.2.18"
+permutohedron = "0.2.4"
+serde_json = "1.0.39"
+indicatif = "0.11.0"
+fnv = "1.0.6"
+bincode = "1.1.4"
+sha3 = "0.8.2"
+byteorder = "1.3.2"

README.md

+# Elite: Dangerous Long Range Router (Rust Version)
+
+## Usage:
+
+1. download `bodies.json` and `systemsWithCoordinates.json` from https://www.edsm.net/en/nightly-dumps/ and place them in the `dumps` folder
+2. run `cargo +nightly install --path .` or `cargo +nightly install --git https://gitlab.com/Earthnuker/ed_lrr.git`
+3. run `ed_lrr_pp --bodies dumps/bodies.json --systems dumps/systemsWithCoordinates.json`
+    - Alternatively run `process.py` in the `dumps` folder
+4. run `ed_lrr --help`
+
+
+
+## Dependencies
+
+- Working nightly Rust Compiler (tested with `rustc 1.37.0-nightly (5d8f59f4b 2019-06-04)`)
+- ~8GB of free RAM
+- Optional:
+  - Python 3.7
+  - Pandas
+  - uJSON

dumps/process.py

+import ujson as json
+from tqdm import tqdm
+from pprint import pprint
+import itertools as ITT
+import os
+import sys
+import csv
+import sqlite3
+import pandas as pd
+from urllib.parse import urljoin
+
+
+def is_scoopable(entry):
+    first = entry.type.split()[0]
+    return first == "Neutron" or first == "White" or first in "KGBFOAM"
+
+
+def get_mult(name):
+    try:
+        first = name.split()[0]
+    except:
+        return 1
+    if first == "Neutron":
+        return 4
+    if first == "White":
+        return 1.5
+    return 1
+
+
+def dict_factory(cursor, row):
+    d = {}
+    for idx, col in enumerate(cursor.description):
+        d[col[0]] = row[idx]
+    return d
+
+
+def blocks(files, size=65536):
+    while True:
+        b = files.read(size)
+        if not b:
+            break
+        yield b
+
+
+def getlines(f, fn, show_progbar=False):
+    f.seek(0, 2)
+    size = f.tell()
+    f.seek(0)
+    progbar = tqdm(
+        desc="Processing " + fn,
+        total=size,
+        unit="b",
+        unit_scale=True,
+        unit_divisor=1024,
+        ascii=True,
+        leave=True,
+        disable=(not show_progbar),
+    )
+    buffer = []
+    for block in blocks(f):
+        progbar.n = f.tell()
+        progbar.update(0)
+        if buffer:
+            buffer += (buffer.pop(0) + block).splitlines(keepends=True)
+        else:
+            buffer += block.splitlines(keepends=True)
+        while buffer and buffer[0].endswith("\n"):
+            try:
+                yield json.loads(buffer.pop(0).strip().rstrip(","))
+            except ValueError:
+                pass
+    while buffer:
+        try:
+            yield json.loads(buffer.pop(0).strip().rstrip(","))
+        except ValueError:
+            pass
+
+
+def process_file(fn, show_progbar=False):
+    with open(fn, "r") as f:
+        for line in tqdm(
+            getlines(f, fn, show_progbar),
+            desc=fn,
+            unit=" lines",
+            unit_scale=True,
+            ascii=True,
+            leave=True,
+            disable=(not show_progbar),
+        ):
+            yield line
+
+
+if not (
+    os.path.isfile("bodies.json") and os.path.isfile("systemsWithCoordinates.json")
+):
+    exit(
+        "Please download bodies.json and systemsWithCoordinates.json from https://www.edsm.net/en/nightly-dumps/"
+    )
+
+if not os.path.isfile("stars.jl"):
+    print("Filtering for Stars")
+    with open("stars.jl", "w") as neut:
+        for body in process_file("bodies.json", True):
+            T = body.get("type") or ""
+            if "Star" in T:
+                neut.write(json.dumps(body) + "\n")
+
+
+def load_systems(load=False):
+    load = not os.path.isfile("systems.db")
+    cache = sqlite3.connect("systems.db")
+    cache.row_factory = dict_factory
+    c = cache.cursor()
+    if load:
+        print("Caching Systems")
+        c.execute("DROP TABLE IF EXISTS systems")
+        c.execute(
+            "CREATE TABLE systems (id64 int primary key, name text, x real, y real, z real)"
+        )
+        cache.commit()
+        recs = []
+        for system in process_file("systemsWithCoordinates.json", True):
+            rec = [
+                system["id64"],
+                system["name"],
+                system["coords"]["x"],
+                system["coords"]["y"],
+                system["coords"]["z"],
+            ]
+            recs.append(rec)
+            if len(recs) % 1024 * 1024 == 0:
+                c.executemany("INSERT INTO systems VALUES (?,?,?,?,?)", recs)
+                recs.clear()
+        c.executemany("INSERT INTO systems VALUES (?,?,?,?,?)", recs)
+        cache.commit()
+    return cache, c
+
+
+if not os.path.isfile("stars.csv"):
+    cache, cur = load_systems()
+    rows = []
+    with open("stars.csv", "w", newline="") as sys_csv:
+        csv_writer = csv.writer(sys_csv, dialect="excel")
+        for neut in process_file("stars.jl", True):
+            cur.execute(
+                "SELECT * FROM systems WHERE id64==?", (neut.get("systemId64"),)
+            )
+            system = cur.fetchone()
+            if not system:
+                continue
+            row = [
+                neut["systemId64"],
+                neut["subType"],
+                neut["name"],
+                get_mult(neut["subType"]),
+                system["x"],
+                system["y"],
+                system["z"],
+            ]
+            rows.append(row)
+            if len(rows) > 1024:
+                csv_writer.writerows(rows)
+                rows.clear()
+        csv_writer.writerows(rows)
+        print()
+    cache.close()
+
+if not os.path.isfile("stars.csv"):
+    tqdm.pandas(ascii=True, leave=True)
+    print("Loading data...")
+    data = pd.read_csv(
+        "stars.csv",
+        encoding="utf-8",
+        names=["id", "type", "name", "mult", "x", "y", "z"],
+    )
+    print("Cleaning data...")
+    data.type.fillna("Unknown", inplace=True)
+    data.drop_duplicates("id", inplace=True)
+    print("Writing CSV...")
+    data.to_csv("stars.csv", header=False, index=False)

dumps/urls.txt

+https://www.edsm.net/dump/systemsWithCoordinates.json
+https://www.edsm.net/dump/bodies.json
\ No newline at end of file

src/common.rs

+use serde::{Deserialize, Serialize};
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SystemSerde {
+    pub id: u32,
+    pub star_type: String,
+    pub system: String,
+    pub body: String,
+    pub mult: f32,
+    pub distance: u32,
+    pub x: f32,
+    pub y: f32,
+    pub z: f32,
+}
+
+impl SystemSerde {
+    pub fn build(&self) -> System {
+        System {
+            id: self.id,
+            star_type: self.star_type.clone(),
+            system: self.system.clone(),
+            body: self.body.clone(),
+            mult: self.mult,
+            distance: self.distance,
+            pos: [self.x, self.y, self.z],
+        }
+    }
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct System {
+    pub id: u32,
+    pub star_type: String,
+    pub system: String,
+    pub body: String,
+    pub mult: f32,
+    pub distance: u32,
+    pub pos: [f32; 3],
+}

src/lib.rs

+pub mod common;
+pub mod preprocess;
+pub mod route;

src/main.rs

+use ed_lrr::preprocess::{preprocess_files, PreprocessOpts};
+use ed_lrr::route::{route, RouteOpts};
+use humantime::format_duration;
+use std::time::Instant;
+use structopt::StructOpt;
+#[derive(Debug, StructOpt)]
+#[structopt(
+    name = "ed_lrr",
+    about = "Elite: Dangerous Long-Range Router",
+    rename_all = "snake_case"
+)]
+enum Opts {
+    /// Plots a route through multiple systems
+    Route(RouteOpts),
+    /// Preprocess EDSM Dump
+    Preprocess(PreprocessOpts),
+}
+
+fn main() -> std::io::Result<()> {
+    let t_start = Instant::now();
+    let opts = Opts::from_args();
+    let ret = match opts {
+        Opts::Route(opts) => route(opts),
+        Opts::Preprocess(opts) => preprocess_files(opts),
+    };
+    println!("Total time: {}", format_duration(t_start.elapsed()));
+    ret
+}

src/preprocess.rs

+use crate::common::SystemSerde;
+use fnv::FnvHashMap;
+use humantime::format_duration;
+use indicatif::{ProgressBar, ProgressStyle};
+use serde::Deserialize;
+use serde_json::Result;
+use std::fs::File;
+use std::io::Seek;
+use std::io::{BufRead, BufReader, BufWriter, SeekFrom};
+use std::path::PathBuf;
+use std::str;
+use std::time::Instant;
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub struct PreprocessOpts {
+    #[structopt(short, long = "bodies")]
+    /// Path to bodies.json
+    pub bodies: PathBuf,
+    #[structopt(short, long = "systems")]
+    /// Path to systemsWithCoordinates.json
+    pub systems: PathBuf,
+    #[structopt(default_value = "stars")]
+    /// outfile prefix
+    pub prefix: String,
+}
+
+#[derive(Debug, Deserialize)]
+#[allow(non_snake_case)]
+struct Body {
+    name: String,
+    subType: String,
+    #[serde(rename = "type")]
+    body_type: String,
+    systemId: i32,
+    systemId64: i64,
+    #[serde(rename = "distanceToArrival")]
+    distance: u32,
+}
+
+#[derive(Debug, Deserialize)]
+struct Coords {
+    x: f32,
+    y: f32,
+    z: f32,
+}
+
+#[derive(Debug, Deserialize)]
+struct System {
+    id: i32,
+    id64: i64,
+    name: String,
+    coords: Coords,
+    date: String,
+}
+
+#[derive(Debug, StructOpt)]
+#[structopt(
+    name = "ed_lrr_pp",
+    about = "Preprocessor for Elite: Dangerous Long-Range Router",
+    rename_all = "snake_case"
+)]
+/// Preprocess data for ed_lrr
+struct Opt {
+    #[structopt(short, long = "bodies")]
+    /// Path to bodies.json
+    bodies: PathBuf,
+    #[structopt(short, long = "systems")]
+    /// Path to systemsWithCoordinates.json
+    systems: PathBuf,
+    #[structopt(default_value = "stars")]
+    /// outfile prefix
+    prefix: String,
+}
+
+fn get_mult(star_type: &str) -> f32 {
+    if star_type.contains("White Dwarf") {
+        return 1.5;
+    }
+    if star_type.contains("Neutron") {
+        return 4.0;
+    }
+    1.0
+}
+
+fn process(path: &PathBuf, func: &mut dyn for<'r> FnMut(&'r str) -> ()) -> std::io::Result<()> {
+    let mut cnt = 0;
+    let mut buffer = String::new();
+    let t_start = Instant::now();
+    let fh = File::open(path)?;
+    let prog_bar = ProgressBar::new(fh.metadata()?.len());
+    prog_bar.set_style(
+        ProgressStyle::default_bar()
+            .template(
+                "[{elapsed_precise}/{eta_precise}]{spinner} [{wide_bar}] {binary_bytes}/{binary_total_bytes} ({percent}%)",
+            )
+            .progress_chars("#9876543210 ")
+            .tick_chars("/-\\|"),
+    );
+    prog_bar.set_draw_delta(1024 * 1024);
+    let mut reader = BufReader::new(fh);
+    println!("Loading {} ...", path.to_str().unwrap());
+    while let Ok(n) = reader.read_line(&mut buffer) {
+        if n == 0 {
+            break;
+        }
+        buffer = buffer.trim_end().trim_end_matches(|c| c == ',').to_string();
+        if !buffer.is_empty() {
+            func(&buffer);
+        }
+        prog_bar.set_position(reader.seek(SeekFrom::Current(0)).unwrap());
+        cnt += 1;
+        buffer.clear();
+    }
+    prog_bar.finish_and_clear();
+    println!(
+        "Processed {} lines in {} ...",
+        cnt,
+        format_duration(t_start.elapsed())
+    );
+    Ok(())
+}
+
+fn process_systems(path: &PathBuf) -> FnvHashMap<i64, System> {
+    let mut ret = FnvHashMap::default();
+    process(path, &mut |line| {
+        let sys_res: Result<System> = serde_json::from_str(&line);
+        if let Ok(sys) = sys_res {
+            ret.insert(sys.id64, sys);
+        } else {
+            eprintln!("\nError parsing: {}\n\t{:?}\n", line, sys_res.unwrap_err());
+        }
+    })
+    .unwrap();
+    ret
+}
+
+fn build_index(path: &PathBuf) -> std::io::Result<()> {
+    let mut wtr = BufWriter::new(File::create(path.with_extension("idx"))?);
+    let mut idx: Vec<u64> = Vec::new();
+    let mut records = (csv::Reader::from_path(path)?).into_deserialize::<SystemSerde>();
+    loop {
+        idx.push(records.reader().position().byte());
+        if records.next().is_none() {
+            break;
+        }
+    }
+    bincode::serialize_into(&mut wtr, &idx).unwrap();
+    Ok(())
+}
+
+fn process_bodies(
+    path: &PathBuf,
+    out_prefix: &str,
+    systems: &mut FnvHashMap<i64, System>,
+) -> std::io::Result<()> {
+    let out_path = PathBuf::from(format!("{}.csv", out_prefix));
+    println!(
+        "Processing {} into {} ...",
+        path.to_str().unwrap(),
+        out_path.to_str().unwrap(),
+    );
+    let mut n: u32 = 0;
+    let mut wtr = csv::Writer::from_path(out_path)?;
+    process(path, &mut |line| {
+        if !line.contains("Star") {
+            return;
+        }
+        let body_res: Result<Body> = serde_json::from_str(&line);
+        if let Ok(body) = body_res {
+            if !body.body_type.contains("Star") {
+                return;
+            }
+            if let Some(sys) = systems.get(&body.systemId64) {
+                let sub_type = body.subType;
+                let mult = get_mult(&sub_type);
+                let sys_name = sys.name.clone();
+                let mut body_name = body.name.replace(&sys_name, "").trim().to_string();
+                if body_name == sys_name {
+                    body_name = "".to_string();
+                }
+                let rec = SystemSerde {
+                    id: n,
+                    star_type: sub_type,
+                    system: sys_name,
+                    body: body_name,
+                    mult,
+                    distance: body.distance,
+                    x: sys.coords.x,
+                    y: sys.coords.y,
+                    z: sys.coords.z,
+                };
+                wtr.serialize(rec).unwrap();
+                n += 1;
+            };
+        } else {
+            eprintln!("\nError parsing: {}\n\t{:?}\n", line, body_res.unwrap_err());
+        }
+    })
+    .unwrap();
+    println!("Total Systems: {}", n);
+    systems.clear();
+    Ok(())
+}
+
+pub fn preprocess_files(opts: PreprocessOpts) -> std::io::Result<()> {
+    let out_path = PathBuf::from(format!("{}.csv", &opts.prefix));
+    if !out_path.exists() {
+        let mut systems = process_systems(&opts.systems);
+        process_bodies(&opts.bodies, &opts.prefix, &mut systems)?;
+    } else {
+        println!(
+            "File '{}' exists, not overwriting it",
+            out_path.to_str().unwrap()
+        );
+    }
+    println!("Building index...");
+    println!("Index result: {:?}", build_index(&out_path));
+    Ok(())
+}