...
 
Commits (10)
DevArticlator
=============
This project provides the ability to pull down all of your Dev.to articles.
The objective is to be able to push your articles from a git repository as
outlined in the Dev.to article:
https://dev.to/jessekphillips/hobby-project-dev-to-api-59ln
Usage
-----
The program utilizes a Dev.to API key stored in an environment variable
`appkey`. At this time there are no commandline arguments as the application
will just pull your articles.
```
export apikey=<https://dev.to/settings/account>
./devarticulator
```
After execution you will find all of your articles and meta data in
'devArticles'.
......@@ -32,6 +32,8 @@ void main()
.until!inPreviousPull
.structureArticles
.tee!(x => x.saveArticle(savePath))
// Don't use unpublished articles to determine newer
.filter!(x => x.meta.published)
.reduceToNewest;
import std.datetime;
......
......@@ -215,14 +215,14 @@ auto reduceToNewest(Range)(Range articles)
} unittest {
PullState ps;
ps.datePulled = SysTime(DateTime(2018, 1, 1, 10, 30, 0), UTC());
ps.lastArticle = 9_998;
alias inPreviousPull = (x) => !x.isNewerArticle(ps);
auto devreq = generate!fakeArticleMe
.seqence(ps.datePulled)
.seqence(ps.datePulled + dur!"days"(1))
.map!(x => x.deserializeJson!(ArticleMe))
.take(3);
auto newestid = devreq.front.id;
assert(devreq.until!inPreviousPull
.structureArticles
.reduceToNewest
.meta.id == 10_000);
.meta.id == newestid);
}
......@@ -6,6 +6,9 @@
* on the items important to this application.
*/
module devto.api;
import std.datetime;
import vibe.data.serialization;
import vibe.data.json;
......@@ -19,6 +22,7 @@ struct ArticleMe {
@optional string description;
@optional string cover_image;
@optional bool published;
@optional SysTime published_timestamp;
@optional string[] tag_list;
@optional string canonical_url;
@optional string body_markdown;
......
......@@ -32,11 +32,11 @@ struct PullState {
*/
@safe pure nothrow @nogc
auto isNewerArticle(const ArticleMe am, const PullState ps) {
return am.id > ps.lastArticle;
if(!am.published) return true;
return am.published_timestamp > ps.datePulled;
} unittest {
PullState ps;
ps.datePulled = SysTime(DateTime(2018, 1, 1, 10, 30, 0), UTC());
ps.lastArticle = 10_000;
alias inPreviousPull = (x) => !x.isNewerArticle(ps);
auto devreq = generate!fakeArticleMe
.seqence(ps.datePulled)
......@@ -46,14 +46,13 @@ auto isNewerArticle(const ArticleMe am, const PullState ps) {
} unittest {
PullState ps;
ps.datePulled = SysTime(DateTime(2018, 1, 1, 10, 30, 0), UTC());
ps.lastArticle = 9_999;
alias inPreviousPull = (x) => !x.isNewerArticle(ps);
auto devreq = generate!fakeArticleMe
.seqence(ps.datePulled)
.seqence(ps.datePulled + dur!"days"(1))
.map!(x => x.deserializeJson!(ArticleMe))
.take(3)
.map!(x => cast(immutable(ArticleMe))x);
assert(devreq.until!inPreviousPull.map!(x => x.id).equal([10_000]));
assert(devreq.until!inPreviousPull.map!(x => x.published_timestamp).equal([ps.datePulled + dur!"days"(1)]));
}
/**
......@@ -79,26 +78,72 @@ struct DevToRange {
private const(ArticleMe)[] function(uint page) @safe articlePage;
const(ArticleMe)[] data;
uint page;
uint page = 1;
@safe nothrow pure
auto front() {
@safe nothrow pure const
const(ArticleMe) front() {
return data.front;
}
@safe nothrow pure
auto empty() {
@safe nothrow pure const
bool empty() {
return data.empty;
}
@safe pure
auto popFront() {
@safe
void popFront() {
data.popFront;
if(data.empty)
data = articlePage(++page);
}
} unittest {
alias fakeArticles = (uint page) @safe {
return () @trusted {
static iteration = 0;
if(iteration++ < 1)
return generate!fakeArticleMe
.map!(x => cast(immutable(ArticleMe))x.deserializeJson!(ArticleMe))
.take(3)
.array;
return generate!fakeArticleMe
.map!(x => cast(immutable(ArticleMe))x.deserializeJson!(ArticleMe))
.take(0)
.array;
}();
};
DevToRange dtv;
dtv.articlePage = fakeArticles;
dtv.data = dtv.articlePage(1);
import std.algorithm : count;
assert(dtv.count == 3);
} unittest {
alias fakeArticles = (uint page) @safe {
return () @trusted {
static iteration = 0;
if(iteration++ < 2)
return generate!fakeArticleMe
.map!(x => cast(immutable(ArticleMe))x.deserializeJson!(ArticleMe))
.take(3)
.array;
return generate!fakeArticleMe
.map!(x => cast(immutable(ArticleMe))x.deserializeJson!(ArticleMe))
.take(0)
.array;
}();
};
DevToRange dtv;
dtv.articlePage = fakeArticles;
dtv.data = dtv.articlePage(1);
import std.algorithm : count;
assert(dtv.count == 6);
}
@safe
auto devtoMyArticles() {
DevToRange devtoMyArticles() {
DevToRange dtv;
dtv.articlePage = &devArticles;
dtv.data = dtv.articlePage(1);
......@@ -106,7 +151,7 @@ auto devtoMyArticles() {
}
@safe
auto devArticles(uint page) {
const(ArticleMe)[] devArticles(uint page) {
import vibe.core.log;
import vibe.http.client;
import std.process;
......@@ -119,12 +164,12 @@ auto devArticles(uint page) {
req.headers.addField("api-key", environment["appkey"]);
},
(scope res) @safe {
logInfo("Response: %d", res.statusCode);
if(res.statusCode != 200)
return;
auto data = res.readJson();
auto savePath = "devArticles";
ret = meArticles(data);
import std.algorithm : each;
auto data = res.readJson().get!(Json[]);
data.each!(x => x["published_timestamp"] = x["published_timestamp"].get!string.empty ? SysTime.init.toISOExtString : x["published_timestamp"].get!string);
ret = meArticles(Json(data));
}
);
......
......@@ -64,37 +64,41 @@ auto seqence(R)(R articles, const SysTime newestDate = Clock.currTime)
if(is(ElementType!R == Json)) {
struct DateSequencing {
DateTime date;
uint id = 10_000;
typeof(iota(3).randomCover) id;
R inner;
this(typeof(id) ids) {
id = ids;
}
auto empty () {
if(inner.empty) return true;
if(id < 2) return true;
if(id.empty) return true;
return false;
}
auto front() {
auto ans = inner.front;
ans["id"] = Json(id);
ans["id"] = Json(id.front);
// DateTime has no timezone, we pretend it is UTC after cast
ans["published_timestamp"] = Json(date.toISOExtString ~ "Z");
ans["published"] = Json(true);
return ans;
}
auto popFront() {
inner.popFront();
id--;
id.popFront;
date -= dur!"days"(uniform(1,12));
}
}
DateSequencing ds;
DateSequencing ds = DateSequencing(iota(10_000).randomCover);
ds.date = cast(DateTime) newestDate;
ds.inner = articles;
return ds;
} unittest {
import std.algorithm : equal, map, isSorted;
auto seq = generate!fakeArticleMe.seqence(Clock.currTime).take(3);
assert(seq.map!(x => x["id"].get!uint).equal([10_000, 9_999, 9_998]));
assert(seq.map!(x => x["published_timestamp"].get!string).array.isSorted!"a > b");
}