README.md

+DevArticlator
+=============
+
+This project provides the ability to pull down all of your Dev.to articles.
+The objective is to be able to push your articles from a git repository as
+outlined in the Dev.to article:
+https://dev.to/jessekphillips/hobby-project-dev-to-api-59ln
+
+
+Usage
+-----
+The program utilizes a Dev.to API key stored in an environment variable
+`appkey`. At this time there are no commandline arguments as the application
+will just pull your articles.
+
+```
+export apikey=<https://dev.to/settings/account>
+./devarticulator
+```
+
+After execution you will find all of your articles and meta data in
+'devArticles'.

source/app.d

@@ -32,6 +32,8 @@ void main()
         .until!inPreviousPull
         .structureArticles
         .tee!(x => x.saveArticle(savePath))
+        // Don't use unpublished articles to determine newer
+        .filter!(x => x.meta.published)
         .reduceToNewest;
 
     import std.datetime;

source/articlewriter.d

@@ -215,14 +215,14 @@ auto reduceToNewest(Range)(Range articles)
 } unittest {
     PullState ps;
     ps.datePulled = SysTime(DateTime(2018, 1, 1, 10, 30, 0), UTC());
-    ps.lastArticle = 9_998;
     alias inPreviousPull = (x) => !x.isNewerArticle(ps);
     auto devreq = generate!fakeArticleMe
-        .seqence(ps.datePulled)
+        .seqence(ps.datePulled + dur!"days"(1))
         .map!(x => x.deserializeJson!(ArticleMe))
         .take(3);
+    auto newestid = devreq.front.id;
     assert(devreq.until!inPreviousPull
            .structureArticles
            .reduceToNewest
-           .meta.id == 10_000);
+           .meta.id == newestid);
 }

source/devto/api.d

@@ -6,6 +6,9 @@
  * on the items important to this application.
  */
 module devto.api;
+
+import std.datetime;
+
 import vibe.data.serialization;
 import vibe.data.json;
 
@@ -19,6 +22,7 @@ struct ArticleMe {
     @optional string description;
     @optional string cover_image;
     @optional bool published;
+    @optional SysTime published_timestamp;
     @optional string[] tag_list;
     @optional string canonical_url;
     @optional string body_markdown;

source/devtopull.d

@@ -32,11 +32,11 @@ struct PullState {
  */
 @safe pure nothrow @nogc
 auto isNewerArticle(const ArticleMe am, const PullState ps) {
-    return am.id > ps.lastArticle;
+    if(!am.published) return true;
+    return am.published_timestamp > ps.datePulled;
 } unittest {
     PullState ps;
     ps.datePulled = SysTime(DateTime(2018, 1, 1, 10, 30, 0), UTC());
-    ps.lastArticle = 10_000;
     alias inPreviousPull = (x) => !x.isNewerArticle(ps);
     auto devreq = generate!fakeArticleMe
         .seqence(ps.datePulled)
@@ -46,14 +46,13 @@ auto isNewerArticle(const ArticleMe am, const PullState ps) {
 } unittest {
     PullState ps;
     ps.datePulled = SysTime(DateTime(2018, 1, 1, 10, 30, 0), UTC());
-    ps.lastArticle = 9_999;
     alias inPreviousPull = (x) => !x.isNewerArticle(ps);
     auto devreq = generate!fakeArticleMe
-        .seqence(ps.datePulled)
+        .seqence(ps.datePulled + dur!"days"(1))
         .map!(x => x.deserializeJson!(ArticleMe))
         .take(3)
         .map!(x => cast(immutable(ArticleMe))x);
-    assert(devreq.until!inPreviousPull.map!(x => x.id).equal([10_000]));
+    assert(devreq.until!inPreviousPull.map!(x => x.published_timestamp).equal([ps.datePulled + dur!"days"(1)]));
 }
 
 /**
@@ -79,26 +78,72 @@ struct DevToRange {
     private const(ArticleMe)[] function(uint page) @safe articlePage;
 
     const(ArticleMe)[] data;
-    uint page;
+    uint page = 1;
 
-    @safe nothrow pure
-    auto front() {
+    @safe nothrow pure const
+    const(ArticleMe) front() {
         return data.front;
     }
 
-    @safe nothrow pure
-    auto empty() {
+    @safe nothrow pure const
+    bool empty() {
         return data.empty;
     }
 
-    @safe pure
-    auto popFront() {
+    @safe
+    void popFront() {
         data.popFront;
+        if(data.empty)
+            data = articlePage(++page);
     }
+} unittest {
+    alias fakeArticles = (uint page) @safe {
+        return () @trusted {
+        static iteration = 0;
+        if(iteration++ < 1)
+        return generate!fakeArticleMe
+            .map!(x => cast(immutable(ArticleMe))x.deserializeJson!(ArticleMe))
+            .take(3)
+            .array;
+        return generate!fakeArticleMe
+            .map!(x => cast(immutable(ArticleMe))x.deserializeJson!(ArticleMe))
+            .take(0)
+            .array;
+        }();
+    };
+
+    DevToRange dtv;
+    dtv.articlePage = fakeArticles;
+    dtv.data = dtv.articlePage(1);
+
+    import std.algorithm : count;
+    assert(dtv.count == 3);
+} unittest {
+    alias fakeArticles = (uint page) @safe {
+        return () @trusted {
+        static iteration = 0;
+        if(iteration++ < 2)
+        return generate!fakeArticleMe
+            .map!(x => cast(immutable(ArticleMe))x.deserializeJson!(ArticleMe))
+            .take(3)
+            .array;
+        return generate!fakeArticleMe
+            .map!(x => cast(immutable(ArticleMe))x.deserializeJson!(ArticleMe))
+            .take(0)
+            .array;
+        }();
+    };
+
+    DevToRange dtv;
+    dtv.articlePage = fakeArticles;
+    dtv.data = dtv.articlePage(1);
+
+    import std.algorithm : count;
+    assert(dtv.count == 6);
 }
 
 @safe
-auto devtoMyArticles() {
+DevToRange devtoMyArticles() {
     DevToRange dtv;
     dtv.articlePage = &devArticles;
     dtv.data = dtv.articlePage(1);
@@ -106,7 +151,7 @@ auto devtoMyArticles() {
 }
 
 @safe
-auto devArticles(uint page) {
+const(ArticleMe)[] devArticles(uint page) {
     import vibe.core.log;
     import vibe.http.client;
     import std.process;
@@ -119,12 +164,12 @@ auto devArticles(uint page) {
             req.headers.addField("api-key", environment["appkey"]);
         },
         (scope res) @safe {
-            logInfo("Response: %d", res.statusCode);
             if(res.statusCode != 200)
                 return;
-            auto data = res.readJson();
-            auto savePath = "devArticles";
-            ret = meArticles(data);
+                import std.algorithm : each;
+            auto data = res.readJson().get!(Json[]);
+            data.each!(x => x["published_timestamp"] = x["published_timestamp"].get!string.empty ?  SysTime.init.toISOExtString : x["published_timestamp"].get!string);
+            ret = meArticles(Json(data));
         }
     );
 

source/unittesting/articlegenerator.d

@@ -64,37 +64,41 @@ auto seqence(R)(R articles, const SysTime newestDate = Clock.currTime)
             if(is(ElementType!R == Json)) {
     struct DateSequencing {
         DateTime date;
-        uint id = 10_000;
+        typeof(iota(3).randomCover) id;
         R inner;
 
+        this(typeof(id) ids) {
+            id = ids;
+        }
+
         auto empty () {
             if(inner.empty) return true;
-            if(id < 2)      return true;
+            if(id.empty) return true;
             return false;
         }
 
         auto front() {
             auto ans = inner.front;
-            ans["id"] = Json(id);
+            ans["id"] = Json(id.front);
             // DateTime has no timezone, we pretend it is UTC after cast
             ans["published_timestamp"] = Json(date.toISOExtString ~ "Z");
+            ans["published"] = Json(true);
             return ans;
         }
 
         auto popFront() {
             inner.popFront();
-            id--;
+            id.popFront;
             date -= dur!"days"(uniform(1,12));
         }
     }
 
-    DateSequencing ds;
+    DateSequencing ds = DateSequencing(iota(10_000).randomCover);
     ds.date = cast(DateTime) newestDate;
     ds.inner = articles;
     return ds;
 } unittest {
     import std.algorithm : equal, map, isSorted;
     auto seq = generate!fakeArticleMe.seqence(Clock.currTime).take(3);
-    assert(seq.map!(x => x["id"].get!uint).equal([10_000, 9_999, 9_998]));
     assert(seq.map!(x => x["published_timestamp"].get!string).array.isSorted!"a > b");
 }