diff --git a/feeds.conf b/feeds.conf index 0ca4786..6439c6e 100644 --- a/feeds.conf +++ b/feeds.conf @@ -1,2 +1,2 @@ Stallman 3h https://stallman.org/rss/rss.xml -Lobsters 3h https://lobste.rs/rss +HN 3h https://news.ycombinator.com/rss diff --git a/source/app.d b/source/app.d index e0feb8e..a431ed4 100644 --- a/source/app.d +++ b/source/app.d @@ -5,20 +5,43 @@ import cartastraccia.actor; import vibe.core.log; +import vibe.http.server; +import vibe.core.core; import pegged.grammar; +import sumtype; import std.stdio; import std.file : readText; +import std.algorithm : each; static immutable string feedsFile = "feeds.conf"; +void handleReq(scope HTTPServerRequest req, scope HTTPServerResponse res) @safe +{ + logInfo("Received request"); +} + void main() { // parse feed list auto pt = ConfigFile(readText(feedsFile)); assert(pt.successful, "Invalid "~feedsFile~" file format, check cartastraccia.config for grammar"); - processFeeds(pt); + auto feeds = processFeeds(pt); - // parse every feed, update if needed - //parseRSS(readText("example.xml")); + // start tasks in charge of updating feeds + feeds.match!( + (InvalidFeeds i) => logFatal(i.msg), + (RSSFeed[] fl) { + fl.each!( + (RSSFeed feed) { + runTask(&updateFeed, feed.url); + }); + }); + + auto settings = HTTPServerSettings(); + settings.port = 8080; + settings.bindAddresses = ["127.0.0.1"]; + listenHTTP!handleReq(settings); + + runApplication; } diff --git a/source/cartastraccia/actor.d b/source/cartastraccia/actor.d index a85b88e..b6c3ec8 100644 --- a/source/cartastraccia/actor.d +++ b/source/cartastraccia/actor.d @@ -1,16 +1,86 @@ module cartastraccia.actor; -import std.algorithm : each; -import std.stdio; +import cartastraccia.rss; + import vibe.core.log; - +import vibe.inet.url; +import vibe.stream.operations : readAllUTF8; +import vibe.http.client; +import vibe.http.common; import pegged.grammar; +import sumtype; -void processFeeds(ParseTree pt) @trusted +import std.algorithm : each, filter; +import std.array; +import std.range; +import std.stdio; +import core.time; +import std.conv : to; + +alias RSSFeedList = SumType!(RSSFeed[], InvalidFeeds); + +struct InvalidFeeds { - foreach(ref conf; pt.children) { - foreach(ref feed; conf.children) { - } - } + string msg; } +struct RSSFeed +{ + string name; + Duration refresh; + URL url; + + this(string[] props) @safe + { + name = props[0]; + url = URL(props[3]); + + switch(props[2][0]) { + case 's': + refresh = dur!"seconds"(props[1].to!uint); + break; + case 'm': + refresh = dur!"minutes"(props[1].to!uint); + break; + case 'h': + refresh = dur!"hours"(props[1].to!uint); + break; + case 'd': + refresh = dur!"days"(props[1].to!uint); + break; + default: + assert(false, "should not get here"); + } + } + +} + +RSSFeedList processFeeds(ParseTree pt) @trusted +{ + RSSFeed[] feeds; + + foreach(ref conf; pt.children) { + foreach(ref feed; conf.children) { + feeds ~= RSSFeed(feed.matches + .filter!((immutable s) => s != "\n" && s != " ") + .array + ); + } + } + if(feeds.empty) return RSSFeedList(InvalidFeeds("No feeds found")); + else return RSSFeedList(feeds); +} + +auto updateFeed(immutable URL url) @safe +{ + RSS rss; + + requestHTTP(url, + (scope HTTPClientRequest req) { + req.method = HTTPMethod.GET; + }, + (scope HTTPClientResponse res) { + rss = res.bodyReader.readAllUTF8.parseRSS; + }); + +} diff --git a/source/cartastraccia/config.d b/source/cartastraccia/config.d index 11e06d6..4caf24b 100644 --- a/source/cartastraccia/config.d +++ b/source/cartastraccia/config.d @@ -7,13 +7,13 @@ /** * Specify grammar for config file in the form: * ... - * [feed_name] [refresh_time] "[feed_address]" + * [feed_name] [refresh_time] [feed_address] * ... */ immutable string ConfigFileParser = ` ConfigFile: - ConfigFile <- Feed (Newline Feed)* + ConfigFile <- Feed* (Newline Feed)* Feed <- Name space* Refresh space* Address diff --git a/source/cartastraccia/rss.d b/source/cartastraccia/rss.d index aeb7cc8..2a84504 100644 --- a/source/cartastraccia/rss.d +++ b/source/cartastraccia/rss.d @@ -4,6 +4,8 @@ import std.experimental.xml; import sumtype; +import std.algorithm.searching : startsWith; + public: alias RSS = SumType!(ValidRSS, InvalidRSS); @@ -23,7 +25,6 @@ * A valid RSS feed is made of various channels */ struct ValidRSS { - string feedName = ""; RSSChannel[string] channels; } @@ -74,7 +75,7 @@ string source; } -void parseRSS(R)(R feed) @trusted +RSS parseRSS(R)(R feed) @trusted { auto cursor = chooseLexer!string .parser @@ -92,6 +93,8 @@ cursor.next(); } } + + return rss; } private: @@ -119,26 +122,33 @@ } else assert(false, "Invalid ElementType provided"); while(cursor.kind != XMLKind.elementEnd && cursor.name != elname) { + immutable name = cursor.name; + if(name == "item") { + static if(is(ElementType == RSSChannel)) { - logInfo("---> Found item:"); cursor.enter(); insertElement!(RSSItem, RSSChannel, C)(rss, newElement, cursor); cursor.exit(); } + + } else if(name.startsWith("atom")){ + + logWarn("Skipping atom link identifier: " ~ name); + } else { + cursor.enter(); immutable content = cursor.content; cursor.exit(); - logInfo("Processing: " ~ name ~ ": " ~ content); - fill: switch(name) { + default: logWarn("Invalid XML entry detected: " ~ name); - rss = InvalidRSS(name, content); break fill; + static if(is(ElementType == RSSChannel)) { static foreach(m; __traits(allMembers, RSSChannel)) { static if(m != "items") { @@ -147,17 +157,20 @@ break fill; } } + } else if(is(ElementType == RSSItem)) { static foreach(m; __traits(allMembers, RSSItem)) { case m: mixin("newElement."~m~" = content;"); break fill; } + } else { assert(false, "Invalid ElementType requested"); } } } + cursor.next(); }