diff --git a/source/cartastraccia/actor.d b/source/cartastraccia/actor.d index c97d1ff..2fc4327 100644 --- a/source/cartastraccia/actor.d +++ b/source/cartastraccia/actor.d @@ -3,6 +3,7 @@ import cartastraccia.rss; import vibe.core.log; +import vibe.core.file : copyFile; import vibe.inet.url; import vibe.stream.operations : readAllUTF8; import vibe.http.client; @@ -21,7 +22,13 @@ alias TaskMap = Task[string]; -void updateFeed(immutable string path) @trusted +/** + * Actor in charge of: + * - parsing a RSS feed + * - dumping news to DB + * - listening for messages from the webserver +*/ +void feedActor(immutable string feedName, immutable string path) @trusted { RSS rss; URL url = URL(path); @@ -35,36 +42,67 @@ parseRSS(rss, res.bodyReader.readAllUTF8()); }); - while(true) { - auto webTask = receiveOnly!Task; + rss.match!( + (ref InvalidRSS i) { + logWarn("Invalid feed at: "~path); + logWarn("Caused by entry \""~i.element~"\": "~i.content); + logWarn("Task exiting."); + }, + (ref ValidRSS vr) { + immutable dtname = "channels/"~feedName ~ ".dt"; + copyFile(NativePath("views/channels/template.dt"), + NativePath("views/"~dtname), true); - receive( - (FeedActorRequest r) { - if(r == FeedActorRequest.DATA) { - logInfo("Received data request from task: "~webTask.getDebugID()); - RequestData data = dumpRSScli(rss); - webTask.dispatch(data); - } else if(r == FeedActorRequest.QUIT){ - logDebug("Task exiting"); - return; - }}, - (Variant v) { - logDebug("Invalid message received"); + busyListen(vr); }); - } } /** * Communication protocol between tasks */ -enum FeedActorRequest { DATA, QUIT } +enum FeedActorRequest { DATA_CLI, DATA_HTML, QUIT } -alias RequestData = string; alias RequestDataLength = ulong; static immutable ulong chunkSize = 4096; -void dispatch(scope Task task, immutable string data) +private: + +/** + * Listen for messages from the webserver +*/ +void busyListen(ref ValidRSS rss) { + while(true) { + // receive the webserver task + auto webTask = receiveOnly!Task; + + // receive the actual request + receive( + (FeedActorRequest r) { + + + if(r == FeedActorRequest.DATA_CLI) { + logInfo("Received CLI request from task: "~webTask.getDebugID()); + immutable string data = dumpRSS!(FeedActorRequest.DATA_CLI)(rss); + webTask.dispatchCLI(data); + + } else if(r == FeedActorRequest.DATA_HTML) { + logInfo("Received HTML request from task: "~webTask.getDebugID()); + webTask.dispatchHTML(rss.channels); + + } else if(r == FeedActorRequest.QUIT){ + logDebug("Task exiting due to quit request."); + return; + + }}, + + (Variant v) { + logFatal("Invalid message received from webserver."); + }); + } +} + +void dispatchCLI(scope Task task, immutable string data) { ulong len = data.length; task.send(len); @@ -77,3 +115,19 @@ b = (b+chunkSize > len) ? len : b + chunkSize; } } + +void dispatchHTML(scope Task task, RSSChannel[string] channels) +{ + ulong len = channels.length; + task.send(len); + + foreach(string cname, RSSChannel ch; channels) { + task.send(cname); + ulong ilen = ch.items.length; + task.send(ilen); + foreach(string iname, RSSItem item; ch.items) { + task.send(item); + } + } +} + diff --git a/source/cartastraccia/endpoint.d b/source/cartastraccia/endpoint.d index 4b6f46d..325b9da 100644 --- a/source/cartastraccia/endpoint.d +++ b/source/cartastraccia/endpoint.d @@ -2,6 +2,7 @@ import cartastraccia.config; import cartastraccia.actor; +import cartastraccia.rss; import vibe.core.log; import vibe.core.task; @@ -31,14 +32,52 @@ tasks = tm; } - @path("/") void getHTMLEndpoint() + @path("/") void getHTMLEndpoint(scope HTTPServerRequest req, scope HTTPServerResponse res) { - //TODO + struct ChannelItems { + string cname; + string fname; + RSSItem[] items; + } + + ChannelItems[] channelItems; + + feedList.tryMatch!( + (RSSFeed[] fl) { + ChannelItems[] tmpCh; + fl.each!( + (RSSFeed f) { + // send task for response from server + tasks[f.name].send(Task.getThis()); + // send data request + tasks[f.name].send(FeedActorRequest.DATA_HTML); + + // receive data length + auto nch = receiveOnly!RequestDataLength; + RequestDataLength chRecv = 0; + + while(chRecv < nch) { + ChannelItems chit; + chit.fname = f.name; + chit.cname = receiveOnly!string; + + RequestDataLength nit = receiveOnly!RequestDataLength; + RequestDataLength iRecv = 0; + while(iRecv < nit) { + chit.items ~= receiveOnly!RSSItem; + iRecv++; + } + channelItems ~= chit; + chRecv++; + } + }); + res.render!("index.dt", req, channelItems, fl); + }); } @path("/cli") void getCLIEndpoint(scope HTTPServerResponse res) { - RequestData data; + string data; feedList.match!( (InvalidFeeds i) {}, (RSSFeed[] fl) { @@ -47,14 +86,14 @@ // send task for response from server tasks[f.name].send(Task.getThis()); // send data request - tasks[f.name].send(FeedActorRequest.DATA); + tasks[f.name].send(FeedActorRequest.DATA_CLI); // receive data length auto totSize = receiveOnly!RequestDataLength; RequestDataLength recSize = 0; while(recSize < totSize) { - data ~= receiveOnly!RequestData; + data ~= receiveOnly!string; recSize += chunkSize; } diff --git a/source/cartastraccia/rss.d b/source/cartastraccia/rss.d index b2235d2..fe8f052 100644 --- a/source/cartastraccia/rss.d +++ b/source/cartastraccia/rss.d @@ -1,6 +1,9 @@ module cartastraccia.rss; +import cartastraccia.actor : FeedActorRequest; + import vibe.core.log; +import vibe.http.server : render; import dxml.parser; import sumtype; @@ -11,7 +14,6 @@ public: alias RSS = SumType!(ValidRSS, InvalidRSS); -alias RSSParent = SumType!(RSS, RSSChannel); /** * In case an element was found @@ -19,6 +21,8 @@ * see: http://www.rssboard.org/rss-specification */ struct InvalidRSS { + // cannot be copied + @disable this(this); string element; string content; } @@ -27,12 +31,14 @@ * A valid RSS feed is made of various channels */ struct ValidRSS { + // cannot be copied + @disable this(this); RSSChannel[string] channels; } /** * Each channel has properties - * and various items (actual news) + * and various RSSItems (actual news) */ struct RSSChannel { // required elements @@ -77,6 +83,43 @@ string source; } +/** + * Entry point for dumping a valid rss feed +*/ +string dumpRSS(FeedActorRequest dataFormat)(ref ValidRSS rss, immutable string feedName = "") +{ + // cli is mainly for testing functionality + static if(dataFormat == FeedActorRequest.DATA_CLI) { + + string res; + + foreach(cname, ref channel; rss.channels) { + res ~= "\n===\n~" + ~ cname ~ "\n" + ~ channel.link ~ "\n" + ~ channel.description ~ "\n" + ~ "\n===\n"; + ulong cnt = 0; + foreach(iname, item; channel.items) { + res ~= " " ~ cnt.to!string ~ ". " + ~ item.title ~ "\n" + ~ item.link ~ "\n" + ~ "---\n" + ~ item.description ~ "\n---\n"; + cnt++; + } + } + return res; + + // generate a valid HTML dump from the given rss struct + } else if(dataFormat == FeedActorRequest.DATA_HTML) { + // TODO + } else logFatal("Invalid data format received from webserver."); +} + +/** + * Entrypoint for parsing a rss feed (repsesented as string) +*/ void parseRSS(ref RSS rss, immutable string feed) @trusted { auto rssRange = parseXML!simpleXML(feed); @@ -95,50 +138,9 @@ insertElement!(RSSChannel, RSS, C)(rss, rss, rssRange); } -// mainly for debugging purposes -string dumpRSScli(ref RSS rss) -{ - string res; - - rss.match!( - (InvalidRSS i) { - res = "Invalid RSS feed"; - }, - (ValidRSS vr) { - foreach(cname, channel; vr.channels) { - res ~= "\n===\n~" - ~ cname ~ "\n" - ~ channel.link ~ "\n" - ~ channel.description ~ "\n" - ~ "\n===\n"; - ulong cnt = 0; - foreach(iname, item; channel.items) { - res ~= " " ~ cnt.to!string ~ ". " - ~ item.title ~ "\n" - ~ item.link ~ "\n" - ~ "---\n" - ~ item.description ~ "\n---\n"; - cnt++; - } - } - }); - return res; -} private: -static immutable string selectElementName = " - string elname; - - static if(is(ElementType == RSSChannel)) { - elname = \"channel\"; - static assert(is(Parent == RSS)); - } else if(is(ElementType == RSSItem)) { - elname = \"item\"; - static assert(is(Parent == RSSChannel)); - } else assert(false, \"Invalid ElementType provided\"); -"; - /** * Insert an element (RSSChannel or RSSItem) which has: * - A parent (be it the RSS xml root (RSSChannel @@ -153,6 +155,7 @@ mixin(selectElementName); + // advance the parser to completion, entry by entry while(rssRange.front.type != EntityType.elementEnd && rssRange.front.type != EntityType.text && rssRange.front.name != elname) { @@ -162,6 +165,7 @@ if(name == "item") { + // recursively insert items static if(is(ElementType == RSSChannel)) { insertElement!(RSSItem, RSSChannel, C)(rss, newElement, rssRange); } else { @@ -171,15 +175,18 @@ } else if(rssRange.front.type == EntityType.text || rssRange.front.type == EntityType.cdata) { + // found a valid text field immutable content = rssRange.front.text; rssRange.popFront(); fill: switch(name) { default: + // we don't care about entries which are not attributes of RSSChannel logDebug("Ignoring XML Entity: " ~ name); break fill; + // inserting a channel static if(is(ElementType == RSSChannel)) { static foreach(m; __traits(allMembers, RSSChannel)) { static if(m != "items") { @@ -189,6 +196,7 @@ } } + // inserting an item } else if(is(ElementType == RSSItem)) { static foreach(m; __traits(allMembers, RSSItem)) { case m: @@ -196,13 +204,15 @@ break fill; } + // should not get here (means function invocation was invalid) } else assert(false, "Invalid ElementType requested"); } } - + // skip elementEnd rssRange.popFront(); } + // finished channel / item parsing. Insert it into rss struct rss.match!( (ref InvalidRSS i) { logWarn("Invalid XML Entity detected: " @@ -213,9 +223,24 @@ (ref ValidRSS v) { static if(is(ElementType == RSSChannel)) parent.tryMatch!( - (ref ValidRSS v) => v.channels[newElement.title] = newElement); + (ref ValidRSS v) { + v.channels[newElement.title] = newElement; + }); else if(is(ElementType == RSSItem)) parent.items[newElement.title] = newElement; logInfo("Inserted " ~ elname ~ ": " ~ newElement.title); }); } + +static immutable string selectElementName = " + string elname; + + static if(is(ElementType == RSSChannel)) { + elname = \"channel\"; + static assert(is(Parent == RSS)); + } else if(is(ElementType == RSSItem)) { + elname = \"item\"; + static assert(is(Parent == RSSChannel)); + } else assert(false, \"Invalid ElementType provided\"); +"; +