diff --git a/dub.sdl b/dub.sdl index 61b0cb1..6d7c5b4 100644 --- a/dub.sdl +++ b/dub.sdl @@ -3,8 +3,7 @@ authors "fra" copyright "Copyright © 2019, fra" license "GPLv3" -/*dependency "vibe-http" path="/home/fra/_progs/saoc/vibe-http"*/ -dependency "vibe-d:web" version="*" +dependency "dxml" version="~>0.4.1" +dependency "pegged" version=">=0.0.0" dependency "sumtype" version="~>0.8.13" -dependency "pegged" version="*" -dependency "std-experimental-xml" version="~>0.1.7" +dependency "vibe-d:web" version=">=0.0.0" diff --git a/source/cartastraccia/actor.d b/source/cartastraccia/actor.d index 6261e8d..dd66c6d 100644 --- a/source/cartastraccia/actor.d +++ b/source/cartastraccia/actor.d @@ -125,7 +125,7 @@ task.send(len); ulong a = 0; - ulong b = chunkSize; + ulong b = (chunkSize > len) ? len : chunkSize; while(a < len) { task.send(data[a..b]); a = b; diff --git a/source/cartastraccia/config.d b/source/cartastraccia/config.d index 804d064..bc332ec 100644 --- a/source/cartastraccia/config.d +++ b/source/cartastraccia/config.d @@ -13,9 +13,9 @@ immutable string ConfigFileParser = ` ConfigFile: - ConfigFile <- Feed* (Newline Feed)* + ConfigFile <- Feed+ - Feed <- Name space* Refresh space* Address + Feed <- Name space* Refresh space* Address Newline Name <- identifier @@ -27,5 +27,5 @@ Timeunit <- [mshd] - Newline <- endOfLine + Newline <- endOfLine / endOfInput `; diff --git a/source/cartastraccia/endpoint.d b/source/cartastraccia/endpoint.d index 8b92522..16e2a18 100644 --- a/source/cartastraccia/endpoint.d +++ b/source/cartastraccia/endpoint.d @@ -37,26 +37,31 @@ @path("/cli") void getCLIEndpoint(scope HTTPServerResponse res) { + RequestData data; feedList.match!( (InvalidFeeds i) {}, (RSSFeed[] fl) { fl.each!( (RSSFeed f) { + // send task for response from server tasks[f.name].send(Task.getThis()); - + // send data request tasks[f.name].send(FeedActorRequest.DATA); - + // receive data length auto totSize = receiveOnly!RequestDataLength; RequestDataLength recSize = 0; - RequestData data; + while(recSize < totSize) { data ~= receiveOnly!RequestData; recSize += chunkSize; } - res.writeBody(data); + data ~= "End of feed: " ~ f.name ~ "\n"; + }); }); + + res.writeBody(data); } } diff --git a/source/cartastraccia/rss.d b/source/cartastraccia/rss.d index 3713482..0c69852 100644 --- a/source/cartastraccia/rss.d +++ b/source/cartastraccia/rss.d @@ -1,10 +1,11 @@ module cartastraccia.rss; import vibe.core.log; -import std.experimental.xml; +import dxml.parser; import sumtype; -import std.algorithm.searching : startsWith; +import std.algorithm : startsWith; +import std.range; import std.conv : to; public: @@ -78,21 +79,20 @@ void parseRSS(ref RSS rss, immutable string feed) @trusted { - auto cursor = chooseLexer!string - .parser - .cursor((CursorError err) {}); - - cursor.setSource(feed); - - cursor.enter(); - cursor.enter(); - if(cursor.name == "channel") { - if(cursor.enter()) { - alias C = typeof(cursor); - insertElement!(RSSChannel, RSS, C)(rss, rss, cursor); - cursor.next(); - } + auto rssRange = parseXML!simpleXML(feed); + if(rssRange.front.name == "html") { + logWarn("Unable to parse HTML file"); + rss = InvalidRSS("html", ""); + return; } + + while(rssRange.front.name != "channel") { + rssRange.popFront(); + } + rssRange.popFront(); + + alias C = typeof(rssRange); + insertElement!(RSSChannel, RSS, C)(rss, rss, rssRange); } // mainly for debugging purposes @@ -132,10 +132,10 @@ * - A parent (be it the RSS xml root (RSSChannel * or the RSSChannel in case of an RSSItem * - Various sub-entries which are processed sequentially - * by advancing cursor + * by advancing rssRange */ void insertElement(ElementType, Parent, C)( - ref RSS rss, ref Parent parent, ref C cursor) @trusted + ref RSS rss, ref Parent parent, ref C rssRange) @trusted { ElementType newElement; @@ -149,32 +149,35 @@ static assert(is(Parent == RSSChannel)); } else assert(false, "Invalid ElementType provided"); - while(cursor.kind != XMLKind.elementEnd && cursor.name != elname) { + while(rssRange.front.type != EntityType.elementEnd + && rssRange.front.type != EntityType.text + && rssRange.front.name != elname) { - immutable name = cursor.name; + immutable name = rssRange.front.name; + rssRange.popFront(); if(name == "item") { static if(is(ElementType == RSSChannel)) { - cursor.enter(); - insertElement!(RSSItem, RSSChannel, C)(rss, newElement, cursor); - cursor.exit(); + insertElement!(RSSItem, RSSChannel, C)(rss, newElement, rssRange); + } else { + rss = InvalidRSS(name, ""); } } else if(name.startsWith("atom")){ logDebug("Skipping atom link identifier: " ~ name); - } else { + } else if(rssRange.front.type == EntityType.text + || rssRange.front.type == EntityType.cdata) { - cursor.enter(); - immutable content = cursor.content; - cursor.exit(); + immutable content = rssRange.front.text; + rssRange.popFront(); fill: switch(name) { default: - logDebug("Invalid XML entry detected: " ~ name); + logDebug("Ignoring XML Entity: " ~ name); break fill; static if(is(ElementType == RSSChannel)) { @@ -199,12 +202,12 @@ } } - cursor.next(); + rssRange.popFront(); } rss.match!( (ref InvalidRSS i) { - logDebug("Invalid XML entry detected: " + logWarn("Invalid XML Entity detected: " ~ i.element ~ ": " ~ i.content);