diff --git a/dub.sdl b/dub.sdl index f5ff662..e41ac72 100644 --- a/dub.sdl +++ b/dub.sdl @@ -3,9 +3,11 @@ authors "fra" copyright "Copyright © 2019, fra" license "GPLv3" - dependency "dxml" version="~>0.4.1" dependency "htmld" version="~>0.3.7" dependency "pegged" version=">=0.0.0" dependency "sumtype" version="~>0.8.13" +dependency "requests" version="~>1.0.9" dependency "vibe-d:web" version=">0.8.5" + +subConfigurations "requests" "vibed" diff --git a/source/app.d b/source/app.d index 594dc08..f170700 100644 --- a/source/app.d +++ b/source/app.d @@ -18,6 +18,7 @@ import vibe.core.concurrency; import pegged.grammar; import sumtype; +import requests; import std.exception; import std.stdio; @@ -77,11 +78,10 @@ (RSSFeed[] fl) { fl.each!( (RSSFeed feed) { - logWarn("Starting: "~feed.name); + logInfo("Starting task: "~feed.name); // start workers to serve RSS data - tasks[feed.name] = - runWorkerTaskH(&feedActor, feed.name, - feed.path, 0); + tasks[feed.name] = runWorkerTaskH( + &feedActor, feed.name, feed.path, 0); }); }); @@ -100,19 +100,12 @@ { if(reloadFeeds) { - - URL url = URL("http://"~bindAddress~":"~bindPort.to!string~"/reload"); - try { - requestHTTP(url, - - (scope HTTPClientRequest req) { - req.method = HTTPMethod.GET; - }, - - (scope HTTPClientResponse res) { - // TODO proper info - }); + string url = "http://"~bindAddress~":"~bindPort.to!string~"/reload"; + auto req = Request(); + req.keepAlive = false; + req.timeout = REQ_TIMEOUT; + req.get(url); } catch (Exception e) { logWarn("ERROR from daemon: "~e.msg~"\nCannot reload feeds file."); @@ -120,19 +113,12 @@ } if(endpoint == EndpointType.cli) { - - URL url = URL("http://"~bindAddress~":"~bindPort.to!string~"/cli"); - try { - requestHTTP(url, - - (scope HTTPClientRequest req) { - req.method = HTTPMethod.GET; - }, - - (scope HTTPClientResponse res) { - writeln(res.bodyReader.readAllUTF8()); - }); + string url = "http://"~bindAddress~":"~bindPort.to!string~"/cli"; + auto req = Request(); + req.keepAlive = false; + req.timeout = REQ_TIMEOUT; + req.get(url); } catch (Exception e) { logWarn("ERROR from daemon: "~e.msg~"\nCheck daemon logs for details (is it running?)"); diff --git a/source/cartastraccia/actor.d b/source/cartastraccia/actor.d index 40e3f39..fc46044 100644 --- a/source/cartastraccia/actor.d +++ b/source/cartastraccia/actor.d @@ -12,6 +12,7 @@ import vibe.core.core; import pegged.grammar; import sumtype; +import requests; import std.algorithm : each, filter; import std.array; @@ -23,6 +24,7 @@ alias TaskMap = Task[string]; immutable uint MAX_RETRIES = 3; +immutable REQ_TIMEOUT = 2.seconds; /** * Actor in charge of: @@ -36,36 +38,37 @@ URL url = URL(path); try { - requestHTTP(url, - (scope HTTPClientRequest req) { - req.method = HTTPMethod.GET; - }, - (scope HTTPClientResponse res) { - parseRSS(rss, res.bodyReader.readAllUTF8()); - }); + auto req = Request(); + req.keepAlive = false; + req.timeout = REQ_TIMEOUT; + + auto res = req.get(path); + parseRSS(rss, cast(immutable string)res.responseBody.data); } catch (Exception e) { - logWarn("Failed connecting to: " ~ path ~ " with error: " ~ e.msg); if(retries < MAX_RETRIES) { - logWarn("Retrying."); feedActor(feedName, path, retries+1); + return; } - return; - + rss = FailedRSS(e.msg); } rss.match!( (ref InvalidRSS i) { logWarn("Invalid feed at: "~path); logWarn("Caused by entry \""~i.element~"\": "~i.content); - listenOnce(feedName, rss); + }, + (ref FailedRSS f) { + logWarn("Failed to load feed: "~ feedName); + logWarn("Error: "~f.msg); }, (ref ValidRSS vr) { immutable fileName = "public/channels/"~feedName~".html"; createHTMLPage(vr, feedName, fileName); - listenOnce(feedName, rss); }); + + listenOnce(feedName, rss); } /** @@ -95,6 +98,11 @@ webTask.send(FeedActorResponse.INVALID); quit = true; }, + (ref FailedRSS f) { + auto webTask = receiveOnly!Task; + webTask.send(FeedActorResponse.INVALID); + quit = true; + }, (ref ValidRSS vr) { try { diff --git a/source/cartastraccia/endpoint.d b/source/cartastraccia/endpoint.d index 0617ce6..606a60d 100644 --- a/source/cartastraccia/endpoint.d +++ b/source/cartastraccia/endpoint.d @@ -5,7 +5,6 @@ import cartastraccia.actor; import cartastraccia.rss; -import core.time; import vibe.core.core; import vibe.core.log; import vibe.core.task; @@ -16,6 +15,7 @@ import std.algorithm : each; import std.datetime; +import core.time; enum EndpointType { cli, diff --git a/source/cartastraccia/rss.d b/source/cartastraccia/rss.d index 0298a04..8f89ee6 100644 --- a/source/cartastraccia/rss.d +++ b/source/cartastraccia/rss.d @@ -14,7 +14,16 @@ public: -alias RSS = SumType!(ValidRSS, InvalidRSS); +alias RSS = SumType!(ValidRSS, InvalidRSS, FailedRSS); + + +/** + * In case the RSS feed couldn't be loaded + */ +struct FailedRSS { + @disable this(this); + string msg; +} /** * In case an element was found @@ -230,6 +239,7 @@ ~ ": " ~ i.content); }, + (ref FailedRSS f) {}, (ref ValidRSS v) { static if(is(ElementType == RSSChannel)) parent.tryMatch!(