diff --git a/dub.sdl b/dub.sdl index 8ca98a4..61b0cb1 100644 --- a/dub.sdl +++ b/dub.sdl @@ -3,7 +3,8 @@ authors "fra" copyright "Copyright © 2019, fra" license "GPLv3" -dependency "vibe-http" path="/home/fra/_progs/saoc/vibe-http" +/*dependency "vibe-http" path="/home/fra/_progs/saoc/vibe-http"*/ +dependency "vibe-d:web" version="*" dependency "sumtype" version="~>0.8.13" dependency "pegged" version="*" dependency "std-experimental-xml" version="~>0.1.7" diff --git a/source/app.d b/source/app.d index a431ed4..8109938 100644 --- a/source/app.d +++ b/source/app.d @@ -1,32 +1,52 @@ module app; -import cartastraccia.rss; import cartastraccia.config; import cartastraccia.actor; +import cartastraccia.endpoint; import vibe.core.log; import vibe.http.server; +import vibe.http.router; +import vibe.inet.url; +import vibe.http.client; +import vibe.web.web; import vibe.core.core; +import vibe.stream.operations : readAllUTF8; +import vibe.core.concurrency; import pegged.grammar; import sumtype; +import std.exception; import std.stdio; import std.file : readText; import std.algorithm : each; +import std.getopt; +import std.conv : to; -static immutable string feedsFile = "feeds.conf"; +immutable string info = "============================================= +CARTASTRACCIA is a news reader for RSS feeds. +============================================= +0. Write a feeds.conf file [feed_name refresh_timeout feed_url] +> echo \"Stallman 3h https://stallman.org/rss/rss.xml\" > feeds.conf +--------------------------------------------- +1. Start the daemon: +> cartastraccia --daemon --endpoint=cli --endpoint=html --feeds=feeds.conf +--------------------------------------------- +2a. Connect to daemon using CLI endpoint: +> cartastraccia +--------------------------------------------- +2b. Connect to daemon using HTML endpoint: +> elinks \"http://localhost:8080\" +---------------------------------------------"; -void handleReq(scope HTTPServerRequest req, scope HTTPServerResponse res) @safe -{ - logInfo("Received request"); -} - -void main() +void runDaemon(EndpointType[] endpoints, immutable string feedsFile, immutable + string bindAddress, immutable ushort bindPort) { // parse feed list auto pt = ConfigFile(readText(feedsFile)); assert(pt.successful, "Invalid "~feedsFile~" file format, check cartastraccia.config for grammar"); auto feeds = processFeeds(pt); + TaskMap tasks; // start tasks in charge of updating feeds feeds.match!( @@ -34,14 +54,62 @@ (RSSFeed[] fl) { fl.each!( (RSSFeed feed) { - runTask(&updateFeed, feed.url); + tasks[feed.name] = runWorkerTaskH(&updateFeed, feed.path); }); }); - auto settings = HTTPServerSettings(); - settings.port = 8080; - settings.bindAddresses = ["127.0.0.1"]; - listenHTTP!handleReq(settings); - runApplication; + // start server to accept client requests + auto router = new URLRouter; + router.registerWebInterface(new EndpointService(feeds, tasks)); + + auto settings = new HTTPServerSettings; + settings.port = bindPort; + settings.bindAddresses = ["127.0.0.1", bindAddress]; + + listenHTTP(settings, router); + runEventLoop(); +} + +void runClient(immutable string bindAddress, immutable ushort bindPort) +{ + URL url = URL("http://"~bindAddress~":"~bindPort.to!string~"/cli"); + try { + requestHTTP(url, + (scope HTTPClientRequest req) { + req.method = HTTPMethod.GET; + }, + (scope HTTPClientResponse res) { + writeln(res.bodyReader.readAllUTF8()); + }); + } catch (Exception e) { + logWarn("ERROR from daemon: "~e.msg~"\nCheck daemon logs for details (is it running?)"); + } +} + +void main(string[] args) +{ + // CLI arguments + bool daemon = false; + EndpointType[] endpoints = [EndpointType.cli]; + string feedsFile = "feeds.conf"; + string bindAddress = "localhost"; + ushort bindPort = 8080; + + auto helpInformation = getopt( + args, + "daemon|d", "Start daemon", &daemon, + "endpoint|e", "Endpoints to register [cli]", &endpoints, + "feeds|f", "File containing feeds to pull [feeds.conf]", &feedsFile, + "host|l", "Bind to this address [localhost]", &bindAddress, + "port|p", "Bind to this port [8080]", &bindPort + ); + + if(helpInformation.helpWanted) { + defaultGetoptPrinter(info, helpInformation.options); + return; + } + + if(daemon) runDaemon(endpoints, feedsFile, bindAddress, bindPort); + else runClient(bindAddress, bindPort); } diff --git a/source/cartastraccia/actor.d b/source/cartastraccia/actor.d index b6c3ec8..6261e8d 100644 --- a/source/cartastraccia/actor.d +++ b/source/cartastraccia/actor.d @@ -7,18 +7,22 @@ import vibe.stream.operations : readAllUTF8; import vibe.http.client; import vibe.http.common; +import vibe.core.concurrency; +import vibe.core.core; import pegged.grammar; import sumtype; import std.algorithm : each, filter; import std.array; import std.range; -import std.stdio; import core.time; import std.conv : to; +import std.variant; alias RSSFeedList = SumType!(RSSFeed[], InvalidFeeds); +alias TaskMap = Task[string]; + struct InvalidFeeds { string msg; @@ -28,12 +32,12 @@ { string name; Duration refresh; - URL url; + string path; this(string[] props) @safe { name = props[0]; - url = URL(props[3]); + path = props[3]; switch(props[2][0]) { case 's': @@ -71,16 +75,60 @@ else return RSSFeedList(feeds); } -auto updateFeed(immutable URL url) @safe +void updateFeed(immutable string path) @trusted { RSS rss; + URL url = URL(path); + requestHTTP(url, (scope HTTPClientRequest req) { req.method = HTTPMethod.GET; }, (scope HTTPClientResponse res) { - rss = res.bodyReader.readAllUTF8.parseRSS; + parseRSS(rss, res.bodyReader.readAllUTF8()); }); + while(true) { + auto webTask = receiveOnly!Task; + + receive( + (FeedActorRequest r) { + if(r == FeedActorRequest.DATA) { + logInfo("Received data request from task: "~webTask.getDebugID()); + RequestData data = dumpRSScli(rss); + webTask.dispatch(data); + } else if(r == FeedActorRequest.QUIT){ + logDebug("Task exiting"); + return; + }}, + (Variant v) { + logDebug("Invalid message received"); + }); + } +} + +/** + * Communication protocol between tasks +*/ + +enum FeedActorRequest { DATA, QUIT } + +alias RequestData = string; +alias RequestDataLength = ulong; + +static immutable ulong chunkSize = 4096; + +void dispatch(scope Task task, immutable string data) +{ + ulong len = data.length; + task.send(len); + + ulong a = 0; + ulong b = chunkSize; + while(a < len) { + task.send(data[a..b]); + a = b; + b = (b+chunkSize > len) ? len : b + chunkSize; + } } diff --git a/source/cartastraccia/config.d b/source/cartastraccia/config.d index 4caf24b..804d064 100644 --- a/source/cartastraccia/config.d +++ b/source/cartastraccia/config.d @@ -28,5 +28,4 @@ Timeunit <- [mshd] Newline <- endOfLine - `; diff --git a/source/cartastraccia/endpoint.d b/source/cartastraccia/endpoint.d new file mode 100644 index 0000000..8b92522 --- /dev/null +++ b/source/cartastraccia/endpoint.d @@ -0,0 +1,62 @@ +module cartastraccia.endpoint; + +import cartastraccia.actor; + +import vibe.core.log; +import vibe.core.task; +import vibe.core.concurrency; +import vibe.http.router; +import vibe.web.web; +import sumtype; + +import std.algorithm : each; + +enum EndpointType { + cli, + xml, + html +} + +class EndpointService { + + private { + RSSFeedList feedList; + TaskMap tasks; + } + + this(RSSFeedList fl, TaskMap tm) + { + feedList = fl; + tasks = tm; + } + + @path("/") void getHTMLEndpoint() + { + //TODO + } + + @path("/cli") void getCLIEndpoint(scope HTTPServerResponse res) + { + feedList.match!( + (InvalidFeeds i) {}, + (RSSFeed[] fl) { + fl.each!( + (RSSFeed f) { + tasks[f.name].send(Task.getThis()); + + tasks[f.name].send(FeedActorRequest.DATA); + + auto totSize = receiveOnly!RequestDataLength; + + RequestDataLength recSize = 0; + RequestData data; + while(recSize < totSize) { + data ~= receiveOnly!RequestData; + recSize += chunkSize; + } + + res.writeBody(data); + }); + }); + } +} diff --git a/source/cartastraccia/rss.d b/source/cartastraccia/rss.d index 2a84504..3713482 100644 --- a/source/cartastraccia/rss.d +++ b/source/cartastraccia/rss.d @@ -5,6 +5,7 @@ import sumtype; import std.algorithm.searching : startsWith; +import std.conv : to; public: @@ -75,15 +76,14 @@ string source; } -RSS parseRSS(R)(R feed) @trusted +void parseRSS(ref RSS rss, immutable string feed) @trusted { auto cursor = chooseLexer!string .parser - .cursor((CursorError err) { logWarn(err); }); + .cursor((CursorError err) {}); cursor.setSource(feed); - RSS rss; cursor.enter(); cursor.enter(); if(cursor.name == "channel") { @@ -93,8 +93,36 @@ cursor.next(); } } +} - return rss; +// mainly for debugging purposes +string dumpRSScli(ref RSS rss) +{ + string res; + + rss.match!( + (InvalidRSS i) { + res = "Invalid RSS feed"; + }, + (ValidRSS vr) { + foreach(cname, channel; vr.channels) { + res ~= "\n===\n~" + ~ cname ~ "\n" + ~ channel.link ~ "\n" + ~ channel.description ~ "\n" + ~ "\n===\n"; + ulong cnt = 0; + foreach(iname, item; channel.items) { + res ~= " " ~ cnt.to!string ~ ". " + ~ item.title ~ "\n" + ~ item.link ~ "\n" + ~ "---\n" + ~ item.description ~ "\n---\n"; + cnt++; + } + } + }); + return res; } private: @@ -135,7 +163,7 @@ } else if(name.startsWith("atom")){ - logWarn("Skipping atom link identifier: " ~ name); + logDebug("Skipping atom link identifier: " ~ name); } else { @@ -146,7 +174,7 @@ fill: switch(name) { default: - logWarn("Invalid XML entry detected: " ~ name); + logDebug("Invalid XML entry detected: " ~ name); break fill; static if(is(ElementType == RSSChannel)) { @@ -176,7 +204,7 @@ rss.match!( (ref InvalidRSS i) { - logWarn("Invalid XML entry detected: " + logDebug("Invalid XML entry detected: " ~ i.element ~ ": " ~ i.content);