diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6cd596f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +public/channels/* +dub.selections.json diff --git a/public/css/news.css b/public/css/news.css index 322075a..025b684 100644 --- a/public/css/news.css +++ b/public/css/news.css @@ -16,8 +16,9 @@ } body .title h1 { - font-size: 60px; + font-size: 1vw; text-align: center; + text-decoration: bold; color: #b2675e; } @@ -38,7 +39,7 @@ } body .index h3 a { - text-align: left; + text-align: center; font-size: 40px; color: #FFFFFF; text-decoration: bold; @@ -46,13 +47,20 @@ } body .index a { - text-align: center; + text-align: left; font-size: 20px; color: #e6e8e6; text-decoration: none; background-color: none; } +body .index b { + text-align: right; + font-size: 20px; + color: #bdd2e1; + background-color: none; +} + body .row:after { content: ""; display: table; diff --git a/source/app.d b/source/app.d index fd6935f..e6d6202 100644 --- a/source/app.d +++ b/source/app.d @@ -1,6 +1,7 @@ module app; import cartastraccia.config; +import cartastraccia.asciiart; import cartastraccia.actor; import cartastraccia.endpoint; @@ -22,12 +23,12 @@ import std.stdio; import std.file : readText; import std.algorithm : each; +import std.datetime : SysTime; import std.getopt; import std.conv : to; import std.process; -immutable string info = "============================================= -CARTASTRACCIA is a news reader for RSS feeds. +immutable string info = asciiArt~" ============================================= 0. Write a feeds.conf file [feed_name refresh_timeout feed_url] > echo \"Stallman 3h https://stallman.org/rss/rss.xml\" > feeds.conf @@ -74,24 +75,10 @@ (RSSFeed[] fl) { fl.each!( (RSSFeed feed) { + // start workers to serve RSS data tasks[feed.name] = runWorkerTaskH(&feedActor, feed.name, feed.path); - // refresh RSS data with a timer - setTimer(feed.refresh, () { - if(feed.name in tasks) - tasks[feed.name].send(Task.getThis()); - else return; - - auto resp = receiveOnly!FeedActorResponse; - if(resp == FeedActorResponse.INVALID) { - tasks.remove(feed.name); - return; - } - - tasks[feed.name].send(FeedActorRequest.QUIT); - tasks[feed.name] = runWorkerTaskH(&feedActor, feed.name, feed.path); - }, true); }); }); @@ -105,21 +92,49 @@ }); } -void runClient(EndpointType endpoint, immutable string browser, immutable string bindAddress, immutable ushort bindPort) +void runClient(EndpointType endpoint, immutable string browser, immutable string + bindAddress, immutable ushort bindPort, immutable bool reloadFeeds) { - if(endpoint == EndpointType.cli) { - URL url = URL("http://"~bindAddress~":"~bindPort.to!string~"/cli"); + + if(reloadFeeds) { + + URL url = URL("http://"~bindAddress~":"~bindPort.to!string~"/reload"); + try { requestHTTP(url, + (scope HTTPClientRequest req) { req.method = HTTPMethod.GET; }, + + (scope HTTPClientResponse res) { + // TODO proper info + }); + + } catch (Exception e) { + logWarn("ERROR from daemon: "~e.msg~"\nCannot reload feeds file."); + } + } + + if(endpoint == EndpointType.cli) { + + URL url = URL("http://"~bindAddress~":"~bindPort.to!string~"/cli"); + + try { + requestHTTP(url, + + (scope HTTPClientRequest req) { + req.method = HTTPMethod.GET; + }, + (scope HTTPClientResponse res) { writeln(res.bodyReader.readAllUTF8()); }); + } catch (Exception e) { logWarn("ERROR from daemon: "~e.msg~"\nCheck daemon logs for details (is it running?)"); } + } else if(endpoint == EndpointType.html) { if(!existsFile(browser)) { @@ -143,6 +158,7 @@ string bindAddress = "localhost"; ushort bindPort = 8080; string browser = "/usr/bin/elinks"; + bool reloadFeeds = false; auto helpInformation = getopt( args, @@ -151,7 +167,8 @@ "feeds|f", "File containing feeds to pull [feeds.conf]", &feedsFile, "host|l", "Bind to this address [localhost]", &bindAddress, "port|p", "Bind to this port [8080]", &bindPort, - "browser|b", "Absolute path to browser for HTML rendering [/usr/bin/elinks]", &browser + "browser|b", "Absolute path to browser for HTML rendering [/usr/bin/elinks]", &browser, + "reload|r", "Reload feeds file", &reloadFeeds ); if(helpInformation.helpWanted) { @@ -160,5 +177,5 @@ } if(daemon) runDaemon(feedsFile, bindAddress, bindPort); - else runClient(endpoint, browser, bindAddress, bindPort); + else runClient(endpoint, browser, bindAddress, bindPort, reloadFeeds); } diff --git a/source/cartastraccia/actor.d b/source/cartastraccia/actor.d index c885e7d..7365655 100644 --- a/source/cartastraccia/actor.d +++ b/source/cartastraccia/actor.d @@ -45,12 +45,12 @@ (ref InvalidRSS i) { logWarn("Invalid feed at: "~path); logWarn("Caused by entry \""~i.element~"\": "~i.content); - busyListen(rss); + busyListen(feedName, rss); }, (ref ValidRSS vr) { immutable fileName = "public/channels/"~feedName~".html"; createHTMLPage(vr, feedName, fileName); - busyListen(rss); + busyListen(feedName, rss); }); } @@ -71,7 +71,7 @@ /** * Listen for messages from the webserver */ -void busyListen(ref RSS rss) { +void busyListen(immutable string feedName, ref RSS rss) { rss.match!( (ref InvalidRSS i) { auto webTask = receiveOnly!Task; @@ -93,25 +93,31 @@ // receive the actual request receive( (FeedActorRequest r) { + switch(r) { - - if(r == FeedActorRequest.DATA_CLI) { + case FeedActorRequest.DATA_CLI: logInfo("Received CLI request from task: "~webTask.getDebugID()); immutable string data = dumpRSS!(FeedActorRequest.DATA_CLI)(vr); webTask.dispatchCLI(data); + break; - } else if(r == FeedActorRequest.DATA_HTML) { - logInfo("Received HTML request from task: "~webTask.getDebugID()); + case FeedActorRequest.DATA_HTML: + logInfo("Received HTML request on feed: "~feedName~"[Task: "~webTask.getDebugID()~"]"); + break; - } else if(r == FeedActorRequest.QUIT){ + case FeedActorRequest.QUIT: logInfo("Task exiting due to QUIT request."); return; - }}, + default: + logFatal("Task received unknown request."); + } + }, (Variant v) { logFatal("Invalid message received from webserver."); }); + } catch (Exception e) { logWarn("Waiting for actors to complete loading feeds."); } diff --git a/source/cartastraccia/asciiart.d b/source/cartastraccia/asciiart.d new file mode 100644 index 0000000..73ffe6d --- /dev/null +++ b/source/cartastraccia/asciiart.d @@ -0,0 +1,15 @@ +module cartastraccia.asciiart; + +static immutable string asciiArt = r" + ▄████▄ ▄▄▄ ██▀███ ▄▄▄█████▓ ▄▄▄ ██████ ▄▄▄█████▓ ██▀███ ▄▄▄ ▄████▄ ▄████▄ ██▓ ▄▄▄ +▒██▀ ▀█ ▒████▄ ▓██ ▒ ██▒▓ ██▒ ▓▒▒████▄ ▒██ ▒ ▓ ██▒ ▓▒▓██ ▒ ██▒▒████▄ ▒██▀ ▀█ ▒██▀ ▀█ ▓██▒▒████▄ +▒▓█ ▄ ▒██ ▀█▄ ▓██ ░▄█ ▒▒ ▓██░ ▒░▒██ ▀█▄ ░ ▓██▄ ▒ ▓██░ ▒░▓██ ░▄█ ▒▒██ ▀█▄ ▒▓█ ▄ ▒▓█ ▄ ▒██▒▒██ ▀█▄ +▒▓▓▄ ▄██▒░██▄▄▄▄██ ▒██▀▀█▄ ░ ▓██▓ ░ ░██▄▄▄▄██ ▒ ██▒░ ▓██▓ ░ ▒██▀▀█▄ ░██▄▄▄▄██ ▒▓▓▄ ▄██▒▒▓▓▄ ▄██▒░██░░██▄▄▄▄██ +▒ ▓███▀ ░ ▓█ ▓██▒░██▓ ▒██▒ ▒██▒ ░ ▓█ ▓██▒ ▒██████▒▒ ▒██▒ ░ ░██▓ ▒██▒ ▓█ ▓██▒▒ ▓███▀ ░▒ ▓███▀ ░░██░ ▓█ ▓██▒ +░ ░▒ ▒ ░ ▒▒ ▓▒█░░ ▒▓ ░▒▓░ ▒ ░░ ▒▒ ▓▒█░ ▒ ▒▓▒ ▒ ░ ▒ ░░ ░ ▒▓ ░▒▓░ ▒▒ ▓▒█░░ ░▒ ▒ ░░ ░▒ ▒ ░░▓ ▒▒ ▓▒█░ + ░ ▒ ▒ ▒▒ ░ ░▒ ░ ▒░ ░ ▒ ▒▒ ░ ░ ░▒ ░ ░ ░ ░▒ ░ ▒░ ▒ ▒▒ ░ ░ ▒ ░ ▒ ▒ ░ ▒ ▒▒ ░ +░ ░ ▒ ░░ ░ ░ ░ ▒ ░ ░ ░ ░ ░░ ░ ░ ▒ ░ ░ ▒ ░ ░ ▒ +░ ░ ░ ░ ░ ░ ░ ░ ░ ░ ░░ ░ ░ ░ ░ ░ ░ +░ ░ ░ + +"; diff --git a/source/cartastraccia/config.d b/source/cartastraccia/config.d index 2d5867a..85134d2 100644 --- a/source/cartastraccia/config.d +++ b/source/cartastraccia/config.d @@ -4,6 +4,7 @@ import sumtype; import core.time; +import std.datetime; import std.conv : to; import std.algorithm : filter; import std.range; @@ -47,10 +48,12 @@ { string name; Duration refresh; + SysTime lastUpdate; string path; this(string[] props) @safe { + lastUpdate = Clock.currTime(); name = props[0]; path = props[3]; @@ -89,4 +92,3 @@ if(feeds.empty) return RSSFeedList(InvalidFeeds("No feeds found")); else return RSSFeedList(feeds); } - diff --git a/source/cartastraccia/endpoint.d b/source/cartastraccia/endpoint.d index 904e604..f9fe282 100644 --- a/source/cartastraccia/endpoint.d +++ b/source/cartastraccia/endpoint.d @@ -1,9 +1,12 @@ module cartastraccia.endpoint; import cartastraccia.config; +import cartastraccia.asciiart; import cartastraccia.actor; import cartastraccia.rss; +import core.time; +import vibe.core.core; import vibe.core.log; import vibe.core.task; import vibe.core.concurrency; @@ -12,6 +15,7 @@ import sumtype; import std.algorithm : each; +import std.datetime; enum EndpointType { cli, @@ -30,16 +34,45 @@ { feedList = fl; tasks = tm; + + // refresh RSS data with a timer + feedList.tryMatch!((ref RSSFeed[] fl) { + + fl.each!((ref RSSFeed feed) { + + setTimer(feed.refresh, () { + + if(feed.name in tasks) + tasks[feed.name].send(Task.getThis()); + else return; + + auto resp = receiveOnly!FeedActorResponse; + if(resp == FeedActorResponse.INVALID) { + tasks.remove(feed.name); + return; + } + + tasks[feed.name].send(FeedActorRequest.QUIT); + tasks[feed.name] = runWorkerTaskH(&feedActor, feed.name, feed.path); + + feed.lastUpdate = Clock.currTime(); + + }, true); + }); + }); } @path("/") void getHTMLEndpoint(scope HTTPServerRequest req, scope HTTPServerResponse res) { RSSFeed[] validFeeds; feedList.match!( + (InvalidFeeds i) {}, + (RSSFeed[] fl) { - fl.each!( - (RSSFeed f) { + + fl.each!((RSSFeed f) { + // send task for response from server if(f.name in tasks) tasks[f.name].send(Task.getThis()); else { @@ -60,11 +93,16 @@ // add valid feed to list validFeeds ~= f; }); + feedList = validFeeds; - res.render!("index.dt", req, validFeeds); + res.render!("index.dt", req, validFeeds, asciiArt); + }); } + /** + * Debug purpose only ATM + */ @path("/cli") void getCLIEndpoint(scope HTTPServerResponse res) { string data; @@ -101,3 +139,4 @@ res.writeBody(data); } } + diff --git a/source/cartastraccia/renderer.d b/source/cartastraccia/renderer.d index 095e8e3..66f75d4 100644 --- a/source/cartastraccia/renderer.d +++ b/source/cartastraccia/renderer.d @@ -7,8 +7,8 @@ import vibe.core.path; import std.conv : to; -import std.stdio; import std.array : appender; +import std.regex; void createHTMLPage(ref ValidRSS rss, immutable string feedName, immutable string pageName) { @@ -19,39 +19,37 @@ Cartastraccia - `~feedName~` `; - foreach(cname, channel; rss.channels) { - auto chCont = doc.createElement("div", doc.root.firstChild); - chCont.attr("class", "channel"); - chCont.attr("id", feedName); - chCont.html = "

"~cname~"

"; + auto chCont = doc.createElement("div", doc.root.firstChild); + chCont.attr("class", "channel"); + chCont.attr("id", feedName); + chCont.html = "

"~rss.channel.title~"

"; - auto row = doc.createElement("div", doc.root.firstChild); - row.attr("class", "row"); + auto row = doc.createElement("div", doc.root.firstChild); + row.attr("class", "row"); - auto icnt = channel.items.length; + auto icnt = rss.channel.items.length; - auto column1 = doc.createElement("div", doc.root.firstChild); - column1.attr("class", "channelitem"); - auto column2= doc.createElement("div", doc.root.firstChild); - column2.attr("class", "channelitem"); + auto column1 = doc.createElement("div", doc.root.firstChild); + column1.attr("class", "channelitem"); + auto column2= doc.createElement("div", doc.root.firstChild); + column2.attr("class", "channelitem"); - uint i=0; - foreach(iname, item; channel.items) { - if(i < icnt/2) { - auto itemCont = doc.createElement("div", column1); - itemCont.html = "

"~iname~"

" - ~ "View Source" - ~ "

"~item.pubDate~"

" - ~ item.description; - } else { - auto itemCont = doc.createElement("div", column2); - itemCont.html = "

"~iname~"

" - ~ "View Source" - ~ "

"~item.pubDate~"

" - ~ item.description; - } - i++; + uint i=0; + foreach(item; rss.channel.items) { + if(i < icnt/2) { + auto itemCont = doc.createElement("div", column1); + itemCont.html = "

"~item.title~"

" + ~ "View Source" + ~ "

"~item.pubDate~"

" + ~ replaceAll(item.description, regex("<.*pre>"), ""); + } else { + auto itemCont = doc.createElement("div", column2); + itemCont.html = "

"~item.title~"

" + ~ "View Source" + ~ "

"~item.pubDate~"

" + ~ replaceAll(item.description, regex("<.*pre>"), ""); } + i++; } auto output = appender!string; diff --git a/source/cartastraccia/rss.d b/source/cartastraccia/rss.d index fe8f052..0298a04 100644 --- a/source/cartastraccia/rss.d +++ b/source/cartastraccia/rss.d @@ -7,7 +7,8 @@ import dxml.parser; import sumtype; -import std.algorithm : startsWith; +import std.algorithm : startsWith, sort; +import std.datetime; import std.range; import std.conv : to; @@ -33,7 +34,7 @@ struct ValidRSS { // cannot be copied @disable this(this); - RSSChannel[string] channels; + RSSChannel channel; } /** @@ -64,7 +65,7 @@ string skipHours; string skipDays; - RSSItem[string] items; + RSSItem[] items; } struct RSSItem { @@ -93,14 +94,14 @@ string res; - foreach(cname, ref channel; rss.channels) { - res ~= "\n===\n~" - ~ cname ~ "\n" - ~ channel.link ~ "\n" - ~ channel.description ~ "\n" - ~ "\n===\n"; - ulong cnt = 0; - foreach(iname, item; channel.items) { + res ~= "\n===\n~" + ~ rss.channel.title ~ "\n" + ~ rss.channel.link ~ "\n" + ~ rss.channel.description ~ "\n" + ~ "\n===\n"; + + uint cnt = 1; + foreach(item; rss.channel.items) { res ~= " " ~ cnt.to!string ~ ". " ~ item.title ~ "\n" ~ item.link ~ "\n" @@ -108,12 +109,9 @@ ~ item.description ~ "\n---\n"; cnt++; } - } return res; // generate a valid HTML dump from the given rss struct - } else if(dataFormat == FeedActorRequest.DATA_HTML) { - // TODO } else logFatal("Invalid data format received from webserver."); } @@ -136,6 +134,18 @@ alias C = typeof(rssRange); insertElement!(RSSChannel, RSS, C)(rss, rss, rssRange); + + // parse date and sort in descending order (newest first) + rss.tryMatch!( + (ref InvalidRSS i) { + logWarn("Invalid RSS for feed: " ~ feed); + return; + }, + (ref ValidRSS vr) { + vr.channel.items.sort!( (i,j) => + (parseRFC822DateTime(i.pubDate) + > parseRFC822DateTime(j.pubDate))); + }); } @@ -224,10 +234,10 @@ static if(is(ElementType == RSSChannel)) parent.tryMatch!( (ref ValidRSS v) { - v.channels[newElement.title] = newElement; + v.channel = newElement; }); else if(is(ElementType == RSSItem)) - parent.items[newElement.title] = newElement; + parent.items ~= newElement; logInfo("Inserted " ~ elname ~ ": " ~ newElement.title); }); } diff --git a/views/index.dt b/views/index.dt index fb89a45..6707d58 100644 --- a/views/index.dt +++ b/views/index.dt @@ -6,18 +6,30 @@ :css body #title(class="title") - h1 Cartastraccia - h2 "...fuck ads" + h1 + pre #{asciiArt} + h2 Never mind the bollocks - ulong flen = validFeeds.length; - foreach(feed; validFeeds[0..flen/2]) + - import std.datetime; + - auto dt = cast(DateTime)feed.lastUpdate; + - immutable updated = dt.toSimpleString; + #chname1(class="index") h3 a(href="channels/"~feed.name~".html") #{feed.name} p a(href=feed.path) external link + b + Last update: #{updated} + - foreach(feed; validFeeds[flen/2..$]) + - import std.datetime; + - auto dt = cast(DateTime)feed.lastUpdate; + - immutable updated = dt.toSimpleString; #chname2(class="index") h3 a(href="channels/"~feed.name~".html") #{feed.name} p a(href=feed.path) external link + b Last update: #{updated}