diff options
| -rwxr-xr-x | src/fead.py | 108 |
1 files changed, 83 insertions, 25 deletions
diff --git a/src/fead.py b/src/fead.py index 4fd2b1b..ac1fcd7 100755 --- a/src/fead.py +++ b/src/fead.py @@ -15,6 +15,9 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>. +__version__ = '0.0.1' + +from argparse import ArgumentParser, FileType from asyncio import gather, open_connection, run from collections import namedtuple from datetime import datetime @@ -22,18 +25,27 @@ from email.utils import parsedate_to_datetime from http.client import HTTPResponse from io import BytesIO from operator import attrgetter -from sys import stdin +from pathlib import Path +from re import compile as regex +from sys import stdin, stdout +from textwrap import shorten from urllib.error import HTTPError from urllib.parse import urljoin, urlsplit from warnings import warn from xml.etree.ElementTree import fromstring as parse_xml REQUEST = 'GET {} HTTP/1.0\r\nHost: {}\r\n\r\n' -Item = namedtuple('Item', ('title', 'link', 'time', 'summary')) +HTML_TAG = regex('<.+?>') + Advert = namedtuple('Advert', ('source_title', 'source_link', 'title', 'link', 'time', 'summary')) +def read_urls(path): + """Read newline-separated URLs from given file path.""" + return Path(path).read_text().splitlines() + + class BytesSocket: """Duck socket for HTTPResponse.""" def __init__(self, response): @@ -47,7 +59,7 @@ class BytesSocket: def parse_rss_item(xml): """Parse given RSS item.""" - time = datetime.fromtimestamp(0).astimezone(None) + time = datetime.fromtimestamp(0) description = '' for child in xml: if child.tag == 'title': @@ -55,14 +67,14 @@ def parse_rss_item(xml): elif child.tag == 'link': link = child.text elif child.tag == 'pubDate': - time = parsedate_to_datetime(child.text).astimezone(None) + time = parsedate_to_datetime(child.text) elif child.tag == 'description': description = child.text elif child.tag == 'content:encoded' and not description: description = child.text if not description: description = xml.text - return Item(title, link, time, description) + return title, link, time, description def parse_rss(xml, title): @@ -75,12 +87,12 @@ def parse_rss(xml, title): link = child.text elif child.tag == 'item': items.append(parse_rss_item(child)) - return Advert(title, link, *max(items, key=attrgetter('time'))) + return title, link, items def parse_atom_entry(xml): """Parse given Atom entry.""" - time = datetime.fromtimestamp(0).astimezone(None) + time = datetime.fromtimestamp(0) summary = '' for child in xml: if child.tag.endswith('Atom}title'): @@ -90,12 +102,12 @@ def parse_atom_entry(xml): if rel == 'alternate' or not rel: link = child.attrib['href'] elif child.tag.endswith('Atom}published'): iso = child.text.replace('Z', '+00:00') # normalized - time = datetime.fromisoformat(iso).astimezone(None) + time = datetime.fromisoformat(iso) elif child.tag.endswith('Atom}summary'): summary = child.text elif child.tag.endswith('Atom}content') and not summary: summary = child.text - return Item(title, link, time, summary) + return title, link, time, summary def parse_atom(xml, title): @@ -109,11 +121,12 @@ def parse_atom(xml, title): if rel == 'alternate' or not rel: link = child.attrib['href'] elif child.tag.endswith('Atom}entry'): entries.append(parse_atom_entry(child)) - return Advert(title, link, *max(entries, key=attrgetter('time'))) + return title, link, entries -async def fetch(url): +async def fetch(raw_url): """Fetch web feed from given URL and return it parsed.""" + url = urlsplit(raw_url) if url.scheme == 'https': reader, writer = await open_connection(url.hostname, 443, ssl=True) elif url.scheme == 'http': @@ -127,29 +140,35 @@ async def fetch(url): response.begin() with response: if response.status >= 400: - raise HTTPError(url.geturl(), response.status, - f'{response.reason}: {url.geturl()}', + raise HTTPError(raw_url, response.status, + f'{response.reason}: {raw_url}', response.getheaders(), response) if response.status >= 300: - location = urljoin(url.geturl(), response.getheader('Location')) - warn(f'{url.geturl()} redirects to {location}') - return await fetch(urlsplit(location)) + location = urljoin(raw_url, response.getheader('Location')) + warn(f'{raw_url} -> {location}', + type('RedirectWarning', (Warning,), {})) + return await fetch(location) if response.status >= 200: xml = parse_xml(response.read()) if xml.tag == 'rss': assert xml[0].tag == 'channel' - return parse_rss(xml[0], url.hostname) - if xml.tag.endswith('Atom}feed'): - return parse_atom(xml, url.hostname) - raise ValueError(f'unsupported feed format at {url.geturl()}') - raise HTTPError(url.geturl(), response.status, - f'{response.reason}: {url.geturl()}', + src_title, src_link, items = parse_rss(xml[0], url.hostname) + elif xml.tag.endswith('Atom}feed'): + src_title, src_link, items = parse_atom(xml, url.hostname) + else: + raise ValueError(f'unsupported feed format at {raw_url}') + return (Advert(src_title, urljoin(raw_url, src_link), + title, urljoin(raw_url, link), + time.astimezone(None), summary) + for title, link, time, summary in items) + raise HTTPError(raw_url, response.status, + f'{response.reason}: {raw_url}', response.getheaders(), response) async def fetch_all(urls): """Fetch all given URLs asynchronously and return them parsed.""" - tasks = gather(*(fetch(urlsplit(url)) for url in urls)) + tasks = gather(*map(fetch, urls)) try: return await tasks except: @@ -157,6 +176,45 @@ async def fetch_all(urls): raise +def select(n, ads): + """Return n most recent ads from given iterable.""" + return sorted(ads, key=attrgetter('time'), reverse=True)[:n] + + +def truncate(ad, summary_length): + """Return ad with truncated summary, whose HTML tags a stripped.""" + return ad._replace(summary=shorten(HTML_TAG.sub(ad.summary, ''), + summary_length, placeholder='…')) + + if __name__ == '__main__': - feeds = sorted(run(fetch_all(stdin.readlines())), - key=attrgetter('time'), reverse=True) + parser = ArgumentParser(description='generate advert from web feeds') + parser.add_argument('-v', '--version', action='version', + version=f'fead {__version__}') + parser.add_argument('-F', '--feeds', metavar='PATH', + type=read_urls, default=[], + help='file containing newline-separated web feed URLs') + parser.add_argument('-f', '--feed', metavar='URL', + action='append', dest='feeds', + help='addtional web feed URL (multiple use)') + parser.add_argument('-n', '--count', metavar='N', type=int, default=3, + help='maximum number of ads in total (default to 3)') + parser.add_argument('-p', '--per-feed', metavar='N', type=int, default=1, + help='maximum number of ads per feed (default to 1)') + parser.add_argument('-l', '--length', metavar='N', + dest='len', type=int, default=256, + help='maximum summary length (default to 256)') + parser.add_argument('-t', '--template', metavar='PATH', + type=FileType('r'), default=stdin, + help='template file (default to stdin)') + parser.add_argument('-o', '--output', metavar='PATH', + type=FileType('w'), default=stdout, + help='output file (default to stdout)') + args = parser.parse_args() + + template = args.template.read() + args.template.close() + for ad in select(args.count, (ad for feed in run(fetch_all(args.feeds)) + for ad in select(args.per_feed, feed))): + args.output.write(template.format(**truncate(ad, args.len)._asdict())) + args.output.close() |