1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
|
commit 277deed03833377bae0ae68cf7db3f0d3d578ccd
Author: Ngô Ngọc Đức Huy <huyngo@disroot.org>
Date: 2023-08-30 11:01:00 +0700
Add option to skip errors
diff --git a/src/fead.py b/src/fead.py
index 370b623..0b35119 100755
--- a/src/fead.py
+++ b/src/fead.py
@@ -185,11 +185,23 @@ async def fetch(raw_url):
response.getheaders(), response)
-async def fetch_all(urls):
+async def fetch_skip_error(url):
+ try:
+ return await fetch(url)
+ except Exception as e:
+ warn(f'fail to fetch {url}: {e}',
+ type('FailureWarning', (Warning,), {}))
+ return None
+
+
+async def fetch_all(urls, skip_error):
"""Fetch all given URLs asynchronously and return them parsed."""
- tasks = gather(*map(fetch, urls))
+ if skip_error:
+ tasks = gather(*map(fetch_skip_error, urls))
+ else:
+ tasks = gather(*map(fetch, urls))
try:
- return await tasks
+ return filter(lambda t: t is not None, await tasks)
except:
tasks.cancel() # structured concurrency
raise
@@ -234,11 +246,15 @@ def main():
parser.add_argument('-o', '--output', metavar='PATH',
type=FileType('w'), default=stdout,
help='output file (default to stdout)')
+ parser.add_argument('-s', '--skip-error', action='store_true',
+ default=False,
+ help="errors not causing failure but logged")
args = parser.parse_args()
template = args.template.read()
args.template.close()
- for ad in select(args.count, (ad for feed in run(fetch_all(args.feeds))
+ for ad in select(args.count, (ad for feed in run(fetch_all(args.feeds,
+ args.skip_error))
for ad in select(args.per_feed, feed))):
args.output.write(template.format(**truncate(ad, args.len)._asdict()))
args.output.close()
|
