zs

Zeitungsschau rss to email converter
git clone git://r-36.net/zs
Log | Files | Refs | LICENSE

commit de38800ce7a8ad243771b74cf84e36a84aebc0d1
parent 4d0d209d7b746599be83f63aad1c61682de36a1e
Author: Christoph Lohmann <20h@r-36.net>
Date:   Mon, 17 Mar 2014 18:08:20 +0100

Add retry handling on error.

Diffstat:
feeddb.py | 23+++++++++++++++++------
zs.py | 26+++++++++++++++++++++++---
2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/feeddb.py b/feeddb.py @@ -106,18 +106,29 @@ class feeddb(object): feed[key] = value self.writefeed(uri, feed) - def pause(self, uri): - self.setfeedval(uri, "pause", True) - - def unpause(self, uri): - self.setfeedval(uri, "pause", False) - def getfeedval(self, uri, key): feed = self.readfeed(uri) if feed == None: return None return feed[key] + def setretry(self, uri, retries): + self.setfeedval(uri, "retry", retries) + + def incretry(self, uri): + retries = self.getfeedval(uri, "retry") + if retries == None: + self.setfeedval(uri, "retry", 1) + else: + self.setfeedval(uri, "retry", retries+1) + + def pause(self, uri): + self.setfeedval(uri, "pause", True) + + def unpause(self, uri): + self.setfeedval(uri, "pause", False) + self.setretry(uri, 0) + def ispaused(self, uri): return self.getfeedval(uri, "pause") diff --git a/zs.py b/zs.py @@ -25,18 +25,33 @@ def run(db, selfeed=None, dryrun=False): print("pause %s" % (feeduri)) continue + retries = db.getretry(feeduri) + estr = None print("fetch %s" % (feeduri)) try: curfeed = feed.fetch(feeduri) except urllib.error.HTTPError as err: if err.code == 404: - print("404 -> pause %s" % (feeduri)) - db.pause(feeduri) - continue + estr = "404" + retries += 1 except socket.gaierror: continue except TimeoutError: continue + except ConnectionResetError: + estr = "connreset" + retries += 1 + + # retry handling + if estr != None: + if retries > 2: + sys.stderr.write("pause %s %s\n" % \ + (estr, feeduri)) + db.pause(feeduri) + db.setretry(feeduri, retries) + continue + elif retries > 0: + db.setretry(feeduri, 0) clen = len(curfeed["articles"]) if clen == 0: @@ -114,6 +129,11 @@ def main(args): usage(args[0]) db.resetarticles(args[1]) + elif args[1] == "retry": + if len(args) < 4: + usage(args[0]) + db.setretry(args[1], int(args[2])) + elif args[1] == "pause": if len(args) < 3: usage(args[0])