zs

commit f29afa30f2666e89c18c0b6c4db3a405bdc9aa13
parent 413f6c943226db4abb04d269d58cac9d6695e26c
Author: Christoph Lohmann <20h@r-36.net>
Date:   Sat, 24 Oct 2015 21:39:25 +0200

Moving from urllib to requests and add testfeed.

Diffstat:
zeitungsschau/feed.py
 | 
10
++++------
zs
 | 
15
+++++++++++++--

2 files changed, 17 insertions(+), 8 deletions(-)
diff --git a/zeitungsschau/feed.py b/zeitungsschau/feed.py
@@ -9,7 +9,7 @@ from lxml import objectify
 from lxml import etree
 from datetime import datetime
 import dateutil.parser
-import urllib.request, urllib.parse, urllib.error
+import requests
 import hashlib
 import pytz
 
@@ -228,10 +228,8 @@ def parse(astr):
 
 	return feed
 
-class feedopener(urllib.request.FancyURLopener):
-	version = "Zeitungsschau/1.0"
-urllib.request._urlopener = feedopener
-
 def fetch(uri):
-	return parse(urllib.request.urlopen(uri, timeout=5).read())
+	return parse(requests.get(uri, timeout=5,\
+			headers={"User-Agent": "Zeitungsschau/1.0"}).text.\
+						encode("utf-8"))
 
diff --git a/zs b/zs
@@ -16,6 +16,7 @@ import socket
 import http.client
 import ssl
 import getopt
+import pprint
 
 dodebug = False
 
@@ -130,6 +131,16 @@ def main(args):
 	if len(largs) < 1:
 		usage(args[0])
 
+	if largs[0] == "testfeed":
+		if len(largs) < 2:
+			print("usage: %s testfeed URI\n" % \
+					(os.path.basename(args[0])))
+			return 1
+
+		fe = feed.fetch(largs[1])
+		pprint.pprint(fe)
+		return 0
+
 	db = feeddb.feeddb()
 	
 	if largs[0] == "run":
@@ -175,8 +186,8 @@ def main(args):
 	elif largs[0] == "listuuids":
 		if len(largs) < 2:
 			usage(args[0])
-		feed = db.readfeed(largs[1])
-		for art in feed["articles"]:
+		rfeed = db.readfeed(largs[1])
+		for art in rfeed["articles"]:
 			print("%s: %s: %s" % (art["uuid"], art["link"],\
 					art["title"]))