zs

Zeitungsschau rss to email converter
git clone git://r-36.net/zs
Log | Files | Refs | LICENSE

zs (6250B)


      1 #!/usr/bin/env python
      2 # coding=utf-8
      3 #
      4 # Copy me if you can.
      5 # by 20h
      6 #
      7 
      8 import sys
      9 import os
     10 import zeitungsschau.feed as feed
     11 import zeitungsschau.feeddb as feeddb
     12 import zeitungsschau.opml as opml
     13 import zeitungsschau.feedemail as feedemail
     14 import socket
     15 import http.client
     16 import ssl
     17 import getopt
     18 import pprint
     19 import requests.exceptions
     20 import requests
     21 import smtplib
     22 import lxml
     23 
     24 dodebug = False
     25 
     26 def debug(msg):
     27 	global dodebug
     28 	if dodebug == True:
     29 		print("debug: %s" % (msg))
     30 
     31 def sendfeed(db, ufeed):
     32 	feedemail.send(ufeed, db.cfg["email"], db.cfg["smtphost"], \
     33 			db.cfg["smtpport"], db.cfg["smtpssl"], \
     34 			db.cfg["smtpstarttls"], db.cfg["smtpuser"], \
     35 			db.cfg["smtppassword"], db.cfg["smtpcmd"], \
     36 			db.cfg["smtpuselocal"])
     37 
     38 def run(db, selfeed=None, dryrun=False, onlychanges=False):
     39 	feeduris = db.listfeeds()
     40 
     41 	if feeduris != None and selfeed in feeduris:
     42 		feeduris = [selfeed] 
     43 
     44 	for feeduri in feeduris:
     45 		if db.ispaused(feeduri):
     46 			print("pause %s" % (feeduri))
     47 			continue
     48 
     49 		retries = db.getretry(feeduri)
     50 		estr = None
     51 		if onlychanges != True:
     52 			print("fetch %s" % (feeduri))
     53 		curfeed = None
     54 		rcode = 0
     55 		try:
     56 			(rcode, curfeed) = feed.fetch(feeduri)
     57 		except socket.gaierror:
     58 			continue
     59 		except socket.timeout:
     60 			continue
     61 		except TimeoutError:
     62 			continue
     63 		except ConnectionResetError:
     64 			estr = "connreset"
     65 			retries += 1
     66 		except requests.exceptions.ConnectionError:
     67 			estr = "connreset"
     68 			retries += 1
     69 		except requests.exceptions.ReadTimeout:
     70 			continue
     71 		except requests.exceptions.ChunkedEncodingError:
     72 			continue
     73 		except lxml.etree.XMLSyntaxError:
     74 			estr = "xml error"
     75 			retries += 1
     76 		except requests.exceptions.TooManyRedirects:
     77 			estr = "redirects"
     78 			retries += 1
     79 		except OSError as err:
     80 			estr = err.strerror
     81 			retries += 1
     82 
     83 		if rcode == 404:
     84 			estr = "404"
     85 			retries += 1
     86 
     87 		if curfeed == None:
     88 			continue
     89 
     90 		# retry handling
     91 		if estr != None:
     92 			if retries > 2:
     93 				sys.stderr.write("pause %s %s\n" % \
     94 						(estr, feeduri))
     95 				db.pause(feeduri)
     96 			db.setretry(feeduri, retries)
     97 			continue
     98 		elif retries > 0:
     99 			db.setretry(feeduri, 0)
    100 
    101 		try:
    102 			clen = len(curfeed["articles"])
    103 		except AttributeError:
    104 			continue
    105 
    106 		if clen == 0:
    107 			# This is no target anymore. Thanks NATO for your
    108 			# crappy RSS feed!
    109 			#print("0 articles -> pause %s" % (feeduri))
    110 			#db.pause(feeduri)
    111 			continue
    112 
    113 		db.mergefeed(feeduri, curfeed)
    114 		ufeed = db.unreadarticles(feeduri)
    115 		if len(ufeed["articles"]) > 0 and onlychanges != True:
    116 			print("cur %d unread %d" % (clen, \
    117 					len(ufeed["articles"])))
    118 		debug(ufeed)
    119 		if dryrun == False:
    120 			try:
    121 				sendfeed(db, ufeed)
    122 				db.setreadarticles(feeduri, ufeed)
    123 			except smtplib.SMTPDataError:
    124 				return
    125 			except smtplib.SMTPSenderRefused:
    126 				return
    127 
    128 class ExceptionHook:
    129 	instance = None
    130 
    131 	def __call__(self, *args, **kwargs):
    132 		if self.instance is None:
    133 			from IPython.core import ultratb
    134 			self.instance = ultratb.FormattedTB(mode='Verbose',
    135 					color_scheme='Linux', call_pdb=1)
    136 		return self.instance(*args, **kwargs)
    137 
    138 def usage(app):
    139 	app = os.path.basename(app)
    140 	sys.stderr.write("usage: %s [-dhs] cmd\n" % (app))
    141 	sys.exit(1)
    142 
    143 def main(args):
    144 	global dodebug
    145 	retval = 0
    146 
    147 	try:
    148 		opts, largs = getopt.getopt(args[1:], "hds")
    149 	except getopt.GetoptError as err:
    150 		print(str(err))
    151 		usage(args[0])
    152 
    153 	silent = False
    154 	for o, a in opts:
    155 		if o == "-h":
    156 			usage(args[0])
    157 		elif o == "-d":
    158 			dodebug = True
    159 		elif o == "-s":
    160 			silent = True
    161 		else:
    162 			usage(args[0])
    163 
    164 	if len(largs) < 1:
    165 		usage(args[0])
    166 
    167 	if dodebug == True:
    168 		sys.excepthook = ExceptionHook()
    169 
    170 	if largs[0] == "testfeed":
    171 		if len(largs) < 2:
    172 			print("usage: %s testfeed URI\n" % \
    173 					(os.path.basename(args[0])))
    174 			return 1
    175 
    176 		fe = feed.fetch(largs[1])
    177 		pprint.pprint(fe)
    178 		return 0
    179 
    180 	db = feeddb.feeddb()
    181 	
    182 	if largs[0] == "run":
    183 		if len(largs) > 1:
    184 			run(db, largs[1], onlychanges=silent)
    185 		else:
    186 			run(db, onlychanges=silent)
    187 	
    188 	elif largs[0] == "dryrun":
    189 		if len(largs) > 1:
    190 			run(db, largs[1], dryrun=True, onlychanges=silent)
    191 		else:
    192 			run(db, dryrun=True, onlychanges=silent)
    193 
    194 	elif largs[0] == "cfg":
    195 		if len(largs) < 2:
    196 			for k in db.cfg:
    197 				print("%s = '%s'" % (k, db.cfg[k]))
    198 		elif len(args) < 3:
    199 			if largs[1] in db.cfg:
    200 				print("%s = '%s'" % (largs[1], \
    201 					db.cfg[largs[1]]))
    202 			else:
    203 				retval = 1
    204 		else:
    205 			db.cfg[largs[1]] = largs[2]
    206 			print("%s = '%s'" % (largs[1], db.cfg[largs[1]]))
    207 			db.default = False
    208 	
    209 	elif largs[0] == "cfgdel":
    210 		if len(largs) < 2:
    211 			usage(args[0])
    212 		if largs[1] in db.cfg:
    213 			del db.cfg[largs[1]]
    214 
    215 	elif largs[0] == "add":
    216 		if len(largs) < 2:
    217 			usage(args[0])
    218 		db.addfeed(largs[1])
    219 
    220 	elif largs[0] == "list":
    221 		print("\n".join(db.listfeeds()))
    222 
    223 	elif largs[0] == "listuuids":
    224 		if len(largs) < 2:
    225 			usage(args[0])
    226 		rfeed = db.readfeed(largs[1])
    227 		for art in rfeed["articles"]:
    228 			print("%s: %s: %s" % (art["uuid"], art["link"],\
    229 					art["title"]))
    230 
    231 	elif largs[0] == "unread":
    232 		if len(largs) < 3:
    233 			usage(args[0])
    234 		db.setarticleunread(largs[1], largs[2])
    235 
    236 	elif largs[0] == "resend":
    237 		if len(largs) < 2:
    238 			usage(args[0])
    239 		ufeed = db.unreadarticles(largs[1])
    240 		sendfeed(db, ufeed)
    241 		db.setreadarticles(largs[1], ufeed)
    242 
    243 	elif largs[0] == "del":
    244 		if len(largs) < 2:
    245 			usage(args[0])
    246 		if db.delfeed(largs[1]) == True:
    247 			print("'%s' has been deleted." % (largs[1]))
    248 
    249 	elif largs[0] == "reset":
    250 		if len(largs) < 2:
    251 			usage(args[0])
    252 		db.resetarticles(largs[1])
    253 
    254 	elif largs[0] == "retry":
    255 		if len(largs) < 3: 
    256 			usage(args[0])
    257 		db.setretry(largs[1], int(largs[2]))
    258 
    259 	elif largs[0] == "pause":
    260 		if len(largs) < 2:
    261 			usage(args[0])
    262 		db.pause(largs[1])
    263 
    264 	elif largs[0] == "unpause":
    265 		if len(largs) < 2:
    266 			usage(args[0])
    267 		db.unpause(largs[1])
    268 
    269 	elif largs[0] == "opmlexport":
    270 		if len(largs) > 1:
    271 			filen = open(largs[1], "w")
    272 		else:
    273 			filen = sys.stdout
    274 		filen.write(opml.write(db.listfeeds()))
    275 
    276 	elif largs[0] == "opmlimport":
    277 		if len(largs) > 1:
    278 			filen = open(largs[1], "r")
    279 		else:
    280 			filen = sys.stdin
    281 		feedlist = db.listfeeds()
    282 		nfeedlist = opml.read(filen.read().encode("utf-8"))
    283 		for f in nfeedlist:
    284 			if not f in feedlist:
    285 				print("import feed: %s" % (f))
    286 				db.addfeed(f)
    287 
    288 	del db
    289 	return retval
    290 
    291 if __name__ == "__main__":
    292 	sys.exit(main(sys.argv))
    293