zs

Zeitungsschau rss to email converter
git clone git://r-36.net/zs
Log | Files | Refs | README | LICENSE

zs (6320B)


      1 #!/usr/bin/env python
      2 # coding=utf-8
      3 #
      4 # Copy me if you can.
      5 # by 20h
      6 #
      7 
      8 import sys
      9 import os
     10 import zeitungsschau.feed as feed
     11 import zeitungsschau.feeddb as feeddb
     12 import zeitungsschau.opml as opml
     13 import zeitungsschau.feedemail as feedemail
     14 import socket
     15 import http.client
     16 import ssl
     17 import getopt
     18 import pprint
     19 import requests.exceptions
     20 import requests
     21 import smtplib
     22 import lxml
     23 
     24 dodebug = False
     25 
     26 def debug(msg):
     27 	global dodebug
     28 	if dodebug == True:
     29 		print("debug: %s" % (msg))
     30 
     31 def sendfeed(db, ufeed):
     32 	feedemail.send(ufeed, db.cfg["email"], db.cfg["smtphost"], \
     33 			db.cfg["smtpport"], db.cfg["smtpssl"], \
     34 			db.cfg["smtpstarttls"], db.cfg["smtpuser"], \
     35 			db.cfg["smtppassword"], db.cfg["smtpcmd"], \
     36 			db.cfg["smtpuselocal"])
     37 
     38 def run(db, selfeed=None, dryrun=False, onlychanges=False):
     39 	feeduris = db.listfeeds()
     40 
     41 	if feeduris != None and selfeed in feeduris:
     42 		feeduris = [selfeed] 
     43 
     44 	for feeduri in feeduris:
     45 		if db.ispaused(feeduri):
     46 			print("pause %s" % (feeduri))
     47 			continue
     48 
     49 		retries = db.getretry(feeduri)
     50 		estr = None
     51 		if onlychanges != True:
     52 			print("fetch %s" % (feeduri))
     53 		curfeed = None
     54 		rcode = 0
     55 
     56 		"""
     57 		# All errors.
     58 		(rcode, curfeed) = feed.fetch(feeduri)
     59 		"""
     60 		try:
     61 			(rcode, curfeed) = feed.fetch(feeduri)
     62 		except socket.gaierror:
     63 			continue
     64 		except socket.timeout:
     65 			continue
     66 		except TimeoutError:
     67 			continue
     68 		except ConnectionResetError:
     69 			estr = "connreset"
     70 			retries += 1
     71 		except requests.exceptions.ConnectionError:
     72 			estr = "connreset"
     73 			retries += 1
     74 		except requests.exceptions.ReadTimeout:
     75 			continue
     76 		except requests.exceptions.ChunkedEncodingError:
     77 			continue
     78 		except lxml.etree.XMLSyntaxError:
     79 			estr = "xml error"
     80 			retries += 1
     81 		except requests.exceptions.TooManyRedirects:
     82 			estr = "redirects"
     83 			retries += 1
     84 		except OSError as err:
     85 			estr = err.strerror
     86 			retries += 1
     87 
     88 		if rcode == 404:
     89 			estr = "404"
     90 			retries += 1
     91 
     92 		if curfeed == None:
     93 			continue
     94 
     95 		# retry handling
     96 		if estr != None:
     97 			if retries > 2:
     98 				sys.stderr.write("pause %s %s\n" % \
     99 						(estr, feeduri))
    100 				db.pause(feeduri)
    101 			db.setretry(feeduri, retries)
    102 			continue
    103 		elif retries > 0:
    104 			db.setretry(feeduri, 0)
    105 
    106 		try:
    107 			clen = len(curfeed["articles"])
    108 		except AttributeError:
    109 			continue
    110 
    111 		if clen == 0:
    112 			# This is no target anymore. Thanks NATO for your
    113 			# crappy RSS feed!
    114 			#print("0 articles -> pause %s" % (feeduri))
    115 			#db.pause(feeduri)
    116 			continue
    117 
    118 		db.mergefeed(feeduri, curfeed)
    119 		ufeed = db.unreadarticles(feeduri)
    120 		if len(ufeed["articles"]) > 0 and onlychanges != True:
    121 			print("cur %d unread %d" % (clen, \
    122 					len(ufeed["articles"])))
    123 		debug(ufeed)
    124 		if dryrun == False:
    125 			try:
    126 				sendfeed(db, ufeed)
    127 				db.setreadarticles(feeduri, ufeed)
    128 			except smtplib.SMTPDataError:
    129 				return
    130 			except smtplib.SMTPSenderRefused:
    131 				return
    132 
    133 class ExceptionHook:
    134 	instance = None
    135 
    136 	def __call__(self, *args, **kwargs):
    137 		if self.instance is None:
    138 			from IPython.core import ultratb
    139 			self.instance = ultratb.FormattedTB(mode='Verbose',
    140 					color_scheme='Linux', call_pdb=1)
    141 		return self.instance(*args, **kwargs)
    142 
    143 def usage(app):
    144 	app = os.path.basename(app)
    145 	sys.stderr.write("usage: %s [-dhs] cmd\n" % (app))
    146 	sys.exit(1)
    147 
    148 def main(args):
    149 	global dodebug
    150 	retval = 0
    151 
    152 	try:
    153 		opts, largs = getopt.getopt(args[1:], "hds")
    154 	except getopt.GetoptError as err:
    155 		print(str(err))
    156 		usage(args[0])
    157 
    158 	silent = False
    159 	for o, a in opts:
    160 		if o == "-h":
    161 			usage(args[0])
    162 		elif o == "-d":
    163 			dodebug = True
    164 		elif o == "-s":
    165 			silent = True
    166 		else:
    167 			usage(args[0])
    168 
    169 	if len(largs) < 1:
    170 		usage(args[0])
    171 
    172 	if dodebug == True:
    173 		sys.excepthook = ExceptionHook()
    174 
    175 	if largs[0] == "testfeed":
    176 		if len(largs) < 2:
    177 			print("usage: %s testfeed URI\n" % \
    178 					(os.path.basename(args[0])))
    179 			return 1
    180 
    181 		fe = feed.fetch(largs[1])
    182 		pprint.pprint(fe)
    183 		return 0
    184 
    185 	db = feeddb.feeddb()
    186 	
    187 	if largs[0] == "run":
    188 		if len(largs) > 1:
    189 			run(db, largs[1], onlychanges=silent)
    190 		else:
    191 			run(db, onlychanges=silent)
    192 	
    193 	elif largs[0] == "dryrun":
    194 		if len(largs) > 1:
    195 			run(db, largs[1], dryrun=True, onlychanges=silent)
    196 		else:
    197 			run(db, dryrun=True, onlychanges=silent)
    198 
    199 	elif largs[0] == "cfg":
    200 		if len(largs) < 2:
    201 			for k in db.cfg:
    202 				print("%s = '%s'" % (k, db.cfg[k]))
    203 		elif len(args) < 3:
    204 			if largs[1] in db.cfg:
    205 				print("%s = '%s'" % (largs[1], \
    206 					db.cfg[largs[1]]))
    207 			else:
    208 				retval = 1
    209 		else:
    210 			db.cfg[largs[1]] = largs[2]
    211 			print("%s = '%s'" % (largs[1], db.cfg[largs[1]]))
    212 			db.default = False
    213 	
    214 	elif largs[0] == "cfgdel":
    215 		if len(largs) < 2:
    216 			usage(args[0])
    217 		if largs[1] in db.cfg:
    218 			del db.cfg[largs[1]]
    219 
    220 	elif largs[0] == "add":
    221 		if len(largs) < 2:
    222 			usage(args[0])
    223 		db.addfeed(largs[1])
    224 
    225 	elif largs[0] == "list":
    226 		print("\n".join(db.listfeeds()))
    227 
    228 	elif largs[0] == "listuuids":
    229 		if len(largs) < 2:
    230 			usage(args[0])
    231 		rfeed = db.readfeed(largs[1])
    232 		for art in rfeed["articles"]:
    233 			print("%s: %s: %s" % (art["uuid"], art["link"],\
    234 					art["title"]))
    235 
    236 	elif largs[0] == "unread":
    237 		if len(largs) < 3:
    238 			usage(args[0])
    239 		db.setarticleunread(largs[1], largs[2])
    240 
    241 	elif largs[0] == "resend":
    242 		if len(largs) < 2:
    243 			usage(args[0])
    244 		ufeed = db.unreadarticles(largs[1])
    245 		sendfeed(db, ufeed)
    246 		db.setreadarticles(largs[1], ufeed)
    247 
    248 	elif largs[0] == "del":
    249 		if len(largs) < 2:
    250 			usage(args[0])
    251 		if db.delfeed(largs[1]) == True:
    252 			print("'%s' has been deleted." % (largs[1]))
    253 
    254 	elif largs[0] == "reset":
    255 		if len(largs) < 2:
    256 			usage(args[0])
    257 		db.resetarticles(largs[1])
    258 
    259 	elif largs[0] == "retry":
    260 		if len(largs) < 3: 
    261 			usage(args[0])
    262 		db.setretry(largs[1], int(largs[2]))
    263 
    264 	elif largs[0] == "pause":
    265 		if len(largs) < 2:
    266 			usage(args[0])
    267 		db.pause(largs[1])
    268 
    269 	elif largs[0] == "unpause":
    270 		if len(largs) < 2:
    271 			usage(args[0])
    272 		db.unpause(largs[1])
    273 
    274 	elif largs[0] == "opmlexport":
    275 		if len(largs) > 1:
    276 			filen = open(largs[1], "w")
    277 		else:
    278 			filen = sys.stdout
    279 		filen.write(opml.write(db.listfeeds()))
    280 
    281 	elif largs[0] == "opmlimport":
    282 		if len(largs) > 1:
    283 			filen = open(largs[1], "r")
    284 		else:
    285 			filen = sys.stdin
    286 		feedlist = db.listfeeds()
    287 		nfeedlist = opml.read(filen.read().encode("utf-8"))
    288 		for f in nfeedlist:
    289 			if not f in feedlist:
    290 				print("import feed: %s" % (f))
    291 				db.addfeed(f)
    292 
    293 	del db
    294 	return retval
    295 
    296 if __name__ == "__main__":
    297 	sys.exit(main(sys.argv))
    298