Context Navigation

email2trac.py.in @ 290

Last change on this file since 290 was 92, checked in by bas, 18 years ago

email2trac.py.in:

Fixed some bugs
Preparing for new release

Property svn:executable set to *
Property svn:keywords set to Id

File size: 20.6 KB

Line
1	#!@PYTHON@
2	# Copyright (C) 2002
3	#
4	# This file is part of the email2trac utils
5	#
6	# This program is free software; you can redistribute it and/or modify it
7	# under the terms of the GNU General Public License as published by the
8	# Free Software Foundation; either version 2, or (at your option) any
9	# later version.
10	#
11	# This program is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with this program; if not, write to the Free Software
18	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
19	#
20	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
21	#
22	"""
23	email2trac.py -- Email tickets to Trac.
24
25	A simple MTA filter to create Trac tickets from inbound emails.
26
27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28	Copyright 2005, Edgewall Software
29
30	Changed By: Bas van der Vlies <basv@sara.nl>
31	Date : 13 September 2005
32	Descr. : Added config file and command line options, spam level
33	detection, reply address and mailto option. Unicode support
34
35	Changed By: Walter de Jong <walter@sara.nl>
36	Descr. : multipart-message code and trac attachments
37
38
39	The scripts reads emails from stdin and inserts directly into a Trac database.
40	MIME headers are mapped as follows:
41
42	* From: => Reporter
43	=> CC (Optional via reply_address option)
44	* Subject: => Summary
45	* Body => Description
46	* Component => Can be set to SPAM via spam_level option
47
48	How to use
49	----------
50	* Create an config file:
51	[DEFAULT] # REQUIRED
52	project : /data/trac/test # REQUIRED
53	debug : 1 # OPTIONAL, if set print some DEBUG info
54	spam_level : 4 # OPTIONAL, if set check for SPAM mail
55	reply_address: 1 # OPTIONAL, if set then fill in ticket CC field
56	umask : 022 # OPTIONAL, if set then use this umask for creation of the attachments
57	mailto_link : 1 # OPTIONAL, if set then [mailto:<>] in description
58	mailto_cc : basv@sara.nl # OPTIONAL, use this address as CC in mailto line
59	ticket_update: 1 # OPTIONAL, if set then check if this is an update for a ticket
60	trac_version : 0.8 # OPTIONAL, default is 0.9
61
62	[jouvin] # OPTIONAL project declaration, if set both fields necessary
63	project : /data/trac/jouvin # use -p\|--project jouvin.
64
65	* default config file is : /etc/email2trac.conf
66
67	* Commandline opions:
68	-h \| --help
69	-c <value> \| --component=<value>
70	-f <config file> \| --file=<config file>
71	-p <project name> \| --project=<project name>
72
73	SVN Info:
74	$Id: email2trac.py.in 92 2006-07-06 10:13:43Z bas $
75	"""
76	import os
77	import sys
78	import string
79	import getopt
80	import stat
81	import time
82	import email
83	import re
84	import urllib
85	import unicodedata
86	import ConfigParser
87	from email import Header
88	from stat import *
89	import mimetypes
90
91	# Fix some errors
92	#import codecs
93
94	trac_default_version = 0.9
95
96	class TicketEmailParser(object):
97	env = None
98	comment = '> '
99
100	def __init__(self, env, parameters, version):
101	self.env = env
102
103	# Database connection
104	#
105	self.db = None
106
107	# Some useful mail constants
108	#
109	self.author = None
110	self.email_addr = None
111	self.email_field = None
112
113	self.VERSION = version
114	if self.VERSION == 0.8:
115	self.get_config = self.env.get_config
116	else:
117	self.get_config = self.env.config.get
118
119	if parameters.has_key('umask'):
120	os.umask(int(parameters['umask'], 8))
121
122	if parameters.has_key('debug'):
123	self.DEBUG = int(parameters['debug'])
124	else:
125	self.DEBUG = 0
126
127	if parameters.has_key('mailto_link'):
128	self.MAILTO = int(parameters['mailto_link'])
129	if parameters.has_key('mailto_cc'):
130	self.MAILTO_CC = parameters['mailto_cc']
131	else:
132	self.MAILTO_CC = ''
133	else:
134	self.MAILTO = 0
135
136	if parameters.has_key('spam_level'):
137	self.SPAM_LEVEL = int(parameters['spam_level'])
138	else:
139	self.SPAM_LEVEL = 0
140
141	if parameters.has_key('email_comment'):
142	self.comment = str(parameters['email_comment'])
143
144	if parameters.has_key('email_header'):
145	self.EMAIL_HEADER = int(parameters['email_header'])
146	else:
147	self.EMAIL_HEADER = 0
148
149	if parameters.has_key('alternate_notify_template'):
150	self.notify_template = str(parameters['alternate_notify_template'])
151	else:
152	self.notify_template = None
153
154	if parameters.has_key('reply_all'):
155	self.REPLY_ALL = int(parameters['reply_all'])
156	else:
157	self.REPLY_ALL = 0
158
159	if parameters.has_key('ticket_update'):
160	self.TICKET_UPDATE = int(parameters['ticket_update'])
161	else:
162	self.TICKET_UPDATE = 0
163
164
165	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
166	# Note if Spam_level then '*' are included
167	def spam(self, message):
168	if message.has_key('X-Spam-Score'):
169	spam_l = string.split(message['X-Spam-Score'])
170	number = spam_l[0].count('*')
171
172	if number >= self.SPAM_LEVEL:
173	return 'Spam'
174
175	elif message.has_key('X-Virus-found'): # treat virus mails as spam
176	return 'Spam'
177
178	return self.get_config('ticket', 'default_component')
179
180	def to_unicode(self, str):
181	"""
182	Email has 7 bit ASCII code, convert it to unicode with the charset
183	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
184	understands it.
185	"""
186	results = Header.decode_header(str)
187	str = None
188	for text,format in results:
189	if format:
190	try:
191	temp = unicode(text, format)
192	except (UnicodeError,LookupError):
193	# This always works
194	#
195	temp = unicode(text, 'iso-8859-15')
196
197	temp = temp.encode('utf-8')
198	else:
199	temp = string.strip(text)
200	temp = unicode(text, 'iso-8859-15')
201
202	if str:
203	str = '%s %s' %(str, temp)
204	else:
205	str = temp
206
207	return str
208
209	def debug_attachments(self, message):
210	n = 0
211	for part in message.walk():
212	if part.get_content_maintype() == 'multipart': # multipart/* is just a container
213	print 'TD: multipart container'
214	continue
215
216	n = n + 1
217	print 'TD: part%d: Content-Type: %s' % (n, part.get_content_type())
218	print 'TD: part%d: filename: %s' % (n, part.get_filename())
219
220	if part.is_multipart():
221	print 'TD: this part is multipart'
222	payload = part.get_payload(decode=1)
223	print 'TD: payload:', payload
224	else:
225	print 'TD: this part is not multipart'
226
227	part_file = '/var/tmp/part%d' % n
228	print 'TD: writing part%d (%s)' % (n,part_file)
229	fx = open(part_file, 'wb')
230	text = part.get_payload(decode=1)
231	if not text:
232	text = '(None)'
233	fx.write(text)
234	fx.close()
235	try:
236	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
237	except OSError:
238	pass
239
240	def email_header_txt(self, m):
241	"""
242	Display To and CC addresses in description field
243	"""
244	str = ''
245	if m['To'] and len(m['To']) > 0 and m['To'] != 'hic@sara.nl':
246	str = "'''To:''' %s [[BR]]" %(m['To'])
247	if m['Cc'] and len(m['Cc']) > 0:
248	str = "%s'''Cc:''' %s [[BR]]" % (str, m['Cc'])
249
250	return self.to_unicode(str)
251
252	def set_owner(self, ticket):
253	"""
254	Select default owner for ticket component
255	"""
256	cursor = self.db.cursor()
257	sql = "SELECT owner FROM component WHERE name='%s'" % ticket['component']
258	cursor.execute(sql)
259	try:
260	ticket['owner'] = cursor.fetchone()[0]
261	except TypeError, detail:
262	ticket['owner'] = "UNKNOWN"
263
264	def get_author_emailaddrs(self, message):
265	"""
266	Get the default author name and email address from the message
267	"""
268	self.author, self.email_addr = email.Utils.parseaddr(message['from'])
269
270	# Look for email address in registered trac users
271	#
272	if self.VERSION == 0.8:
273	users = []
274	else:
275	users = [ u for (u, n, e) in self.env.get_known_users(self.db)
276	if e == self.email_addr ]
277
278	if len(users) == 1:
279	self.email_field = users[0]
280	else:
281	self.email_field = self.to_unicode(message['from'])
282
283	def set_reply_fields(self, ticket, message):
284	"""
285	Set all the right fields for a new ticket
286	"""
287	ticket['reporter'] = self.email_field
288
289	# Put all CC-addresses in ticket CC field
290	#
291	if self.REPLY_ALL:
292	#tos = message.get_all('to', [])
293	ccs = message.get_all('cc', [])
294
295	addrs = email.Utils.getaddresses(ccs)
296
297	# Remove reporter email address if notification is
298	# on
299	#
300	if self.notification:
301	try:
302	addrs.remove((self.author, self.email_addr))
303	except ValueError, detail:
304	pass
305
306	for name,mail in addrs:
307	try:
308	ticket['cc'] = '%s,%s' %(ticket['cc'], mail)
309	except:
310	ticket['cc'] = mail
311
312	def save_email_for_debug(self, message):
313	msg_file = '/var/tmp/msg.txt'
314	print 'TD: saving email to %s' % msg_file
315	fx = open(msg_file, 'wb')
316	fx.write('%s' % message)
317	fx.close()
318	try:
319	os.chmod(msg_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
320	except OSError:
321	pass
322
323	def ticket_update(self, m):
324	"""
325	If the current email is a reply to an existing ticket, this function
326	will append the contents of this email to that ticket, instead of
327	creating a new one.
328	"""
329	if not m['Subject']:
330	return False
331	else:
332	subject = self.to_unicode(m['Subject'])
333
334	TICKET_RE = re.compile(r"""
335	(?P<ticketnr>[#][0-9]+:)
336	""", re.VERBOSE)
337
338	result = TICKET_RE.search(subject)
339	if not result:
340	return False
341
342	body_text = self.get_body_text(m)
343	body_text = '{{{\n%s\n}}}' %body_text
344
345	# Strip '#' and ':' from ticket_id
346	#
347	ticket_id = result.group('ticketnr')
348	ticket_id = int(ticket_id[1:-1])
349
350	# Get current time
351	#
352	when = int(time.time())
353
354	if self.VERSION == 0.8:
355	tkt = Ticket(self.db, ticket_id)
356	tkt.save_changes(self.db, self.author, body_text, when)
357	else:
358	tkt = Ticket(self.env, ticket_id, self.db)
359	tkt.save_changes(self.author, body_text, when)
360	tkt['id'] = ticket_id
361
362	self.attachments(m, tkt)
363
364	if self.notification:
365	self.notify(tkt, False, when)
366
367	return True
368
369	def new_ticket(self, msg):
370	"""
371	Create a new ticket
372	"""
373	tkt = Ticket(self.env)
374	tkt['status'] = 'new'
375
376	# Some defaults
377	#
378	tkt['milestone'] = self.get_config('ticket', 'default_milestone')
379	tkt['priority'] = self.get_config('ticket', 'default_priority')
380	tkt['severity'] = self.get_config('ticket', 'default_severity')
381	tkt['version'] = self.get_config('ticket', 'default_version')
382
383	if not msg['Subject']:
384	tkt['summary'] = '(geen subject)'
385	else:
386	tkt['summary'] = self.to_unicode(msg['Subject'])
387
388
389	if settings.has_key('component'):
390	tkt['component'] = settings['component']
391	else:
392	tkt['component'] = self.spam(msg)
393
394	# Must make this an option or so, discard SPAM messages or save then
395	# and delete later
396	#
397	#if self.SPAM_LEVEL and self.spam(msg):
398	# print 'This message is a SPAM. Automatic ticket insertion refused (SPAM level > %d' % self.SPAM_LEVEL
399	# sys.exit(1)
400
401	# Set default owner for component
402	#
403	self.set_owner(tkt)
404	self.set_reply_fields(tkt, msg)
405
406	# produce e-mail like header
407	#
408	head = ''
409	if self.EMAIL_HEADER > 0:
410	head = self.email_header_txt(msg)
411
412
413	body_text = self.get_body_text(msg)
414	body_text = ('\r\n'.join(body_text.splitlines()))
415
416	# A patch so that the web-interface will not update the description
417	# field of a ticket
418	#
419	tkt['description'] = 'email2trac:%s\r\n{{{\r\n%s\r\n}}}' \
420	%(head, body_text)
421
422	when = int(time.time())
423	if self.VERSION == 0.8:
424	ticket_id = tkt.insert(self.db)
425	else:
426	ticket_id = tkt.insert()
427
428	changed = False
429	comment = ''
430
431	# Rewrite the description if we have mailto enabled
432	#
433	if self.MAILTO:
434	mailto = self.html_mailto_link(self.to_unicode(msg['subject']), ticket_id, body_text)
435	tkt['description'] = 'email2trac:%s%s\r\n{{{\r\n%s\r\n}}}' %(head, mailto, body_text)
436	comment = '\nadded mailto line\n'
437	changed = True
438
439	n = self.attachments(msg, tkt)
440	if n:
441	comment = '%s\nThis message has %d attachment(s)\n' %(comment, n)
442	changed = True
443
444	if changed:
445	if self.VERSION == 0.8:
446	tkt.save_changes(self.db, self.author, comment)
447	else:
448	tkt.save_changes(self.author, comment)
449
450	#print tkt.get_changelog(self.db, when)
451
452	if self.notification:
453	self.notify(tkt, True)
454	#self.notify(tkt, False)
455
456	def parse(self, fp):
457	m = email.message_from_file(fp)
458	if not m:
459	return
460
461	if self.DEBUG > 1: # save the entire e-mail message text
462	self.save_email_for_debug(m)
463	self.debug_attachments(m)
464
465	self.db = self.env.get_db_cnx()
466	self.get_author_emailaddrs(m)
467
468	if self.get_config('notification', 'smtp_enabled') in ['true']:
469	self.notification = 1
470	else:
471	self.notification = 0
472
473	# Must we update existing tickets
474	#
475	if self.TICKET_UPDATE > 0:
476	if self.ticket_update(m):
477	return True
478
479	self.new_ticket(m)
480
481	def get_body_text(self, msg):
482	"""
483	put the message text in the ticket description or in the changes field.
484	message text can be plain text or html or something else
485	"""
486	has_description = 0
487	ubody_text = '\r\n{{{\nNo plain text message\n}}}\r\n'
488	for part in msg.walk():
489
490	# 'multipart/*' is a container for multipart messages
491	#
492	if part.get_content_maintype() == 'multipart':
493	continue
494
495	if part.get_content_type() == 'text/plain':
496	# Try to decode, if fails then do not decode
497	#
498	body_text = part.get_payload(decode=1)
499	if not body_text:
500	body_text = part.get_payload(decode=0)
501
502	# Get contents charset (iso-8859-15 if not defined in mail headers)
503	# UTF-8 encode body_text
504	#
505	charset = part.get_content_charset('iso-8859-15')
506	try:
507	temp = unicode(body_text, charset)
508	except (UnicodeError,LookupError):
509	temp = unicode(body_text, 'iso-8859-15')
510
511	ubody_text = temp.encode('utf-8')
512
513	elif part.get_content_type() == 'text/html':
514	ubody_text = '\r\n(see attachment for HTML mail message)\r\n'
515
516	else:
517	ubody_text = '\r\n(see attachment for message)\r\n'
518
519	has_description = 1
520	break # we have the description, so break
521
522	if not has_description:
523	ubody_text = '\r\n(see attachment for message)\r\n'
524
525	return ubody_text
526
527	def notify(self, tkt , new=True, modtime=0):
528	"""
529	A wrapper for the TRAC notify function. So we can use templates
530	"""
531	try:
532	# create false {abs_}href properties, to trick Notify()
533	#
534	self.env.abs_href = Href(self.get_config('project', 'url'))
535	self.env.href = Href(self.get_config('project', 'url'))
536
537	tn = TicketNotifyEmail(self.env)
538	if self.notify_template:
539	tn.template_name = self.notify_template;
540
541	tn.notify(tkt, new, modtime)
542
543	except Exception, e:
544	print 'TD: Failure sending notification on creation of ticket #%s: %s' %(tkt['id'], e)
545
546	def mail_line(self, str):
547	return '%s %s' % (self.comment, str)
548
549
550	def html_mailto_link(self, subject, id, body):
551	if not self.author:
552	author = self.email_addr
553	else:
554	author = self.to_unicode(self.author)
555
556	# Must find a fix
557	#
558	#arr = string.split(body, '\n')
559	#arr = map(self.mail_line, arr)
560	#body = string.join(arr, '\n')
561	#body = '%s wrote:\n%s' %(author, body)
562
563	# Temporary fix
564	str = 'mailto:%s?Subject=%s&Cc=%s' %(
565	urllib.quote(self.email_addr),
566	urllib.quote('Re: #%s: %s' %(id, subject)),
567	urllib.quote(self.MAILTO_CC)
568	)
569
570	str = '\r\n{{{\r\n#!html\r\n<a href="%s">Reply to: %s</a>\r\n}}}\r\n' %(str, author)
571
572	return str
573
574	def attachments(self, message, ticket):
575	'''
576	save any attachments as files in the ticket's directory
577	'''
578	count = 0
579	first = 0
580	number = 0
581	for part in message.walk():
582	if part.get_content_maintype() == 'multipart': # multipart/* is just a container
583	continue
584
585	if not first: # first content is the message
586	first = 1
587	if part.get_content_type() == 'text/plain': # if first is text, is was already put in the description
588	continue
589
590	filename = part.get_filename()
591	count = count + 1
592	if not filename:
593	number = number + 1
594	filename = 'part%04d' % number
595
596	ext = mimetypes.guess_extension(part.get_content_type())
597	if not ext:
598	ext = '.bin'
599
600	filename = '%s%s' % (filename, ext)
601	else:
602	filename = self.to_unicode(filename)
603
604	# From the trac code
605	#
606	filename = filename.replace('\\', '/').replace(':', '/')
607	filename = os.path.basename(filename)
608
609	# We try to normalize the filename to utf-8 NFC if we can.
610	# Files uploaded from OS X might be in NFD.
611	# Check python version and then try it
612	#
613	if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
614	try:
615	filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
616	except TypeError:
617	pass
618
619	url_filename = urllib.quote(filename)
620	if self.VERSION == 0.8:
621	dir = os.path.join(self.env.get_attachments_dir(), 'ticket',
622	urllib.quote(str(ticket['id'])))
623	if not os.path.exists(dir):
624	mkdir_p(dir, 0755)
625	else:
626	dir = '/tmp'
627
628	path, fd = util.create_unique_file(os.path.join(dir, url_filename))
629	text = part.get_payload(decode=1)
630	if not text:
631	text = '(None)'
632	fd.write(text)
633	fd.close()
634
635	# get the filesize
636	#
637	stats = os.lstat(path)
638	filesize = stats[stat.ST_SIZE]
639
640	# Insert the attachment it differs for the different TRAC versions
641	#
642	if self.VERSION == 0.8:
643	cursor = self.db.cursor()
644	try:
645	cursor.execute('INSERT INTO attachment VALUES("%s","%s","%s",%d,%d,"%s","%s","%s")'
646	%('ticket', urllib.quote(str(ticket['id'])), filename + '?format=raw', filesize,
647	int(time.time()),'', self.author, 'e-mail') )
648
649	# Attachment is already known
650	#
651	except sqlite.IntegrityError:
652	#self.db.close()
653	return count
654
655	self.db.commit()
656
657	else:
658	fd = open(path)
659	att = attachment.Attachment(self.env, 'ticket', ticket['id'])
660	att.insert(url_filename, fd, filesize)
661	fd.close()
662
663	# Return how many attachments
664	#
665	return count
666
667
668	def mkdir_p(dir, mode):
669	'''do a mkdir -p'''
670
671	arr = string.split(dir, '/')
672	path = ''
673	for part in arr:
674	path = '%s/%s' % (path, part)
675	try:
676	stats = os.stat(path)
677	except OSError:
678	os.mkdir(path, mode)
679
680
681	def ReadConfig(file, name):
682	"""
683	Parse the config file
684	"""
685
686	if not os.path.isfile(file):
687	print 'File %s does not exist' %file
688	sys.exit(1)
689
690	config = ConfigParser.ConfigParser()
691	try:
692	config.read(file)
693	except ConfigParser.MissingSectionHeaderError,detail:
694	print detail
695	sys.exit(1)
696
697
698	# Use given project name else use defaults
699	#
700	if name:
701	if not config.has_section(name):
702	print "Not a valid project name: %s" %name
703	print "Valid names: %s" %config.sections()
704	sys.exit(1)
705
706	project = dict()
707	for option in config.options(name):
708	project[option] = config.get(name, option)
709
710	else:
711	project = config.defaults()
712
713	return project
714
715
716	if __name__ == '__main__':
717	# Default config file
718	#
719	configfile = '@email2trac_conf@'
720	project = ''
721	component = ''
722	ENABLE_SYSLOG = 0
723
724	try:
725	opts, args = getopt.getopt(sys.argv[1:], 'chf:p:', ['component=','help', 'file=', 'project='])
726	except getopt.error,detail:
727	print __doc__
728	print detail
729	sys.exit(1)
730
731	project_name = None
732	for opt,value in opts:
733	if opt in [ '-h', '--help']:
734	print __doc__
735	sys.exit(0)
736	elif opt in ['-c', '--component']:
737	component = value
738	elif opt in ['-f', '--file']:
739	configfile = value
740	elif opt in ['-p', '--project']:
741	project_name = value
742
743	settings = ReadConfig(configfile, project_name)
744	if not settings.has_key('project'):
745	print __doc__
746	print 'No Trac project is defined in the email2trac config file.'
747	sys.exit(1)
748
749	if component:
750	settings['component'] = component
751
752	if settings.has_key('trac_version'):
753	version = float(settings['trac_version'])
754	else:
755	version = trac_default_version
756
757	if settings.has_key('enable_syslog'):
758	ENABLE_SYSLOG = float(settings['enable_syslog'])
759
760	#debug HvB
761	#print settings
762
763	try:
764	if version == 0.8:
765	from trac.Environment import Environment
766	from trac.Ticket import Ticket
767	from trac.Notify import TicketNotifyEmail
768	from trac.Href import Href
769	from trac import util
770	import sqlite
771	elif version == 0.9:
772	from trac import attachment
773	from trac.env import Environment
774	from trac.ticket import Ticket
775	from trac.web.href import Href
776	from trac import util
777	from trac.Notify import TicketNotifyEmail
778	elif version == 0.10:
779	from trac import attachment
780	from trac.env import Environment
781	from trac.ticket import Ticket
782	from trac.web.href import Href
783	from trac import util
784	# see http://projects.edgewall.com/trac/changeset/2799
785	from trac.ticket.notification import TicketNotifyEmail
786
787	env = Environment(settings['project'], create=0)
788	tktparser = TicketEmailParser(env, settings, version)
789	tktparser.parse(sys.stdin)
790
791	# Catch all errors ans log to SYSLOG if we have enabled this
792	# else stdout
793	#
794	except Exception, error:
795	import syslog, traceback
796
797	if ENABLE_SYSLOG:
798	syslog.openlog('email2trac', syslog.LOG_NOWAIT)
799	etype, evalue, etb = sys.exc_info()
800	for e in traceback.format_exception(etype, evalue, etb):
801	syslog.syslog(e)
802	syslog.closelog()
803	else:
804	traceback.print_exc()
805
806	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format