Context Navigation

email2trac.py.in @ 73

Last change on this file since 73 was 73, checked in by bas, 18 years ago

email2trac.py.in:

Fixed an error with duplicate attachments Thanks Walter

Property svn:executable set to *
Property svn:keywords set to Id

File size: 18.0 KB

Line
1	#!@PYTHON@
2	# Copyright (C) 2002
3	#
4	# This file is part of the email2trac utils
5	#
6	# This program is free software; you can redistribute it and/or modify it
7	# under the terms of the GNU General Public License as published by the
8	# Free Software Foundation; either version 2, or (at your option) any
9	# later version.
10	#
11	# This program is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with this program; if not, write to the Free Software
18	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
19	#
20	"""
21	email2trac.py -- Email tickets to Trac.
22
23	A simple MTA filter to create Trac tickets from inbound emails.
24
25	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
26	Copyright 2005, Edgewall Software
27
28	Changed By: Bas van der Vlies <basv@sara.nl>
29	Date : 13 September 2005
30	Descr. : Added config file and command line options, spam level
31	detection, reply address and mailto option. Unicode support
32
33	Changed By: Walter de Jong <walter@sara.nl>
34	Descr. : multipart-message code and trac attachments
35
36
37	The scripts reads emails from stdin and inserts directly into a Trac database.
38	MIME headers are mapped as follows:
39
40	* From: => Reporter
41	=> CC (Optional via reply_address option)
42	* Subject: => Summary
43	* Body => Description
44	* Component => Can be set to SPAM via spam_level option
45
46	How to use
47	----------
48	* Create an config file:
49	[DEFAULT] # REQUIRED
50	project : /data/trac/test # REQUIRED
51	debug : 1 # OPTIONAL, if set print some DEBUG info
52	spam_level : 4 # OPTIONAL, if set check for SPAM mail
53	reply_address: 1 # OPTIONAL, if set then fill in ticket CC field
54	umask : 022 # OPTIONAL, if set then use this umask for creation of the attachments
55	mailto_link : 1 # OPTIONAL, if set then [mailto:<CC>] in description
56	trac_version : 0.8 # OPTIONAL, default is 0.9
57
58	[jouvin] # OPTIONAL project declaration, if set both fields necessary
59	project : /data/trac/jouvin # use -p\|--project jouvin.
60
61	* default config file is : /etc/email2trac.conf
62
63	* Commandline opions:
64	-h \| --help
65	-c <value> \| --component=<value>
66	-f <config file> \| --file=<config file>
67	-p <project name> \| --project=<project name>
68
69	SVN Info:
70	$Id: email2trac.py.in 73 2006-05-22 10:46:09Z bas $
71	"""
72
73	import os
74	import sys
75	import string
76	import getopt
77	import stat
78	import time
79	import email
80	import re
81	import urllib
82	import unicodedata
83	import ConfigParser
84	from email import Header
85	from stat import *
86	import mimetypes
87
88	trac_default_version = 0.9
89
90	class TicketEmailParser(object):
91	env = None
92	comment = '> '
93
94	def __init__(self, env, parameters, version):
95	self.env = env
96
97	# Database connection
98	#
99	self.db = None
100
101	# Some useful mail constants
102	#
103	self.author = None
104	self.email_addr = None
105	self.email_field = None
106
107	self.VERSION = version
108	if self.VERSION == 0.8:
109	self.get_config = self.env.get_config
110	else:
111	self.get_config = self.env.config.get
112
113	if parameters.has_key('umask'):
114	os.umask(int(parameters['umask'], 8))
115
116	if parameters.has_key('debug'):
117	self.DEBUG = int(parameters['debug'])
118	else:
119	self.DEBUG = 0
120
121	if parameters.has_key('mailto_link'):
122	self.MAILTO = int(parameters['mailto_link'])
123	else:
124	self.MAILTO = 0
125
126	if parameters.has_key('spam_level'):
127	self.SPAM_LEVEL = int(parameters['spam_level'])
128	else:
129	self.SPAM_LEVEL = 0
130
131	if parameters.has_key('email_comment'):
132	self.comment = str(parameters['email_comment'])
133
134	if parameters.has_key('email_header'):
135	self.EMAIL_HEADER = int(parameters['email_header'])
136	else:
137	self.EMAIL_HEADER = 0
138
139	if parameters.has_key('alternate_notify_template'):
140	self.notify_template = str(parameters['alternate_notify_template'])
141	else:
142	self.notify_template = None
143
144	if parameters.has_key('reply_all'):
145	self.REPLY_ALL = int(parameters['reply_all'])
146	else:
147	self.REPLY_ALL = 0
148
149
150	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
151	# Note if Spam_level then '*' are included
152	def spam(self, message):
153	if message.has_key('X-Spam-Score'):
154	spam_l = string.split(message['X-Spam-Score'])
155	number = spam_l[0].count('*')
156
157	if number >= self.SPAM_LEVEL:
158	return 'Spam'
159
160	elif message.has_key('X-Virus-found'): # treat virus mails as spam
161	return 'Spam'
162
163	return self.get_config('ticket', 'default_component')
164
165	def to_unicode(self, str):
166	"""
167	Email has 7 bit ASCII code, convert it to unicode with the charset
168	that is encoded in 7-bit ASCII code and encode it as utf-8 so TRAC
169	understands it.
170	"""
171	results = Header.decode_header(str)
172	str = None
173	for text,format in results:
174	if format:
175	try:
176	temp = unicode(text, format)
177	except UnicodeError:
178	# This always works
179	#
180	temp = unicode(text, 'iso-8859-15')
181	temp = temp.encode('utf-8')
182	else:
183	temp = string.strip(text)
184
185	if str:
186	str = '%s %s' %(str, temp)
187	else:
188	str = temp
189
190	return str
191
192	def debug_attachments(self, message):
193	n = 0
194	for part in message.walk():
195	if part.get_content_maintype() == 'multipart': # multipart/* is just a container
196	print 'TD: multipart container'
197	continue
198
199	n = n + 1
200	print 'TD: part%d: Content-Type: %s' % (n, part.get_content_type())
201	print 'TD: part%d: filename: %s' % (n, part.get_filename())
202
203	if part.is_multipart():
204	print 'TD: this part is multipart'
205	payload = part.get_payload(decode=1)
206	print 'TD: payload:', payload
207	else:
208	print 'TD: this part is not multipart'
209
210	part_file = '/var/tmp/part%d' % n
211	print 'TD: writing part%d (%s)' % (n,part_file)
212	fx = open(part_file, 'wb')
213	text = part.get_payload(decode=1)
214	if not text:
215	text = '(None)'
216	fx.write(text)
217	fx.close()
218	try:
219	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
220	except OSError:
221	pass
222
223	def email_header_txt(self, m):
224	"""
225	Display To and CC addresses in description field
226	"""
227	str = ''
228	if m['To'] and len(m['To']) > 0 and m['To'] != 'hic@sara.nl':
229	str = "'''To:''' %s [[BR]]" %(m['To'])
230	if m['Cc'] and len(m['Cc']) > 0:
231	str = "%s'''Cc:''' %s [[BR]]" % (str, m['Cc'])
232
233	return str
234
235	def set_owner(self, ticket):
236	"""
237	Select default owner for ticket component
238	"""
239	cursor = self.db.cursor()
240	sql = "SELECT owner FROM component WHERE name='%s'" % ticket['component']
241	cursor.execute(sql)
242	try:
243	ticket['owner'] = cursor.fetchone()[0]
244	except TypeError, detail:
245	ticket['owner'] = "UNKNOWN"
246
247	def get_author_emailaddrs(self, message):
248	"""
249	Get the default author name and email address from the message
250	"""
251	self.author, self.email_addr = email.Utils.parseaddr(message['from'])
252
253	# Look for email address in registered trac users
254	#
255	if self.VERSION == 0.8:
256	users = []
257	else:
258	users = [ u for (u, n, e) in self.env.get_known_users(self.db)
259	if e == self.email_addr ]
260
261	if len(users) == 1:
262	self.email_field = users[0]
263	else:
264	self.email_field = self.to_unicode(message['from'])
265
266	def set_reply_fields(self, ticket, message):
267	"""
268	Set all the right fields for a new ticket
269	"""
270	ticket['reporter'] = self.email_field
271
272	# Put all CC-addresses in ticket CC field
273	#
274	if self.REPLY_ALL:
275	#tos = message.get_all('to', [])
276	ccs = message.get_all('cc', [])
277
278	addrs = email.Utils.getaddresses(ccs)
279
280	# Remove reporter email address if notification is
281	# on
282	#
283	if self.notification:
284	try:
285	addrs.remove((self.author, self.email_addr))
286	except ValueError, detail:
287	pass
288
289	for name,mail in addrs:
290	try:
291	ticket['cc'] = '%s,%s' %(ticket['cc'], mail)
292	except:
293	ticket['cc'] = mail
294
295	def save_email_for_debug(self, message):
296
297	msg_file = '/var/tmp/msg.txt'
298	print 'TD: saving email to %s' % msg_file
299	fx = open(msg_file, 'wb')
300	fx.write('%s' % message)
301	fx.close()
302	try:
303	os.chmod(msg_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
304	except OSError:
305	pass
306
307	def ticket_update(self, m):
308
309	if not m['Subject']:
310	return False
311	else:
312	subject = self.to_unicode(m['Subject'])
313	print subject
314
315	TICKET_RE = re.compile(r"""
316	(?P<ticketnr>[#][0-9]+:)
317	""", re.VERBOSE)
318
319	result = TICKET_RE.search(subject)
320	if not result:
321	return False
322
323	# Strip '#' and ':' from ticket_id
324	ticket_id = result.group('ticketnr')
325	ticket_id = int(ticket_id[1:-1])
326
327	tkt = Ticket(self.db, ticket_id)
328
329	body_text = self.get_body_text(m)
330	body_text = '{{{\n%s\n}}}' %body_text
331	tkt.save_changes(self.db, self.author, body_text)
332
333	self.attachments(m, tkt)
334	if self.notification:
335	self.notify(tkt, False)
336
337	return True
338
339
340	def parse(self, fp):
341	msg = email.message_from_file(fp)
342	if not msg:
343	return
344
345	if self.DEBUG > 1: # save the entire e-mail message text
346	self.save_email_for_debug(msg)
347	self.debug_attachments(msg)
348
349	self.db = self.env.get_db_cnx()
350	self.get_author_emailaddrs(msg)
351
352	if self.get_config('notification', 'smtp_enabled') in ['true']:
353	self.notification = 1
354	else:
355	self.notification = 0
356
357	# Is this a previous ticket
358	#
359	if self.ticket_update(msg):
360	return True
361
362	tkt = Ticket(self.env)
363	tkt['status'] = 'new'
364
365	# Some defaults
366	#
367	tkt['milestone'] = self.get_config('ticket', 'default_milestone')
368	tkt['priority'] = self.get_config('ticket', 'default_priority')
369	tkt['severity'] = self.get_config('ticket', 'default_severity')
370	tkt['version'] = self.get_config('ticket', 'default_version')
371
372	if not msg['Subject']:
373	tkt['summary'] = '(geen subject)'
374	else:
375	tkt['summary'] = self.to_unicode(msg['Subject'])
376
377
378	if settings.has_key('component'):
379	tkt['component'] = settings['component']
380	else:
381	tkt['component'] = self.spam(msg)
382
383	# Must make this an option or so, discard SPAM messages or save then
384	# and delete later
385	#
386	#if self.SPAM_LEVEL and self.spam(msg):
387	# print 'This message is a SPAM. Automatic ticket insertion refused (SPAM level > %d' % self.SPAM_LEVEL
388	# sys.exit(1)
389
390	# Set default owner for component
391	#
392	self.set_owner(tkt)
393	self.set_reply_fields(tkt, msg)
394
395	# produce e-mail like header
396	#
397	head = ''
398	if self.EMAIL_HEADER > 0:
399	head = self.email_header_txt(msg)
400
401
402	body_text = self.get_body_text(msg)
403	tkt['description'] = '\n{{{\n%s\n}}}\n' %(body_text)
404
405	# Insert ticket in database
406	#
407	if self.VERSION == 0.8:
408	tkt['id'] = tkt.insert(self.db)
409	else:
410	tkt['id'] = tkt.insert()
411
412	# Else we do no have a ticket id for the subject line
413	#
414	mailto = ''
415	if self.MAILTO:
416	mailto = self.html_mailto_link(self.to_unicode(msg['subject']), tkt['id'], body_text)
417	tkt['description'] = '%s%s\n{{{\n%s\n}}}\n' %(head, mailto, body_text)
418	tkt.save_changes(self.db, self.author, "")
419
420	self.attachments(msg, tkt)
421	if self.notification:
422	self.notify(tkt)
423
424	def get_body_text(self, msg):
425	"""
426	put the message text in the ticket description or in the changes field
427	message text can be plain text or html or something else
428	"""
429	has_description = 0
430	ubody_text = '\n{{{\nNo plain text message\n}}}\n'
431	for part in msg.walk():
432
433	# 'multipart/*' is a container for multipart messages
434	#
435	if part.get_content_maintype() == 'multipart':
436	continue
437
438	if part.get_content_type() == 'text/plain':
439	# Try to decode, if fails then do not decode
440	#
441	body_text = part.get_payload(decode=1)
442	if not body_text:
443	body_text = part.get_payload(decode=0)
444
445	# Get contents charset (iso-8859-15 if not defined in mail headers)
446	# UTF-8 encode body_text
447	#
448	charset = msg.get_content_charset('iso-8859-15')
449	ubody_text = unicode(body_text, charset).encode('utf-8')
450
451	elif part.get_content_type() == 'text/html':
452	ubody_text = '\n\n(see attachment for HTML mail message)\n'
453
454	else:
455	ubody_text = '\n\n(see attachment for message)\n'
456
457	has_description = 1
458	break # we have the description, so break
459
460	if not has_description:
461	ubody_text = '\n\n(see attachment for message)\n'
462
463	return ubody_text
464
465	def notify(self, tkt , new=True):
466	try:
467	# create false {abs_}href properties, to trick Notify()
468	#
469	self.env.abs_href = Href(self.get_config('project', 'url'))
470	self.env.href = Href(self.get_config('project', 'url'))
471
472	tn = TicketNotifyEmail(self.env)
473	if self.notify_template:
474	tn.template_name = self.notify_template;
475
476	tn.notify(tkt, new)
477
478	except Exception, e:
479	print 'TD: Failure sending notification on creation of ticket #%s: %s' \
480	% (tkt['id'], e)
481
482	def mail_line(self, str):
483	return '%s %s' % (self.comment, str)
484
485
486	def html_mailto_link(self, subject, id, body):
487	if not self.author:
488	author = self.mail_addr
489	else:
490	author = self.to_unicode(self.author)
491
492	# Must find a fix
493	#
494	#arr = string.split(body, '\n')
495	#arr = map(self.mail_line, arr)
496	#body = string.join(arr, '\n')
497	#body = '%s wrote:\n%s' %(author, body)
498
499	# Temporary fix
500	str = 'mailto:%s?subject=%s' %(urllib.quote(self.email_addr), urllib.quote('Re: #%s: %s' %(id, subject)))
501	str = '\n{{{\n#!html\n<a href="%s">Reply to: %s</a>\n}}}\n' %(str, author)
502
503	return str
504
505	def attachments(self, message, ticket):
506	'''save any attachments as file in the ticket's directory'''
507
508	count = 0
509	first = 0
510	for part in message.walk():
511	if part.get_content_maintype() == 'multipart': # multipart/* is just a container
512	continue
513
514	if not first: # first content is the message
515	first = 1
516	if part.get_content_type() == 'text/plain': # if first is text, is was already put in the description
517	continue
518
519	filename = part.get_filename()
520	if not filename:
521	count = count + 1
522	filename = 'part%04d' % count
523
524	ext = mimetypes.guess_extension(part.get_content_type())
525	if not ext:
526	ext = '.bin'
527
528	filename = '%s%s' % (filename, ext)
529	else:
530	filename = self.to_unicode(filename)
531
532	# From the trac code
533	#
534	filename = filename.replace('\\', '/').replace(':', '/')
535	filename = os.path.basename(filename)
536
537	# We try to normalize the filename to utf-8 NFC if we can.
538	# Files uploaded from OS X might be in NFD.
539	#
540	if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
541	filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
542
543	url_filename = urllib.quote(filename)
544	if self.VERSION == 0.8:
545	dir = os.path.join(self.env.get_attachments_dir(), 'ticket',
546	urllib.quote(str(ticket['id'])))
547	if not os.path.exists(dir):
548	mkdir_p(dir, 0755)
549	else:
550	dir = '/tmp'
551
552	path, fd = util.create_unique_file(os.path.join(dir, url_filename))
553	text = part.get_payload(decode=1)
554	if not text:
555	text = '(None)'
556	fd.write(text)
557	fd.close()
558
559	# get the filesize
560	#
561	stats = os.lstat(path)
562	filesize = stats[stat.ST_SIZE]
563
564	# Insert the attachment it differs for the different TRAC versions
565	#
566	if self.VERSION == 0.8:
567	cursor = self.db.cursor()
568	try:
569	cursor.execute('INSERT INTO attachment VALUES("%s","%s","%s",%d,%d,"%s","%s","%s")'
570	%('ticket', urllib.quote(str(ticket['id'])), filename + '?format=raw', filesize,
571	int(time.time()),'', self.author, 'e-mail') )
572
573	# Attachment is already known
574	#
575	except sqlite.IntegrityError:
576	self.db.close()
577	return
578
579	self.db.commit()
580
581	else:
582	fd = open(path)
583	att = attachment.Attachment(self.env, 'ticket', ticket['id'])
584	att.insert(url_filename, fd, filesize)
585	fd.close()
586
587
588	def mkdir_p(dir, mode):
589	'''do a mkdir -p'''
590
591	arr = string.split(dir, '/')
592	path = ''
593	for part in arr:
594	path = '%s/%s' % (path, part)
595	try:
596	stats = os.stat(path)
597	except OSError:
598	os.mkdir(path, mode)
599
600
601	def ReadConfig(file, name):
602	"""
603	Parse the config file
604	"""
605
606	if not os.path.isfile(file):
607	print 'File %s does not exists' %file
608	sys.exit(1)
609
610	config = ConfigParser.ConfigParser()
611	try:
612	config.read(file)
613	except ConfigParser.MissingSectionHeaderError,detail:
614	print detail
615	sys.exit(1)
616
617
618	# Use given project name else use defaults
619	#
620	if name:
621	if not config.has_section(name):
622	print "Not an valid project name: %s" %name
623	print "Valid names: %s" %config.sections()
624	sys.exit(1)
625
626	project = dict()
627	for option in config.options(name):
628	project[option] = config.get(name, option)
629
630	else:
631	project = config.defaults()
632
633	return project
634
635	if __name__ == '__main__':
636	# Default config file
637	#
638	configfile = '@email2trac_conf@'
639	project = ''
640	component = ''
641
642	try:
643	opts, args = getopt.getopt(sys.argv[1:], 'chf:p:', ['component=','help', 'file=', 'project='])
644	except getopt.error,detail:
645	print __doc__
646	print detail
647	sys.exit(1)
648
649	project_name = None
650	for opt,value in opts:
651	if opt in [ '-h', '--help']:
652	print __doc__
653	sys.exit(0)
654	elif opt in ['-c', '--component']:
655	component = value
656	elif opt in ['-f', '--file']:
657	configfile = value
658	elif opt in ['-p', '--project']:
659	project_name = value
660
661	settings = ReadConfig(configfile, project_name)
662	if not settings.has_key('project'):
663	print __doc__
664	print 'No project defined in config file, eg:\n\t project: /data/trac/bas'
665	sys.exit(1)
666
667	if component:
668	settings['component'] = component
669
670	if settings.has_key('trac_version'):
671	version = float(settings['trac_version'])
672	else:
673	version = trac_default_version
674
675	#debug HvB
676	#print settings
677
678	if version == 0.8:
679	from trac.Environment import Environment
680	from trac.Ticket import Ticket
681	from trac.Notify import TicketNotifyEmail
682	from trac.Href import Href
683	from trac import util
684	import sqlite
685	elif version == 0.9:
686	from trac import attachment
687	from trac.env import Environment
688	from trac.ticket import Ticket
689	from trac.web.href import Href
690	from trac import util
691	from trac.Notify import TicketNotifyEmail
692	elif version == 0.10:
693	from trac import attachment
694	from trac.env import Environment
695	from trac.ticket import Ticket
696	from trac.web.href import Href
697	from trac import util
698	# see http://projects.edgewall.com/trac/changeset/2799
699	from trac.ticket.notification import TicketNotifyEmail
700
701	env = Environment(settings['project'], create=0)
702	tktparser = TicketEmailParser(env, settings, version)
703	tktparser.parse(sys.stdin)
704
705	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format