Context Navigation

source: trunk/email2trac.py.in @ 296

Last change on this file since 296 was 296, checked in by bas, 14 years ago

email2trac.py.in:

Fixed an error when adding attachments with filenames that needed to be escaped, see #149

Property svn:executable set to *
Property svn:keywords set to Id

File size: 39.5 KB

Line
1	#!@PYTHON@
2	# Copyright (C) 2002
3	#
4	# This file is part of the email2trac utils
5	#
6	# This program is free software; you can redistribute it and/or modify it
7	# under the terms of the GNU General Public License as published by the
8	# Free Software Foundation; either version 2, or (at your option) any
9	# later version.
10	#
11	# This program is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with this program; if not, write to the Free Software
18	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
19	#
20	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
21	#
22	"""
23	email2trac.py -- Email tickets to Trac.
24
25	A simple MTA filter to create Trac tickets from inbound emails.
26
27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28	Copyright 2005, Edgewall Software
29
30	Authors:
31	Bas van der Vlies <basv@sara.nl>
32	Walter de Jong <walter@sara.nl>
33
34	The scripts reads emails from stdin and inserts directly into a Trac database.
35
36	How to use
37	----------
38	* See https://subtrac.sara.nl/oss/email2trac/
39
40	* Create an config file:
41	[DEFAULT] # REQUIRED
42	project : /data/trac/test # REQUIRED
43	debug : 1 # OPTIONAL, if set print some DEBUG info
44	trac_version : 0.9 # OPTIONAL, default is 0.11
45
46	[jouvin] # OPTIONAL project declaration, if set both fields necessary
47	project : /data/trac/jouvin # use -p\|--project jouvin.
48
49	* default config file is : /etc/email2trac.conf
50
51	* Commandline opions:
52	-h,--help
53	-f,--file <configuration file>
54	-n,--dry-run
55	-p, --project <project name>
56	-t, --ticket_prefix <name>
57
58	SVN Info:
59	$Id: email2trac.py.in 296 2010-01-07 11:54:44Z bas $
60	"""
61	import os
62	import sys
63	import string
64	import getopt
65	import stat
66	import time
67	import email
68	import email.Iterators
69	import email.Header
70	import re
71	import urllib
72	import unicodedata
73	from stat import *
74	import mimetypes
75	import traceback
76
77
78	# Will fail where unavailable, e.g. Windows
79	#
80	try:
81	import syslog
82	SYSLOG_AVAILABLE = True
83	except ImportError:
84	SYSLOG_AVAILABLE = False
85
86	from datetime import tzinfo, timedelta, datetime
87	from trac import config as trac_config
88
89	# Some global variables
90	#
91	trac_default_version = '0.11'
92	m = None
93
94	# A UTC class needed for trac version 0.11, added by
95	# tbaschak at ktc dot mb dot ca
96	#
97	class UTC(tzinfo):
98	"""UTC"""
99	ZERO = timedelta(0)
100	HOUR = timedelta(hours=1)
101
102	def utcoffset(self, dt):
103	return self.ZERO
104
105	def tzname(self, dt):
106	return "UTC"
107
108	def dst(self, dt):
109	return self.ZERO
110
111
112	class TicketEmailParser(object):
113	env = None
114	comment = '> '
115
116	def __init__(self, env, parameters, version):
117	self.env = env
118
119	# Database connection
120	#
121	self.db = None
122
123	# Save parameters
124	#
125	self.parameters = parameters
126
127	# Some useful mail constants
128	#
129	self.email_name = None
130	self.email_addr = None
131	self.email_from = None
132	self.author = None
133	self.id = None
134
135	self.STRIP_CONTENT_TYPES = list()
136
137	self.VERSION = version
138	self.DRY_RUN = parameters['dry_run']
139
140	self.get_config = self.env.config.get
141
142	if parameters.has_key('umask'):
143	os.umask(int(parameters['umask'], 8))
144
145	if parameters.has_key('debug'):
146	self.DEBUG = int(parameters['debug'])
147	else:
148	self.DEBUG = 0
149
150	if parameters.has_key('mailto_link'):
151	self.MAILTO = int(parameters['mailto_link'])
152	if parameters.has_key('mailto_cc'):
153	self.MAILTO_CC = parameters['mailto_cc']
154	else:
155	self.MAILTO_CC = ''
156	else:
157	self.MAILTO = 0
158
159	if parameters.has_key('spam_level'):
160	self.SPAM_LEVEL = int(parameters['spam_level'])
161	else:
162	self.SPAM_LEVEL = 0
163
164	if parameters.has_key('spam_header'):
165	self.SPAM_HEADER = parameters['spam_header']
166	else:
167	self.SPAM_HEADER = 'X-Spam-Score'
168
169	if parameters.has_key('email_quote'):
170	self.EMAIL_QUOTE = str(parameters['email_quote'])
171	else:
172	self.EMAIL_QUOTE = '> '
173
174	if parameters.has_key('email_header'):
175	self.EMAIL_HEADER = int(parameters['email_header'])
176	else:
177	self.EMAIL_HEADER = 0
178
179	if parameters.has_key('alternate_notify_template'):
180	self.notify_template = str(parameters['alternate_notify_template'])
181	else:
182	self.notify_template = None
183
184	if parameters.has_key('alternate_notify_template_update'):
185	self.notify_template_update = str(parameters['alternate_notify_template_update'])
186	else:
187	self.notify_template_update = None
188
189	if parameters.has_key('reply_all'):
190	self.REPLY_ALL = int(parameters['reply_all'])
191	else:
192	self.REPLY_ALL = 0
193
194	if parameters.has_key('ticket_update'):
195	self.TICKET_UPDATE = int(parameters['ticket_update'])
196	else:
197	self.TICKET_UPDATE = 0
198
199	if parameters.has_key('drop_spam'):
200	self.DROP_SPAM = int(parameters['drop_spam'])
201	else:
202	self.DROP_SPAM = 0
203
204	if parameters.has_key('verbatim_format'):
205	self.VERBATIM_FORMAT = int(parameters['verbatim_format'])
206	else:
207	self.VERBATIM_FORMAT = 1
208
209	if parameters.has_key('reflow'):
210	self.REFLOW = int(parameters['reflow'])
211	else:
212	self.REFLOW = 1
213
214	if parameters.has_key('drop_alternative_html_version'):
215	self.DROP_ALTERNATIVE_HTML_VERSION = int(parameters['drop_alternative_html_version'])
216	else:
217	self.DROP_ALTERNATIVE_HTML_VERSION = 0
218
219	if parameters.has_key('strip_signature'):
220	self.STRIP_SIGNATURE = int(parameters['strip_signature'])
221	else:
222	self.STRIP_SIGNATURE = 0
223
224	if parameters.has_key('strip_quotes'):
225	self.STRIP_QUOTES = int(parameters['strip_quotes'])
226	else:
227	self.STRIP_QUOTES = 0
228
229	if parameters.has_key('use_textwrap'):
230	self.USE_TEXTWRAP = int(parameters['use_textwrap'])
231	else:
232	self.USE_TEXTWRAP = 0
233
234	if parameters.has_key('binhex'):
235	self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
236
237	if parameters.has_key('applesingle'):
238	self.STRIP_CONTENT_TYPES.append('application/applefile')
239
240	if parameters.has_key('appledouble'):
241	self.STRIP_CONTENT_TYPES.append('application/applefile')
242
243	if parameters.has_key('strip_content_types'):
244	items = parameters['strip_content_types'].split(',')
245	for item in items:
246	self.STRIP_CONTENT_TYPES.append(item.strip())
247
248	self.WORKFLOW = None
249	if parameters.has_key('workflow'):
250	self.WORKFLOW = parameters['workflow']
251
252	# Use OS independend functions
253	#
254	self.TMPDIR = os.path.normcase('/tmp')
255	if parameters.has_key('tmpdir'):
256	self.TMPDIR = os.path.normcase(str(parameters['tmpdir']))
257
258	if parameters.has_key('ignore_trac_user_settings'):
259	self.IGNORE_TRAC_USER_SETTINGS = int(parameters['ignore_trac_user_settings'])
260	else:
261	self.IGNORE_TRAC_USER_SETTINGS = 0
262
263	def spam(self, message):
264	"""
265	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
266	# Note if Spam_level then '*' are included
267	"""
268	spam = False
269	if message.has_key(self.SPAM_HEADER):
270	spam_l = string.split(message[self.SPAM_HEADER])
271
272	try:
273	number = spam_l[0].count('*')
274	except IndexError, detail:
275	number = 0
276
277	if number >= self.SPAM_LEVEL:
278	spam = True
279
280	# treat virus mails as spam
281	#
282	elif message.has_key('X-Virus-found'):
283	spam = True
284
285	# How to handle SPAM messages
286	#
287	if self.DROP_SPAM and spam:
288	if self.DEBUG > 2 :
289	print 'This message is a SPAM. Automatic ticket insertion refused (SPAM level > %d' % self.SPAM_LEVEL
290
291	return 'drop'
292
293	elif spam:
294
295	return 'Spam'
296
297	else:
298
299	return False
300
301	def email_header_acl(self, keyword, header_field, default):
302	"""
303	This function wil check if the email address is allowed or denied
304	to send mail to the ticket list
305	"""
306	try:
307	mail_addresses = self.parameters[keyword]
308
309	# Check if we have an empty string
310	#
311	if not mail_addresses:
312	return default
313
314	except KeyError, detail:
315	if self.DEBUG > 2 :
316	print 'TD: %s not defined, all messages are allowed.' %(keyword)
317
318	return default
319
320	mail_addresses = string.split(mail_addresses, ',')
321
322	for entry in mail_addresses:
323	entry = entry.strip()
324	TO_RE = re.compile(entry, re.VERBOSE\|re.IGNORECASE)
325	result = TO_RE.search(header_field)
326	if result:
327	return True
328
329	return False
330
331	def email_to_unicode(self, message_str):
332	"""
333	Email has 7 bit ASCII code, convert it to unicode with the charset
334	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
335	understands it.
336	"""
337	results = email.Header.decode_header(message_str)
338	s = None
339	for text,format in results:
340	if format:
341	try:
342	temp = unicode(text, format)
343	except UnicodeError, detail:
344	# This always works
345	#
346	temp = unicode(text, 'iso-8859-15')
347	except LookupError, detail:
348	#text = 'ERROR: Could not find charset: %s, please install' %format
349	#temp = unicode(text, 'iso-8859-15')
350	temp = message_str
351
352	else:
353	temp = string.strip(text)
354	temp = unicode(text, 'iso-8859-15')
355
356	if s:
357	s = '%s %s' %(s, temp)
358	else:
359	s = '%s' %temp
360
361	#s = s.encode('utf-8')
362	return s
363
364	def debug_body(self, message_body, tempfile=False):
365	if tempfile:
366	import tempfile
367	body_file = tempfile.mktemp('.email2trac')
368	else:
369	body_file = os.path.join(self.TMPDIR, 'body.txt')
370
371	print 'TD: writing body (%s)' % body_file
372	fx = open(body_file, 'wb')
373	if not message_body:
374	message_body = '(None)'
375
376	message_body = message_body.encode('utf-8')
377	#message_body = unicode(message_body, 'iso-8859-15')
378
379	fx.write(message_body)
380	fx.close()
381	try:
382	os.chmod(body_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
383	except OSError:
384	pass
385
386	def debug_attachments(self, message_parts):
387	n = 0
388	for part in message_parts:
389	# Skip inline text parts
390	if not isinstance(part, tuple):
391	continue
392
393	(original, filename, part) = part
394
395	n = n + 1
396	print 'TD: part%d: Content-Type: %s' % (n, part.get_content_type())
397	print 'TD: part%d: filename: %s' % (n, part.get_filename())
398
399	part_file = os.path.join(self.TMPDIR, filename)
400	#part_file = '/var/tmp/part%d' % n
401	print 'TD: writing part%d (%s)' % (n,part_file)
402	fx = open(part_file, 'wb')
403	text = part.get_payload(decode=1)
404	if not text:
405	text = '(None)'
406	fx.write(text)
407	fx.close()
408	try:
409	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
410	except OSError:
411	pass
412
413	def email_header_txt(self, m):
414	"""
415	Display To and CC addresses in description field
416	"""
417	s = ''
418	#if m['To'] and len(m['To']) > 0 and m['To'] != 'hic@sara.nl':
419	if m['To'] and len(m['To']) > 0:
420	s = "'''To:''' %s\r\n" %(m['To'])
421	if m['Cc'] and len(m['Cc']) > 0:
422	s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
423
424	return self.email_to_unicode(s)
425
426
427	def get_sender_info(self, message):
428	"""
429	Get the default author name and email address from the message
430	"""
431
432	self.email_to = self.email_to_unicode(message['to'])
433	self.to_name, self.to_email_addr = email.Utils.parseaddr (self.email_to)
434
435	self.email_from = self.email_to_unicode(message['from'])
436	self.email_name, self.email_addr = email.Utils.parseaddr(self.email_from)
437
438	# Trac can not handle author's name that contains spaces
439	#
440	self.author = self.email_addr
441
442	if self.IGNORE_TRAC_USER_SETTINGS:
443	return
444
445	# Is this a registered user, use email address as search key:
446	# result:
447	# u : login name
448	# n : Name that the user has set in the settings tab
449	# e : email address that the user has set in the settings tab
450	#
451	users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
452	if e and (e.lower() == self.email_addr.lower()) ]
453
454	if len(users) == 1:
455	self.email_from = users[0][0]
456	self.author = users[0][0]
457
458	def set_reply_fields(self, ticket, message):
459	"""
460	Set all the right fields for a new ticket
461	"""
462
463	## Only use name or email adress
464	#ticket['reporter'] = self.email_from
465	ticket['reporter'] = self.author
466
467
468	# Put all CC-addresses in ticket CC field
469	#
470	if self.REPLY_ALL:
471	#tos = message.get_all('to', [])
472	ccs = message.get_all('cc', [])
473
474	addrs = email.Utils.getaddresses(ccs)
475	if not addrs:
476	return
477
478	# Remove reporter email address if notification is
479	# on
480	#
481	if self.notification:
482	try:
483	addrs.remove((self.author, self.email_addr))
484	except ValueError, detail:
485	pass
486
487	for name,mail in addrs:
488	try:
489	mail_list = '%s, %s' %(mail_list, mail)
490	except UnboundLocalError, detail:
491	mail_list = mail
492
493	if mail_list:
494	ticket['cc'] = self.email_to_unicode(mail_list)
495
496	def save_email_for_debug(self, message, tempfile=False):
497	if tempfile:
498	import tempfile
499	msg_file = tempfile.mktemp('.email2trac')
500	else:
501	#msg_file = '/var/tmp/msg.txt'
502	msg_file = os.path.join(self.TMPDIR, 'msg.txt')
503
504	print 'TD: saving email to %s' % msg_file
505	fx = open(msg_file, 'wb')
506	fx.write('%s' % message)
507	fx.close()
508	try:
509	os.chmod(msg_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
510	except OSError:
511	pass
512
513	def str_to_dict(self, s):
514	"""
515	Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
516	"""
517
518	fields = string.split(s, ',')
519
520	result = dict()
521	for field in fields:
522	try:
523	index, value = string.split(field, '=')
524
525	# We can not change the description of a ticket via the subject
526	# line. The description is the body of the email
527	#
528	if index.lower() in ['description']:
529	continue
530
531	if value:
532	result[index.lower()] = value
533
534	except ValueError:
535	pass
536
537	return result
538
539	def update_ticket_fields(self, ticket, user_dict, use_default=None):
540	"""
541	This will update the ticket fields. It will check if the
542	given fields are known and if the right values are specified
543	It will only update the ticket field value:
544	- If the field is known
545	- If the value supplied is valid for the ticket field.
546	If not then there are two options:
547	1) Skip the value (use_default=None)
548	2) Set default value for field (use_default=1)
549	"""
550
551	# Build a system dictionary from the ticket fields
552	# with field as index and option as value
553	#
554	sys_dict = dict()
555	for field in ticket.fields:
556	try:
557	sys_dict[field['name']] = field['options']
558
559	except KeyError:
560	sys_dict[field['name']] = None
561	pass
562
563	# Check user supplied fields an compare them with the
564	# system one's
565	#
566	for field,value in user_dict.items():
567	if self.DEBUG >= 10:
568	print 'user_field\t %s = %s' %(field,value)
569
570	if sys_dict.has_key(field):
571
572	# Check if value is an allowed system option, if TypeError then
573	# every value is allowed
574	#
575	try:
576	if value in sys_dict[field]:
577	ticket[field] = value
578	else:
579	# Must we set a default if value is not allowed
580	#
581	if use_default:
582	value = self.get_config('ticket', 'default_%s' %(field) )
583	ticket[field] = value
584
585	except TypeError:
586	ticket[field] = value
587
588	if self.DEBUG >= 10:
589	print 'ticket_field\t %s = %s' %(field, ticket[field])
590
591	def ticket_update(self, m, id, spam):
592	"""
593	If the current email is a reply to an existing ticket, this function
594	will append the contents of this email to that ticket, instead of
595	creating a new one.
596	"""
597	if self.DEBUG:
598	print "TD: ticket_update: %s" %id
599
600	# Must we update ticket fields
601	#
602	update_fields = dict()
603	try:
604	id, keywords = string.split(id, '?')
605
606	# Skip the last ':' character
607	#
608	keywords = keywords[:-1]
609	update_fields = self.str_to_dict(keywords)
610
611	# Strip '#'
612	#
613	self.id = int(id[1:])
614
615	except ValueError:
616	# Strip '#' and ':'
617	#
618	self.id = int(id[1:-1])
619
620
621	# When is the change committed
622	#
623	if self.VERSION == 0.11:
624	utc = UTC()
625	when = datetime.now(utc)
626	else:
627	when = int(time.time())
628
629	try:
630	tkt = Ticket(self.env, self.id, self.db)
631	except util.TracError, detail:
632	# Not a valid ticket
633	self.id = None
634	return False
635
636	# How many changes has this ticket
637	cnum = len(tkt.get_changelog())
638
639
640	# reopen the ticket if it is was closed
641	# We must use the ticket workflow framework
642	#
643	if tkt['status'] in ['closed']:
644
645	#print controller.actions['reopen']
646	#
647	# As reference
648	# req = Mock(href=Href('/'), abs_href=Href('http://www.example.com/'), authname='anonymous', perm=MockPerm(), args={})
649	#
650	#a = controller.render_ticket_action_control(req, tkt, 'reopen')
651	#print 'controller : ', a
652	#
653	#b = controller.get_all_status()
654	#print 'get all status: ', b
655	#
656	#b = controller.get_ticket_changes(req, tkt, 'reopen')
657	#print 'get_ticket_changes :', b
658
659	if self.WORKFLOW and (self.VERSION in [0.11]) :
660	from trac.ticket.default_workflow import ConfigurableTicketWorkflow
661	from trac.test import Mock, MockPerm
662
663	req = Mock(authname='anonymous', perm=MockPerm(), args={})
664
665	controller = ConfigurableTicketWorkflow(self.env)
666	fields = controller.get_ticket_changes(req, tkt, self.WORKFLOW)
667
668	if self.DEBUG:
669	print 'TD: Workflow ticket update fields: ', fields
670
671	for key in fields.keys():
672	tkt[key] = fields[key]
673
674	else:
675	tkt['status'] = 'reopened'
676	tkt['resolution'] = ''
677
678	# Must we update some ticket fields properties
679	#
680	if update_fields:
681	self.update_ticket_fields(tkt, update_fields)
682
683	message_parts = self.get_message_parts(m)
684	message_parts = self.unique_attachment_names(message_parts)
685
686	if self.EMAIL_HEADER:
687	message_parts.insert(0, self.email_header_txt(m))
688
689	body_text = self.body_text(message_parts)
690
691	if body_text.strip() or update_fields:
692	if self.DRY_RUN:
693	print 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) ', self.author, cnum
694	else:
695	tkt.save_changes(self.author, body_text, when, None, str(cnum))
696
697
698	if self.VERSION == 0.9:
699	s = self.attachments(message_parts, True)
700	else:
701	s = self.attachments(message_parts)
702
703	if self.notification and not spam:
704	self.notify(tkt, False, when)
705
706	return True
707
708	def set_ticket_fields(self, ticket):
709	"""
710	set the ticket fields to value specified
711	- /etc/email2trac.conf with <prefix>_<field>
712	- trac default values, trac.ini
713	"""
714	user_dict = dict()
715
716	for field in ticket.fields:
717
718	name = field['name']
719
720	# skip some fields like resolution
721	#
722	if name in [ 'resolution' ]:
723	continue
724
725	# default trac value
726	#
727	if not field.get('custom'):
728	value = self.get_config('ticket', 'default_%s' %(name) )
729	else:
730	value = field.get('value')
731	options = field.get('options')
732	if value and options and value not in options:
733	value = options[int(value)]
734
735	if self.DEBUG > 10:
736	print 'trac.ini name %s = %s' %(name, value)
737
738	prefix = self.parameters['ticket_prefix']
739	try:
740	value = self.parameters['%s_%s' %(prefix, name)]
741	if self.DEBUG > 10:
742	print 'email2trac.conf %s = %s ' %(name, value)
743
744	except KeyError, detail:
745	pass
746
747	if self.DEBUG:
748	print 'user_dict[%s] = %s' %(name, value)
749
750	user_dict[name] = value
751
752	self.update_ticket_fields(ticket, user_dict, use_default=1)
753
754	# Set status ticket
755	#`
756	ticket['status'] = 'new'
757
758
759
760	def new_ticket(self, msg, subject, spam, set_fields = None):
761	"""
762	Create a new ticket
763	"""
764	if self.DEBUG:
765	print "TD: new_ticket"
766
767	tkt = Ticket(self.env)
768	self.set_ticket_fields(tkt)
769
770	# Old style setting for component, will be removed
771	#
772	if spam:
773	tkt['component'] = 'Spam'
774
775	elif self.parameters.has_key('component'):
776	tkt['component'] = self.parameters['component']
777
778	if not msg['Subject']:
779	tkt['summary'] = u'(No subject)'
780	else:
781	tkt['summary'] = subject
782
783	self.set_reply_fields(tkt, msg)
784
785	if set_fields:
786	rest, keywords = string.split(set_fields, '?')
787
788	if keywords:
789	update_fields = self.str_to_dict(keywords)
790	self.update_ticket_fields(tkt, update_fields)
791
792	# produce e-mail like header
793	#
794	head = ''
795	if self.EMAIL_HEADER > 0:
796	head = self.email_header_txt(msg)
797
798
799	message_parts = self.get_message_parts(msg)
800	if self.DEBUG:
801	print 'TD: self.get_message_parts ',
802	print message_parts
803
804	message_parts = self.unique_attachment_names(message_parts)
805	if self.DEBUG:
806	print 'TD: self.unique_attachment_names',
807	print message_parts
808
809	if self.EMAIL_HEADER > 0:
810	message_parts.insert(0, self.email_header_txt(msg))
811
812	body_text = self.body_text(message_parts)
813
814	tkt['description'] = body_text
815
816	#when = int(time.time())
817	#
818	utc = UTC()
819	when = datetime.now(utc)
820
821	if not self.DRY_RUN:
822	self.id = tkt.insert()
823
824	changed = False
825	comment = ''
826
827	# some routines in trac are dependend on ticket id
828	# like alternate notify template
829	#
830	if self.notify_template:
831	tkt['id'] = self.id
832	changed = True
833
834	## Rewrite the description if we have mailto enabled
835	#
836	if self.MAILTO:
837	changed = True
838	comment = u'\nadded mailto line\n'
839	mailto = self.html_mailto_link( m['Subject'], body_text)
840
841	tkt['description'] = u'%s\r\n%s%s\r\n' \
842	%(head, mailto, body_text)
843
844	## Save the attachments to the ticket
845	#
846	has_attachments = self.attachments(message_parts)
847
848	# Disabled
849	#if has_attachments:
850	# changed = True
851	# comment = '%s\n%s\n' %(comment, has_attachments)
852
853	if changed:
854	if self.DRY_RUN:
855	print 'DRY_RUN: tkt.save_changes(self.author, comment) ', self.author
856	else:
857	tkt.save_changes(self.author, comment)
858	#print tkt.get_changelog(self.db, when)
859
860	if self.notification and not spam:
861	self.notify(tkt, True)
862
863
864	def blog(self, id):
865	"""
866	The blog create/update function
867	"""
868	# import the modules
869	#
870	from tracfullblog.core import FullBlogCore
871	from tracfullblog.model import BlogPost, BlogCommen
872
873	# instantiate blog core
874	blog = FullBlogCore(self.env)
875	req = ''
876
877	if id:
878
879	# update blog
880	#
881	comment = BlogComment(self.env, id)
882	comment.author = self.author
883	comment.comment = self.get_body_text(m)
884	blog.create_comment(req, comment)
885
886	else:
887	# create blog
888	#
889	import time
890	post = BlogPost(self.env, 'blog_'+time.strftime("%Y%m%d%H%M%S", time.gmtime()))
891
892	#post = BlogPost(self.env, blog._get_default_postname(self.env))
893
894	post.author = self.author
895	post.title = self.email_to_unicode(m['Subject'])
896	post.body = self.get_body_text(m)
897
898	blog.create_post(req, post, self.author, u'Created by email2trac', False)
899
900
901	def parse(self, fp):
902	global m
903
904	m = email.message_from_file(fp)
905
906	if not m:
907	if self.DEBUG:
908	print "TD: This is not a valid email message format"
909	return
910
911	# Work around lack of header folding in Python; see http://bugs.python.org/issue4696
912	m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
913
914	if self.DEBUG > 1: # save the entire e-mail message text
915	message_parts = self.get_message_parts(m)
916	message_parts = self.unique_attachment_names(message_parts)
917	self.save_email_for_debug(m, True)
918	body_text = self.body_text(message_parts)
919	self.debug_body(body_text, True)
920	self.debug_attachments(message_parts)
921
922	self.db = self.env.get_db_cnx()
923	self.get_sender_info(m)
924
925	if not self.email_header_acl('white_list', self.email_addr, True):
926	if self.DEBUG > 1 :
927	print 'Message rejected : %s not in white list' %(self.email_addr)
928	return False
929
930	if self.email_header_acl('black_list', self.email_addr, False):
931	if self.DEBUG > 1 :
932	print 'Message rejected : %s in black list' %(self.email_addr)
933	return False
934
935	if not self.email_header_acl('recipient_list', self.to_email_addr, True):
936	if self.DEBUG > 1 :
937	print 'Message rejected : %s not in recipient list' %(self.to_email_addr)
938	return False
939
940	# If drop the message
941	#
942	if self.spam(m) == 'drop':
943	return False
944
945	elif self.spam(m) == 'spam':
946	spam_msg = True
947	else:
948	spam_msg = False
949
950	if self.get_config('notification', 'smtp_enabled') in ['true']:
951	self.notification = 1
952	else:
953	self.notification = 0
954
955	# Check if FullBlogPlugin is installed
956	#
957	blog_enabled = None
958	if self.get_config('components', 'tracfullblog.*') in ['enabled']:
959	blog_enabled = True
960
961	if not m['Subject']:
962	return False
963	else:
964	subject = self.email_to_unicode(m['Subject'])
965
966	#
967	# [hic] #1529: Re: LRZ
968	# [hic] #1529?owner=bas,priority=medium: Re: LRZ
969	#
970	TICKET_RE = re.compile(r"""
971	(?P<blog>blog:(?P<blog_id>\w*))
972	\|(?P<new_fields>[#][?].*)
973	\|(?P<reply>[#][\d]+:)
974	\|(?P<reply_fields>[#][\d]+\?.*?:)
975	""", re.VERBOSE)
976
977	# Find out if this is a ticket or a blog
978	#
979	result = TICKET_RE.search(subject)
980
981	if result:
982	if result.group('blog') and blog_enabled:
983	self.blog(result.group('blog_id'))
984
985	# update ticket + fields
986	#
987	if result.group('reply_fields') and self.TICKET_UPDATE:
988	self.ticket_update(m, result.group('reply_fields'), spam_msg)
989
990	# Update ticket
991	#
992	elif result.group('reply') and self.TICKET_UPDATE:
993	self.ticket_update(m, result.group('reply'), spam_msg)
994
995	# New ticket + fields
996	#
997	elif result.group('new_fields'):
998	self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
999
1000	# Create ticket
1001	#
1002	else:
1003	self.new_ticket(m, subject, spam_msg)
1004
1005	def strip_signature(self, text):
1006	"""
1007	Strip signature from message, inspired by Mailman software
1008	"""
1009	body = []
1010	for line in text.splitlines():
1011	if line == '-- ':
1012	break
1013	body.append(line)
1014
1015	return ('\n'.join(body))
1016
1017	def reflow(self, text, delsp = 0):
1018	"""
1019	Reflow the message based on the format="flowed" specification (RFC 3676)
1020	"""
1021	flowedlines = []
1022	quotelevel = 0
1023	prevflowed = 0
1024
1025	for line in text.splitlines():
1026	from re import match
1027
1028	# Figure out the quote level and the content of the current line
1029	m = match('(>)( ?)(.)', line)
1030	linequotelevel = len(m.group(1))
1031	line = m.group(3)
1032
1033	# Determine whether this line is flowed
1034	if line and line != '-- ' and line[-1] == ' ':
1035	flowed = 1
1036	else:
1037	flowed = 0
1038
1039	if flowed and delsp and line and line[-1] == ' ':
1040	line = line[:-1]
1041
1042	# If the previous line is flowed, append this line to it
1043	if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1044	flowedlines[-1] += line
1045	# Otherwise, start a new line
1046	else:
1047	flowedlines.append('>' * linequotelevel + line)
1048
1049	prevflowed = flowed
1050
1051
1052	return '\n'.join(flowedlines)
1053
1054	def strip_quotes(self, text):
1055	"""
1056	Strip quotes from message by Nicolas Mendoza
1057	"""
1058	body = []
1059	for line in text.splitlines():
1060	if line.startswith(self.EMAIL_QUOTE):
1061	continue
1062	body.append(line)
1063
1064	return ('\n'.join(body))
1065
1066	def wrap_text(self, text, replace_whitespace = False):
1067	"""
1068	Will break a lines longer then given length into several small
1069	lines of size given length
1070	"""
1071	import textwrap
1072
1073	LINESEPARATOR = '\n'
1074	reformat = ''
1075
1076	for s in text.split(LINESEPARATOR):
1077	tmp = textwrap.fill(s,self.USE_TEXTWRAP)
1078	if tmp:
1079	reformat = '%s\n%s' %(reformat,tmp)
1080	else:
1081	reformat = '%s\n' %reformat
1082
1083	return reformat
1084
1085	# Python2.4 and higher
1086	#
1087	#return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
1088	#
1089
1090
1091	def get_message_parts(self, msg):
1092	"""
1093	parses the email message and returns a list of body parts and attachments
1094	body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
1095	"""
1096	message_parts = []
1097
1098	ALTERNATIVE_MULTIPART = False
1099
1100	for part in msg.walk():
1101	if self.DEBUG:
1102	print 'TD: Message part: Main-Type: %s' % part.get_content_maintype()
1103	print 'TD: Message part: Content-Type: %s' % part.get_content_type()
1104
1105	## Check content type
1106	#
1107	if part.get_content_type() in self.STRIP_CONTENT_TYPES:
1108
1109	if self.DEBUG:
1110	print "TD: A %s attachment named '%s' was skipped" %(part.get_content_type(), part.get_filename())
1111
1112	continue
1113
1114	## Catch some mulitpart execptions
1115	#
1116	if part.get_content_type() == 'multipart/alternative':
1117	ALTERNATIVE_MULTIPART = True
1118	continue
1119
1120	## Skip multipart containers
1121	#
1122	if part.get_content_maintype() == 'multipart':
1123	if self.DEBUG:
1124	print "TD: Skipping multipart container"
1125	continue
1126
1127	## Check if this is an inline part. It's inline if there is co Cont-Disp header, or if there is one and it says "inline"
1128	#
1129	inline = self.inline_part(part)
1130
1131	## Drop HTML message
1132	#
1133	if ALTERNATIVE_MULTIPART and self.DROP_ALTERNATIVE_HTML_VERSION:
1134	if part.get_content_type() == 'text/html':
1135	if self.DEBUG:
1136	print "TD: Skipping alternative HTML message"
1137
1138	ALTERNATIVE_MULTIPART = False
1139	continue
1140
1141	## Inline text parts are where the body is
1142	#
1143	if part.get_content_type() == 'text/plain' and inline:
1144	if self.DEBUG:
1145	print 'TD: Inline body part'
1146
1147	# Try to decode, if fails then do not decode
1148	#
1149	body_text = part.get_payload(decode=1)
1150	if not body_text:
1151	body_text = part.get_payload(decode=0)
1152
1153	format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
1154	delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
1155
1156	if self.REFLOW and not self.VERBATIM_FORMAT and format == 'flowed':
1157	body_text = self.reflow(body_text, delsp == 'yes')
1158
1159	if self.STRIP_SIGNATURE:
1160	body_text = self.strip_signature(body_text)
1161
1162	if self.STRIP_QUOTES:
1163	body_text = self.strip_quotes(body_text)
1164
1165	if self.USE_TEXTWRAP:
1166	body_text = self.wrap_text(body_text)
1167
1168	## Get contents charset (iso-8859-15 if not defined in mail headers)
1169	#
1170	charset = part.get_content_charset()
1171	if not charset:
1172	charset = 'iso-8859-15'
1173
1174	try:
1175	ubody_text = unicode(body_text, charset)
1176
1177	except UnicodeError, detail:
1178	ubody_text = unicode(body_text, 'iso-8859-15')
1179
1180	except LookupError, detail:
1181	ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
1182
1183	if self.VERBATIM_FORMAT:
1184	message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
1185	else:
1186	message_parts.append('%s' %ubody_text)
1187	else:
1188	if self.DEBUG:
1189	print 'TD: Filename: %s' % part.get_filename()
1190
1191	message_parts.append((part.get_filename(), part))
1192
1193	return message_parts
1194
1195	def unique_attachment_names(self, message_parts):
1196	"""
1197	"""
1198	renamed_parts = []
1199	attachment_names = set()
1200
1201	for part in message_parts:
1202
1203	## If not an attachment, leave it alone
1204	#
1205	if not isinstance(part, tuple):
1206	renamed_parts.append(part)
1207	continue
1208
1209	(filename, part) = part
1210
1211	## If no filename, use a default one
1212	#
1213	if not filename:
1214	filename = 'untitled-part'
1215
1216	# Guess the extension from the content type, use non strict mode
1217	# some additional non-standard but commonly used MIME types
1218	# are also recognized
1219	#
1220	ext = mimetypes.guess_extension(part.get_content_type(), False)
1221	if not ext:
1222	ext = '.bin'
1223
1224	filename = '%s%s' % (filename, ext)
1225
1226	# We now use the attachment insert function
1227	#
1228	## Discard relative paths in attachment names
1229	#
1230	#filename = filename.replace('\\', '/').replace(':', '/')
1231	#filename = os.path.basename(filename)
1232	#
1233	# We try to normalize the filename to utf-8 NFC if we can.
1234	# Files uploaded from OS X might be in NFD.
1235	# Check python version and then try it
1236	#
1237	#if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
1238	# try:
1239	# filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
1240	# except TypeError:
1241	# pass
1242
1243	# Make the filename unique for this ticket
1244	num = 0
1245	unique_filename = filename
1246	dummy_filename, ext = os.path.splitext(filename)
1247
1248	while unique_filename in attachment_names or self.attachment_exists(unique_filename):
1249	num += 1
1250	unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
1251
1252	if self.DEBUG:
1253	print 'TD: Attachment with filename %s will be saved as %s' % (filename, unique_filename)
1254
1255	attachment_names.add(unique_filename)
1256
1257	renamed_parts.append((filename, unique_filename, part))
1258
1259	return renamed_parts
1260
1261	def inline_part(self, part):
1262	return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
1263
1264
1265	def attachment_exists(self, filename):
1266
1267	if self.DEBUG:
1268	print "TD: attachment_exists: Ticket number : %s, Filename : %s" %(self.id, filename)
1269
1270	# We have no valid ticket id
1271	#
1272	if not self.id:
1273	return False
1274
1275	try:
1276	att = attachment.Attachment(self.env, 'ticket', self.id, filename)
1277	return True
1278	except attachment.ResourceNotFound:
1279	return False
1280
1281	def body_text(self, message_parts):
1282	body_text = []
1283
1284	for part in message_parts:
1285	# Plain text part, append it
1286	if not isinstance(part, tuple):
1287	body_text.extend(part.strip().splitlines())
1288	body_text.append("")
1289	continue
1290
1291	(original, filename, part) = part
1292	inline = self.inline_part(part)
1293
1294	if part.get_content_maintype() == 'image' and inline:
1295	body_text.append('[[Image(%s)]]' % filename)
1296	body_text.append("")
1297	else:
1298	body_text.append('[attachment:"%s"]' % filename)
1299	body_text.append("")
1300
1301	body_text = '\r\n'.join(body_text)
1302	return body_text
1303
1304	def notify(self, tkt, new=True, modtime=0):
1305	"""
1306	A wrapper for the TRAC notify function. So we can use templates
1307	"""
1308	if self.DRY_RUN:
1309	print 'DRY_RUN: self.notify(tkt, True) ', self.author
1310	return
1311	try:
1312	# create false {abs_}href properties, to trick Notify()
1313	#
1314	if not self.VERSION == 0.11:
1315	self.env.abs_href = Href(self.get_config('project', 'url'))
1316	self.env.href = Href(self.get_config('project', 'url'))
1317
1318	tn = TicketNotifyEmail(self.env)
1319
1320	if self.notify_template:
1321
1322	if self.VERSION == 0.11:
1323
1324	from trac.web.chrome import Chrome
1325
1326	if self.notify_template_update and not new:
1327	tn.template_name = self.notify_template_update
1328	else:
1329	tn.template_name = self.notify_template
1330
1331	tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
1332
1333	else:
1334
1335	tn.template_name = self.notify_template;
1336
1337	tn.notify(tkt, new, modtime)
1338
1339	except Exception, e:
1340	print 'TD: Failure sending notification on creation of ticket #%s: %s' %(self.id, e)
1341
1342	def html_mailto_link(self, subject, body):
1343	"""
1344	This function returns a HTML mailto tag with the ticket id and author email address
1345	"""
1346	if not self.author:
1347	author = self.email_addr
1348	else:
1349	author = self.author
1350
1351	# use urllib to escape the chars
1352	#
1353	s = 'mailto:%s?Subject=%s&Cc=%s' %(
1354	urllib.quote(self.email_addr),
1355	urllib.quote('Re: #%s: %s' %(self.id, subject)),
1356	urllib.quote(self.MAILTO_CC)
1357	)
1358
1359	s = '\r\n{{{\r\n#!html\r\n<a\r\n href="%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
1360	return s
1361
1362	def attachments(self, message_parts, update=False):
1363	'''
1364	save any attachments as files in the ticket's directory
1365	'''
1366	if self.DRY_RUN:
1367	print "DRY_RUN: no attachments saved"
1368	return ''
1369
1370	count = 0
1371
1372	# Get Maxium attachment size
1373	#
1374	max_size = int(self.get_config('attachment', 'max_size'))
1375	status = ''
1376
1377	for part in message_parts:
1378	# Skip body parts
1379	if not isinstance(part, tuple):
1380	continue
1381
1382	(original, filename, part) = part
1383	#
1384	# Must be tuneables HvB
1385	#
1386	path, fd = util.create_unique_file(os.path.join(self.TMPDIR, filename))
1387	text = part.get_payload(decode=1)
1388	if not text:
1389	text = '(None)'
1390	fd.write(text)
1391	fd.close()
1392
1393	# get the file_size
1394	#
1395	stats = os.lstat(path)
1396	file_size = stats[stat.ST_SIZE]
1397
1398	# Check if the attachment size is allowed
1399	#
1400	if (max_size != -1) and (file_size > max_size):
1401	status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1402	%(status, original, file_size, max_size)
1403
1404	os.unlink(path)
1405	continue
1406	else:
1407	count = count + 1
1408
1409	# Insert the attachment
1410	#
1411	fd = open(path, 'rb')
1412	att = attachment.Attachment(self.env, 'ticket', self.id)
1413
1414	# This will break the ticket_update system, the body_text is vaporized
1415	# ;-(
1416	#
1417	if not update:
1418	att.author = self.author
1419	att.description = self.email_to_unicode('Added by email2trac')
1420
1421	att.insert(filename, fd, file_size)
1422
1423	#except util.TracError, detail:
1424	# print detail
1425
1426	# Remove the created temporary filename
1427	#
1428	fd.close()
1429	os.unlink(path)
1430
1431	# Return how many attachments
1432	#
1433	status = 'This message has %d attachment(s)\n%s' %(count, status)
1434	return status
1435
1436
1437	def mkdir_p(dir, mode):
1438	'''do a mkdir -p'''
1439
1440	arr = string.split(dir, '/')
1441	path = ''
1442	for part in arr:
1443	path = '%s/%s' % (path, part)
1444	try:
1445	stats = os.stat(path)
1446	except OSError:
1447	os.mkdir(path, mode)
1448
1449	def ReadConfig(file, name):
1450	"""
1451	Parse the config file
1452	"""
1453	if not os.path.isfile(file):
1454	print 'File %s does not exist' %file
1455	sys.exit(1)
1456
1457	config = trac_config.Configuration(file)
1458
1459	# Use given project name else use defaults
1460	#
1461	if name:
1462	sections = config.sections()
1463	if not name in sections:
1464	print "Not a valid project name: %s" %name
1465	print "Valid names: %s" %sections
1466	sys.exit(1)
1467
1468	project = dict()
1469	for option, value in config.options(name):
1470	project[option] = value
1471
1472	else:
1473	# use some trac internals to get the defaults
1474	#
1475	project = config.parser.defaults()
1476
1477	return project
1478
1479
1480	if __name__ == '__main__':
1481	# Default config file
1482	#
1483	configfile = '@email2trac_conf@'
1484	project = ''
1485	component = ''
1486	ticket_prefix = 'default'
1487	dry_run = None
1488
1489	ENABLE_SYSLOG = 0
1490
1491
1492	SHORT_OPT = 'chf:np:t:'
1493	LONG_OPT = ['component=', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=']
1494
1495	try:
1496	opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
1497	except getopt.error,detail:
1498	print __doc__
1499	print detail
1500	sys.exit(1)
1501
1502	project_name = None
1503	for opt,value in opts:
1504	if opt in [ '-h', '--help']:
1505	print __doc__
1506	sys.exit(0)
1507	elif opt in ['-c', '--component']:
1508	component = value
1509	elif opt in ['-f', '--file']:
1510	configfile = value
1511	elif opt in ['-n', '--dry-run']:
1512	dry_run = True
1513	elif opt in ['-p', '--project']:
1514	project_name = value
1515	elif opt in ['-t', '--ticket_prefix']:
1516	ticket_prefix = value
1517
1518	settings = ReadConfig(configfile, project_name)
1519	if not settings.has_key('project'):
1520	print __doc__
1521	print 'No Trac project is defined in the email2trac config file.'
1522	sys.exit(1)
1523
1524	if component:
1525	settings['component'] = component
1526
1527	# The default prefix for ticket values in email2trac.conf
1528	#
1529	settings['ticket_prefix'] = ticket_prefix
1530	settings['dry_run'] = dry_run
1531
1532	if settings.has_key('trac_version'):
1533	version = settings['trac_version']
1534	else:
1535	version = trac_default_version
1536
1537
1538	#debug HvB
1539	#print settings
1540
1541	try:
1542	if version == '0.9':
1543	from trac import attachment
1544	from trac.env import Environment
1545	from trac.ticket import Ticket
1546	from trac.web.href import Href
1547	from trac import util
1548	from trac.Notify import TicketNotifyEmail
1549	elif version == '0.10':
1550	from trac import attachment
1551	from trac.env import Environment
1552	from trac.ticket import Ticket
1553	from trac.web.href import Href
1554	from trac import util
1555	#
1556	# return util.text.to_unicode(str)
1557	#
1558	# see http://projects.edgewall.com/trac/changeset/2799
1559	from trac.ticket.notification import TicketNotifyEmail
1560	from trac import config as trac_config
1561	elif version == '0.11':
1562	from trac import attachment
1563	from trac.env import Environment
1564	from trac.ticket import Ticket
1565	from trac.web.href import Href
1566	from trac import config as trac_config
1567	from trac import util
1568
1569
1570	#
1571	# return util.text.to_unicode(str)
1572	#
1573	# see http://projects.edgewall.com/trac/changeset/2799
1574	from trac.ticket.notification import TicketNotifyEmail
1575	else:
1576	print 'TRAC version %s is not supported' %version
1577	sys.exit(1)
1578
1579	if settings.has_key('enable_syslog'):
1580	if SYSLOG_AVAILABLE:
1581	ENABLE_SYSLOG = float(settings['enable_syslog'])
1582
1583
1584	# Must be set before environment is created
1585	#
1586	if settings.has_key('python_egg_cache'):
1587	python_egg_cache = str(settings['python_egg_cache'])
1588	os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
1589
1590	env = Environment(settings['project'], create=0)
1591	tktparser = TicketEmailParser(env, settings, float(version))
1592	tktparser.parse(sys.stdin)
1593
1594	# Catch all errors ans log to SYSLOG if we have enabled this
1595	# else stdout
1596	#
1597	except Exception, error:
1598	if ENABLE_SYSLOG:
1599	syslog.openlog('email2trac', syslog.LOG_NOWAIT)
1600
1601	etype, evalue, etb = sys.exc_info()
1602	for e in traceback.format_exception(etype, evalue, etb):
1603	syslog.syslog(e)
1604
1605	syslog.closelog()
1606	else:
1607	traceback.print_exc()
1608
1609	if m:
1610	tktparser.save_email_for_debug(m, True)
1611
1612	sys.exit(1)
1613	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format