Context Navigation

source: trunk/email2trac.py.in @ 294

Last change on this file since 294 was 294, checked in by bas, 14 years ago

email2trac.py.in:

added new configuration parameter, strip_content_types thanks to Otto. eg:

strip_content_types: application/pgp-signature, application/mac-binhex40

Will skip pgp signature attachments and apple binhex40 attachments, closes #68

Property svn:executable set to *
Property svn:keywords set to Id

File size: 39.6 KB

Line
1	#!@PYTHON@
2	# Copyright (C) 2002
3	#
4	# This file is part of the email2trac utils
5	#
6	# This program is free software; you can redistribute it and/or modify it
7	# under the terms of the GNU General Public License as published by the
8	# Free Software Foundation; either version 2, or (at your option) any
9	# later version.
10	#
11	# This program is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with this program; if not, write to the Free Software
18	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
19	#
20	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
21	#
22	"""
23	email2trac.py -- Email tickets to Trac.
24
25	A simple MTA filter to create Trac tickets from inbound emails.
26
27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28	Copyright 2005, Edgewall Software
29
30	Authors:
31	Bas van der Vlies <basv@sara.nl>
32	Walter de Jong <walter@sara.nl>
33
34	The scripts reads emails from stdin and inserts directly into a Trac database.
35
36	How to use
37	----------
38	* See https://subtrac.sara.nl/oss/email2trac/
39
40	* Create an config file:
41	[DEFAULT] # REQUIRED
42	project : /data/trac/test # REQUIRED
43	debug : 1 # OPTIONAL, if set print some DEBUG info
44	trac_version : 0.9 # OPTIONAL, default is 0.11
45
46	[jouvin] # OPTIONAL project declaration, if set both fields necessary
47	project : /data/trac/jouvin # use -p\|--project jouvin.
48
49	* default config file is : /etc/email2trac.conf
50
51	* Commandline opions:
52	-h,--help
53	-f,--file <configuration file>
54	-n,--dry-run
55	-p, --project <project name>
56	-t, --ticket_prefix <name>
57
58	SVN Info:
59	$Id: email2trac.py.in 294 2010-01-06 12:18:48Z bas $
60	"""
61	import os
62	import sys
63	import string
64	import getopt
65	import stat
66	import time
67	import email
68	import email.Iterators
69	import email.Header
70	import re
71	import urllib
72	import unicodedata
73	from stat import *
74	import mimetypes
75	import traceback
76
77
78	# Will fail where unavailable, e.g. Windows
79	#
80	try:
81	import syslog
82	SYSLOG_AVAILABLE = True
83	except ImportError:
84	SYSLOG_AVAILABLE = False
85
86	from datetime import tzinfo, timedelta, datetime
87	from trac import config as trac_config
88
89	# Some global variables
90	#
91	trac_default_version = '0.11'
92	m = None
93
94	# A UTC class needed for trac version 0.11, added by
95	# tbaschak at ktc dot mb dot ca
96	#
97	class UTC(tzinfo):
98	"""UTC"""
99	ZERO = timedelta(0)
100	HOUR = timedelta(hours=1)
101
102	def utcoffset(self, dt):
103	return self.ZERO
104
105	def tzname(self, dt):
106	return "UTC"
107
108	def dst(self, dt):
109	return self.ZERO
110
111
112	class TicketEmailParser(object):
113	env = None
114	comment = '> '
115
116	def __init__(self, env, parameters, version):
117	self.env = env
118
119	# Database connection
120	#
121	self.db = None
122
123	# Save parameters
124	#
125	self.parameters = parameters
126
127	# Some useful mail constants
128	#
129	self.email_name = None
130	self.email_addr = None
131	self.email_from = None
132	self.author = None
133	self.id = None
134
135	self.STRIP_CONTENT_TYPES = list()
136
137	self.VERSION = version
138	self.DRY_RUN = parameters['dry_run']
139
140	self.get_config = self.env.config.get
141
142	if parameters.has_key('umask'):
143	os.umask(int(parameters['umask'], 8))
144
145	if parameters.has_key('quote_attachment_filenames'):
146	self.QUOTE_ATTACHMENT_FILENAMES = int(parameters['quote_attachment_filenames'])
147	else:
148	self.QUOTE_ATTACHMENT_FILENAMES = 1
149
150	if parameters.has_key('debug'):
151	self.DEBUG = int(parameters['debug'])
152	else:
153	self.DEBUG = 0
154
155	if parameters.has_key('mailto_link'):
156	self.MAILTO = int(parameters['mailto_link'])
157	if parameters.has_key('mailto_cc'):
158	self.MAILTO_CC = parameters['mailto_cc']
159	else:
160	self.MAILTO_CC = ''
161	else:
162	self.MAILTO = 0
163
164	if parameters.has_key('spam_level'):
165	self.SPAM_LEVEL = int(parameters['spam_level'])
166	else:
167	self.SPAM_LEVEL = 0
168
169	if parameters.has_key('spam_header'):
170	self.SPAM_HEADER = parameters['spam_header']
171	else:
172	self.SPAM_HEADER = 'X-Spam-Score'
173
174	if parameters.has_key('email_quote'):
175	self.EMAIL_QUOTE = str(parameters['email_quote'])
176	else:
177	self.EMAIL_QUOTE = '> '
178
179	if parameters.has_key('email_header'):
180	self.EMAIL_HEADER = int(parameters['email_header'])
181	else:
182	self.EMAIL_HEADER = 0
183
184	if parameters.has_key('alternate_notify_template'):
185	self.notify_template = str(parameters['alternate_notify_template'])
186	else:
187	self.notify_template = None
188
189	if parameters.has_key('alternate_notify_template_update'):
190	self.notify_template_update = str(parameters['alternate_notify_template_update'])
191	else:
192	self.notify_template_update = None
193
194	if parameters.has_key('reply_all'):
195	self.REPLY_ALL = int(parameters['reply_all'])
196	else:
197	self.REPLY_ALL = 0
198
199	if parameters.has_key('ticket_update'):
200	self.TICKET_UPDATE = int(parameters['ticket_update'])
201	else:
202	self.TICKET_UPDATE = 0
203
204	if parameters.has_key('drop_spam'):
205	self.DROP_SPAM = int(parameters['drop_spam'])
206	else:
207	self.DROP_SPAM = 0
208
209	if parameters.has_key('verbatim_format'):
210	self.VERBATIM_FORMAT = int(parameters['verbatim_format'])
211	else:
212	self.VERBATIM_FORMAT = 1
213
214	if parameters.has_key('reflow'):
215	self.REFLOW = int(parameters['reflow'])
216	else:
217	self.REFLOW = 1
218
219	if parameters.has_key('drop_alternative_html_version'):
220	self.DROP_ALTERNATIVE_HTML_VERSION = int(parameters['drop_alternative_html_version'])
221	else:
222	self.DROP_ALTERNATIVE_HTML_VERSION = 0
223
224	if parameters.has_key('strip_signature'):
225	self.STRIP_SIGNATURE = int(parameters['strip_signature'])
226	else:
227	self.STRIP_SIGNATURE = 0
228
229	if parameters.has_key('strip_quotes'):
230	self.STRIP_QUOTES = int(parameters['strip_quotes'])
231	else:
232	self.STRIP_QUOTES = 0
233
234	if parameters.has_key('use_textwrap'):
235	self.USE_TEXTWRAP = int(parameters['use_textwrap'])
236	else:
237	self.USE_TEXTWRAP = 0
238
239	if parameters.has_key('binhex'):
240	self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
241
242	if parameters.has_key('applesingle'):
243	self.STRIP_CONTENT_TYPES.append('application/applefile')
244
245	if parameters.has_key('appledouble'):
246	self.STRIP_CONTENT_TYPES.append('application/applefile')
247
248	if parameters.has_key('strip_content_types'):
249	items = parameters['strip_content_types'].split(',')
250	for item in items:
251	self.STRIP_CONTENT_TYPES.append(item.strip())
252
253	self.WORKFLOW = None
254	if parameters.has_key('workflow'):
255	self.WORKFLOW = parameters['workflow']
256
257	# Use OS independend functions
258	#
259	self.TMPDIR = os.path.normcase('/tmp')
260	if parameters.has_key('tmpdir'):
261	self.TMPDIR = os.path.normcase(str(parameters['tmpdir']))
262
263	if parameters.has_key('ignore_trac_user_settings'):
264	self.IGNORE_TRAC_USER_SETTINGS = int(parameters['ignore_trac_user_settings'])
265	else:
266	self.IGNORE_TRAC_USER_SETTINGS = 0
267
268	def spam(self, message):
269	"""
270	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
271	# Note if Spam_level then '*' are included
272	"""
273	spam = False
274	if message.has_key(self.SPAM_HEADER):
275	spam_l = string.split(message[self.SPAM_HEADER])
276
277	try:
278	number = spam_l[0].count('*')
279	except IndexError, detail:
280	number = 0
281
282	if number >= self.SPAM_LEVEL:
283	spam = True
284
285	# treat virus mails as spam
286	#
287	elif message.has_key('X-Virus-found'):
288	spam = True
289
290	# How to handle SPAM messages
291	#
292	if self.DROP_SPAM and spam:
293	if self.DEBUG > 2 :
294	print 'This message is a SPAM. Automatic ticket insertion refused (SPAM level > %d' % self.SPAM_LEVEL
295
296	return 'drop'
297
298	elif spam:
299
300	return 'Spam'
301
302	else:
303
304	return False
305
306	def email_header_acl(self, keyword, header_field, default):
307	"""
308	This function wil check if the email address is allowed or denied
309	to send mail to the ticket list
310	"""
311	try:
312	mail_addresses = self.parameters[keyword]
313
314	# Check if we have an empty string
315	#
316	if not mail_addresses:
317	return default
318
319	except KeyError, detail:
320	if self.DEBUG > 2 :
321	print 'TD: %s not defined, all messages are allowed.' %(keyword)
322
323	return default
324
325	mail_addresses = string.split(mail_addresses, ',')
326
327	for entry in mail_addresses:
328	entry = entry.strip()
329	TO_RE = re.compile(entry, re.VERBOSE\|re.IGNORECASE)
330	result = TO_RE.search(header_field)
331	if result:
332	return True
333
334	return False
335
336	def email_to_unicode(self, message_str):
337	"""
338	Email has 7 bit ASCII code, convert it to unicode with the charset
339	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
340	understands it.
341	"""
342	results = email.Header.decode_header(message_str)
343	s = None
344	for text,format in results:
345	if format:
346	try:
347	temp = unicode(text, format)
348	except UnicodeError, detail:
349	# This always works
350	#
351	temp = unicode(text, 'iso-8859-15')
352	except LookupError, detail:
353	#text = 'ERROR: Could not find charset: %s, please install' %format
354	#temp = unicode(text, 'iso-8859-15')
355	temp = message_str
356
357	else:
358	temp = string.strip(text)
359	temp = unicode(text, 'iso-8859-15')
360
361	if s:
362	s = '%s %s' %(s, temp)
363	else:
364	s = '%s' %temp
365
366	#s = s.encode('utf-8')
367	return s
368
369	def debug_body(self, message_body, tempfile=False):
370	if tempfile:
371	import tempfile
372	body_file = tempfile.mktemp('.email2trac')
373	else:
374	body_file = os.path.join(self.TMPDIR, 'body.txt')
375
376	print 'TD: writing body (%s)' % body_file
377	fx = open(body_file, 'wb')
378	if not message_body:
379	message_body = '(None)'
380
381	message_body = message_body.encode('utf-8')
382	#message_body = unicode(message_body, 'iso-8859-15')
383
384	fx.write(message_body)
385	fx.close()
386	try:
387	os.chmod(body_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
388	except OSError:
389	pass
390
391	def debug_attachments(self, message_parts):
392	n = 0
393	for part in message_parts:
394	# Skip inline text parts
395	if not isinstance(part, tuple):
396	continue
397
398	(original, filename, part) = part
399
400	n = n + 1
401	print 'TD: part%d: Content-Type: %s' % (n, part.get_content_type())
402	print 'TD: part%d: filename: %s' % (n, part.get_filename())
403
404	part_file = os.path.join(self.TMPDIR, filename)
405	#part_file = '/var/tmp/part%d' % n
406	print 'TD: writing part%d (%s)' % (n,part_file)
407	fx = open(part_file, 'wb')
408	text = part.get_payload(decode=1)
409	if not text:
410	text = '(None)'
411	fx.write(text)
412	fx.close()
413	try:
414	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
415	except OSError:
416	pass
417
418	def email_header_txt(self, m):
419	"""
420	Display To and CC addresses in description field
421	"""
422	s = ''
423	#if m['To'] and len(m['To']) > 0 and m['To'] != 'hic@sara.nl':
424	if m['To'] and len(m['To']) > 0:
425	s = "'''To:''' %s\r\n" %(m['To'])
426	if m['Cc'] and len(m['Cc']) > 0:
427	s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
428
429	return self.email_to_unicode(s)
430
431
432	def get_sender_info(self, message):
433	"""
434	Get the default author name and email address from the message
435	"""
436
437	self.email_to = self.email_to_unicode(message['to'])
438	self.to_name, self.to_email_addr = email.Utils.parseaddr (self.email_to)
439
440	self.email_from = self.email_to_unicode(message['from'])
441	self.email_name, self.email_addr = email.Utils.parseaddr(self.email_from)
442
443	# Trac can not handle author's name that contains spaces
444	#
445	self.author = self.email_addr
446
447	if self.IGNORE_TRAC_USER_SETTINGS:
448	return
449
450	# Is this a registered user, use email address as search key:
451	# result:
452	# u : login name
453	# n : Name that the user has set in the settings tab
454	# e : email address that the user has set in the settings tab
455	#
456	users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
457	if e and (e.lower() == self.email_addr.lower()) ]
458
459	if len(users) == 1:
460	self.email_from = users[0][0]
461	self.author = users[0][0]
462
463	def set_reply_fields(self, ticket, message):
464	"""
465	Set all the right fields for a new ticket
466	"""
467
468	## Only use name or email adress
469	#ticket['reporter'] = self.email_from
470	ticket['reporter'] = self.author
471
472
473	# Put all CC-addresses in ticket CC field
474	#
475	if self.REPLY_ALL:
476	#tos = message.get_all('to', [])
477	ccs = message.get_all('cc', [])
478
479	addrs = email.Utils.getaddresses(ccs)
480	if not addrs:
481	return
482
483	# Remove reporter email address if notification is
484	# on
485	#
486	if self.notification:
487	try:
488	addrs.remove((self.author, self.email_addr))
489	except ValueError, detail:
490	pass
491
492	for name,mail in addrs:
493	try:
494	mail_list = '%s, %s' %(mail_list, mail)
495	except UnboundLocalError, detail:
496	mail_list = mail
497
498	if mail_list:
499	ticket['cc'] = self.email_to_unicode(mail_list)
500
501	def save_email_for_debug(self, message, tempfile=False):
502	if tempfile:
503	import tempfile
504	msg_file = tempfile.mktemp('.email2trac')
505	else:
506	#msg_file = '/var/tmp/msg.txt'
507	msg_file = os.path.join(self.TMPDIR, 'msg.txt')
508
509	print 'TD: saving email to %s' % msg_file
510	fx = open(msg_file, 'wb')
511	fx.write('%s' % message)
512	fx.close()
513	try:
514	os.chmod(msg_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
515	except OSError:
516	pass
517
518	def str_to_dict(self, s):
519	"""
520	Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
521	"""
522
523	fields = string.split(s, ',')
524
525	result = dict()
526	for field in fields:
527	try:
528	index, value = string.split(field, '=')
529
530	# We can not change the description of a ticket via the subject
531	# line. The description is the body of the email
532	#
533	if index.lower() in ['description']:
534	continue
535
536	if value:
537	result[index.lower()] = value
538
539	except ValueError:
540	pass
541
542	return result
543
544	def update_ticket_fields(self, ticket, user_dict, use_default=None):
545	"""
546	This will update the ticket fields. It will check if the
547	given fields are known and if the right values are specified
548	It will only update the ticket field value:
549	- If the field is known
550	- If the value supplied is valid for the ticket field.
551	If not then there are two options:
552	1) Skip the value (use_default=None)
553	2) Set default value for field (use_default=1)
554	"""
555
556	# Build a system dictionary from the ticket fields
557	# with field as index and option as value
558	#
559	sys_dict = dict()
560	for field in ticket.fields:
561	try:
562	sys_dict[field['name']] = field['options']
563
564	except KeyError:
565	sys_dict[field['name']] = None
566	pass
567
568	# Check user supplied fields an compare them with the
569	# system one's
570	#
571	for field,value in user_dict.items():
572	if self.DEBUG >= 10:
573	print 'user_field\t %s = %s' %(field,value)
574
575	if sys_dict.has_key(field):
576
577	# Check if value is an allowed system option, if TypeError then
578	# every value is allowed
579	#
580	try:
581	if value in sys_dict[field]:
582	ticket[field] = value
583	else:
584	# Must we set a default if value is not allowed
585	#
586	if use_default:
587	value = self.get_config('ticket', 'default_%s' %(field) )
588	ticket[field] = value
589
590	except TypeError:
591	ticket[field] = value
592
593	if self.DEBUG >= 10:
594	print 'ticket_field\t %s = %s' %(field, ticket[field])
595
596	def ticket_update(self, m, id, spam):
597	"""
598	If the current email is a reply to an existing ticket, this function
599	will append the contents of this email to that ticket, instead of
600	creating a new one.
601	"""
602	if self.DEBUG:
603	print "TD: ticket_update: %s" %id
604
605	# Must we update ticket fields
606	#
607	update_fields = dict()
608	try:
609	id, keywords = string.split(id, '?')
610
611	# Skip the last ':' character
612	#
613	keywords = keywords[:-1]
614	update_fields = self.str_to_dict(keywords)
615
616	# Strip '#'
617	#
618	self.id = int(id[1:])
619
620	except ValueError:
621	# Strip '#' and ':'
622	#
623	self.id = int(id[1:-1])
624
625
626	# When is the change committed
627	#
628	if self.VERSION == 0.11:
629	utc = UTC()
630	when = datetime.now(utc)
631	else:
632	when = int(time.time())
633
634	try:
635	tkt = Ticket(self.env, self.id, self.db)
636	except util.TracError, detail:
637	# Not a valid ticket
638	self.id = None
639	return False
640
641	# How many changes has this ticket
642	cnum = len(tkt.get_changelog())
643
644
645	# reopen the ticket if it is was closed
646	# We must use the ticket workflow framework
647	#
648	if tkt['status'] in ['closed']:
649
650	#print controller.actions['reopen']
651	#
652	# As reference
653	# req = Mock(href=Href('/'), abs_href=Href('http://www.example.com/'), authname='anonymous', perm=MockPerm(), args={})
654	#
655	#a = controller.render_ticket_action_control(req, tkt, 'reopen')
656	#print 'controller : ', a
657	#
658	#b = controller.get_all_status()
659	#print 'get all status: ', b
660	#
661	#b = controller.get_ticket_changes(req, tkt, 'reopen')
662	#print 'get_ticket_changes :', b
663
664	if self.WORKFLOW and (self.VERSION in [0.11]) :
665	from trac.ticket.default_workflow import ConfigurableTicketWorkflow
666	from trac.test import Mock, MockPerm
667
668	req = Mock(authname='anonymous', perm=MockPerm(), args={})
669
670	controller = ConfigurableTicketWorkflow(self.env)
671	fields = controller.get_ticket_changes(req, tkt, self.WORKFLOW)
672
673	if self.DEBUG:
674	print 'TD: Workflow ticket update fields: ', fields
675
676	for key in fields.keys():
677	tkt[key] = fields[key]
678
679	else:
680	tkt['status'] = 'reopened'
681	tkt['resolution'] = ''
682
683	# Must we update some ticket fields properties
684	#
685	if update_fields:
686	self.update_ticket_fields(tkt, update_fields)
687
688	message_parts = self.get_message_parts(m)
689	message_parts = self.unique_attachment_names(message_parts)
690
691	if self.EMAIL_HEADER:
692	message_parts.insert(0, self.email_header_txt(m))
693
694	body_text = self.body_text(message_parts)
695
696	if body_text.strip() or update_fields:
697	if self.DRY_RUN:
698	print 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) ', self.author, cnum
699	else:
700	tkt.save_changes(self.author, body_text, when, None, str(cnum))
701
702
703	if self.VERSION == 0.9:
704	s = self.attachments(message_parts, True)
705	else:
706	s = self.attachments(message_parts)
707
708	if self.notification and not spam:
709	self.notify(tkt, False, when)
710
711	return True
712
713	def set_ticket_fields(self, ticket):
714	"""
715	set the ticket fields to value specified
716	- /etc/email2trac.conf with <prefix>_<field>
717	- trac default values, trac.ini
718	"""
719	user_dict = dict()
720
721	for field in ticket.fields:
722
723	name = field['name']
724
725	# skip some fields like resolution
726	#
727	if name in [ 'resolution' ]:
728	continue
729
730	# default trac value
731	#
732	if not field.get('custom'):
733	value = self.get_config('ticket', 'default_%s' %(name) )
734	else:
735	value = field.get('value')
736	options = field.get('options')
737	if value and options and value not in options:
738	value = options[int(value)]
739
740	if self.DEBUG > 10:
741	print 'trac.ini name %s = %s' %(name, value)
742
743	prefix = self.parameters['ticket_prefix']
744	try:
745	value = self.parameters['%s_%s' %(prefix, name)]
746	if self.DEBUG > 10:
747	print 'email2trac.conf %s = %s ' %(name, value)
748
749	except KeyError, detail:
750	pass
751
752	if self.DEBUG:
753	print 'user_dict[%s] = %s' %(name, value)
754
755	user_dict[name] = value
756
757	self.update_ticket_fields(ticket, user_dict, use_default=1)
758
759	# Set status ticket
760	#`
761	ticket['status'] = 'new'
762
763
764
765	def new_ticket(self, msg, subject, spam, set_fields = None):
766	"""
767	Create a new ticket
768	"""
769	if self.DEBUG:
770	print "TD: new_ticket"
771
772	tkt = Ticket(self.env)
773	self.set_ticket_fields(tkt)
774
775	# Old style setting for component, will be removed
776	#
777	if spam:
778	tkt['component'] = 'Spam'
779
780	elif self.parameters.has_key('component'):
781	tkt['component'] = self.parameters['component']
782
783	if not msg['Subject']:
784	tkt['summary'] = u'(No subject)'
785	else:
786	tkt['summary'] = subject
787
788	self.set_reply_fields(tkt, msg)
789
790	if set_fields:
791	rest, keywords = string.split(set_fields, '?')
792
793	if keywords:
794	update_fields = self.str_to_dict(keywords)
795	self.update_ticket_fields(tkt, update_fields)
796
797	# produce e-mail like header
798	#
799	head = ''
800	if self.EMAIL_HEADER > 0:
801	head = self.email_header_txt(msg)
802
803	message_parts = self.get_message_parts(msg)
804	message_parts = self.unique_attachment_names(message_parts)
805
806	if self.EMAIL_HEADER > 0:
807	message_parts.insert(0, self.email_header_txt(msg))
808
809	body_text = self.body_text(message_parts)
810
811	tkt['description'] = body_text
812
813	#when = int(time.time())
814	#
815	utc = UTC()
816	when = datetime.now(utc)
817
818	if not self.DRY_RUN:
819	self.id = tkt.insert()
820
821	changed = False
822	comment = ''
823
824	# some routines in trac are dependend on ticket id
825	# like alternate notify template
826	#
827	if self.notify_template:
828	tkt['id'] = self.id
829	changed = True
830
831	# Rewrite the description if we have mailto enabled
832	#
833	if self.MAILTO:
834	changed = True
835	comment = u'\nadded mailto line\n'
836	mailto = self.html_mailto_link( m['Subject'], body_text)
837
838	tkt['description'] = u'%s\r\n%s%s\r\n' \
839	%(head, mailto, body_text)
840
841	# Disabled
842	#has_attachments = self.attachments(message_parts)
843	#if has_attachments:
844	# changed = True
845	# comment = '%s\n%s\n' %(comment, has_attachments)
846
847	if changed:
848	if self.DRY_RUN:
849	print 'DRY_RUN: tkt.save_changes(self.author, comment) ', self.author
850	else:
851	tkt.save_changes(self.author, comment)
852	#print tkt.get_changelog(self.db, when)
853
854	if self.notification and not spam:
855	self.notify(tkt, True)
856
857
858	def blog(self, id):
859	"""
860	The blog create/update function
861	"""
862	# import the modules
863	#
864	from tracfullblog.core import FullBlogCore
865	from tracfullblog.model import BlogPost, BlogCommen
866
867	# instantiate blog core
868	blog = FullBlogCore(self.env)
869	req = ''
870
871	if id:
872
873	# update blog
874	#
875	comment = BlogComment(self.env, id)
876	comment.author = self.author
877	comment.comment = self.get_body_text(m)
878	blog.create_comment(req, comment)
879
880	else:
881	# create blog
882	#
883	import time
884	post = BlogPost(self.env, 'blog_'+time.strftime("%Y%m%d%H%M%S", time.gmtime()))
885
886	#post = BlogPost(self.env, blog._get_default_postname(self.env))
887
888	post.author = self.author
889	post.title = self.email_to_unicode(m['Subject'])
890	post.body = self.get_body_text(m)
891
892	blog.create_post(req, post, self.author, u'Created by email2trac', False)
893
894
895	def parse(self, fp):
896	global m
897
898	m = email.message_from_file(fp)
899
900	if not m:
901	if self.DEBUG:
902	print "TD: This is not a valid email message format"
903	return
904
905	# Work around lack of header folding in Python; see http://bugs.python.org/issue4696
906	m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
907
908	if self.DEBUG > 1: # save the entire e-mail message text
909	message_parts = self.get_message_parts(m)
910	message_parts = self.unique_attachment_names(message_parts)
911	self.save_email_for_debug(m, True)
912	body_text = self.body_text(message_parts)
913	self.debug_body(body_text, True)
914	self.debug_attachments(message_parts)
915
916	self.db = self.env.get_db_cnx()
917	self.get_sender_info(m)
918
919	if not self.email_header_acl('white_list', self.email_addr, True):
920	if self.DEBUG > 1 :
921	print 'Message rejected : %s not in white list' %(self.email_addr)
922	return False
923
924	if self.email_header_acl('black_list', self.email_addr, False):
925	if self.DEBUG > 1 :
926	print 'Message rejected : %s in black list' %(self.email_addr)
927	return False
928
929	if not self.email_header_acl('recipient_list', self.to_email_addr, True):
930	if self.DEBUG > 1 :
931	print 'Message rejected : %s not in recipient list' %(self.to_email_addr)
932	return False
933
934	# If drop the message
935	#
936	if self.spam(m) == 'drop':
937	return False
938
939	elif self.spam(m) == 'spam':
940	spam_msg = True
941	else:
942	spam_msg = False
943
944	if self.get_config('notification', 'smtp_enabled') in ['true']:
945	self.notification = 1
946	else:
947	self.notification = 0
948
949	# Check if FullBlogPlugin is installed
950	#
951	blog_enabled = None
952	if self.get_config('components', 'tracfullblog.*') in ['enabled']:
953	blog_enabled = True
954
955	if not m['Subject']:
956	return False
957	else:
958	subject = self.email_to_unicode(m['Subject'])
959
960	#
961	# [hic] #1529: Re: LRZ
962	# [hic] #1529?owner=bas,priority=medium: Re: LRZ
963	#
964	TICKET_RE = re.compile(r"""
965	(?P<blog>blog:(?P<blog_id>\w*))
966	\|(?P<new_fields>[#][?].*)
967	\|(?P<reply>[#][\d]+:)
968	\|(?P<reply_fields>[#][\d]+\?.*?:)
969	""", re.VERBOSE)
970
971	# Find out if this is a ticket or a blog
972	#
973	result = TICKET_RE.search(subject)
974
975	if result:
976	if result.group('blog') and blog_enabled:
977	self.blog(result.group('blog_id'))
978
979	# update ticket + fields
980	#
981	if result.group('reply_fields') and self.TICKET_UPDATE:
982	self.ticket_update(m, result.group('reply_fields'), spam_msg)
983
984	# Update ticket
985	#
986	elif result.group('reply') and self.TICKET_UPDATE:
987	self.ticket_update(m, result.group('reply'), spam_msg)
988
989	# New ticket + fields
990	#
991	elif result.group('new_fields'):
992	self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
993
994	# Create ticket
995	#
996	else:
997	self.new_ticket(m, subject, spam_msg)
998
999	def strip_signature(self, text):
1000	"""
1001	Strip signature from message, inspired by Mailman software
1002	"""
1003	body = []
1004	for line in text.splitlines():
1005	if line == '-- ':
1006	break
1007	body.append(line)
1008
1009	return ('\n'.join(body))
1010
1011	def reflow(self, text, delsp = 0):
1012	"""
1013	Reflow the message based on the format="flowed" specification (RFC 3676)
1014	"""
1015	flowedlines = []
1016	quotelevel = 0
1017	prevflowed = 0
1018
1019	for line in text.splitlines():
1020	from re import match
1021
1022	# Figure out the quote level and the content of the current line
1023	m = match('(>)( ?)(.)', line)
1024	linequotelevel = len(m.group(1))
1025	line = m.group(3)
1026
1027	# Determine whether this line is flowed
1028	if line and line != '-- ' and line[-1] == ' ':
1029	flowed = 1
1030	else:
1031	flowed = 0
1032
1033	if flowed and delsp and line and line[-1] == ' ':
1034	line = line[:-1]
1035
1036	# If the previous line is flowed, append this line to it
1037	if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1038	flowedlines[-1] += line
1039	# Otherwise, start a new line
1040	else:
1041	flowedlines.append('>' * linequotelevel + line)
1042
1043	prevflowed = flowed
1044
1045
1046	return '\n'.join(flowedlines)
1047
1048	def strip_quotes(self, text):
1049	"""
1050	Strip quotes from message by Nicolas Mendoza
1051	"""
1052	body = []
1053	for line in text.splitlines():
1054	if line.startswith(self.EMAIL_QUOTE):
1055	continue
1056	body.append(line)
1057
1058	return ('\n'.join(body))
1059
1060	def wrap_text(self, text, replace_whitespace = False):
1061	"""
1062	Will break a lines longer then given length into several small
1063	lines of size given length
1064	"""
1065	import textwrap
1066
1067	LINESEPARATOR = '\n'
1068	reformat = ''
1069
1070	for s in text.split(LINESEPARATOR):
1071	tmp = textwrap.fill(s,self.USE_TEXTWRAP)
1072	if tmp:
1073	reformat = '%s\n%s' %(reformat,tmp)
1074	else:
1075	reformat = '%s\n' %reformat
1076
1077	return reformat
1078
1079	# Python2.4 and higher
1080	#
1081	#return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
1082	#
1083
1084
1085	def get_message_parts(self, msg):
1086	"""
1087	parses the email message and returns a list of body parts and attachments
1088	body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
1089	"""
1090	message_parts = []
1091
1092	ALTERNATIVE_MULTIPART = False
1093
1094	for part in msg.walk():
1095	if self.DEBUG:
1096	print 'TD: Message part: Main-Type: %s' % part.get_content_maintype()
1097	print 'TD: Message part: Content-Type: %s' % part.get_content_type()
1098
1099	## Check content type
1100	#
1101	if part.get_content_type() in self.STRIP_CONTENT_TYPES:
1102
1103	if self.DEBUG:
1104	print "TD: A %s attachment named '%s' was skipped" %(part.get_content_type(), part.get_filename())
1105
1106	continue
1107
1108	## Catch some mulitpart execptions
1109	#
1110	if part.get_content_type() == 'multipart/alternative':
1111	ALTERNATIVE_MULTIPART = True
1112	continue
1113
1114	## Skip multipart containers
1115	#
1116	if part.get_content_maintype() == 'multipart':
1117	if self.DEBUG:
1118	print "TD: Skipping multipart container"
1119	continue
1120
1121	## Check if this is an inline part. It's inline if there is co Cont-Disp header, or if there is one and it says "inline"
1122	#
1123	inline = self.inline_part(part)
1124
1125	## Drop HTML message
1126	#
1127	if ALTERNATIVE_MULTIPART and self.DROP_ALTERNATIVE_HTML_VERSION:
1128	if part.get_content_type() == 'text/html':
1129	if self.DEBUG:
1130	print "TD: Skipping alternative HTML message"
1131
1132	ALTERNATIVE_MULTIPART = False
1133	continue
1134
1135	## Inline text parts are where the body is
1136	#
1137	if part.get_content_type() == 'text/plain' and inline:
1138	if self.DEBUG:
1139	print 'TD: Inline body part'
1140
1141	# Try to decode, if fails then do not decode
1142	#
1143	body_text = part.get_payload(decode=1)
1144	if not body_text:
1145	body_text = part.get_payload(decode=0)
1146
1147	format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
1148	delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
1149
1150	if self.REFLOW and not self.VERBATIM_FORMAT and format == 'flowed':
1151	body_text = self.reflow(body_text, delsp == 'yes')
1152
1153	if self.STRIP_SIGNATURE:
1154	body_text = self.strip_signature(body_text)
1155
1156	if self.STRIP_QUOTES:
1157	body_text = self.strip_quotes(body_text)
1158
1159	if self.USE_TEXTWRAP:
1160	body_text = self.wrap_text(body_text)
1161
1162	## Get contents charset (iso-8859-15 if not defined in mail headers)
1163	#
1164	charset = part.get_content_charset()
1165	if not charset:
1166	charset = 'iso-8859-15'
1167
1168	try:
1169	ubody_text = unicode(body_text, charset)
1170
1171	except UnicodeError, detail:
1172	ubody_text = unicode(body_text, 'iso-8859-15')
1173
1174	except LookupError, detail:
1175	ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
1176
1177	if self.VERBATIM_FORMAT:
1178	message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
1179	else:
1180	message_parts.append('%s' %ubody_text)
1181	else:
1182	if self.DEBUG:
1183	print 'TD: Filename: %s' % part.get_filename()
1184
1185	message_parts.append((part.get_filename(), part))
1186
1187	return message_parts
1188
1189	def unique_attachment_names(self, message_parts):
1190	renamed_parts = []
1191	attachment_names = set()
1192	for part in message_parts:
1193
1194	# If not an attachment, leave it alone
1195	if not isinstance(part, tuple):
1196	renamed_parts.append(part)
1197	continue
1198
1199	(filename, part) = part
1200	# Decode the filename
1201	if filename:
1202	filename = self.email_to_unicode(filename)
1203	# If no name, use a default one
1204	else:
1205	filename = 'untitled-part'
1206
1207	# Guess the extension from the content type, use non strict mode
1208	# some additional non-standard but commonly used MIME types
1209	# are also recognized
1210	#
1211	ext = mimetypes.guess_extension(part.get_content_type(), False)
1212	if not ext:
1213	ext = '.bin'
1214
1215	filename = '%s%s' % (filename, ext)
1216
1217	# Discard relative paths in attachment names
1218	filename = filename.replace('\\', '/').replace(':', '/')
1219	filename = os.path.basename(filename)
1220
1221	# We try to normalize the filename to utf-8 NFC if we can.
1222	# Files uploaded from OS X might be in NFD.
1223	# Check python version and then try it
1224	#
1225	if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
1226	try:
1227	filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
1228	except TypeError:
1229	pass
1230
1231	if self.QUOTE_ATTACHMENT_FILENAMES:
1232	try:
1233	filename = urllib.quote(filename)
1234	except KeyError, detail:
1235	filename = urllib.quote(filename.encode('utf-8'))
1236
1237	# Make the filename unique for this ticket
1238	num = 0
1239	unique_filename = filename
1240	filename, ext = os.path.splitext(filename)
1241
1242	while unique_filename in attachment_names or self.attachment_exists(unique_filename):
1243	num += 1
1244	unique_filename = "%s-%s%s" % (filename, num, ext)
1245
1246	if self.DEBUG:
1247	print 'TD: Attachment with filename %s will be saved as %s' % (filename, unique_filename)
1248
1249	attachment_names.add(unique_filename)
1250
1251	renamed_parts.append((filename, unique_filename, part))
1252
1253	return renamed_parts
1254
1255	def inline_part(self, part):
1256	return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
1257
1258
1259	def attachment_exists(self, filename):
1260
1261	if self.DEBUG:
1262	print "TD: attachment_exists: Ticket number : %s, Filename : %s" %(self.id, filename)
1263
1264	# We have no valid ticket id
1265	#
1266	if not self.id:
1267	return False
1268
1269	try:
1270	att = attachment.Attachment(self.env, 'ticket', self.id, filename)
1271	return True
1272	except attachment.ResourceNotFound:
1273	return False
1274
1275	def body_text(self, message_parts):
1276	body_text = []
1277
1278	for part in message_parts:
1279	# Plain text part, append it
1280	if not isinstance(part, tuple):
1281	body_text.extend(part.strip().splitlines())
1282	body_text.append("")
1283	continue
1284
1285	(original, filename, part) = part
1286	inline = self.inline_part(part)
1287
1288	if part.get_content_maintype() == 'image' and inline:
1289	body_text.append('[[Image(%s)]]' % filename)
1290	body_text.append("")
1291	else:
1292	body_text.append('[attachment:"%s"]' % filename)
1293	body_text.append("")
1294
1295	body_text = '\r\n'.join(body_text)
1296	return body_text
1297
1298	def notify(self, tkt, new=True, modtime=0):
1299	"""
1300	A wrapper for the TRAC notify function. So we can use templates
1301	"""
1302	if self.DRY_RUN:
1303	print 'DRY_RUN: self.notify(tkt, True) ', self.author
1304	return
1305	try:
1306	# create false {abs_}href properties, to trick Notify()
1307	#
1308	if not self.VERSION == 0.11:
1309	self.env.abs_href = Href(self.get_config('project', 'url'))
1310	self.env.href = Href(self.get_config('project', 'url'))
1311
1312	tn = TicketNotifyEmail(self.env)
1313
1314	if self.notify_template:
1315
1316	if self.VERSION == 0.11:
1317
1318	from trac.web.chrome import Chrome
1319
1320	if self.notify_template_update and not new:
1321	tn.template_name = self.notify_template_update
1322	else:
1323	tn.template_name = self.notify_template
1324
1325	tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
1326
1327	else:
1328
1329	tn.template_name = self.notify_template;
1330
1331	tn.notify(tkt, new, modtime)
1332
1333	except Exception, e:
1334	print 'TD: Failure sending notification on creation of ticket #%s: %s' %(self.id, e)
1335
1336	def html_mailto_link(self, subject, body):
1337	"""
1338	This function returns a HTML mailto tag with the ticket id and author email address
1339	"""
1340	if not self.author:
1341	author = self.email_addr
1342	else:
1343	author = self.author
1344
1345	# use urllib to escape the chars
1346	#
1347	s = 'mailto:%s?Subject=%s&Cc=%s' %(
1348	urllib.quote(self.email_addr),
1349	urllib.quote('Re: #%s: %s' %(self.id, subject)),
1350	urllib.quote(self.MAILTO_CC)
1351	)
1352
1353	s = '\r\n{{{\r\n#!html\r\n<a\r\n href="%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
1354	return s
1355
1356	def attachments(self, message_parts, update=False):
1357	'''
1358	save any attachments as files in the ticket's directory
1359	'''
1360	if self.DRY_RUN:
1361	print "DRY_RUN: no attachments saved"
1362	return ''
1363
1364	count = 0
1365
1366	# Get Maxium attachment size
1367	#
1368	max_size = int(self.get_config('attachment', 'max_size'))
1369	status = ''
1370
1371	for part in message_parts:
1372	# Skip body parts
1373	if not isinstance(part, tuple):
1374	continue
1375
1376	(original, filename, part) = part
1377	#
1378	# Must be tuneables HvB
1379	#
1380	path, fd = util.create_unique_file(os.path.join(self.TMPDIR, filename))
1381	text = part.get_payload(decode=1)
1382	if not text:
1383	text = '(None)'
1384	fd.write(text)
1385	fd.close()
1386
1387	# get the file_size
1388	#
1389	stats = os.lstat(path)
1390	file_size = stats[stat.ST_SIZE]
1391
1392	# Check if the attachment size is allowed
1393	#
1394	if (max_size != -1) and (file_size > max_size):
1395	status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1396	%(status, original, file_size, max_size)
1397
1398	os.unlink(path)
1399	continue
1400	else:
1401	count = count + 1
1402
1403	# Insert the attachment
1404	#
1405	fd = open(path, 'rb')
1406	att = attachment.Attachment(self.env, 'ticket', self.id)
1407
1408	# This will break the ticket_update system, the body_text is vaporized
1409	# ;-(
1410	#
1411	if not update:
1412	att.author = self.author
1413	att.description = self.email_to_unicode('Added by email2trac')
1414
1415	att.insert(filename, fd, file_size)
1416	#except util.TracError, detail:
1417	# print detail
1418
1419	# Remove the created temporary filename
1420	#
1421	fd.close()
1422	os.unlink(path)
1423
1424	# Return how many attachments
1425	#
1426	status = 'This message has %d attachment(s)\n%s' %(count, status)
1427	return status
1428
1429
1430	def mkdir_p(dir, mode):
1431	'''do a mkdir -p'''
1432
1433	arr = string.split(dir, '/')
1434	path = ''
1435	for part in arr:
1436	path = '%s/%s' % (path, part)
1437	try:
1438	stats = os.stat(path)
1439	except OSError:
1440	os.mkdir(path, mode)
1441
1442	def ReadConfig(file, name):
1443	"""
1444	Parse the config file
1445	"""
1446	if not os.path.isfile(file):
1447	print 'File %s does not exist' %file
1448	sys.exit(1)
1449
1450	config = trac_config.Configuration(file)
1451
1452	# Use given project name else use defaults
1453	#
1454	if name:
1455	sections = config.sections()
1456	if not name in sections:
1457	print "Not a valid project name: %s" %name
1458	print "Valid names: %s" %sections
1459	sys.exit(1)
1460
1461	project = dict()
1462	for option, value in config.options(name):
1463	project[option] = value
1464
1465	else:
1466	# use some trac internals to get the defaults
1467	#
1468	project = config.parser.defaults()
1469
1470	return project
1471
1472
1473	if __name__ == '__main__':
1474	# Default config file
1475	#
1476	configfile = '@email2trac_conf@'
1477	project = ''
1478	component = ''
1479	ticket_prefix = 'default'
1480	dry_run = None
1481
1482	ENABLE_SYSLOG = 0
1483
1484
1485	SHORT_OPT = 'chf:np:t:'
1486	LONG_OPT = ['component=', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=']
1487
1488	try:
1489	opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
1490	except getopt.error,detail:
1491	print __doc__
1492	print detail
1493	sys.exit(1)
1494
1495	project_name = None
1496	for opt,value in opts:
1497	if opt in [ '-h', '--help']:
1498	print __doc__
1499	sys.exit(0)
1500	elif opt in ['-c', '--component']:
1501	component = value
1502	elif opt in ['-f', '--file']:
1503	configfile = value
1504	elif opt in ['-n', '--dry-run']:
1505	dry_run = True
1506	elif opt in ['-p', '--project']:
1507	project_name = value
1508	elif opt in ['-t', '--ticket_prefix']:
1509	ticket_prefix = value
1510
1511	settings = ReadConfig(configfile, project_name)
1512	if not settings.has_key('project'):
1513	print __doc__
1514	print 'No Trac project is defined in the email2trac config file.'
1515	sys.exit(1)
1516
1517	if component:
1518	settings['component'] = component
1519
1520	# The default prefix for ticket values in email2trac.conf
1521	#
1522	settings['ticket_prefix'] = ticket_prefix
1523	settings['dry_run'] = dry_run
1524
1525	if settings.has_key('trac_version'):
1526	version = settings['trac_version']
1527	else:
1528	version = trac_default_version
1529
1530
1531	#debug HvB
1532	#print settings
1533
1534	try:
1535	if version == '0.9':
1536	from trac import attachment
1537	from trac.env import Environment
1538	from trac.ticket import Ticket
1539	from trac.web.href import Href
1540	from trac import util
1541	from trac.Notify import TicketNotifyEmail
1542	elif version == '0.10':
1543	from trac import attachment
1544	from trac.env import Environment
1545	from trac.ticket import Ticket
1546	from trac.web.href import Href
1547	from trac import util
1548	#
1549	# return util.text.to_unicode(str)
1550	#
1551	# see http://projects.edgewall.com/trac/changeset/2799
1552	from trac.ticket.notification import TicketNotifyEmail
1553	from trac import config as trac_config
1554	elif version == '0.11':
1555	from trac import attachment
1556	from trac.env import Environment
1557	from trac.ticket import Ticket
1558	from trac.web.href import Href
1559	from trac import config as trac_config
1560	from trac import util
1561
1562
1563	#
1564	# return util.text.to_unicode(str)
1565	#
1566	# see http://projects.edgewall.com/trac/changeset/2799
1567	from trac.ticket.notification import TicketNotifyEmail
1568	else:
1569	print 'TRAC version %s is not supported' %version
1570	sys.exit(1)
1571
1572	if settings.has_key('enable_syslog'):
1573	if SYSLOG_AVAILABLE:
1574	ENABLE_SYSLOG = float(settings['enable_syslog'])
1575
1576
1577	# Must be set before environment is created
1578	#
1579	if settings.has_key('python_egg_cache'):
1580	python_egg_cache = str(settings['python_egg_cache'])
1581	os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
1582
1583	env = Environment(settings['project'], create=0)
1584	tktparser = TicketEmailParser(env, settings, float(version))
1585	tktparser.parse(sys.stdin)
1586
1587	# Catch all errors ans log to SYSLOG if we have enabled this
1588	# else stdout
1589	#
1590	except Exception, error:
1591	if ENABLE_SYSLOG:
1592	syslog.openlog('email2trac', syslog.LOG_NOWAIT)
1593
1594	etype, evalue, etb = sys.exc_info()
1595	for e in traceback.format_exception(etype, evalue, etb):
1596	syslog.syslog(e)
1597
1598	syslog.closelog()
1599	else:
1600	traceback.print_exc()
1601
1602	if m:
1603	tktparser.save_email_for_debug(m, True)
1604
1605	sys.exit(1)
1606	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format