Context Navigation

source: trunk/email2trac.py.in @ 264

Last change on this file since 264 was 264, checked in by bas, 15 years ago

email2trac.py.in,:

fixed an unicode error

debian/changelog:

is now a proper changelog

Property svn:executable set to *
Property svn:keywords set to Id

File size: 40.7 KB

Line
1	#!@PYTHON@
2	# Copyright (C) 2002
3	#
4	# This file is part of the email2trac utils
5	#
6	# This program is free software; you can redistribute it and/or modify it
7	# under the terms of the GNU General Public License as published by the
8	# Free Software Foundation; either version 2, or (at your option) any
9	# later version.
10	#
11	# This program is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with this program; if not, write to the Free Software
18	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
19	#
20	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
21	#
22	"""
23	email2trac.py -- Email tickets to Trac.
24
25	A simple MTA filter to create Trac tickets from inbound emails.
26
27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28	Copyright 2005, Edgewall Software
29
30	Changed By: Bas van der Vlies <basv@sara.nl>
31	Date : 13 September 2005
32	Descr. : Added config file and command line options, spam level
33	detection, reply address and mailto option. Unicode support
34
35	Changed By: Walter de Jong <walter@sara.nl>
36	Descr. : multipart-message code and trac attachments
37
38
39	The scripts reads emails from stdin and inserts directly into a Trac database.
40	MIME headers are mapped as follows:
41
42	* From: => Reporter
43	=> CC (Optional via reply_all option)
44	* Subject: => Summary
45	* Body => Description
46	* Component => Can be set to SPAM via spam_level option
47
48	How to use
49	----------
50	* Create an config file:
51	[DEFAULT] # REQUIRED
52	project : /data/trac/test # REQUIRED
53	debug : 1 # OPTIONAL, if set print some DEBUG info
54	spam_level : 4 # OPTIONAL, if set check for SPAM mail
55	reply_all : 1 # OPTIONAL, if set then fill in ticket CC field
56	umask : 022 # OPTIONAL, if set then use this umask for creation of the attachments
57	mailto_link : 1 # OPTIONAL, if set then [mailto:<>] in description
58	mailto_cc : basv@sara.nl # OPTIONAL, use this address as CC in mailto line
59	ticket_update: 1 # OPTIONAL, if set then check if this is an update for a ticket
60	trac_version : 0.9 # OPTIONAL, default is 0.10
61
62	[jouvin] # OPTIONAL project declaration, if set both fields necessary
63	project : /data/trac/jouvin # use -p\|--project jouvin.
64
65	* default config file is : /etc/email2trac.conf
66
67	* Commandline opions:
68	-h,--help
69	-f,--file <configuration file>
70	-n,--dry-run
71	-p, --project <project name>
72	-t, --ticket_prefix <name>
73
74	SVN Info:
75	$Id: email2trac.py.in 264 2009-04-09 07:55:23Z bas $
76	"""
77	import os
78	import sys
79	import string
80	import getopt
81	import stat
82	import time
83	import email
84	import email.Iterators
85	import email.Header
86	import re
87	import urllib
88	import unicodedata
89	from stat import *
90	import mimetypes
91	import traceback
92
93
94	# Will fail where unavailable, e.g. Windows
95	#
96	try:
97	import syslog
98	SYSLOG_AVAILABLE = True
99	except ImportError:
100	SYSLOG_AVAILABLE = False
101
102	from datetime import tzinfo, timedelta, datetime
103	from trac import config as trac_config
104
105	# Some global variables
106	#
107	trac_default_version = '0.10'
108	m = None
109
110	# A UTC class needed for trac version 0.11, added by
111	# tbaschak at ktc dot mb dot ca
112	#
113	class UTC(tzinfo):
114	"""UTC"""
115	ZERO = timedelta(0)
116	HOUR = timedelta(hours=1)
117
118	def utcoffset(self, dt):
119	return self.ZERO
120
121	def tzname(self, dt):
122	return "UTC"
123
124	def dst(self, dt):
125	return self.ZERO
126
127
128	class TicketEmailParser(object):
129	env = None
130	comment = '> '
131
132	def __init__(self, env, parameters, version):
133	self.env = env
134
135	# Database connection
136	#
137	self.db = None
138
139	# Save parameters
140	#
141	self.parameters = parameters
142
143	# Some useful mail constants
144	#
145	self.author = None
146	self.email_addr = None
147	self.email_from = None
148	self.id = None
149
150	self.VERSION = version
151	self.DRY_RUN = parameters['dry_run']
152
153	self.get_config = self.env.config.get
154
155	if parameters.has_key('umask'):
156	os.umask(int(parameters['umask'], 8))
157
158	if parameters.has_key('quote_attachment_filenames'):
159	self.QUOTE_ATTACHMENT_FILENAMES = int(parameters['quote_attachment_filenames'])
160	else:
161	self.QUOTE_ATTACHMENT_FILENAMES = 1
162
163	if parameters.has_key('debug'):
164	self.DEBUG = int(parameters['debug'])
165	else:
166	self.DEBUG = 0
167
168	if parameters.has_key('mailto_link'):
169	self.MAILTO = int(parameters['mailto_link'])
170	if parameters.has_key('mailto_cc'):
171	self.MAILTO_CC = parameters['mailto_cc']
172	else:
173	self.MAILTO_CC = ''
174	else:
175	self.MAILTO = 0
176
177	if parameters.has_key('spam_level'):
178	self.SPAM_LEVEL = int(parameters['spam_level'])
179	else:
180	self.SPAM_LEVEL = 0
181
182	if parameters.has_key('spam_header'):
183	self.SPAM_HEADER = parameters['spam_header']
184	else:
185	self.SPAM_HEADER = 'X-Spam-Score'
186
187	if parameters.has_key('email_quote'):
188	self.EMAIL_QUOTE = str(parameters['email_quote'])
189	else:
190	self.EMAIL_QUOTE = '> '
191
192	if parameters.has_key('email_header'):
193	self.EMAIL_HEADER = int(parameters['email_header'])
194	else:
195	self.EMAIL_HEADER = 0
196
197	if parameters.has_key('alternate_notify_template'):
198	self.notify_template = str(parameters['alternate_notify_template'])
199	else:
200	self.notify_template = None
201
202	if parameters.has_key('alternate_notify_template_update'):
203	self.notify_template_update = str(parameters['alternate_notify_template_update'])
204	else:
205	self.notify_template_update = None
206
207	if parameters.has_key('reply_all'):
208	self.REPLY_ALL = int(parameters['reply_all'])
209	else:
210	self.REPLY_ALL = 0
211
212	if parameters.has_key('ticket_update'):
213	self.TICKET_UPDATE = int(parameters['ticket_update'])
214	else:
215	self.TICKET_UPDATE = 0
216
217	if parameters.has_key('drop_spam'):
218	self.DROP_SPAM = int(parameters['drop_spam'])
219	else:
220	self.DROP_SPAM = 0
221
222	if parameters.has_key('verbatim_format'):
223	self.VERBATIM_FORMAT = int(parameters['verbatim_format'])
224	else:
225	self.VERBATIM_FORMAT = 1
226
227	if parameters.has_key('reflow'):
228	self.REFLOW = int(parameters['reflow'])
229	else:
230	self.REFLOW = 1
231
232	if parameters.has_key('strip_signature'):
233	self.STRIP_SIGNATURE = int(parameters['strip_signature'])
234	else:
235	self.STRIP_SIGNATURE = 0
236
237	if parameters.has_key('strip_quotes'):
238	self.STRIP_QUOTES = int(parameters['strip_quotes'])
239	else:
240	self.STRIP_QUOTES = 0
241
242	if parameters.has_key('use_textwrap'):
243	self.USE_TEXTWRAP = int(parameters['use_textwrap'])
244	else:
245	self.USE_TEXTWRAP = 0
246
247	if parameters.has_key('binhex'):
248	self.BINHEX = parameters['binhex']
249	else:
250	self.BINHEX = 'warn'
251
252	if parameters.has_key('applesingle'):
253	self.APPLESINGLE = parameters['applesingle']
254	else:
255	self.APPLESINGLE = 'warn'
256
257	if parameters.has_key('appledouble'):
258	self.APPLEDOUBLE = parameters['appledouble']
259	else:
260	self.APPLEDOUBLE = 'warn'
261
262	if parameters.has_key('python_egg_cache'):
263	self.python_egg_cache = str(parameters['python_egg_cache'])
264	os.environ['PYTHON_EGG_CACHE'] = self.python_egg_cache
265
266	self.WORKFLOW = None
267	if parameters.has_key('workflow'):
268	self.WORKFLOW = parameters['workflow']
269
270	# Use OS independend functions
271	#
272	self.TMPDIR = os.path.normcase('/tmp')
273	if parameters.has_key('tmpdir'):
274	self.TMPDIR = os.path.normcase(str(parameters['tmpdir']))
275
276	if parameters.has_key('ignore_trac_user_settings'):
277	self.IGNORE_TRAC_USER_SETTINGS = int(parameters['ignore_trac_user_settings'])
278	else:
279	self.IGNORE_TRAC_USER_SETTINGS = 0
280
281	def spam(self, message):
282	"""
283	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
284	# Note if Spam_level then '*' are included
285	"""
286	spam = False
287	if message.has_key(self.SPAM_HEADER):
288	spam_l = string.split(message[self.SPAM_HEADER])
289
290	try:
291	number = spam_l[0].count('*')
292	except IndexError, detail:
293	number = 0
294
295	if number >= self.SPAM_LEVEL:
296	spam = True
297
298	# treat virus mails as spam
299	#
300	elif message.has_key('X-Virus-found'):
301	spam = True
302
303	# How to handle SPAM messages
304	#
305	if self.DROP_SPAM and spam:
306	if self.DEBUG > 2 :
307	print 'This message is a SPAM. Automatic ticket insertion refused (SPAM level > %d' % self.SPAM_LEVEL
308
309	return 'drop'
310
311	elif spam:
312
313	return 'Spam'
314
315	else:
316
317	return False
318
319	def email_header_acl(self, keyword, header_field, default):
320	"""
321	This function wil check if the email address is allowed or denied
322	to send mail to the ticket list
323	"""
324	try:
325	mail_addresses = self.parameters[keyword]
326
327	# Check if we have an empty string
328	#
329	if not mail_addresses:
330	return default
331
332	except KeyError, detail:
333	if self.DEBUG > 2 :
334	print 'TD: %s not defined, all messages are allowed.' %(keyword)
335
336	return default
337
338	mail_addresses = string.split(mail_addresses, ',')
339
340	for entry in mail_addresses:
341	entry = entry.strip()
342	TO_RE = re.compile(entry, re.VERBOSE\|re.IGNORECASE)
343	result = TO_RE.search(header_field)
344	if result:
345	return True
346
347	return False
348
349	def email_to_unicode(self, message_str):
350	"""
351	Email has 7 bit ASCII code, convert it to unicode with the charset
352	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
353	understands it.
354	"""
355	results = email.Header.decode_header(message_str)
356	str = None
357	for text,format in results:
358	if format:
359	try:
360	temp = unicode(text, format)
361	except UnicodeError, detail:
362	# This always works
363	#
364	temp = unicode(text, 'iso-8859-15')
365	except LookupError, detail:
366	#text = 'ERROR: Could not find charset: %s, please install' %format
367	#temp = unicode(text, 'iso-8859-15')
368	temp = message_str
369
370	else:
371	temp = string.strip(text)
372	temp = unicode(text, 'iso-8859-15')
373
374	if str:
375	str = '%s %s' %(str, temp)
376	else:
377	str = '%s' %temp
378
379	#str = str.encode('utf-8')
380	return str
381
382	def debug_body(self, message_body, tempfile=False):
383	if tempfile:
384	import tempfile
385	body_file = tempfile.mktemp('.email2trac')
386	else:
387	body_file = os.path.join(self.TMPDIR, 'body.txt')
388
389	print 'TD: writing body (%s)' % body_file
390	fx = open(body_file, 'wb')
391	if not message_body:
392	message_body = '(None)'
393	fx.write(message_body)
394	fx.close()
395	try:
396	os.chmod(body_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
397	except OSError:
398	pass
399
400	def debug_attachments(self, message_parts):
401	n = 0
402	for part in message_parts:
403	# Skip inline text parts
404	if not isinstance(part, tuple):
405	continue
406
407	(original, filename, part) = part
408
409	n = n + 1
410	print 'TD: part%d: Content-Type: %s' % (n, part.get_content_type())
411	print 'TD: part%d: filename: %s' % (n, part.get_filename())
412
413	part_file = os.path.join(self.TMPDIR, filename)
414	#part_file = '/var/tmp/part%d' % n
415	print 'TD: writing part%d (%s)' % (n,part_file)
416	fx = open(part_file, 'wb')
417	text = part.get_payload(decode=1)
418	if not text:
419	text = '(None)'
420	fx.write(text)
421	fx.close()
422	try:
423	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
424	except OSError:
425	pass
426
427	def email_header_txt(self, m):
428	"""
429	Display To and CC addresses in description field
430	"""
431	str = ''
432	#if m['To'] and len(m['To']) > 0 and m['To'] != 'hic@sara.nl':
433	if m['To'] and len(m['To']) > 0:
434	str = "'''To:''' %s\r\n" %(m['To'])
435	if m['Cc'] and len(m['Cc']) > 0:
436	str = "%s'''Cc:''' %s\r\n" % (str, m['Cc'])
437
438	return self.email_to_unicode(str)
439
440
441	def get_sender_info(self, message):
442	"""
443	Get the default author name and email address from the message
444	"""
445
446	self.email_to = self.email_to_unicode(message['to'])
447	self.to_name, self.to_email_addr = email.Utils.parseaddr (self.email_to)
448
449	self.email_from = self.email_to_unicode(message['from'])
450	self.author, self.email_addr = email.Utils.parseaddr(self.email_from)
451
452	# Trac can not handle author's name that contains spaces
453	#
454	self.author = self.email_addr
455
456	if self.IGNORE_TRAC_USER_SETTINGS:
457	return
458
459	# Is this a registered user, use email address as search key:
460	# result:
461	# u : login name
462	# n : Name that the user has set in the settings tab
463	# e : email address that the user has set in the settings tab
464	#
465	users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
466	if e and (e.lower() == self.email_addr.lower()) ]
467
468	if len(users) == 1:
469	self.email_from = users[0][0]
470	self.author = users[0][0]
471
472	def set_reply_fields(self, ticket, message):
473	"""
474	Set all the right fields for a new ticket
475	"""
476	ticket['reporter'] = self.email_from
477
478	# Put all CC-addresses in ticket CC field
479	#
480	if self.REPLY_ALL:
481	#tos = message.get_all('to', [])
482	ccs = message.get_all('cc', [])
483
484	addrs = email.Utils.getaddresses(ccs)
485	if not addrs:
486	return
487
488	# Remove reporter email address if notification is
489	# on
490	#
491	if self.notification:
492	try:
493	addrs.remove((self.author, self.email_addr))
494	except ValueError, detail:
495	pass
496
497	for name,mail in addrs:
498	try:
499	mail_list = '%s, %s' %(mail_list, mail)
500	except UnboundLocalError, detail:
501	mail_list = mail
502
503	if mail_list:
504	ticket['cc'] = self.email_to_unicode(mail_list)
505
506	def save_email_for_debug(self, message, tempfile=False):
507	if tempfile:
508	import tempfile
509	msg_file = tempfile.mktemp('.email2trac')
510	else:
511	#msg_file = '/var/tmp/msg.txt'
512	msg_file = os.path.join(self.TMPDIR, 'msg.txt')
513
514	print 'TD: saving email to %s' % msg_file
515	fx = open(msg_file, 'wb')
516	fx.write('%s' % message)
517	fx.close()
518	try:
519	os.chmod(msg_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
520	except OSError:
521	pass
522
523	def str_to_dict(self, str):
524	"""
525	Transfrom a str of the form [<key>=<value>]+ to dict[<key>] = <value>
526	"""
527
528	fields = string.split(str, ',')
529
530	result = dict()
531	for field in fields:
532	try:
533	index, value = string.split(field, '=')
534
535	# We can not change the description of a ticket via the subject
536	# line. The description is the body of the email
537	#
538	if index.lower() in ['description']:
539	continue
540
541	if value:
542	result[index.lower()] = value
543
544	except ValueError:
545	pass
546
547	return result
548
549	def update_ticket_fields(self, ticket, user_dict, use_default=None):
550	"""
551	This will update the ticket fields. It will check if the
552	given fields are known and if the right values are specified
553	It will only update the ticket field value:
554	- If the field is known
555	- If the value supplied is valid for the ticket field.
556	If not then there are two options:
557	1) Skip the value (use_default=None)
558	2) Set default value for field (use_default=1)
559	"""
560
561	# Build a system dictionary from the ticket fields
562	# with field as index and option as value
563	#
564	sys_dict = dict()
565	for field in ticket.fields:
566	try:
567	sys_dict[field['name']] = field['options']
568
569	except KeyError:
570	sys_dict[field['name']] = None
571	pass
572
573	# Check user supplied fields an compare them with the
574	# system one's
575	#
576	for field,value in user_dict.items():
577	if self.DEBUG >= 10:
578	print 'user_field\t %s = %s' %(field,value)
579
580	if sys_dict.has_key(field):
581
582	# Check if value is an allowed system option, if TypeError then
583	# every value is allowed
584	#
585	try:
586	if value in sys_dict[field]:
587	ticket[field] = value
588	else:
589	# Must we set a default if value is not allowed
590	#
591	if use_default:
592	value = self.get_config('ticket', 'default_%s' %(field) )
593	ticket[field] = value
594
595	except TypeError:
596	ticket[field] = value
597
598	if self.DEBUG >= 10:
599	print 'ticket_field\t %s = %s' %(field, ticket[field])
600
601	def ticket_update(self, m, id, spam):
602	"""
603	If the current email is a reply to an existing ticket, this function
604	will append the contents of this email to that ticket, instead of
605	creating a new one.
606	"""
607	if self.DEBUG:
608	print "TD: ticket_update: %s" %id
609
610	if not m['Subject']:
611	return False
612	else:
613	subject = self.email_to_unicode(m['Subject'])
614
615	# Must we update ticket fields
616	#
617	update_fields = dict()
618	try:
619	id, keywords = string.split(id, '?')
620
621	# Skip the last ':' character
622	#
623	keywords = keywords[:-1]
624	update_fields = self.str_to_dict(keywords)
625
626	# Strip '#'
627	#
628	self.id = int(id[1:])
629
630	except ValueError:
631	# Strip '#' and ':'
632	#
633	self.id = int(id[1:-1])
634
635
636	# When is the change committed
637	#
638	#
639	if self.VERSION == 0.11:
640	utc = UTC()
641	when = datetime.now(utc)
642	else:
643	when = int(time.time())
644
645	try:
646	tkt = Ticket(self.env, self.id, self.db)
647	except util.TracError, detail:
648	# Not a valid ticket
649	self.id = None
650	return False
651
652	# reopen the ticket if it is was closed
653	# We must use the ticket workflow framework
654	#
655	if tkt['status'] in ['closed']:
656
657	#print controller.actions['reopen']
658	#
659	# As reference
660	# req = Mock(href=Href('/'), abs_href=Href('http://www.example.com/'), authname='anonymous', perm=MockPerm(), args={})
661	#
662	#a = controller.render_ticket_action_control(req, tkt, 'reopen')
663	#print 'controller : ', a
664	#
665	#b = controller.get_all_status()
666	#print 'get all status: ', b
667	#
668	#b = controller.get_ticket_changes(req, tkt, 'reopen')
669	#print 'get_ticket_changes :', b
670
671	if self.WORKFLOW and (self.VERSION in ['0.11']) :
672	from trac.ticket.default_workflow import ConfigurableTicketWorkflow
673	from trac.test import Mock, MockPerm
674
675	req = Mock(authname='anonymous', perm=MockPerm(), args={})
676
677	controller = ConfigurableTicketWorkflow(self.env)
678	fields = controller.get_ticket_changes(req, tkt, self.WORKFLOW)
679
680	if self.DEBUG:
681	print 'TD: Workflow ticket update fields: ', fields
682
683	for key in fields.keys():
684	tkt[key] = fields[key]
685
686	else:
687	tkt['status'] = 'reopened'
688	tkt['resolution'] = ''
689
690	# Must we update some ticket fields properties
691	#
692	if update_fields:
693	self.update_ticket_fields(tkt, update_fields)
694
695	message_parts = self.get_message_parts(m)
696	message_parts = self.unique_attachment_names(message_parts)
697
698	if self.EMAIL_HEADER:
699	message_parts.insert(0, self.email_header_txt(m))
700
701	body_text = self.body_text(message_parts)
702
703	if body_text.strip() or update_fields:
704	if self.DRY_RUN:
705	print 'DRY_RUN: tkt.save_changes(self.author, comment) ', self.author
706	else:
707	tkt.save_changes(self.author, body_text, when)
708
709	if self.VERSION == 0.9:
710	str = self.attachments(message_parts, True)
711	else:
712	str = self.attachments(message_parts)
713
714	if self.notification and not spam:
715	self.notify(tkt, False, when)
716
717	return True
718
719	def set_ticket_fields(self, ticket):
720	"""
721	set the ticket fields to value specified
722	- /etc/email2trac.conf with <prefix>_<field>
723	- trac default values, trac.ini
724	"""
725	user_dict = dict()
726
727	for field in ticket.fields:
728
729	name = field['name']
730
731	# skip some fields like resolution
732	#
733	if name in [ 'resolution' ]:
734	continue
735
736	# default trac value
737	#
738	if not field.get('custom'):
739	value = self.get_config('ticket', 'default_%s' %(name) )
740	else:
741	value = field.get('value')
742	options = field.get('options')
743	if value and options and value not in options:
744	value = options[int(value)]
745
746	if self.DEBUG > 10:
747	print 'trac.ini name %s = %s' %(name, value)
748
749	prefix = self.parameters['ticket_prefix']
750	try:
751	value = self.parameters['%s_%s' %(prefix, name)]
752	if self.DEBUG > 10:
753	print 'email2trac.conf %s = %s ' %(name, value)
754
755	except KeyError, detail:
756	pass
757
758	if self.DEBUG:
759	print 'user_dict[%s] = %s' %(name, value)
760
761	user_dict[name] = value
762
763	self.update_ticket_fields(ticket, user_dict, use_default=1)
764
765	# Set status ticket
766	#`
767	ticket['status'] = 'new'
768
769
770
771	def new_ticket(self, msg, subject, spam, set_fields = None):
772	"""
773	Create a new ticket
774	"""
775	if self.DEBUG:
776	print "TD: new_ticket"
777
778	tkt = Ticket(self.env)
779	self.set_ticket_fields(tkt)
780
781	# Old style setting for component, will be removed
782	#
783	if spam:
784	tkt['component'] = 'Spam'
785
786	elif self.parameters.has_key('component'):
787	tkt['component'] = self.parameters['component']
788
789	if not msg['Subject']:
790	tkt['summary'] = u'(No subject)'
791	else:
792	tkt['summary'] = subject
793
794	self.set_reply_fields(tkt, msg)
795
796	if set_fields:
797	rest, keywords = string.split(set_fields, '?')
798
799	if keywords:
800	update_fields = self.str_to_dict(keywords)
801	self.update_ticket_fields(tkt, update_fields)
802
803	# produce e-mail like header
804	#
805	head = ''
806	if self.EMAIL_HEADER > 0:
807	head = self.email_header_txt(msg)
808
809	message_parts = self.get_message_parts(msg)
810	message_parts = self.unique_attachment_names(message_parts)
811
812	if self.EMAIL_HEADER > 0:
813	message_parts.insert(0, self.email_header_txt(msg))
814
815	body_text = self.body_text(message_parts)
816
817	tkt['description'] = body_text
818
819	#when = int(time.time())
820	#
821	utc = UTC()
822	when = datetime.now(utc)
823
824	if not self.DRY_RUN:
825	self.id = tkt.insert()
826
827	changed = False
828	comment = ''
829
830	# Rewrite the description if we have mailto enabled
831	#
832	if self.MAILTO:
833	changed = True
834	comment = u'\nadded mailto line\n'
835	mailto = self.html_mailto_link( m['Subject'], body_text)
836
837	tkt['description'] = u'%s\r\n%s%s\r\n' \
838	%(head, mailto, body_text)
839
840	str = self.attachments(message_parts)
841	if str:
842	changed = True
843	comment = '%s\n%s\n' %(comment, str)
844
845	if changed:
846	if self.DRY_RUN:
847	print 'DRY_RUN: tkt.save_changes(self.author, comment) ', self.author
848	else:
849	tkt.save_changes(self.author, comment)
850	#print tkt.get_changelog(self.db, when)
851
852	if self.notification and not spam:
853	self.notify(tkt, True)
854
855
856	def blog(self, id):
857	"""
858	The blog create/update function
859	"""
860	# import the modules
861	#
862	from tracfullblog.core import FullBlogCore
863	from tracfullblog.model import BlogPost, BlogCommen
864
865	# instantiate blog core
866	blog = FullBlogCore(self.env)
867	req = ''
868
869	if id:
870
871	# update blog
872	#
873	comment = BlogComment(self.env, result.group('blog_id'))
874	comment.author = self.author
875	comment.comment = self.get_body_text(m)
876	blog.create_comment(req, comment)
877
878	else:
879	# create blog
880	#
881	import time
882	post = BlogPost(self.env, 'blog_'+time.strftime("%Y%m%d%H%M%S", time.gmtime()))
883
884	#post = BlogPost(self.env, blog._get_default_postname(self.env))
885
886	post.author = self.author
887	post.title = self.email_to_unicode(m['Subject'])
888	post.body = self.get_body_text(m)
889
890	blog.create_post(req, post, self.author, u'Created by email2trac', False)
891
892
893	def parse(self, fp):
894	global m
895
896	m = email.message_from_file(fp)
897
898	if not m:
899	if self.DEBUG:
900	print "TD: This is not a valid email message format"
901	return
902
903	# Work around lack of header folding in Python; see http://bugs.python.org/issue4696
904	m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
905
906	if self.DEBUG > 1: # save the entire e-mail message text
907	message_parts = self.get_message_parts(m)
908	message_parts = self.unique_attachment_names(message_parts)
909	self.save_email_for_debug(m, True)
910	body_text = self.body_text(message_parts)
911	self.debug_body(body_text, True)
912	self.debug_attachments(message_parts)
913
914	self.db = self.env.get_db_cnx()
915	self.get_sender_info(m)
916
917	if not self.email_header_acl('white_list', self.email_addr, True):
918	if self.DEBUG > 1 :
919	print 'Message rejected : %s not in white list' %(self.email_addr)
920	return False
921
922	if self.email_header_acl('black_list', self.email_addr, False):
923	if self.DEBUG > 1 :
924	print 'Message rejected : %s in black list' %(self.email_addr)
925	return False
926
927	if not self.email_header_acl('recipient_list', self.to_email_addr, True):
928	if self.DEBUG > 1 :
929	print 'Message rejected : %s not in recipient list' %(self.to_email_addr)
930	return False
931
932	# If drop the message
933	#
934	if self.spam(m) == 'drop':
935	return False
936
937	elif self.spam(m) == 'spam':
938	spam_msg = True
939
940	else:
941	spam_msg = False
942
943	if self.get_config('notification', 'smtp_enabled') in ['true']:
944	self.notification = 1
945	else:
946	self.notification = 0
947
948	# Check if FullBlogPlugin is installed
949	#
950	blog_enabled = None
951	if self.get_config('components', 'tracfullblog.*') in ['enabled']:
952	blog_enabled = True
953
954	# Find out if this is a ticket or a blog
955	if not m['Subject']:
956	return False
957	else:
958	subject = self.email_to_unicode(m['Subject'])
959
960	#
961	# [hic] #1529: Re: LRZ
962	# [hic] #1529?owner=bas,priority=medium: Re: LRZ
963	#
964	TICKET_RE = re.compile(r"""
965	(?P<blog>blog:(?P<blog_id>\w*))
966	\|(?P<new_fields>[#][?].*)
967	\|(?P<reply>[#][\d]+:)
968	\|(?P<reply_fields>[#][\d]+\?.*?:)
969	""", re.VERBOSE)
970
971	result = TICKET_RE.search(subject)
972
973	if result:
974	if result.group('blog') and blog_enabled:
975	self.blog(result.group('blog_id'))
976
977	# update ticket + fields
978	#
979	if result.group('reply_fields') and self.TICKET_UPDATE:
980	self.ticket_update(m, result.group('reply_fields'), spam_msg)
981
982	# Update ticket
983	#
984	elif result.group('reply') and self.TICKET_UPDATE:
985	self.ticket_update(m, result.group('reply'), spam_msg)
986
987	# New ticket + fields
988	#
989	elif result.group('new_fields'):
990	self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
991
992	# Create ticket
993	#
994	else:
995	self.new_ticket(m, subject, spam_msg)
996
997	def strip_signature(self, text):
998	"""
999	Strip signature from message, inspired by Mailman software
1000	"""
1001	body = []
1002	for line in text.splitlines():
1003	if line == '-- ':
1004	break
1005	body.append(line)
1006
1007	return ('\n'.join(body))
1008
1009	def reflow(self, text, delsp = 0):
1010	"""
1011	Reflow the message based on the format="flowed" specification (RFC 3676)
1012	"""
1013	flowedlines = []
1014	quotelevel = 0
1015	prevflowed = 0
1016
1017	for line in text.splitlines():
1018	from re import match
1019
1020	# Figure out the quote level and the content of the current line
1021	m = match('(>)( ?)(.)', line)
1022	linequotelevel = len(m.group(1))
1023	line = m.group(3)
1024
1025	# Determine whether this line is flowed
1026	if line and line != '-- ' and line[-1] == ' ':
1027	flowed = 1
1028	else:
1029	flowed = 0
1030
1031	if flowed and delsp and line and line[-1] == ' ':
1032	line = line[:-1]
1033
1034	# If the previous line is flowed, append this line to it
1035	if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1036	flowedlines[-1] += line
1037	# Otherwise, start a new line
1038	else:
1039	flowedlines.append('>' * linequotelevel + line)
1040
1041	prevflowed = flowed
1042
1043
1044	return '\n'.join(flowedlines)
1045
1046	def strip_quotes(self, text):
1047	"""
1048	Strip quotes from message by Nicolas Mendoza
1049	"""
1050	body = []
1051	for line in text.splitlines():
1052	if line.startswith(self.EMAIL_QUOTE):
1053	continue
1054	body.append(line)
1055
1056	return ('\n'.join(body))
1057
1058	def wrap_text(self, text, replace_whitespace = False):
1059	"""
1060	Will break a lines longer then given length into several small
1061	lines of size given length
1062	"""
1063	import textwrap
1064
1065	LINESEPARATOR = '\n'
1066	reformat = ''
1067
1068	for s in text.split(LINESEPARATOR):
1069	tmp = textwrap.fill(s,self.USE_TEXTWRAP)
1070	if tmp:
1071	reformat = '%s\n%s' %(reformat,tmp)
1072	else:
1073	reformat = '%s\n' %reformat
1074
1075	return reformat
1076
1077	# Python2.4 and higher
1078	#
1079	#return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
1080	#
1081
1082
1083	def get_message_parts(self, msg):
1084	"""
1085	parses the email message and returns a list of body parts and attachments
1086	body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
1087	"""
1088	message_parts = []
1089
1090	# This is used to figure out when we are inside an AppleDouble container
1091	# AppleDouble containers consists of two parts: Mac-specific file data, and platform-independent data
1092	# We strip away Mac-specific stuff
1093	appledouble_parts = []
1094
1095	for part in msg.walk():
1096	if self.DEBUG:
1097	print 'TD: Message part: Content-Type: %s' % part.get_content_type()
1098
1099	# Check whether we just finished processing an AppleDouble container
1100	if part not in appledouble_parts:
1101	appledouble_parts = []
1102
1103	# Special handling for BinHex attachments. Options are drop (leave out with no warning), warn (and leave out), and keep
1104	if part.get_content_type() == 'application/mac-binhex40':
1105	if self.BINHEX == 'warn':
1106	message_parts.append("'''A BinHex attachment named '%s' was ignored (use MIME encoding instead).'''" % part.get_filename())
1107	continue
1108	elif self.BINHEX == 'drop':
1109	continue
1110
1111	# Special handling for AppleSingle attachments. Options are drop (leave out with no warning), warn (and leave out), and keep
1112	if part.get_content_type() == 'application/applefile' and not part in appledouble_parts:
1113	if self.APPLESINGLE == 'warn':
1114	message_parts.append("'''An AppleSingle attachment named '%s' was ignored (use MIME encoding instead).'''" % part.get_filename())
1115	continue
1116	elif self.APPLESINGLE == 'drop':
1117	continue
1118
1119	# Special handling for the Mac-specific part of AppleDouble attachments. Options are strip (leave out with no warning), warn (and leave out), and keep
1120	if part.get_content_type() == 'application/applefile':
1121	if self.APPLEDOUBLE == 'warn':
1122	message_parts.append("'''The resource fork of an attachment named '%s' was removed.'''" % part.get_filename())
1123	continue
1124	elif self.APPLEDOUBLE == 'strip':
1125	continue
1126
1127	# If we entering an AppleDouble container, set up appledouble_parts so that we know what to do with its subparts
1128	if part.get_content_type() == 'multipart/appledouble':
1129	appledouble_parts = part.get_payload()
1130	continue
1131
1132	# Any other multipart/* is just a container for multipart messages
1133	if part.get_content_maintype() == 'multipart':
1134	continue
1135
1136	# Check if this is an inline part. It's inline if there is co Cont-Disp header, or if there is one and it says "inline"
1137	inline = self.inline_part(part)
1138
1139	# Inline text parts are where the body is
1140	if part.get_content_type() == 'text/plain' and inline:
1141	if self.DEBUG:
1142	print 'TD: Inline body part'
1143
1144	# Try to decode, if fails then do not decode
1145	#
1146	body_text = part.get_payload(decode=1)
1147	if not body_text:
1148	body_text = part.get_payload(decode=0)
1149
1150	format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
1151	delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
1152
1153	if self.REFLOW and not self.VERBATIM_FORMAT and format == 'flowed':
1154	body_text = self.reflow(body_text, delsp == 'yes')
1155
1156	if self.STRIP_SIGNATURE:
1157	body_text = self.strip_signature(body_text)
1158
1159	if self.STRIP_QUOTES:
1160	body_text = self.strip_quotes(body_text)
1161
1162	if self.USE_TEXTWRAP:
1163	body_text = self.wrap_text(body_text)
1164
1165	# Get contents charset (iso-8859-15 if not defined in mail headers)
1166	#
1167	charset = part.get_content_charset()
1168	if not charset:
1169	charset = 'iso-8859-15'
1170
1171	try:
1172	ubody_text = unicode(body_text, charset)
1173
1174	except UnicodeError, detail:
1175	ubody_text = unicode(body_text, 'iso-8859-15')
1176
1177	except LookupError, detail:
1178	ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
1179
1180	if self.VERBATIM_FORMAT:
1181	message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
1182	else:
1183	message_parts.append('%s' %ubody_text)
1184	else:
1185	if self.DEBUG:
1186	print 'TD: Filename: %s' % part.get_filename()
1187
1188	message_parts.append((part.get_filename(), part))
1189
1190	return message_parts
1191
1192	def unique_attachment_names(self, message_parts):
1193	renamed_parts = []
1194	attachment_names = set()
1195	for part in message_parts:
1196
1197	# If not an attachment, leave it alone
1198	if not isinstance(part, tuple):
1199	renamed_parts.append(part)
1200	continue
1201
1202	(filename, part) = part
1203	# Decode the filename
1204	if filename:
1205	filename = self.email_to_unicode(filename)
1206	# If no name, use a default one
1207	else:
1208	filename = 'untitled-part'
1209
1210	# Guess the extension from the content type, use non strict mode
1211	# some additional non-standard but commonly used MIME types
1212	# are also recognized
1213	#
1214	ext = mimetypes.guess_extension(part.get_content_type(), False)
1215	if not ext:
1216	ext = '.bin'
1217
1218	filename = '%s%s' % (filename, ext)
1219
1220	# Discard relative paths in attachment names
1221	filename = filename.replace('\\', '/').replace(':', '/')
1222	filename = os.path.basename(filename)
1223
1224	# We try to normalize the filename to utf-8 NFC if we can.
1225	# Files uploaded from OS X might be in NFD.
1226	# Check python version and then try it
1227	#
1228	if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
1229	try:
1230	filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
1231	except TypeError:
1232	pass
1233
1234	if self.QUOTE_ATTACHMENT_FILENAMES:
1235	filename = urllib.quote(filename)
1236
1237	# Make the filename unique for this ticket
1238	num = 0
1239	unique_filename = filename
1240	filename, ext = os.path.splitext(filename)
1241
1242	while unique_filename in attachment_names or self.attachment_exists(unique_filename):
1243	num += 1
1244	unique_filename = "%s-%s%s" % (filename, num, ext)
1245
1246	if self.DEBUG:
1247	print 'TD: Attachment with filename %s will be saved as %s' % (filename, unique_filename)
1248
1249	attachment_names.add(unique_filename)
1250
1251	renamed_parts.append((filename, unique_filename, part))
1252
1253	return renamed_parts
1254
1255	def inline_part(self, part):
1256	return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
1257
1258
1259	def attachment_exists(self, filename):
1260
1261	if self.DEBUG:
1262	print "TD: attachment_exists: Ticket number : %s, Filename : %s" %(self.id, filename)
1263
1264	# We have no valid ticket id
1265	#
1266	if not self.id:
1267	return False
1268
1269	try:
1270	att = attachment.Attachment(self.env, 'ticket', self.id, filename)
1271	return True
1272	except attachment.ResourceNotFound:
1273	return False
1274
1275	def body_text(self, message_parts):
1276	body_text = []
1277
1278	for part in message_parts:
1279	# Plain text part, append it
1280	if not isinstance(part, tuple):
1281	body_text.extend(part.strip().splitlines())
1282	body_text.append("")
1283	continue
1284
1285	(original, filename, part) = part
1286	inline = self.inline_part(part)
1287
1288	if part.get_content_maintype() == 'image' and inline:
1289	body_text.append('[[Image(%s)]]' % filename)
1290	body_text.append("")
1291	else:
1292	body_text.append('[attachment:"%s"]' % filename)
1293	body_text.append("")
1294
1295	body_text = '\r\n'.join(body_text)
1296	return body_text
1297
1298	def notify(self, tkt, new=True, modtime=0):
1299	"""
1300	A wrapper for the TRAC notify function. So we can use templates
1301	"""
1302	if self.DRY_RUN:
1303	print 'DRY_RUN: self.notify(tkt, True) ', self.author
1304	return
1305	try:
1306	# create false {abs_}href properties, to trick Notify()
1307	#
1308	if not self.VERSION == 0.11:
1309	self.env.abs_href = Href(self.get_config('project', 'url'))
1310	self.env.href = Href(self.get_config('project', 'url'))
1311
1312	tn = TicketNotifyEmail(self.env)
1313
1314	if self.notify_template:
1315
1316	if self.VERSION == 0.11:
1317
1318	from trac.web.chrome import Chrome
1319
1320	if self.notify_template_update and not new:
1321	tn.template_name = self.notify_template_update
1322	else:
1323	tn.template_name = self.notify_template
1324
1325	tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
1326
1327	else:
1328
1329	tn.template_name = self.notify_template;
1330
1331	tn.notify(tkt, new, modtime)
1332
1333	except Exception, e:
1334	print 'TD: Failure sending notification on creation of ticket #%s: %s' %(self.id, e)
1335
1336	def html_mailto_link(self, subject, body):
1337	"""
1338	This function returns a HTML mailto tag with the ticket id and author email address
1339	"""
1340	if not self.author:
1341	author = self.email_addr
1342	else:
1343	author = self.author
1344
1345	# use urllib to escape the chars
1346	#
1347	str = 'mailto:%s?Subject=%s&Cc=%s' %(
1348	urllib.quote(self.email_addr),
1349	urllib.quote('Re: #%s: %s' %(self.id, subject)),
1350	urllib.quote(self.MAILTO_CC)
1351	)
1352
1353	str = '\r\n{{{\r\n#!html\r\n<a\r\n href="%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(str, author)
1354	return str
1355
1356	def attachments(self, message_parts, update=False):
1357	'''
1358	save any attachments as files in the ticket's directory
1359	'''
1360	if self.DRY_RUN:
1361	print "DRY_RUN: no attachments saved"
1362	return ''
1363
1364	count = 0
1365
1366	# Get Maxium attachment size
1367	#
1368	max_size = int(self.get_config('attachment', 'max_size'))
1369	status = ''
1370
1371	for part in message_parts:
1372	# Skip body parts
1373	if not isinstance(part, tuple):
1374	continue
1375
1376	(original, filename, part) = part
1377	#
1378	# Must be tuneables HvB
1379	#
1380	path, fd = util.create_unique_file(os.path.join(self.TMPDIR, filename))
1381	text = part.get_payload(decode=1)
1382	if not text:
1383	text = '(None)'
1384	fd.write(text)
1385	fd.close()
1386
1387	# get the file_size
1388	#
1389	stats = os.lstat(path)
1390	file_size = stats[stat.ST_SIZE]
1391
1392	# Check if the attachment size is allowed
1393	#
1394	if (max_size != -1) and (file_size > max_size):
1395	status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1396	%(status, original, file_size, max_size)
1397
1398	os.unlink(path)
1399	continue
1400	else:
1401	count = count + 1
1402
1403	# Insert the attachment
1404	#
1405	fd = open(path, 'rb')
1406	att = attachment.Attachment(self.env, 'ticket', self.id)
1407
1408	# This will break the ticket_update system, the body_text is vaporized
1409	# ;-(
1410	#
1411	if not update:
1412	att.author = self.author
1413	att.description = self.email_to_unicode('Added by email2trac')
1414
1415	att.insert(filename, fd, file_size)
1416	#except util.TracError, detail:
1417	# print detail
1418
1419	# Remove the created temporary filename
1420	#
1421	fd.close()
1422	os.unlink(path)
1423
1424	# Return how many attachments
1425	#
1426	status = 'This message has %d attachment(s)\n%s' %(count, status)
1427	return status
1428
1429
1430	def mkdir_p(dir, mode):
1431	'''do a mkdir -p'''
1432
1433	arr = string.split(dir, '/')
1434	path = ''
1435	for part in arr:
1436	path = '%s/%s' % (path, part)
1437	try:
1438	stats = os.stat(path)
1439	except OSError:
1440	os.mkdir(path, mode)
1441
1442	def ReadConfig(file, name):
1443	"""
1444	Parse the config file
1445	"""
1446	if not os.path.isfile(file):
1447	print 'File %s does not exist' %file
1448	sys.exit(1)
1449
1450	config = trac_config.Configuration(file)
1451
1452	# Use given project name else use defaults
1453	#
1454	if name:
1455	sections = config.sections()
1456	if not name in sections:
1457	print "Not a valid project name: %s" %name
1458	print "Valid names: %s" %sections
1459	sys.exit(1)
1460
1461	project = dict()
1462	for option, value in config.options(name):
1463	project[option] = value
1464
1465	else:
1466	# use some trac internales to get the defaults
1467	#
1468	project = config.parser.defaults()
1469
1470	return project
1471
1472
1473	if __name__ == '__main__':
1474	# Default config file
1475	#
1476	configfile = '@email2trac_conf@'
1477	project = ''
1478	component = ''
1479	ticket_prefix = 'default'
1480	dry_run = None
1481
1482	ENABLE_SYSLOG = 0
1483
1484
1485	SHORT_OPT = 'chf:np:t:'
1486	LONG_OPT = ['component=', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=']
1487
1488	try:
1489	opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
1490	except getopt.error,detail:
1491	print __doc__
1492	print detail
1493	sys.exit(1)
1494
1495	project_name = None
1496	for opt,value in opts:
1497	if opt in [ '-h', '--help']:
1498	print __doc__
1499	sys.exit(0)
1500	elif opt in ['-c', '--component']:
1501	component = value
1502	elif opt in ['-f', '--file']:
1503	configfile = value
1504	elif opt in ['-n', '--dry-run']:
1505	dry_run = True
1506	elif opt in ['-p', '--project']:
1507	project_name = value
1508	elif opt in ['-t', '--ticket_prefix']:
1509	ticket_prefix = value
1510
1511	settings = ReadConfig(configfile, project_name)
1512	if not settings.has_key('project'):
1513	print __doc__
1514	print 'No Trac project is defined in the email2trac config file.'
1515	sys.exit(1)
1516
1517	if component:
1518	settings['component'] = component
1519
1520	# The default prefix for ticket values in email2trac.conf
1521	#
1522	settings['ticket_prefix'] = ticket_prefix
1523	settings['dry_run'] = dry_run
1524
1525	if settings.has_key('trac_version'):
1526	version = settings['trac_version']
1527	else:
1528	version = trac_default_version
1529
1530
1531	#debug HvB
1532	#print settings
1533
1534	try:
1535	if version == '0.9':
1536	from trac import attachment
1537	from trac.env import Environment
1538	from trac.ticket import Ticket
1539	from trac.web.href import Href
1540	from trac import util
1541	from trac.Notify import TicketNotifyEmail
1542	elif version == '0.10':
1543	from trac import attachment
1544	from trac.env import Environment
1545	from trac.ticket import Ticket
1546	from trac.web.href import Href
1547	from trac import util
1548	#
1549	# return util.text.to_unicode(str)
1550	#
1551	# see http://projects.edgewall.com/trac/changeset/2799
1552	from trac.ticket.notification import TicketNotifyEmail
1553	from trac import config as trac_config
1554	elif version == '0.11':
1555	from trac import attachment
1556	from trac.env import Environment
1557	from trac.ticket import Ticket
1558	from trac.web.href import Href
1559	from trac import config as trac_config
1560	from trac import util
1561
1562
1563	#
1564	# return util.text.to_unicode(str)
1565	#
1566	# see http://projects.edgewall.com/trac/changeset/2799
1567	from trac.ticket.notification import TicketNotifyEmail
1568	else:
1569	print 'TRAC version %s is not supported' %version
1570	sys.exit(1)
1571
1572	if settings.has_key('enable_syslog'):
1573	if SYSLOG_AVAILABLE:
1574	ENABLE_SYSLOG = float(settings['enable_syslog'])
1575
1576	env = Environment(settings['project'], create=0)
1577	tktparser = TicketEmailParser(env, settings, float(version))
1578	tktparser.parse(sys.stdin)
1579
1580	# Catch all errors ans log to SYSLOG if we have enabled this
1581	# else stdout
1582	#
1583	except Exception, error:
1584	if ENABLE_SYSLOG:
1585	syslog.openlog('email2trac', syslog.LOG_NOWAIT)
1586
1587	etype, evalue, etb = sys.exc_info()
1588	for e in traceback.format_exception(etype, evalue, etb):
1589	syslog.syslog(e)
1590
1591	syslog.closelog()
1592	else:
1593	traceback.print_exc()
1594
1595	if m:
1596	tktparser.save_email_for_debug(m, True)
1597
1598	sys.exit(1)
1599	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format