Context Navigation

source: trunk/email2trac.py.in @ 260

Last change on this file since 260 was 260, checked in by bas, 15 years ago

email2trac.py.in:

added FullBlogPlugin? support

Property svn:executable set to *
Property svn:keywords set to Id

File size: 40.4 KB

Line
1	#!@PYTHON@
2	# Copyright (C) 2002
3	#
4	# This file is part of the email2trac utils
5	#
6	# This program is free software; you can redistribute it and/or modify it
7	# under the terms of the GNU General Public License as published by the
8	# Free Software Foundation; either version 2, or (at your option) any
9	# later version.
10	#
11	# This program is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with this program; if not, write to the Free Software
18	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
19	#
20	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
21	#
22	"""
23	email2trac.py -- Email tickets to Trac.
24
25	A simple MTA filter to create Trac tickets from inbound emails.
26
27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28	Copyright 2005, Edgewall Software
29
30	Changed By: Bas van der Vlies <basv@sara.nl>
31	Date : 13 September 2005
32	Descr. : Added config file and command line options, spam level
33	detection, reply address and mailto option. Unicode support
34
35	Changed By: Walter de Jong <walter@sara.nl>
36	Descr. : multipart-message code and trac attachments
37
38
39	The scripts reads emails from stdin and inserts directly into a Trac database.
40	MIME headers are mapped as follows:
41
42	* From: => Reporter
43	=> CC (Optional via reply_all option)
44	* Subject: => Summary
45	* Body => Description
46	* Component => Can be set to SPAM via spam_level option
47
48	How to use
49	----------
50	* Create an config file:
51	[DEFAULT] # REQUIRED
52	project : /data/trac/test # REQUIRED
53	debug : 1 # OPTIONAL, if set print some DEBUG info
54	spam_level : 4 # OPTIONAL, if set check for SPAM mail
55	reply_all : 1 # OPTIONAL, if set then fill in ticket CC field
56	umask : 022 # OPTIONAL, if set then use this umask for creation of the attachments
57	mailto_link : 1 # OPTIONAL, if set then [mailto:<>] in description
58	mailto_cc : basv@sara.nl # OPTIONAL, use this address as CC in mailto line
59	ticket_update: 1 # OPTIONAL, if set then check if this is an update for a ticket
60	trac_version : 0.9 # OPTIONAL, default is 0.10
61
62	[jouvin] # OPTIONAL project declaration, if set both fields necessary
63	project : /data/trac/jouvin # use -p\|--project jouvin.
64
65	* default config file is : /etc/email2trac.conf
66
67	* Commandline opions:
68	-h,--help
69	-f,--file <configuration file>
70	-n,--dry-run
71	-p, --project <project name>
72	-t, --ticket_prefix <name>
73
74	SVN Info:
75	$Id: email2trac.py.in 260 2009-03-25 13:37:24Z bas $
76	"""
77	import os
78	import sys
79	import string
80	import getopt
81	import stat
82	import time
83	import email
84	import email.Iterators
85	import email.Header
86	import re
87	import urllib
88	import unicodedata
89	from stat import *
90	import mimetypes
91	import traceback
92
93
94	# Will fail where unavailable, e.g. Windows
95	#
96	try:
97	import syslog
98	SYSLOG_AVAILABLE = True
99	except ImportError:
100	SYSLOG_AVAILABLE = False
101
102	from datetime import tzinfo, timedelta, datetime
103	from trac import config as trac_config
104
105	# Some global variables
106	#
107	trac_default_version = '0.10'
108	m = None
109
110	# A UTC class needed for trac version 0.11, added by
111	# tbaschak at ktc dot mb dot ca
112	#
113	class UTC(tzinfo):
114	"""UTC"""
115	ZERO = timedelta(0)
116	HOUR = timedelta(hours=1)
117
118	def utcoffset(self, dt):
119	return self.ZERO
120
121	def tzname(self, dt):
122	return "UTC"
123
124	def dst(self, dt):
125	return self.ZERO
126
127
128	class TicketEmailParser(object):
129	env = None
130	comment = '> '
131
132	def __init__(self, env, parameters, version):
133	self.env = env
134
135	# Database connection
136	#
137	self.db = None
138
139	# Save parameters
140	#
141	self.parameters = parameters
142
143	# Some useful mail constants
144	#
145	self.author = None
146	self.email_addr = None
147	self.email_from = None
148	self.id = None
149
150	self.VERSION = version
151	self.DRY_RUN = parameters['dry_run']
152
153	self.get_config = self.env.config.get
154
155	if parameters.has_key('umask'):
156	os.umask(int(parameters['umask'], 8))
157
158	if parameters.has_key('quote_attachment_filenames'):
159	self.QUOTE_ATTACHMENT_FILENAMES = int(parameters['quote_attachment_filenames'])
160	else:
161	self.QUOTE_ATTACHMENT_FILENAMES = 1
162
163	if parameters.has_key('debug'):
164	self.DEBUG = int(parameters['debug'])
165	else:
166	self.DEBUG = 0
167
168	if parameters.has_key('mailto_link'):
169	self.MAILTO = int(parameters['mailto_link'])
170	if parameters.has_key('mailto_cc'):
171	self.MAILTO_CC = parameters['mailto_cc']
172	else:
173	self.MAILTO_CC = ''
174	else:
175	self.MAILTO = 0
176
177	if parameters.has_key('spam_level'):
178	self.SPAM_LEVEL = int(parameters['spam_level'])
179	else:
180	self.SPAM_LEVEL = 0
181
182	if parameters.has_key('spam_header'):
183	self.SPAM_HEADER = parameters['spam_header']
184	else:
185	self.SPAM_HEADER = 'X-Spam-Score'
186
187	if parameters.has_key('email_quote'):
188	self.EMAIL_QUOTE = str(parameters['email_quote'])
189	else:
190	self.EMAIL_QUOTE = '> '
191
192	if parameters.has_key('email_header'):
193	self.EMAIL_HEADER = int(parameters['email_header'])
194	else:
195	self.EMAIL_HEADER = 0
196
197	if parameters.has_key('alternate_notify_template'):
198	self.notify_template = str(parameters['alternate_notify_template'])
199	else:
200	self.notify_template = None
201
202	if parameters.has_key('alternate_notify_template_update'):
203	self.notify_template_update = str(parameters['alternate_notify_template_update'])
204	else:
205	self.notify_template_update = None
206
207	if parameters.has_key('reply_all'):
208	self.REPLY_ALL = int(parameters['reply_all'])
209	else:
210	self.REPLY_ALL = 0
211
212	if parameters.has_key('ticket_update'):
213	self.TICKET_UPDATE = int(parameters['ticket_update'])
214	else:
215	self.TICKET_UPDATE = 0
216
217	if parameters.has_key('drop_spam'):
218	self.DROP_SPAM = int(parameters['drop_spam'])
219	else:
220	self.DROP_SPAM = 0
221
222	if parameters.has_key('verbatim_format'):
223	self.VERBATIM_FORMAT = int(parameters['verbatim_format'])
224	else:
225	self.VERBATIM_FORMAT = 1
226
227	if parameters.has_key('reflow'):
228	self.REFLOW = int(parameters['reflow'])
229	else:
230	self.REFLOW = 1
231
232	if parameters.has_key('strip_signature'):
233	self.STRIP_SIGNATURE = int(parameters['strip_signature'])
234	else:
235	self.STRIP_SIGNATURE = 0
236
237	if parameters.has_key('strip_quotes'):
238	self.STRIP_QUOTES = int(parameters['strip_quotes'])
239	else:
240	self.STRIP_QUOTES = 0
241
242	if parameters.has_key('use_textwrap'):
243	self.USE_TEXTWRAP = int(parameters['use_textwrap'])
244	else:
245	self.USE_TEXTWRAP = 0
246
247	if parameters.has_key('binhex'):
248	self.BINHEX = parameters['binhex']
249	else:
250	self.BINHEX = 'warn'
251
252	if parameters.has_key('applesingle'):
253	self.APPLESINGLE = parameters['applesingle']
254	else:
255	self.APPLESINGLE = 'warn'
256
257	if parameters.has_key('appledouble'):
258	self.APPLEDOUBLE = parameters['appledouble']
259	else:
260	self.APPLEDOUBLE = 'warn'
261
262	if parameters.has_key('python_egg_cache'):
263	self.python_egg_cache = str(parameters['python_egg_cache'])
264	os.environ['PYTHON_EGG_CACHE'] = self.python_egg_cache
265
266	self.WORKFLOW = None
267	if parameters.has_key('workflow'):
268	self.WORKFLOW = parameters['workflow']
269
270	# Use OS independend functions
271	#
272	self.TMPDIR = os.path.normcase('/tmp')
273	if parameters.has_key('tmpdir'):
274	self.TMPDIR = os.path.normcase(str(parameters['tmpdir']))
275
276	if parameters.has_key('ignore_trac_user_settings'):
277	self.IGNORE_TRAC_USER_SETTINGS = int(parameters['ignore_trac_user_settings'])
278	else:
279	self.IGNORE_TRAC_USER_SETTINGS = 0
280
281	def spam(self, message):
282	"""
283	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
284	# Note if Spam_level then '*' are included
285	"""
286	spam = False
287	if message.has_key(self.SPAM_HEADER):
288	spam_l = string.split(message[self.SPAM_HEADER])
289
290	try:
291	number = spam_l[0].count('*')
292	except IndexError, detail:
293	number = 0
294
295	if number >= self.SPAM_LEVEL:
296	spam = True
297
298	# treat virus mails as spam
299	#
300	elif message.has_key('X-Virus-found'):
301	spam = True
302
303	# How to handle SPAM messages
304	#
305	if self.DROP_SPAM and spam:
306	if self.DEBUG > 2 :
307	print 'This message is a SPAM. Automatic ticket insertion refused (SPAM level > %d' % self.SPAM_LEVEL
308
309	return 'drop'
310
311	elif spam:
312
313	return 'Spam'
314
315	else:
316
317	return False
318
319	def email_header_acl(self, keyword, header_field, default):
320	"""
321	This function wil check if the email address is allowed or denied
322	to send mail to the ticket list
323	"""
324	try:
325	mail_addresses = self.parameters[keyword]
326
327	# Check if we have an empty string
328	#
329	if not mail_addresses:
330	return default
331
332	except KeyError, detail:
333	if self.DEBUG > 2 :
334	print 'TD: %s not defined, all messages are allowed.' %(keyword)
335
336	return default
337
338	mail_addresses = string.split(mail_addresses, ',')
339
340	for entry in mail_addresses:
341	entry = entry.strip()
342	TO_RE = re.compile(entry, re.VERBOSE\|re.IGNORECASE)
343	result = TO_RE.search(header_field)
344	if result:
345	return True
346
347	return False
348
349	def email_to_unicode(self, message_str):
350	"""
351	Email has 7 bit ASCII code, convert it to unicode with the charset
352	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
353	understands it.
354	"""
355	results = email.Header.decode_header(message_str)
356	str = None
357	for text,format in results:
358	if format:
359	try:
360	temp = unicode(text, format)
361	except UnicodeError, detail:
362	# This always works
363	#
364	temp = unicode(text, 'iso-8859-15')
365	except LookupError, detail:
366	#text = 'ERROR: Could not find charset: %s, please install' %format
367	#temp = unicode(text, 'iso-8859-15')
368	temp = message_str
369
370	else:
371	temp = string.strip(text)
372	temp = unicode(text, 'iso-8859-15')
373
374	if str:
375	str = '%s %s' %(str, temp)
376	else:
377	str = '%s' %temp
378
379	#str = str.encode('utf-8')
380	return str
381
382	def debug_body(self, message_body, tempfile=False):
383	if tempfile:
384	import tempfile
385	body_file = tempfile.mktemp('.email2trac')
386	else:
387	body_file = os.path.join(self.TMPDIR, 'body.txt')
388
389	print 'TD: writing body (%s)' % body_file
390	fx = open(body_file, 'wb')
391	if not message_body:
392	message_body = '(None)'
393	fx.write(message_body)
394	fx.close()
395	try:
396	os.chmod(body_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
397	except OSError:
398	pass
399
400	def debug_attachments(self, message_parts):
401	n = 0
402	for part in message_parts:
403	# Skip inline text parts
404	if not isinstance(part, tuple):
405	continue
406
407	(original, filename, part) = part
408
409	n = n + 1
410	print 'TD: part%d: Content-Type: %s' % (n, part.get_content_type())
411	print 'TD: part%d: filename: %s' % (n, part.get_filename())
412
413	part_file = os.path.join(self.TMPDIR, filename)
414	#part_file = '/var/tmp/part%d' % n
415	print 'TD: writing part%d (%s)' % (n,part_file)
416	fx = open(part_file, 'wb')
417	text = part.get_payload(decode=1)
418	if not text:
419	text = '(None)'
420	fx.write(text)
421	fx.close()
422	try:
423	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
424	except OSError:
425	pass
426
427	def email_header_txt(self, m):
428	"""
429	Display To and CC addresses in description field
430	"""
431	str = ''
432	#if m['To'] and len(m['To']) > 0 and m['To'] != 'hic@sara.nl':
433	if m['To'] and len(m['To']) > 0:
434	str = "'''To:''' %s\r\n" %(m['To'])
435	if m['Cc'] and len(m['Cc']) > 0:
436	str = "%s'''Cc:''' %s\r\n" % (str, m['Cc'])
437
438	return self.email_to_unicode(str)
439
440
441	def get_sender_info(self, message):
442	"""
443	Get the default author name and email address from the message
444	"""
445
446	self.email_to = self.email_to_unicode(message['to'])
447	self.to_name, self.to_email_addr = email.Utils.parseaddr (self.email_to)
448
449	self.email_from = self.email_to_unicode(message['from'])
450	self.author, self.email_addr = email.Utils.parseaddr(self.email_from)
451
452	# Trac can not handle author's name that contains spaces
453	#
454	self.author = self.email_addr
455
456	if self.IGNORE_TRAC_USER_SETTINGS:
457	return
458
459	# Is this a registered user, use email address as search key:
460	# result:
461	# u : login name
462	# n : Name that the user has set in the settings tab
463	# e : email address that the user has set in the settings tab
464	#
465	users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
466	if e and (e.lower() == self.email_addr.lower()) ]
467
468	if len(users) == 1:
469	self.email_from = users[0][0]
470	self.author = users[0][0]
471
472	def set_reply_fields(self, ticket, message):
473	"""
474	Set all the right fields for a new ticket
475	"""
476	ticket['reporter'] = self.email_from
477
478	# Put all CC-addresses in ticket CC field
479	#
480	if self.REPLY_ALL:
481	#tos = message.get_all('to', [])
482	ccs = message.get_all('cc', [])
483
484	addrs = email.Utils.getaddresses(ccs)
485	if not addrs:
486	return
487
488	# Remove reporter email address if notification is
489	# on
490	#
491	if self.notification:
492	try:
493	addrs.remove((self.author, self.email_addr))
494	except ValueError, detail:
495	pass
496
497	for name,mail in addrs:
498	try:
499	mail_list = '%s, %s' %(mail_list, mail)
500	except UnboundLocalError, detail:
501	mail_list = mail
502
503	if mail_list:
504	ticket['cc'] = self.email_to_unicode(mail_list)
505
506	def save_email_for_debug(self, message, tempfile=False):
507	if tempfile:
508	import tempfile
509	msg_file = tempfile.mktemp('.email2trac')
510	else:
511	#msg_file = '/var/tmp/msg.txt'
512	msg_file = os.path.join(self.TMPDIR, 'msg.txt')
513
514	print 'TD: saving email to %s' % msg_file
515	fx = open(msg_file, 'wb')
516	fx.write('%s' % message)
517	fx.close()
518	try:
519	os.chmod(msg_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
520	except OSError:
521	pass
522
523	def str_to_dict(self, str):
524	"""
525	Transfrom a str of the form [<key>=<value>]+ to dict[<key>] = <value>
526	"""
527	# Skip the last ':' character
528	#
529	fields = string.split(str[:-1], ',')
530
531	result = dict()
532	for field in fields:
533	try:
534	index, value = string.split(field,'=')
535
536	# We can not change the description of a ticket via the subject
537	# line. The description is the body of the email
538	#
539	if index.lower() in ['description']:
540	continue
541
542	if value:
543	result[index.lower()] = value
544
545	except ValueError:
546	pass
547
548	return result
549
550	def update_ticket_fields(self, ticket, user_dict, use_default=None):
551	"""
552	This will update the ticket fields. It will check if the
553	given fields are known and if the right values are specified
554	It will only update the ticket field value:
555	- If the field is known
556	- If the value supplied is valid for the ticket field.
557	If not then there are two options:
558	1) Skip the value (use_default=None)
559	2) Set default value for field (use_default=1)
560	"""
561
562	# Build a system dictionary from the ticket fields
563	# with field as index and option as value
564	#
565	sys_dict = dict()
566	for field in ticket.fields:
567	try:
568	sys_dict[field['name']] = field['options']
569
570	except KeyError:
571	sys_dict[field['name']] = None
572	pass
573
574	# Check user supplied fields an compare them with the
575	# system one's
576	#
577	for field,value in user_dict.items():
578	if self.DEBUG >= 10:
579	print 'user_field\t %s = %s' %(field,value)
580
581	if sys_dict.has_key(field):
582
583	# Check if value is an allowed system option, if TypeError then
584	# every value is allowed
585	#
586	try:
587	if value in sys_dict[field]:
588	ticket[field] = value
589	else:
590	# Must we set a default if value is not allowed
591	#
592	if use_default:
593	value = self.get_config('ticket', 'default_%s' %(field) )
594	ticket[field] = value
595
596	except TypeError:
597	ticket[field] = value
598
599	if self.DEBUG >= 10:
600	print 'ticket_field\t %s = %s' %(field, ticket[field])
601
602	def ticket_update(self, m, id, spam):
603	"""
604	If the current email is a reply to an existing ticket, this function
605	will append the contents of this email to that ticket, instead of
606	creating a new one.
607	"""
608	if self.DEBUG:
609	print "TD: ticket_update: %s" %id
610
611	if not m['Subject']:
612	return False
613	else:
614	subject = self.email_to_unicode(m['Subject'])
615
616	# Must we update ticket fields
617	#
618	update_fields = dict()
619	try:
620	id, keywords = string.split(id, '?')
621	update_fields = self.str_to_dict(keywords)
622
623	# Strip '#'
624	#
625	self.id = int(id[1:])
626
627	except ValueError:
628	# Strip '#' and ':'
629	#
630	self.id = int(id[1:-1])
631
632
633	# When is the change committed
634	#
635	#
636	if self.VERSION == 0.11:
637	utc = UTC()
638	when = datetime.now(utc)
639	else:
640	when = int(time.time())
641
642	try:
643	tkt = Ticket(self.env, self.id, self.db)
644	except util.TracError, detail:
645	# Not a valid ticket
646	self.id = None
647	return False
648
649	# reopen the ticket if it is was closed
650	# We must use the ticket workflow framework
651	#
652	if tkt['status'] in ['closed']:
653
654	#print controller.actions['reopen']
655	#
656	# As reference
657	# req = Mock(href=Href('/'), abs_href=Href('http://www.example.com/'), authname='anonymous', perm=MockPerm(), args={})
658	#
659	#a = controller.render_ticket_action_control(req, tkt, 'reopen')
660	#print 'controller : ', a
661	#
662	#b = controller.get_all_status()
663	#print 'get all status: ', b
664	#
665	#b = controller.get_ticket_changes(req, tkt, 'reopen')
666	#print 'get_ticket_changes :', b
667
668	if self.WORKFLOW and (self.VERSION in ['0.11']) :
669	from trac.ticket.default_workflow import ConfigurableTicketWorkflow
670	from trac.test import Mock, MockPerm
671
672	req = Mock(authname='anonymous', perm=MockPerm(), args={})
673
674	controller = ConfigurableTicketWorkflow(self.env)
675	fields = controller.get_ticket_changes(req, tkt, self.WORKFLOW)
676
677	if self.DEBUG:
678	print 'TD: Workflow ticket update fields: ', fields
679
680	for key in fields.keys():
681	tkt[key] = fields[key]
682
683	else:
684	tkt['status'] = 'reopened'
685	tkt['resolution'] = ''
686
687	# Must we update some ticket fields properties
688	#
689	if update_fields:
690	self.update_ticket_fields(tkt, update_fields)
691
692	message_parts = self.get_message_parts(m)
693	message_parts = self.unique_attachment_names(message_parts)
694
695	if self.EMAIL_HEADER:
696	message_parts.insert(0, self.email_header_txt(m))
697
698	body_text = self.body_text(message_parts)
699
700	if body_text.strip() or update_fields:
701	if self.DRY_RUN:
702	print 'DRY_RUN: tkt.save_changes(self.author, comment) ', self.author
703	else:
704	tkt.save_changes(self.author, body_text, when)
705
706	if self.VERSION == 0.9:
707	str = self.attachments(message_parts, True)
708	else:
709	str = self.attachments(message_parts)
710
711	if self.notification and not spam:
712	self.notify(tkt, False, when)
713
714	return True
715
716	def set_ticket_fields(self, ticket):
717	"""
718	set the ticket fields to value specified
719	- /etc/email2trac.conf with <prefix>_<field>
720	- trac default values, trac.ini
721	"""
722	user_dict = dict()
723
724	for field in ticket.fields:
725
726	name = field['name']
727
728	# skip some fields like resolution
729	#
730	if name in [ 'resolution' ]:
731	continue
732
733	# default trac value
734	#
735	if not field.get('custom'):
736	value = self.get_config('ticket', 'default_%s' %(name) )
737	else:
738	value = field.get('value')
739	options = field.get('options')
740	if value and options and value not in options:
741	value = options[int(value)]
742
743	if self.DEBUG > 10:
744	print 'trac.ini name %s = %s' %(name, value)
745
746	prefix = self.parameters['ticket_prefix']
747	try:
748	value = self.parameters['%s_%s' %(prefix, name)]
749	if self.DEBUG > 10:
750	print 'email2trac.conf %s = %s ' %(name, value)
751
752	except KeyError, detail:
753	pass
754
755	if self.DEBUG:
756	print 'user_dict[%s] = %s' %(name, value)
757
758	user_dict[name] = value
759
760	self.update_ticket_fields(ticket, user_dict, use_default=1)
761
762	# Set status ticket
763	#`
764	ticket['status'] = 'new'
765
766
767
768	def new_ticket(self, msg, spam):
769	"""
770	Create a new ticket
771	"""
772	if self.DEBUG:
773	print "TD: new_ticket"
774
775	tkt = Ticket(self.env)
776	self.set_ticket_fields(tkt)
777
778	# Old style setting for component, will be removed
779	#
780	if spam:
781	tkt['component'] = 'Spam'
782
783	elif self.parameters.has_key('component'):
784	tkt['component'] = self.parameters['component']
785
786	if not msg['Subject']:
787	tkt['summary'] = u'(No subject)'
788	else:
789	tkt['summary'] = self.email_to_unicode(msg['Subject'])
790
791	self.set_reply_fields(tkt, msg)
792
793	# produce e-mail like header
794	#
795	head = ''
796	if self.EMAIL_HEADER > 0:
797	head = self.email_header_txt(msg)
798
799	message_parts = self.get_message_parts(msg)
800	message_parts = self.unique_attachment_names(message_parts)
801
802	if self.EMAIL_HEADER > 0:
803	message_parts.insert(0, self.email_header_txt(msg))
804
805	body_text = self.body_text(message_parts)
806
807	tkt['description'] = body_text
808
809	#when = int(time.time())
810	#
811	utc = UTC()
812	when = datetime.now(utc)
813
814	if not self.DRY_RUN:
815	self.id = tkt.insert()
816
817	changed = False
818	comment = ''
819
820	# Rewrite the description if we have mailto enabled
821	#
822	if self.MAILTO:
823	changed = True
824	comment = u'\nadded mailto line\n'
825	mailto = self.html_mailto_link( m['Subject'], body_text)
826
827	tkt['description'] = u'%s\r\n%s%s\r\n' \
828	%(head, mailto, body_text)
829
830	str = self.attachments(message_parts)
831	if str:
832	changed = True
833	comment = '%s\n%s\n' %(comment, str)
834
835	if changed:
836	if self.DRY_RUN:
837	print 'DRY_RUN: tkt.save_changes(self.author, comment) ', self.author
838	else:
839	tkt.save_changes(self.author, comment)
840	#print tkt.get_changelog(self.db, when)
841
842	if self.notification and not spam:
843	self.notify(tkt, True)
844
845
846	def blog(self, id):
847	"""
848	The blog create/update function
849	"""
850	# import the modules
851	#
852	from tracfullblog.core import FullBlogCore
853	from tracfullblog.model import BlogPost, BlogCommen
854
855	# instantiate blog core
856	blog = FullBlogCore(self.env)
857	req = ''
858
859	if id:
860
861	# update blog
862	#
863	comment = BlogComment(self.env, result.group('blog_id'))
864	comment.author = self.author
865	comment.comment = self.get_body_text(m)
866	blog.create_comment(req, comment)
867
868	else:
869	# create blog
870	#
871	import time
872	post = BlogPost(self.env, 'blog_'+time.strftime("%Y%m%d%H%M%S", time.gmtime()))
873
874	#post = BlogPost(self.env, blog._get_default_postname(self.env))
875
876	post.author = self.author
877	post.title = self.email_to_unicode(m['Subject'])
878	post.body = self.get_body_text(m)
879
880	blog.create_post(req, post, self.author, u'Created by email2trac', False)
881
882
883	def parse(self, fp):
884	global m
885
886	m = email.message_from_file(fp)
887
888	if not m:
889	if self.DEBUG:
890	print "TD: This is not a valid email message format"
891	return
892
893	# Work around lack of header folding in Python; see http://bugs.python.org/issue4696
894	m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
895
896	if self.DEBUG > 1: # save the entire e-mail message text
897	message_parts = self.get_message_parts(m)
898	message_parts = self.unique_attachment_names(message_parts)
899	self.save_email_for_debug(m, True)
900	body_text = self.body_text(message_parts)
901	self.debug_body(body_text, True)
902	self.debug_attachments(message_parts)
903
904	self.db = self.env.get_db_cnx()
905	self.get_sender_info(m)
906
907	if not self.email_header_acl('white_list', self.email_addr, True):
908	if self.DEBUG > 1 :
909	print 'Message rejected : %s not in white list' %(self.email_addr)
910	return False
911
912	if self.email_header_acl('black_list', self.email_addr, False):
913	if self.DEBUG > 1 :
914	print 'Message rejected : %s in black list' %(self.email_addr)
915	return False
916
917	if not self.email_header_acl('recipient_list', self.to_email_addr, True):
918	if self.DEBUG > 1 :
919	print 'Message rejected : %s not in recipient list' %(self.to_email_addr)
920	return False
921
922	# If drop the message
923	#
924	if self.spam(m) == 'drop':
925	return False
926
927	elif self.spam(m) == 'spam':
928	spam_msg = True
929
930	else:
931	spam_msg = False
932
933	if self.get_config('notification', 'smtp_enabled') in ['true']:
934	self.notification = 1
935	else:
936	self.notification = 0
937
938	print self.get_config('components', 'tracext.google.ads.*')
939
940	# Check if FullBlogPlugin is installed
941	#
942	blog_enabled = None
943	if self.get_config('components', 'tracfullblog.*') in ['enabled']:
944	blog_enabled = True
945
946	# Find out if this is a ticket or a blog
947	if not m['Subject']:
948	return False
949	else:
950	subject = self.email_to_unicode(m['Subject'])
951
952	#
953	# [hic] #1529: Re: LRZ
954	# [hic] #1529?owner=bas,priority=medium: Re: LRZ
955	#
956	TICKET_RE = re.compile(r"""
957	(?P<blog>blog:(?P<blog_id>\w*))
958	\|(?P<ticket_id>[#][\d]+:)
959	\|(?P<ticket_fields>[#][\d]+\?.*?:)
960	""", re.VERBOSE)
961
962	result = TICKET_RE.search(subject)
963
964	if result:
965	if result.group('blog') and blog_enabled:
966	self.blog(result.group('blog_id'))
967
968	# update ticket + fields
969	#
970	if result.group('ticket_fields') and self.TICKET_UPDATE:
971	self.ticket_update(m, result.group('ticket_fields'), spam_msg)
972
973	# Update ticket
974	#
975	elif result.group('ticket_id') and self.TICKET_UPDATE:
976	self.ticket_update(m, result.group('ticket_id'), spam_msg)
977
978	# Create ticket
979	#
980	else:
981	self.new_ticket(m, spam_msg)
982
983	def strip_signature(self, text):
984	"""
985	Strip signature from message, inspired by Mailman software
986	"""
987	body = []
988	for line in text.splitlines():
989	if line == '-- ':
990	break
991	body.append(line)
992
993	return ('\n'.join(body))
994
995	def reflow(self, text, delsp = 0):
996	"""
997	Reflow the message based on the format="flowed" specification (RFC 3676)
998	"""
999	flowedlines = []
1000	quotelevel = 0
1001	prevflowed = 0
1002
1003	for line in text.splitlines():
1004	from re import match
1005
1006	# Figure out the quote level and the content of the current line
1007	m = match('(>)( ?)(.)', line)
1008	linequotelevel = len(m.group(1))
1009	line = m.group(3)
1010
1011	# Determine whether this line is flowed
1012	if line and line != '-- ' and line[-1] == ' ':
1013	flowed = 1
1014	else:
1015	flowed = 0
1016
1017	if flowed and delsp and line and line[-1] == ' ':
1018	line = line[:-1]
1019
1020	# If the previous line is flowed, append this line to it
1021	if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1022	flowedlines[-1] += line
1023	# Otherwise, start a new line
1024	else:
1025	flowedlines.append('>' * linequotelevel + line)
1026
1027	prevflowed = flowed
1028
1029
1030	return '\n'.join(flowedlines)
1031
1032	def strip_quotes(self, text):
1033	"""
1034	Strip quotes from message by Nicolas Mendoza
1035	"""
1036	body = []
1037	for line in text.splitlines():
1038	if line.startswith(self.EMAIL_QUOTE):
1039	continue
1040	body.append(line)
1041
1042	return ('\n'.join(body))
1043
1044	def wrap_text(self, text, replace_whitespace = False):
1045	"""
1046	Will break a lines longer then given length into several small
1047	lines of size given length
1048	"""
1049	import textwrap
1050
1051	LINESEPARATOR = '\n'
1052	reformat = ''
1053
1054	for s in text.split(LINESEPARATOR):
1055	tmp = textwrap.fill(s,self.USE_TEXTWRAP)
1056	if tmp:
1057	reformat = '%s\n%s' %(reformat,tmp)
1058	else:
1059	reformat = '%s\n' %reformat
1060
1061	return reformat
1062
1063	# Python2.4 and higher
1064	#
1065	#return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
1066	#
1067
1068
1069	def get_message_parts(self, msg):
1070	"""
1071	parses the email message and returns a list of body parts and attachments
1072	body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
1073	"""
1074	message_parts = []
1075
1076	# This is used to figure out when we are inside an AppleDouble container
1077	# AppleDouble containers consists of two parts: Mac-specific file data, and platform-independent data
1078	# We strip away Mac-specific stuff
1079	appledouble_parts = []
1080
1081	for part in msg.walk():
1082	if self.DEBUG:
1083	print 'TD: Message part: Content-Type: %s' % part.get_content_type()
1084
1085	# Check whether we just finished processing an AppleDouble container
1086	if part not in appledouble_parts:
1087	appledouble_parts = []
1088
1089	# Special handling for BinHex attachments. Options are drop (leave out with no warning), warn (and leave out), and keep
1090	if part.get_content_type() == 'application/mac-binhex40':
1091	if self.BINHEX == 'warn':
1092	message_parts.append("'''A BinHex attachment named '%s' was ignored (use MIME encoding instead).'''" % part.get_filename())
1093	continue
1094	elif self.BINHEX == 'drop':
1095	continue
1096
1097	# Special handling for AppleSingle attachments. Options are drop (leave out with no warning), warn (and leave out), and keep
1098	if part.get_content_type() == 'application/applefile' and not part in appledouble_parts:
1099	if self.APPLESINGLE == 'warn':
1100	message_parts.append("'''An AppleSingle attachment named '%s' was ignored (use MIME encoding instead).'''" % part.get_filename())
1101	continue
1102	elif self.APPLESINGLE == 'drop':
1103	continue
1104
1105	# Special handling for the Mac-specific part of AppleDouble attachments. Options are strip (leave out with no warning), warn (and leave out), and keep
1106	if part.get_content_type() == 'application/applefile':
1107	if self.APPLEDOUBLE == 'warn':
1108	message_parts.append("'''The resource fork of an attachment named '%s' was removed.'''" % part.get_filename())
1109	continue
1110	elif self.APPLEDOUBLE == 'strip':
1111	continue
1112
1113	# If we entering an AppleDouble container, set up appledouble_parts so that we know what to do with its subparts
1114	if part.get_content_type() == 'multipart/appledouble':
1115	appledouble_parts = part.get_payload()
1116	continue
1117
1118	# Any other multipart/* is just a container for multipart messages
1119	if part.get_content_maintype() == 'multipart':
1120	continue
1121
1122	# Check if this is an inline part. It's inline if there is co Cont-Disp header, or if there is one and it says "inline"
1123	inline = self.inline_part(part)
1124
1125	# Inline text parts are where the body is
1126	if part.get_content_type() == 'text/plain' and inline:
1127	if self.DEBUG:
1128	print 'TD: Inline body part'
1129
1130	# Try to decode, if fails then do not decode
1131	#
1132	body_text = part.get_payload(decode=1)
1133	if not body_text:
1134	body_text = part.get_payload(decode=0)
1135
1136	format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
1137	delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
1138
1139	if self.REFLOW and not self.VERBATIM_FORMAT and format == 'flowed':
1140	body_text = self.reflow(body_text, delsp == 'yes')
1141
1142	if self.STRIP_SIGNATURE:
1143	body_text = self.strip_signature(body_text)
1144
1145	if self.STRIP_QUOTES:
1146	body_text = self.strip_quotes(body_text)
1147
1148	if self.USE_TEXTWRAP:
1149	body_text = self.wrap_text(body_text)
1150
1151	# Get contents charset (iso-8859-15 if not defined in mail headers)
1152	#
1153	charset = part.get_content_charset()
1154	if not charset:
1155	charset = 'iso-8859-15'
1156
1157	try:
1158	ubody_text = unicode(body_text, charset)
1159
1160	except UnicodeError, detail:
1161	ubody_text = unicode(body_text, 'iso-8859-15')
1162
1163	except LookupError, detail:
1164	ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
1165
1166	if self.VERBATIM_FORMAT:
1167	message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
1168	else:
1169	message_parts.append('%s' %ubody_text)
1170	else:
1171	if self.DEBUG:
1172	print 'TD: Filename: %s' % part.get_filename()
1173
1174	message_parts.append((part.get_filename(), part))
1175
1176	return message_parts
1177
1178	def unique_attachment_names(self, message_parts):
1179	renamed_parts = []
1180	attachment_names = set()
1181	for part in message_parts:
1182
1183	# If not an attachment, leave it alone
1184	if not isinstance(part, tuple):
1185	renamed_parts.append(part)
1186	continue
1187
1188	(filename, part) = part
1189	# Decode the filename
1190	if filename:
1191	filename = self.email_to_unicode(filename)
1192	# If no name, use a default one
1193	else:
1194	filename = 'untitled-part'
1195
1196	# Guess the extension from the content type, use non strict mode
1197	# some additional non-standard but commonly used MIME types
1198	# are also recognized
1199	#
1200	ext = mimetypes.guess_extension(part.get_content_type(), False)
1201	if not ext:
1202	ext = '.bin'
1203
1204	filename = '%s%s' % (filename, ext)
1205
1206	# Discard relative paths in attachment names
1207	filename = filename.replace('\\', '/').replace(':', '/')
1208	filename = os.path.basename(filename)
1209
1210	# We try to normalize the filename to utf-8 NFC if we can.
1211	# Files uploaded from OS X might be in NFD.
1212	# Check python version and then try it
1213	#
1214	if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
1215	try:
1216	filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
1217	except TypeError:
1218	pass
1219
1220	if self.QUOTE_ATTACHMENT_FILENAMES:
1221	filename = urllib.quote(filename)
1222
1223	# Make the filename unique for this ticket
1224	num = 0
1225	unique_filename = filename
1226	filename, ext = os.path.splitext(filename)
1227
1228	while unique_filename in attachment_names or self.attachment_exists(unique_filename):
1229	num += 1
1230	unique_filename = "%s-%s%s" % (filename, num, ext)
1231
1232	if self.DEBUG:
1233	print 'TD: Attachment with filename %s will be saved as %s' % (filename, unique_filename)
1234
1235	attachment_names.add(unique_filename)
1236
1237	renamed_parts.append((filename, unique_filename, part))
1238
1239	return renamed_parts
1240
1241	def inline_part(self, part):
1242	return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
1243
1244
1245	def attachment_exists(self, filename):
1246
1247	if self.DEBUG:
1248	print "TD: attachment_exists: Ticket number : %s, Filename : %s" %(self.id, filename)
1249
1250	# We have no valid ticket id
1251	#
1252	if not self.id:
1253	return False
1254
1255	try:
1256	att = attachment.Attachment(self.env, 'ticket', self.id, filename)
1257	return True
1258	except attachment.ResourceNotFound:
1259	return False
1260
1261	def body_text(self, message_parts):
1262	body_text = []
1263
1264	for part in message_parts:
1265	# Plain text part, append it
1266	if not isinstance(part, tuple):
1267	body_text.extend(part.strip().splitlines())
1268	body_text.append("")
1269	continue
1270
1271	(original, filename, part) = part
1272	inline = self.inline_part(part)
1273
1274	if part.get_content_maintype() == 'image' and inline:
1275	body_text.append('[[Image(%s)]]' % filename)
1276	body_text.append("")
1277	else:
1278	body_text.append('[attachment:"%s"]' % filename)
1279	body_text.append("")
1280
1281	body_text = '\r\n'.join(body_text)
1282	return body_text
1283
1284	def notify(self, tkt, new=True, modtime=0):
1285	"""
1286	A wrapper for the TRAC notify function. So we can use templates
1287	"""
1288	if self.DRY_RUN:
1289	print 'DRY_RUN: self.notify(tkt, True) ', self.author
1290	return
1291	try:
1292	# create false {abs_}href properties, to trick Notify()
1293	#
1294	if not self.VERSION == 0.11:
1295	self.env.abs_href = Href(self.get_config('project', 'url'))
1296	self.env.href = Href(self.get_config('project', 'url'))
1297
1298	tn = TicketNotifyEmail(self.env)
1299
1300	if self.notify_template:
1301
1302	if self.VERSION == 0.11:
1303
1304	from trac.web.chrome import Chrome
1305
1306	if self.notify_template_update and not new:
1307	tn.template_name = self.notify_template_update
1308	else:
1309	tn.template_name = self.notify_template
1310
1311	tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
1312
1313	else:
1314
1315	tn.template_name = self.notify_template;
1316
1317	tn.notify(tkt, new, modtime)
1318
1319	except Exception, e:
1320	print 'TD: Failure sending notification on creation of ticket #%s: %s' %(self.id, e)
1321
1322	def html_mailto_link(self, subject, body):
1323	"""
1324	This function returns a HTML mailto tag with the ticket id and author email address
1325	"""
1326	if not self.author:
1327	author = self.email_addr
1328	else:
1329	author = self.author
1330
1331	# use urllib to escape the chars
1332	#
1333	str = 'mailto:%s?Subject=%s&Cc=%s' %(
1334	urllib.quote(self.email_addr),
1335	urllib.quote('Re: #%s: %s' %(self.id, subject)),
1336	urllib.quote(self.MAILTO_CC)
1337	)
1338
1339	str = '\r\n{{{\r\n#!html\r\n<a\r\n href="%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(str, author)
1340	return str
1341
1342	def attachments(self, message_parts, update=False):
1343	'''
1344	save any attachments as files in the ticket's directory
1345	'''
1346	if self.DRY_RUN:
1347	print "DRY_RUN: no attachments saved"
1348	return ''
1349
1350	count = 0
1351
1352	# Get Maxium attachment size
1353	#
1354	max_size = int(self.get_config('attachment', 'max_size'))
1355	status = ''
1356
1357	for part in message_parts:
1358	# Skip body parts
1359	if not isinstance(part, tuple):
1360	continue
1361
1362	(original, filename, part) = part
1363	#
1364	# Must be tuneables HvB
1365	#
1366	path, fd = util.create_unique_file(os.path.join(self.TMPDIR, filename))
1367	text = part.get_payload(decode=1)
1368	if not text:
1369	text = '(None)'
1370	fd.write(text)
1371	fd.close()
1372
1373	# get the file_size
1374	#
1375	stats = os.lstat(path)
1376	file_size = stats[stat.ST_SIZE]
1377
1378	# Check if the attachment size is allowed
1379	#
1380	if (max_size != -1) and (file_size > max_size):
1381	status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1382	%(status, original, file_size, max_size)
1383
1384	os.unlink(path)
1385	continue
1386	else:
1387	count = count + 1
1388
1389	# Insert the attachment
1390	#
1391	fd = open(path, 'rb')
1392	att = attachment.Attachment(self.env, 'ticket', self.id)
1393
1394	# This will break the ticket_update system, the body_text is vaporized
1395	# ;-(
1396	#
1397	if not update:
1398	att.author = self.author
1399	att.description = self.email_to_unicode('Added by email2trac')
1400
1401	att.insert(filename, fd, file_size)
1402	#except util.TracError, detail:
1403	# print detail
1404
1405	# Remove the created temporary filename
1406	#
1407	fd.close()
1408	os.unlink(path)
1409
1410	# Return how many attachments
1411	#
1412	status = 'This message has %d attachment(s)\n%s' %(count, status)
1413	return status
1414
1415
1416	def mkdir_p(dir, mode):
1417	'''do a mkdir -p'''
1418
1419	arr = string.split(dir, '/')
1420	path = ''
1421	for part in arr:
1422	path = '%s/%s' % (path, part)
1423	try:
1424	stats = os.stat(path)
1425	except OSError:
1426	os.mkdir(path, mode)
1427
1428	def ReadConfig(file, name):
1429	"""
1430	Parse the config file
1431	"""
1432	if not os.path.isfile(file):
1433	print 'File %s does not exist' %file
1434	sys.exit(1)
1435
1436	config = trac_config.Configuration(file)
1437
1438	# Use given project name else use defaults
1439	#
1440	if name:
1441	sections = config.sections()
1442	if not name in sections:
1443	print "Not a valid project name: %s" %name
1444	print "Valid names: %s" %sections
1445	sys.exit(1)
1446
1447	project = dict()
1448	for option, value in config.options(name):
1449	project[option] = value
1450
1451	else:
1452	# use some trac internales to get the defaults
1453	#
1454	project = config.parser.defaults()
1455
1456	return project
1457
1458
1459	if __name__ == '__main__':
1460	# Default config file
1461	#
1462	configfile = '@email2trac_conf@'
1463	project = ''
1464	component = ''
1465	ticket_prefix = 'default'
1466	dry_run = None
1467
1468	ENABLE_SYSLOG = 0
1469
1470
1471	SHORT_OPT = 'chf:np:t:'
1472	LONG_OPT = ['component=', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=']
1473
1474	try:
1475	opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
1476	except getopt.error,detail:
1477	print __doc__
1478	print detail
1479	sys.exit(1)
1480
1481	project_name = None
1482	for opt,value in opts:
1483	if opt in [ '-h', '--help']:
1484	print __doc__
1485	sys.exit(0)
1486	elif opt in ['-c', '--component']:
1487	component = value
1488	elif opt in ['-f', '--file']:
1489	configfile = value
1490	elif opt in ['-n', '--dry-run']:
1491	dry_run = True
1492	elif opt in ['-p', '--project']:
1493	project_name = value
1494	elif opt in ['-t', '--ticket_prefix']:
1495	ticket_prefix = value
1496
1497	settings = ReadConfig(configfile, project_name)
1498	if not settings.has_key('project'):
1499	print __doc__
1500	print 'No Trac project is defined in the email2trac config file.'
1501	sys.exit(1)
1502
1503	if component:
1504	settings['component'] = component
1505
1506	# The default prefix for ticket values in email2trac.conf
1507	#
1508	settings['ticket_prefix'] = ticket_prefix
1509	settings['dry_run'] = dry_run
1510
1511	if settings.has_key('trac_version'):
1512	version = settings['trac_version']
1513	else:
1514	version = trac_default_version
1515
1516
1517	#debug HvB
1518	#print settings
1519
1520	try:
1521	if version == '0.9':
1522	from trac import attachment
1523	from trac.env import Environment
1524	from trac.ticket import Ticket
1525	from trac.web.href import Href
1526	from trac import util
1527	from trac.Notify import TicketNotifyEmail
1528	elif version == '0.10':
1529	from trac import attachment
1530	from trac.env import Environment
1531	from trac.ticket import Ticket
1532	from trac.web.href import Href
1533	from trac import util
1534	#
1535	# return util.text.to_unicode(str)
1536	#
1537	# see http://projects.edgewall.com/trac/changeset/2799
1538	from trac.ticket.notification import TicketNotifyEmail
1539	from trac import config as trac_config
1540	elif version == '0.11':
1541	from trac import attachment
1542	from trac.env import Environment
1543	from trac.ticket import Ticket
1544	from trac.web.href import Href
1545	from trac import config as trac_config
1546	from trac import util
1547
1548
1549	#
1550	# return util.text.to_unicode(str)
1551	#
1552	# see http://projects.edgewall.com/trac/changeset/2799
1553	from trac.ticket.notification import TicketNotifyEmail
1554	else:
1555	print 'TRAC version %s is not supported' %version
1556	sys.exit(1)
1557
1558	if settings.has_key('enable_syslog'):
1559	if SYSLOG_AVAILABLE:
1560	ENABLE_SYSLOG = float(settings['enable_syslog'])
1561
1562	env = Environment(settings['project'], create=0)
1563	tktparser = TicketEmailParser(env, settings, float(version))
1564	tktparser.parse(sys.stdin)
1565
1566	# Catch all errors ans log to SYSLOG if we have enabled this
1567	# else stdout
1568	#
1569	except Exception, error:
1570	if ENABLE_SYSLOG:
1571	syslog.openlog('email2trac', syslog.LOG_NOWAIT)
1572
1573	etype, evalue, etb = sys.exc_info()
1574	for e in traceback.format_exception(etype, evalue, etb):
1575	syslog.syslog(e)
1576
1577	syslog.closelog()
1578	else:
1579	traceback.print_exc()
1580
1581	if m:
1582	tktparser.save_email_for_debug(m, True)
1583
1584	sys.exit(1)
1585	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format