Context Navigation

source: trunk/email2trac.py.in @ 434

Last change on this file since 434 was 434, checked in by bas, 14 years ago
converted mailto_link, mailto_cc and umask to new UserDict?
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 59.7 KB

Line
1	#!@PYTHON@
2	# Copyright (C) 2002
3	#
4	# This file is part of the email2trac utils
5	#
6	# This program is free software; you can redistribute it and/or modify it
7	# under the terms of the GNU General Public License as published by the
8	# Free Software Foundation; either version 2, or (at your option) any
9	# later version.
10	#
11	# This program is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with this program; if not, write to the Free Software
18	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
19	#
20	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
21	#
22	"""
23	email2trac.py -- Email tickets to Trac.
24
25	A simple MTA filter to create Trac tickets from inbound emails.
26
27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28	Copyright 2005, Edgewall Software
29
30	Authors:
31	Bas van der Vlies <basv@sara.nl>
32	Walter de Jong <walter@sara.nl>
33
34	The scripts reads emails from stdin and inserts directly into a Trac database.
35
36	How to use
37	----------
38	* See https://subtrac.sara.nl/oss/email2trac/
39
40	* Create an config file:
41	[DEFAULT] # REQUIRED
42	project : /data/trac/test # REQUIRED
43	debug : 1 # OPTIONAL, if set print some DEBUG info
44
45	[jouvin] # OPTIONAL project declaration, if set both fields necessary
46	project : /data/trac/jouvin # use -p\|--project jouvin.
47
48	* default config file is : /etc/email2trac.conf
49
50	* Commandline opions:
51	-h,--help
52	-d, --debug
53	-f,--file <configuration file>
54	-n,--dry-run
55	-p, --project <project name>
56	-t, --ticket_prefix <name>
57
58	SVN Info:
59	$Id: email2trac.py.in 434 2010-07-22 12:20:32Z bas $
60	"""
61	import os
62	import sys
63	import string
64	import getopt
65	import time
66	import email
67	import email.Iterators
68	import email.Header
69	import re
70	import urllib
71	import unicodedata
72	import mimetypes
73	import traceback
74	import logging
75	import logging.handlers
76	import UserDict
77
78	from datetime import tzinfo, timedelta, datetime
79	from stat import *
80
81
82	from trac import __version__ as trac_version
83	from trac import config as trac_config
84
85	# Some global variables
86	#
87	trac_default_version = '0.11'
88	m = None
89
90	class SaraDict(UserDict.UserDict):
91	def __init__(self, dictin = None):
92	UserDict.UserDict.__init__(self)
93	self.name = None
94
95	if dictin:
96	if dictin.has_key('name'):
97	self.name = dictin['name']
98	del dictin['name']
99	self.data = dictin
100
101	def get_value(self, key):
102	if self.has_key(key):
103	return self[key]
104	else:
105	return None
106
107	def __repr__(self):
108	return repr(self.data)
109
110	def __str__(self):
111	return str(self.data)
112
113	def __getattr__(self, name):
114	"""
115	override the class attribute get method. Return the value
116	from the dictionary
117	"""
118	if self.data.has_key(name):
119	return self.data[name]
120	else:
121	return None
122
123	def __setattr__(self, name, value):
124	"""
125	override the class attribute set method only when the UserDict
126	has set its class attribute
127	"""
128	if self.__dict__.has_key('data'):
129	self.data[name] = value
130	else:
131	self.__dict__[name] = value
132
133	def __iter__(self):
134	return iter(self.data.keys())
135
136	class TicketEmailParser(object):
137	env = None
138	comment = '> '
139
140	def __init__(self, env, parameters, logger, version):
141	self.env = env
142
143	# Database connection
144	#
145	self.db = None
146
147	# Save parameters
148	#
149	self.parameters = parameters
150	self.logger = logger
151
152	# Some useful mail constants
153	#
154	self.email_name = None
155	self.email_addr = None
156	self.email_from = None
157	self.author = None
158	self.id = None
159
160	self.STRIP_CONTENT_TYPES = list()
161
162	self.VERSION = version
163
164	self.get_config = self.env.config.get
165
166	## init function ##
167	#
168	self.setup_parameters()
169
170	def setup_parameters(self):
171	if self.parameters.umask:
172	os.umask(int(self.parameters.umask, 8))
173
174	if self.parameters.spam_level:
175	self.SPAM_LEVEL = int(self.parameters.spam_level)
176	else:
177	self.SPAM_LEVEL = 0
178
179	if not self.parameters.spam_header:
180	self.parameters.spam_header = 'X-Spam-Score'
181
182	if self.parameters.has_key('email_quote'):
183	self.EMAIL_QUOTE = str(self.parameters['email_quote'])
184	else:
185	self.EMAIL_QUOTE = '> '
186
187	if self.parameters.has_key('email_header'):
188	self.EMAIL_HEADER = int(self.parameters['email_header'])
189	else:
190	self.EMAIL_HEADER = 0
191
192	if self.parameters.has_key('alternate_notify_template'):
193	self.notify_template = str(self.parameters['alternate_notify_template'])
194	else:
195	self.notify_template = None
196
197	if self.parameters.has_key('alternate_notify_template_update'):
198	self.notify_template_update = str(self.parameters['alternate_notify_template_update'])
199	else:
200	self.notify_template_update = None
201
202	if self.parameters.has_key('reply_all'):
203	self.REPLY_ALL = int(self.parameters['reply_all'])
204	else:
205	self.REPLY_ALL = 0
206
207	if self.parameters.has_key('ticket_permission_system'):
208	self.TICKET_PERMISSION_SYSTEM = str(self.parameters['ticket_permission_system'])
209	else:
210	self.TICKET_PERMISSION_SYSTEM = None
211
212	if self.parameters.has_key('ticket_update'):
213	self.TICKET_UPDATE = int(self.parameters['ticket_update'])
214	else:
215	self.TICKET_UPDATE = 0
216
217	if self.parameters.has_key('ticket_update_by_subject'):
218	self.TICKET_UPDATE_BY_SUBJECT = int(self.parameters['ticket_update_by_subject'])
219	else:
220	self.TICKET_UPDATE_BY_SUBJECT = 0
221
222	if self.parameters.has_key('ticket_update_by_subject_lookback'):
223	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK = int(self.parameters['ticket_update_by_subject_lookback'])
224	else:
225	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK = 30
226
227	if self.parameters.has_key('drop_spam'):
228	self.DROP_SPAM = int(self.parameters['drop_spam'])
229	else:
230	self.DROP_SPAM = 0
231
232	if self.parameters.has_key('verbatim_format'):
233	self.VERBATIM_FORMAT = int(self.parameters['verbatim_format'])
234	else:
235	self.VERBATIM_FORMAT = 1
236
237	if self.parameters.has_key('reflow'):
238	self.REFLOW = int(self.parameters['reflow'])
239	else:
240	self.REFLOW = 1
241
242	if self.parameters.has_key('drop_alternative_html_version'):
243	self.DROP_ALTERNATIVE_HTML_VERSION = int(self.parameters['drop_alternative_html_version'])
244	else:
245	self.DROP_ALTERNATIVE_HTML_VERSION = 0
246
247	if self.parameters.has_key('strip_signature'):
248	self.STRIP_SIGNATURE = int(self.parameters['strip_signature'])
249	else:
250	self.STRIP_SIGNATURE = 0
251
252	if self.parameters.has_key('strip_quotes'):
253	self.STRIP_QUOTES = int(self.parameters['strip_quotes'])
254	else:
255	self.STRIP_QUOTES = 0
256
257	self.properties = dict()
258	if self.parameters.has_key('inline_properties'):
259	self.INLINE_PROPERTIES = int(self.parameters['inline_properties'])
260	else:
261	self.INLINE_PROPERTIES = 0
262
263	if self.parameters.has_key('use_textwrap'):
264	self.USE_TEXTWRAP = int(self.parameters['use_textwrap'])
265	else:
266	self.USE_TEXTWRAP = 0
267
268	if self.parameters.has_key('binhex'):
269	self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
270
271	if self.parameters.has_key('applesingle'):
272	self.STRIP_CONTENT_TYPES.append('application/applefile')
273
274	if self.parameters.has_key('appledouble'):
275	self.STRIP_CONTENT_TYPES.append('application/applefile')
276
277	if self.parameters.has_key('strip_content_types'):
278	items = self.parameters['strip_content_types'].split(',')
279	for item in items:
280	self.STRIP_CONTENT_TYPES.append(item.strip())
281
282	self.WORKFLOW = None
283	if self.parameters.has_key('workflow'):
284	self.WORKFLOW = self.parameters['workflow']
285
286	# Use OS independend functions
287	#
288	self.TMPDIR = os.path.normcase('/tmp')
289	if self.parameters.has_key('tmpdir'):
290	self.TMPDIR = os.path.normcase(str(self.parameters['tmpdir']))
291
292	if self.parameters.has_key('ignore_trac_user_settings'):
293	self.IGNORE_TRAC_USER_SETTINGS = int(self.parameters['ignore_trac_user_settings'])
294	else:
295	self.IGNORE_TRAC_USER_SETTINGS = 0
296
297	if self.parameters.has_key('email_triggers_workflow'):
298	self.EMAIL_TRIGGERS_WORKFLOW = int(self.parameters['email_triggers_workflow'])
299	else:
300	self.EMAIL_TRIGGERS_WORKFLOW = 1
301
302	if self.parameters.has_key('subject_field_separator'):
303	self.SUBJECT_FIELD_SEPARATOR = self.parameters['subject_field_separator'].strip()
304	else:
305	self.SUBJECT_FIELD_SEPARATOR = '&'
306
307	self.trac_smtp_from = self.get_config('notification', 'smtp_from')
308
309	self.system = None
310
311	########## Email Header Functions ###########################################################
312
313	def spam(self, message):
314	"""
315	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
316	# Note if Spam_level then '*' are included
317	"""
318	spam = False
319	if message.has_key(self.parameters.spam_header):
320	spam_l = string.split(message[self.parameters.spam_header])
321
322	try:
323	number = spam_l[0].count('*')
324	except IndexError, detail:
325	number = 0
326
327	if number >= self.SPAM_LEVEL:
328	spam = True
329
330	# treat virus mails as spam
331	#
332	elif message.has_key('X-Virus-found'):
333	spam = True
334
335	# How to handle SPAM messages
336	#
337	if self.DROP_SPAM and spam:
338
339	self.logger.info('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' % self.SPAM_LEVEL)
340	return 'drop'
341
342	elif spam:
343
344	return 'Spam'
345	else:
346
347	return False
348
349	def email_header_acl(self, keyword, header_field, default):
350	"""
351	This function wil check if the email address is allowed or denied
352	to send mail to the ticket list
353	"""
354	self.logger.debug('function email_header_acl: %s' %keyword)
355
356	try:
357	mail_addresses = self.parameters[keyword]
358
359	# Check if we have an empty string
360	#
361	if not mail_addresses:
362	return default
363
364	except KeyError, detail:
365	self.logger.debug('%s not defined, all messages are allowed.' %(keyword))
366
367	return default
368
369	mail_addresses = string.split(mail_addresses, ',')
370
371	for entry in mail_addresses:
372	entry = entry.strip()
373	TO_RE = re.compile(entry, re.VERBOSE\|re.IGNORECASE)
374	result = TO_RE.search(header_field)
375	if result:
376	return True
377
378	return False
379
380	def email_header_txt(self, m):
381	"""
382	Display To and CC addresses in description field
383	"""
384	s = ''
385
386	if m['To'] and len(m['To']) > 0:
387	s = "'''To:''' %s\r\n" %(m['To'])
388	if m['Cc'] and len(m['Cc']) > 0:
389	s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
390
391	return self.email_to_unicode(s)
392
393
394	def get_sender_info(self, message):
395	"""
396	Get the default author name and email address from the message
397	"""
398
399	self.email_to = self.email_to_unicode(message['to'])
400	self.to_name, self.to_email_addr = email.Utils.parseaddr (self.email_to)
401
402	self.email_from = self.email_to_unicode(message['from'])
403	self.email_name, self.email_addr = email.Utils.parseaddr(self.email_from)
404
405	## Trac can not handle author's name that contains spaces
406	# and forbid the ticket email address as author field
407
408	if self.email_addr == self.trac_smtp_from:
409	if self.email_name:
410	self.author = self.email_name
411	else:
412	self.author = "email2trac"
413	else:
414	self.author = self.email_addr
415
416	if self.IGNORE_TRAC_USER_SETTINGS:
417	return
418
419	# Is this a registered user, use email address as search key:
420	# result:
421	# u : login name
422	# n : Name that the user has set in the settings tab
423	# e : email address that the user has set in the settings tab
424	#
425	users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
426	if e and (e.lower() == self.email_addr.lower()) ]
427
428	if len(users) == 1:
429	self.email_from = users[0][0]
430	self.author = users[0][0]
431
432	def set_reply_fields(self, ticket, message):
433	"""
434	Set all the right fields for a new ticket
435	"""
436	self.logger.debug('function set_reply_fields')
437
438	## Only use name or email adress
439	#ticket['reporter'] = self.email_from
440	ticket['reporter'] = self.author
441
442
443	# Put all CC-addresses in ticket CC field
444	#
445	if self.REPLY_ALL:
446
447	email_cc = ''
448
449	cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
450
451	if not cc_addrs:
452	return
453
454	## Build a list of forbidden CC addresses
455	#
456	#to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
457	#to_list = list()
458	#for n,e in to_addrs:
459	# to_list.append(e)
460
461	# Always Remove reporter email address from cc-list
462	#
463	try:
464	cc_addrs.remove((self.author, self.email_addr))
465	except ValueError, detail:
466	pass
467
468	for name,addr in cc_addrs:
469
470	## Prevent mail loop
471	#
472	#if addr in to_list:
473
474	if addr == self.trac_smtp_from:
475	self.logger.debug("Skipping %s mail address for CC-field" %(addr))
476	continue
477
478	if email_cc:
479	email_cc = '%s, %s' %(email_cc, addr)
480	else:
481	email_cc = addr
482
483	if email_cc:
484	self.logger.debug('set_reply_fields: %s' %email_cc)
485
486	ticket['cc'] = self.email_to_unicode(email_cc)
487
488
489	########## DEBUG functions ###########################################################
490
491	def debug_body(self, message_body, tempfile=False):
492	if tempfile:
493	import tempfile
494	body_file = tempfile.mktemp('.email2trac')
495	else:
496	body_file = os.path.join(self.TMPDIR, 'body.txt')
497
498	if self.parameters.dry_run:
499	print 'DRY-RUN: not saving body to %s' %(body_file)
500	return
501
502	print 'writing body to %s' %(body_file)
503	fx = open(body_file, 'wb')
504	if not message_body:
505	message_body = '(None)'
506
507	message_body = message_body.encode('utf-8')
508	#message_body = unicode(message_body, 'iso-8859-15')
509
510	fx.write(message_body)
511	fx.close()
512	try:
513	os.chmod(body_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
514	except OSError:
515	pass
516
517	def debug_attachments(self, message_parts):
518	"""
519	"""
520	self.logger.debug('function debug_attachments')
521
522	n = 0
523	for item in message_parts:
524	# Skip inline text parts
525	if not isinstance(item, tuple):
526	continue
527
528	(original, filename, part) = item
529
530	n = n + 1
531	print 'part%d: Content-Type: %s' % (n, part.get_content_type())
532
533	s = 'part%d: filename: %s' %(n, filename)
534	self.print_unicode(s)
535
536	## Forbidden chars
537	#
538	filename = filename.replace('\\', '_')
539	filename = filename.replace('/', '_')
540
541
542	part_file = os.path.join(self.TMPDIR, filename)
543	s = 'writing part%d (%s)' % (n,part_file)
544	self.print_unicode(s)
545
546	if self.parameters.dry_run:
547	print 'DRY_RUN: NOT saving attachments'
548	continue
549
550	part_file = util.text.unicode_quote(part_file)
551
552	fx = open(part_file, 'wb')
553	text = part.get_payload(decode=1)
554
555	if not text:
556	text = '(None)'
557
558	fx.write(text)
559	fx.close()
560
561	try:
562	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
563	except OSError:
564	pass
565
566	def save_email_for_debug(self, message, tempfile=False):
567
568	if tempfile:
569	import tempfile
570	msg_file = tempfile.mktemp('.email2trac')
571	else:
572	#msg_file = '/var/tmp/msg.txt'
573	msg_file = os.path.join(self.TMPDIR, 'msg.txt')
574
575	if self.parameters.dry_run:
576	print 'DRY_RUN: NOT saving email message to %s' %(msg_file)
577	else:
578	print 'saving email to %s' %(msg_file)
579
580	fx = open(msg_file, 'wb')
581	fx.write('%s' % message)
582	fx.close()
583
584	try:
585	os.chmod(msg_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
586	except OSError:
587	pass
588
589	message_parts = self.get_message_parts(message)
590	message_parts = self.unique_attachment_names(message_parts)
591	body_text = self.body_text(message_parts)
592	self.debug_body(body_text, True)
593	self.debug_attachments(message_parts)
594
595	########## Conversion functions ###########################################################
596
597	def email_to_unicode(self, message_str):
598	"""
599	Email has 7 bit ASCII code, convert it to unicode with the charset
600	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
601	understands it.
602	"""
603	self.logger.debug("function email_to_unicode")
604
605	results = email.Header.decode_header(message_str)
606
607	s = None
608	for text,format in results:
609	if format:
610	try:
611	temp = unicode(text, format)
612	except UnicodeError, detail:
613	# This always works
614	#
615	temp = unicode(text, 'iso-8859-15')
616	except LookupError, detail:
617	#text = 'ERROR: Could not find charset: %s, please install' %format
618	#temp = unicode(text, 'iso-8859-15')
619	temp = message_str
620
621	else:
622	temp = string.strip(text)
623	temp = unicode(text, 'iso-8859-15')
624
625	if s:
626	s = '%s %s' %(s, temp)
627	else:
628	s = '%s' %temp
629
630	#s = s.encode('utf-8')
631	return s
632
633	def str_to_dict(self, s):
634	"""
635	Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
636	"""
637	self.logger.debug("function str_to_dict")
638
639	fields = string.split(s, self.SUBJECT_FIELD_SEPARATOR)
640
641	result = dict()
642	for field in fields:
643	try:
644	index, value = string.split(field, '=')
645
646	# We can not change the description of a ticket via the subject
647	# line. The description is the body of the email
648	#
649	if index.lower() in ['description']:
650	continue
651
652	if value:
653	result[index.lower()] = value
654
655	except ValueError:
656	pass
657	return result
658
659	def print_unicode(self,s):
660	"""
661	This function prints unicode strings uif possible else it will quote it
662	"""
663	try:
664	self.logger.debug(s)
665	except UnicodeEncodeError, detail:
666	self.logger.debug(util.text.unicode_quote(s))
667
668	########## TRAC ticket functions ###########################################################
669
670	def check_permission_participants(self, tkt):
671	"""
672	Check if the mailer is allowed to update the ticket
673	"""
674	self.logger.debug('function check_permission_participants')
675
676	if tkt['reporter'].lower() in [self.author, self.email_addr]:
677	self.logger.debug('ALLOW, %s is the ticket reporter' %(self.email_addr))
678
679	return True
680
681	perm = PermissionSystem(self.env)
682	if perm.check_permission('TICKET_MODIFY', self.author):
683	self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
684
685	return True
686
687	else:
688	return False
689
690
691	# Is the updater in the CC?
692	try:
693	cc_list = tkt['cc'].split(',')
694	for cc in cc_list:
695	if self.email_addr.lower() in cc.strip():
696	self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
697
698	return True
699
700	except KeyError:
701	return False
702
703	def check_permission(self, tkt, action):
704	"""
705	check if the reporter has the right permission for the action:
706	- TICKET_CREATE
707	- TICKET_MODIFY
708
709	There are three models:
710	- None : no checking at all
711	- trac : check the permission via trac permission model
712	- email2trac: ....
713	"""
714	self.logger.debug("function check_permission")
715
716	if self.TICKET_PERMISSION_SYSTEM in ['trac']:
717
718	perm = PermissionSystem(self.env)
719	if perm.check_permission(action, self.author):
720	return True
721	else:
722	return False
723
724	elif self.TICKET_PERMISSION_SYSTEM in ['update_restricted_to_participants']:
725	if action in ['TICKET_MODIFY']:
726	return (self.check_permission_participants(tkt))
727	else:
728	return True
729
730	# Default is to allow everybody ticket updates and ticket creation
731	else:
732	return True
733
734
735	def update_ticket_fields(self, ticket, user_dict, use_default=None):
736	"""
737	This will update the ticket fields. It will check if the
738	given fields are known and if the right values are specified
739	It will only update the ticket field value:
740	- If the field is known
741	- If the value supplied is valid for the ticket field.
742	If not then there are two options:
743	1) Skip the value (use_default=None)
744	2) Set default value for field (use_default=1)
745	"""
746	self.logger.debug("function update_ticket_fields")
747
748	# Build a system dictionary from the ticket fields
749	# with field as index and option as value
750	#
751	sys_dict = dict()
752	for field in ticket.fields:
753	try:
754	sys_dict[field['name']] = field['options']
755
756	except KeyError:
757	sys_dict[field['name']] = None
758	pass
759
760	## Check user supplied fields an compare them with the
761	# system one's
762	#
763	for field,value in user_dict.items():
764	if self.parameters.debug:
765	s = 'user_field\t %s = %s' %(field,value)
766	self.print_unicode(s)
767
768	## To prevent mail loop
769	#
770	if field == 'cc':
771
772	cc_list = user_dict['cc'].split(',')
773
774	if self.trac_smtp_from in cc_list:
775	self.logger.debug('MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
776
777	cc_list.remove(self.trac_smtp_from)
778
779	value = ','.join(cc_list)
780
781
782	if sys_dict.has_key(field):
783
784	# Check if value is an allowed system option, if TypeError then
785	# every value is allowed
786	#
787	try:
788	if value in sys_dict[field]:
789	ticket[field] = value
790	else:
791	# Must we set a default if value is not allowed
792	#
793	if use_default:
794	value = self.get_config('ticket', 'default_%s' %(field) )
795
796	except TypeError:
797	pass
798
799	## Only set if we have a value
800	#
801	if value:
802	ticket[field] = value
803
804	if self.parameters.debug:
805	s = 'ticket_field\t %s = %s' %(field, ticket[field])
806	self.print_unicode(s)
807
808	def ticket_update(self, m, id, spam):
809	"""
810	If the current email is a reply to an existing ticket, this function
811	will append the contents of this email to that ticket, instead of
812	creating a new one.
813	"""
814	self.logger.debug("function ticket_update")
815
816	# Must we update ticket fields
817	#
818	update_fields = dict()
819	try:
820	id, keywords = string.split(id, '?')
821
822	update_fields = self.str_to_dict(keywords)
823
824	# Strip '#'
825	#
826	self.id = int(id[1:])
827
828	except ValueError:
829
830	# Strip '#'
831	#
832	self.id = int(id[1:])
833
834	self.logger.debug("function ticket_update id %s" %id)
835
836	# When is the change committed
837	#
838	if self.VERSION < 0.11:
839	when = int(time.time())
840	else:
841	when = datetime.now(util.datefmt.utc)
842
843	try:
844	tkt = Ticket(self.env, self.id, self.db)
845
846	except util.TracError, detail:
847
848	# Not a valid ticket
849
850	self.id = None
851	return False
852
853	# Check the permission of the reporter
854	#
855	if self.TICKET_PERMISSION_SYSTEM:
856	if not self.check_permission(tkt, 'TICKET_MODIFY'):
857	self.logger.info('Reporter: %s has no permission to modify tickets' %self.author)
858	return False
859
860	# How many changes has this ticket
861	cnum = len(tkt.get_changelog())
862
863
864	# reopen the ticket if it is was closed
865	# We must use the ticket workflow framework
866	#
867	if tkt['status'] in ['closed'] and self.EMAIL_TRIGGERS_WORKFLOW:
868
869	#print controller.actions['reopen']
870	#
871	# As reference
872	# req = Mock(href=Href('/'), abs_href=Href('http://www.example.com/'), authname='anonymous', perm=MockPerm(), args={})
873	#
874	#a = controller.render_ticket_action_control(req, tkt, 'reopen')
875	#print 'controller : ', a
876	#
877	#b = controller.get_all_status()
878	#print 'get all status: ', b
879	#
880	#b = controller.get_ticket_changes(req, tkt, 'reopen')
881	#print 'get_ticket_changes :', b
882
883	if self.WORKFLOW and (self.VERSION >= 0.11 ) :
884	from trac.ticket.default_workflow import ConfigurableTicketWorkflow
885	from trac.test import Mock, MockPerm
886
887	req = Mock(authname='anonymous', perm=MockPerm(), args={})
888
889	controller = ConfigurableTicketWorkflow(self.env)
890	fields = controller.get_ticket_changes(req, tkt, self.WORKFLOW)
891
892	self.logger.debug('Workflow ticket update fields: ')
893
894	for key in fields.keys():
895	self.logger.debug('\t %s : %s' %(key, fields[key]))
896	tkt[key] = fields[key]
897
898	else:
899	tkt['status'] = 'reopened'
900	tkt['resolution'] = ''
901
902	# Must we update some ticket fields properties via subjectline
903	#
904	if update_fields:
905	self.update_ticket_fields(tkt, update_fields)
906
907	message_parts = self.get_message_parts(m)
908	message_parts = self.unique_attachment_names(message_parts)
909
910	# Must we update some ticket fields properties via body_text
911	#
912	if self.properties:
913	self.update_ticket_fields(tkt, self.properties)
914
915	if self.EMAIL_HEADER:
916	message_parts.insert(0, self.email_header_txt(m))
917
918	body_text = self.body_text(message_parts)
919
920	error_with_attachments = self.attach_attachments(message_parts)
921
922	if body_text.strip() or update_fields or self.properties:
923	if self.parameters.dry_run:
924	print 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) ', self.author, cnum
925	else:
926	if error_with_attachments:
927	body_text = '%s\\%s' %(error_with_attachments, body_text)
928	self.logger.debug('tkt.save_changes(%s, %d)' %(self.author, cnum))
929	tkt.save_changes(self.author, body_text, when, None, str(cnum))
930
931
932	if not spam:
933	self.notify(tkt, False, when)
934
935	return True
936
937	def set_ticket_fields(self, ticket):
938	"""
939	set the ticket fields to value specified
940	- /etc/email2trac.conf with <prefix>_<field>
941	- trac default values, trac.ini
942	"""
943	self.logger.debug('function set_ticket_fields')
944
945	user_dict = dict()
946
947	for field in ticket.fields:
948
949	name = field['name']
950
951	## default trac value
952	#
953	if not field.get('custom'):
954	value = self.get_config('ticket', 'default_%s' %(name) )
955	else:
956	## Else we get the default value for reporter
957	#
958	value = field.get('value')
959	options = field.get('options')
960
961	if value and options and (value not in options):
962	value = options[int(value)]
963
964	if self.parameters.debug:
965	s = 'trac[%s] = %s' %(name, value)
966	self.print_unicode(s)
967
968	## email2trac.conf settings
969	#
970	prefix = self.parameters['ticket_prefix']
971	try:
972	value = self.parameters['%s_%s' %(prefix, name)]
973	if self.parameters.debug > 10:
974	s = 'email2trac[%s] = %s ' %(name, value)
975	self.print_unicode(s)
976
977	except KeyError, detail:
978	pass
979
980	if self.parameters.debug:
981	s = 'used %s = %s' %(name, value)
982	self.print_unicode(s)
983
984	if value:
985	user_dict[name] = value
986
987	self.update_ticket_fields(ticket, user_dict, use_default=1)
988
989	if 'status' not in user_dict.keys():
990	ticket['status'] = 'new'
991
992
993	def ticket_update_by_subject(self, subject):
994	"""
995	This list of Re: prefixes is probably incomplete. Taken from
996	wikipedia. Here is how the subject is matched
997	- Re: <subject>
998	- Re: (<Mail list label>:)+ <subject>
999
1000	So we must have the last column
1001	"""
1002	self.logger.debug('function ticket_update_by_subject')
1003
1004	matched_id = None
1005	if self.TICKET_UPDATE and self.TICKET_UPDATE_BY_SUBJECT:
1006
1007	SUBJECT_RE = re.compile(r'^(RE\|AW\|VS\|SV):(.:)\s(.)', re.IGNORECASE)
1008	result = SUBJECT_RE.search(subject)
1009
1010	if result:
1011	# This is a reply
1012	orig_subject = result.group(3)
1013
1014	self.logger.debug('subject search string: %s' %(orig_subject))
1015
1016	cursor = self.db.cursor()
1017	summaries = [orig_subject, '%%: %s' % orig_subject]
1018
1019	##
1020	# Convert days to seconds
1021	lookback = int(time.mktime(time.gmtime())) - \
1022	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK * 24 * 3600
1023
1024
1025	for summary in summaries:
1026	self.logger.debug('Looking for summary matching: "%s"' % summary)
1027
1028	sql = """SELECT id FROM ticket
1029	WHERE changetime >= %s AND summary LIKE %s
1030	ORDER BY changetime DESC"""
1031	cursor.execute(sql, [lookback, summary.strip()])
1032
1033	for row in cursor:
1034	(matched_id,) = row
1035
1036	self.logger.debug('Found matching ticket id: %d' % matched_id)
1037
1038	break
1039
1040	if matched_id:
1041	matched_id = '#%d' % matched_id
1042	return matched_id
1043
1044	return matched_id
1045
1046
1047	def new_ticket(self, msg, subject, spam, set_fields = None):
1048	"""
1049	Create a new ticket
1050	"""
1051	self.logger.debug('function new_ticket')
1052
1053	tkt = Ticket(self.env)
1054
1055	self.set_reply_fields(tkt, msg)
1056
1057	self.set_ticket_fields(tkt)
1058
1059	# Check the permission of the reporter
1060	#
1061	if self.TICKET_PERMISSION_SYSTEM:
1062	if not self.check_permission(tkt, 'TICKET_CREATE'):
1063	self.logger.info('Reporter: %s has no permission to create tickets' %self.author)
1064	return False
1065
1066	# Old style setting for component, will be removed
1067	#
1068	if spam:
1069	tkt['component'] = 'Spam'
1070
1071	elif self.parameters.has_key('component'):
1072	tkt['component'] = self.parameters['component']
1073
1074	if not msg['Subject']:
1075	tkt['summary'] = u'(No subject)'
1076	else:
1077	tkt['summary'] = subject
1078
1079
1080	if set_fields:
1081	rest, keywords = string.split(set_fields, '?')
1082
1083	if keywords:
1084	update_fields = self.str_to_dict(keywords)
1085	self.update_ticket_fields(tkt, update_fields)
1086
1087	# produce e-mail like header
1088	#
1089	head = ''
1090	if self.EMAIL_HEADER > 0:
1091	head = self.email_header_txt(msg)
1092
1093	message_parts = self.get_message_parts(msg)
1094
1095	# Must we update some ticket fields properties via body_text
1096	#
1097	if self.properties:
1098	self.update_ticket_fields(tkt, self.properties)
1099
1100	message_parts = self.unique_attachment_names(message_parts)
1101
1102	if self.EMAIL_HEADER > 0:
1103	message_parts.insert(0, self.email_header_txt(msg))
1104
1105	body_text = self.body_text(message_parts)
1106
1107	tkt['description'] = body_text
1108
1109	# When is the change committed
1110	#
1111	if self.VERSION < 0.11:
1112	when = int(time.time())
1113	else:
1114	when = datetime.now(util.datefmt.utc)
1115
1116	if self.parameters.dry_run:
1117	print 'DRY_RUN: tkt.insert()'
1118	else:
1119	self.id = tkt.insert()
1120
1121	changed = False
1122	comment = ''
1123
1124	# some routines in trac are dependend on ticket id
1125	# like alternate notify template
1126	#
1127	if self.notify_template:
1128	tkt['id'] = self.id
1129	changed = True
1130
1131	## Rewrite the description if we have mailto enabled
1132	#
1133	if self.parameters.mailto_link:
1134	changed = True
1135	comment = u'\nadded mailto line\n'
1136	mailto = self.html_mailto_link( m['Subject'])
1137
1138	tkt['description'] = u'%s\r\n%s%s\r\n' \
1139	%(head, mailto, body_text)
1140
1141	## Save the attachments to the ticket
1142	#
1143	error_with_attachments = self.attach_attachments(message_parts)
1144
1145	if error_with_attachments:
1146	changed = True
1147	comment = '%s\n%s\n' %(comment, error_with_attachments)
1148
1149	if changed:
1150	if self.parameters.dry_run:
1151	print 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
1152	else:
1153	tkt.save_changes(tkt['reporter'], comment)
1154	#print tkt.get_changelog(self.db, when)
1155
1156	if not spam:
1157	self.notify(tkt, True)
1158
1159
1160	def attach_attachments(self, message_parts, update=False):
1161	'''
1162	save any attachments as files in the ticket's directory
1163	'''
1164	self.logger.debug('function attach_attachments()')
1165
1166	if self.parameters.dry_run:
1167	print "DRY_RUN: no attachments attached to tickets"
1168	return ''
1169
1170	count = 0
1171
1172	# Get Maxium attachment size
1173	#
1174	max_size = int(self.get_config('attachment', 'max_size'))
1175	status = None
1176
1177	for item in message_parts:
1178	# Skip body parts
1179	if not isinstance(item, tuple):
1180	continue
1181
1182	(original, filename, part) = item
1183	#
1184	# We have to determine the size so we use this temporary solution. we must escape it
1185	# else we get UnicodeErrors.
1186	#
1187	path, fd = util.create_unique_file(os.path.join(self.TMPDIR, util.text.unicode_quote(filename)))
1188	text = part.get_payload(decode=1)
1189	if not text:
1190	text = '(None)'
1191	fd.write(text)
1192	fd.close()
1193
1194	# get the file_size
1195	#
1196	stats = os.lstat(path)
1197	file_size = stats[ST_SIZE]
1198
1199	# Check if the attachment size is allowed
1200	#
1201	if (max_size != -1) and (file_size > max_size):
1202	status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1203	%(status, original, file_size, max_size)
1204
1205	os.unlink(path)
1206	continue
1207	else:
1208	count = count + 1
1209
1210	# Insert the attachment
1211	#
1212	fd = open(path, 'rb')
1213	if self.system == 'discussion':
1214	att = attachment.Attachment(self.env, 'discussion', 'topic/%s'
1215	% (self.id,))
1216	else:
1217	self.logger.debug('Attach %s to ticket %d' %(util.text.unicode_quote(filename), self.id))
1218	att = attachment.Attachment(self.env, 'ticket', self.id)
1219
1220	# This will break the ticket_update system, the body_text is vaporized
1221	# ;-(
1222	#
1223	if not update:
1224	att.author = self.author
1225	att.description = self.email_to_unicode('Added by email2trac')
1226
1227	try:
1228	self.logger.debug('Insert atachment')
1229	att.insert(filename, fd, file_size)
1230	except OSError, detail:
1231	self.logger.info('%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail))
1232	status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
1233
1234	# Remove the created temporary filename
1235	#
1236	fd.close()
1237	os.unlink(path)
1238
1239	## return error
1240	#
1241	return status
1242
1243	########## Fullblog functions #################################################
1244
1245	def blog(self, id):
1246	"""
1247	The blog create/update function
1248	"""
1249	# import the modules
1250	#
1251	from tracfullblog.core import FullBlogCore
1252	from tracfullblog.model import BlogPost, BlogComment
1253	from trac.test import Mock, MockPerm
1254
1255	# instantiate blog core
1256	blog = FullBlogCore(self.env)
1257	req = Mock(authname='anonymous', perm=MockPerm(), args={})
1258
1259	if id:
1260
1261	# update blog
1262	#
1263	comment = BlogComment(self.env, id)
1264	comment.author = self.author
1265
1266	message_parts = self.get_message_parts(m)
1267	comment.comment = self.body_text(message_parts)
1268
1269	blog.create_comment(req, comment)
1270
1271	else:
1272	# create blog
1273	#
1274	import time
1275	post = BlogPost(self.env, 'blog_'+time.strftime("%Y%m%d%H%M%S", time.gmtime()))
1276
1277	#post = BlogPost(self.env, blog._get_default_postname(self.env))
1278
1279	post.author = self.author
1280	post.title = self.email_to_unicode(m['Subject'])
1281
1282	message_parts = self.get_message_parts(m)
1283	post.body = self.body_text(message_parts)
1284
1285	blog.create_post(req, post, self.author, u'Created by email2trac', False)
1286
1287
1288	########## Discussion functions ##############################################
1289
1290	def discussion_topic(self, content, subject):
1291
1292	# Import modules.
1293	from tracdiscussion.api import DiscussionApi
1294	from trac.util.datefmt import to_timestamp, utc
1295
1296	self.logger.debug('Creating a new topic in forum:', self.id)
1297
1298	# Get dissussion API component.
1299	api = self.env[DiscussionApi]
1300	context = self._create_context(content, subject)
1301
1302	# Get forum for new topic.
1303	forum = api.get_forum(context, self.id)
1304
1305	if not forum:
1306	self.logger.error("ERROR: Replied forum doesn't exist")
1307
1308	# Prepare topic.
1309	topic = {'forum' : forum['id'],
1310	'subject' : context.subject,
1311	'time': to_timestamp(datetime.now(utc)),
1312	'author' : self.author,
1313	'subscribers' : [self.email_addr],
1314	'body' : self.body_text(context.content_parts)}
1315
1316	# Add topic to DB and commit it.
1317	self._add_topic(api, context, topic)
1318	self.db.commit()
1319
1320	def discussion_topic_reply(self, content, subject):
1321
1322	# Import modules.
1323	from tracdiscussion.api import DiscussionApi
1324	from trac.util.datefmt import to_timestamp, utc
1325
1326	self.logger.debug('Replying to discussion topic', self.id)
1327
1328	# Get dissussion API component.
1329	api = self.env[DiscussionApi]
1330	context = self._create_context(content, subject)
1331
1332	# Get replied topic.
1333	topic = api.get_topic(context, self.id)
1334
1335	if not topic:
1336	self.logger.error("ERROR: Replied topic doesn't exist")
1337
1338	# Prepare message.
1339	message = {'forum' : topic['forum'],
1340	'topic' : topic['id'],
1341	'replyto' : -1,
1342	'time' : to_timestamp(datetime.now(utc)),
1343	'author' : self.author,
1344	'body' : self.body_text(context.content_parts)}
1345
1346	# Add message to DB and commit it.
1347	self._add_message(api, context, message)
1348	self.db.commit()
1349
1350	def discussion_message_reply(self, content, subject):
1351
1352	# Import modules.
1353	from tracdiscussion.api import DiscussionApi
1354	from trac.util.datefmt import to_timestamp, utc
1355
1356	self.loggger.debug('Replying to discussion message', self.id)
1357
1358	# Get dissussion API component.
1359	api = self.env[DiscussionApi]
1360	context = self._create_context(content, subject)
1361
1362	# Get replied message.
1363	message = api.get_message(context, self.id)
1364
1365	if not message:
1366	self.logger.error("ERROR: Replied message doesn't exist")
1367
1368	# Prepare message.
1369	message = {'forum' : message['forum'],
1370	'topic' : message['topic'],
1371	'replyto' : message['id'],
1372	'time' : to_timestamp(datetime.now(utc)),
1373	'author' : self.author,
1374	'body' : self.body_text(context.content_parts)}
1375
1376	# Add message to DB and commit it.
1377	self._add_message(api, context, message)
1378	self.db.commit()
1379
1380	def _create_context(self, content, subject):
1381
1382	# Import modules.
1383	from trac.mimeview import Context
1384	from trac.web.api import Request
1385	from trac.perm import PermissionCache
1386
1387	# TODO: Read server base URL from config.
1388	# Create request object to mockup context creation.
1389	#
1390	environ = {'SERVER_PORT' : 80,
1391	'SERVER_NAME' : 'test',
1392	'REQUEST_METHOD' : 'POST',
1393	'wsgi.url_scheme' : 'http',
1394	'wsgi.input' : sys.stdin}
1395	chrome = {'links': {},
1396	'scripts': [],
1397	'ctxtnav': [],
1398	'warnings': [],
1399	'notices': []}
1400
1401	if self.env.base_url_for_redirect:
1402	environ['trac.base_url'] = self.env.base_url
1403
1404	req = Request(environ, None)
1405	req.chrome = chrome
1406	req.tz = 'missing'
1407	req.authname = self.author
1408	req.perm = PermissionCache(self.env, self.author)
1409
1410	# Create and return context.
1411	context = Context.from_request(req)
1412	context.realm = 'discussion-email2trac'
1413	context.cursor = self.db.cursor()
1414	context.content = content
1415	context.subject = subject
1416
1417	# Read content parts from content.
1418	context.content_parts = self.get_message_parts(content)
1419	context.content_parts = self.unique_attachment_names(
1420	context.content_parts)
1421
1422	return context
1423
1424	def _add_topic(self, api, context, topic):
1425	context.req.perm.assert_permission('DISCUSSION_APPEND')
1426
1427	# Filter topic.
1428	for discussion_filter in api.discussion_filters:
1429	accept, topic_or_error = discussion_filter.filter_topic(
1430	context, topic)
1431	if accept:
1432	topic = topic_or_error
1433	else:
1434	raise TracError(topic_or_error)
1435
1436	# Add a new topic.
1437	api.add_topic(context, topic)
1438
1439	# Get inserted topic with new ID.
1440	topic = api.get_topic_by_time(context, topic['time'])
1441
1442	# Attach attachments.
1443	self.id = topic['id']
1444	self.attach_attachments(context.content_parts, True)
1445
1446	# Notify change listeners.
1447	for listener in api.topic_change_listeners:
1448	listener.topic_created(context, topic)
1449
1450	def _add_message(self, api, context, message):
1451	context.req.perm.assert_permission('DISCUSSION_APPEND')
1452
1453	# Filter message.
1454	for discussion_filter in api.discussion_filters:
1455	accept, message_or_error = discussion_filter.filter_message(
1456	context, message)
1457	if accept:
1458	message = message_or_error
1459	else:
1460	raise TracError(message_or_error)
1461
1462	# Add message.
1463	api.add_message(context, message)
1464
1465	# Get inserted message with new ID.
1466	message = api.get_message_by_time(context, message['time'])
1467
1468	# Attach attachments.
1469	self.id = message['topic']
1470	self.attach_attachments(context.content_parts, True)
1471
1472	# Notify change listeners.
1473	for listener in api.message_change_listeners:
1474	listener.message_created(context, message)
1475
1476	########## MAIN function ######################################################
1477
1478	def parse(self, fp):
1479	"""
1480	"""
1481	self.logger.debug('Main function parse')
1482	global m
1483
1484	m = email.message_from_file(fp)
1485
1486	if not m:
1487	self.logger.debug('This is not a valid email message format')
1488	return
1489
1490	# Work around lack of header folding in Python; see http://bugs.python.org/issue4696
1491	try:
1492	m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
1493	except AttributeError, detail:
1494	pass
1495
1496	if self.parameters.debug: # save the entire e-mail message text
1497	self.save_email_for_debug(m, True)
1498
1499	self.db = self.env.get_db_cnx()
1500	self.get_sender_info(m)
1501
1502	if not self.email_header_acl('white_list', self.email_addr, True):
1503	self.logger.info('Message rejected : %s not in white list' %(self.email_addr))
1504	return False
1505
1506	if self.email_header_acl('black_list', self.email_addr, False):
1507	self.logger.info('Message rejected : %s in black list' %(self.email_addr))
1508	return False
1509
1510	if not self.email_header_acl('recipient_list', self.to_email_addr, True):
1511	self.logger.info('Message rejected : %s not in recipient list' %(self.to_email_addr))
1512	return False
1513
1514	# If spam drop the message
1515	#
1516	if self.spam(m) == 'drop':
1517	return False
1518
1519	elif self.spam(m) == 'spam':
1520	spam_msg = True
1521	else:
1522	spam_msg = False
1523
1524	if not m['Subject']:
1525	subject = 'No Subject'
1526	else:
1527	subject = self.email_to_unicode(m['Subject'])
1528
1529	self.logger.debug('subject: %s' %subject)
1530
1531	#
1532	# [hic] #1529: Re: LRZ
1533	# [hic] #1529?owner=bas,priority=medium: Re: LRZ
1534	#
1535	ticket_regex = r'''
1536	(?P<new_fields>[#][?].*)
1537	\|(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?.*)?:)
1538	'''
1539	# Check if FullBlogPlugin is installed
1540	#
1541	blog_enabled = None
1542	blog_regex = ''
1543	if self.get_config('components', 'tracfullblog.*') in ['enabled']:
1544	blog_enabled = True
1545	blog_regex = '''\|(?P<blog>blog:(?P<blog_id>\w*))'''
1546
1547
1548	# Check if DiscussionPlugin is installed
1549	#
1550	discussion_enabled = None
1551	discussion_regex = ''
1552	if self.get_config('components', 'tracdiscussion.api.*') in ['enabled']:
1553	discussion_enabled = True
1554	discussion_regex = r'''
1555	\|(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
1556	\|(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
1557	\|(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
1558	'''
1559
1560
1561	regex_str = ticket_regex + blog_regex + discussion_regex
1562	SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
1563
1564	# Find out if this is a ticket, a blog or a discussion
1565	#
1566	result = SYSTEM_RE.search(subject)
1567
1568	if result:
1569	# update ticket + fields
1570	#
1571	if result.group('reply') and self.TICKET_UPDATE:
1572	self.system = 'ticket'
1573
1574	# Skip the last ':' character
1575	#
1576	if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
1577	self.new_ticket(m, subject, spam_msg)
1578
1579	# New ticket + fields
1580	#
1581	elif result.group('new_fields'):
1582	self.system = 'ticket'
1583	self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
1584
1585	if blog_enabled:
1586	if result.group('blog'):
1587	self.system = 'blog'
1588	self.blog(result.group('blog_id'))
1589
1590	if discussion_enabled:
1591	# New topic.
1592	#
1593	if result.group('forum'):
1594	self.system = 'discussion'
1595	self.id = int(result.group('forum_id'))
1596	self.discussion_topic(m, subject[result.end('forum'):])
1597
1598	# Reply to topic.
1599	#
1600	elif result.group('topic'):
1601	self.system = 'discussion'
1602	self.id = int(result.group('topic_id'))
1603	self.discussion_topic_reply(m, subject[result.end('topic'):])
1604
1605	# Reply to topic message.
1606	#
1607	elif result.group('message'):
1608	self.system = 'discussion'
1609	self.id = int(result.group('message_id'))
1610	self.discussion_message_reply(m, subject[result.end('message'):])
1611
1612	else:
1613	self.system = 'ticket'
1614	result = self.ticket_update_by_subject(subject)
1615	if result:
1616	if not self.ticket_update(m, result, spam_msg):
1617	self.new_ticket(m, subject, spam_msg)
1618	else:
1619	# No update by subject, so just create a new ticket
1620	self.new_ticket(m, subject, spam_msg)
1621
1622
1623	########## BODY TEXT functions ###########################################################
1624
1625	def strip_signature(self, text):
1626	"""
1627	Strip signature from message, inspired by Mailman software
1628	"""
1629	body = []
1630	for line in text.splitlines():
1631	if line == '-- ':
1632	break
1633	body.append(line)
1634
1635	return ('\n'.join(body))
1636
1637	def reflow(self, text, delsp = 0):
1638	"""
1639	Reflow the message based on the format="flowed" specification (RFC 3676)
1640	"""
1641	flowedlines = []
1642	quotelevel = 0
1643	prevflowed = 0
1644
1645	for line in text.splitlines():
1646	from re import match
1647
1648	# Figure out the quote level and the content of the current line
1649	m = match('(>)( ?)(.)', line)
1650	linequotelevel = len(m.group(1))
1651	line = m.group(3)
1652
1653	# Determine whether this line is flowed
1654	if line and line != '-- ' and line[-1] == ' ':
1655	flowed = 1
1656	else:
1657	flowed = 0
1658
1659	if flowed and delsp and line and line[-1] == ' ':
1660	line = line[:-1]
1661
1662	# If the previous line is flowed, append this line to it
1663	if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1664	flowedlines[-1] += line
1665	# Otherwise, start a new line
1666	else:
1667	flowedlines.append('>' * linequotelevel + line)
1668
1669	prevflowed = flowed
1670
1671
1672	return '\n'.join(flowedlines)
1673
1674	def strip_quotes(self, text):
1675	"""
1676	Strip quotes from message by Nicolas Mendoza
1677	"""
1678	body = []
1679	for line in text.splitlines():
1680	if line.startswith(self.EMAIL_QUOTE):
1681	continue
1682	body.append(line)
1683
1684	return ('\n'.join(body))
1685
1686	def inline_properties(self, text):
1687	"""
1688	Parse text if we use inline keywords to set ticket fields
1689	"""
1690	self.logger.debug('function inline_properties')
1691
1692	properties = dict()
1693	body = list()
1694
1695	INLINE_EXP = re.compile('\s[@]\s([a-zA-Z]+)\s:(.)$')
1696
1697	for line in text.splitlines():
1698	match = INLINE_EXP.match(line)
1699	if match:
1700	keyword, value = match.groups()
1701	self.properties[keyword] = value.strip()
1702
1703	self.logger.debug('inline properties: %s : %s' %(keyword,value))
1704
1705	else:
1706	body.append(line)
1707
1708	return '\n'.join(body)
1709
1710
1711	def wrap_text(self, text, replace_whitespace = False):
1712	"""
1713	Will break a lines longer then given length into several small
1714	lines of size given length
1715	"""
1716	import textwrap
1717
1718	LINESEPARATOR = '\n'
1719	reformat = ''
1720
1721	for s in text.split(LINESEPARATOR):
1722	tmp = textwrap.fill(s,self.USE_TEXTWRAP)
1723	if tmp:
1724	reformat = '%s\n%s' %(reformat,tmp)
1725	else:
1726	reformat = '%s\n' %reformat
1727
1728	return reformat
1729
1730	# Python2.4 and higher
1731	#
1732	#return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
1733	#
1734
1735	########## EMAIL attachements functions ###########################################################
1736
1737	def inline_part(self, part):
1738	"""
1739	"""
1740	self.logger.debug('function inline_part()')
1741
1742	return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
1743
1744	def get_message_parts(self, msg):
1745	"""
1746	parses the email message and returns a list of body parts and attachments
1747	body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
1748	"""
1749	self.logger.debug('function get_message_parts()')
1750
1751	message_parts = list()
1752
1753	ALTERNATIVE_MULTIPART = False
1754
1755	for part in msg.walk():
1756	self.logger.debug('Message part: Main-Type: %s' % part.get_content_maintype())
1757	self.logger.debug('Message part: Content-Type: %s' % part.get_content_type())
1758
1759	## Check content type
1760	#
1761	if part.get_content_type() in self.STRIP_CONTENT_TYPES:
1762	self.logger.debug("A %s attachment named '%s' was skipped" %(part.get_content_type(), part.get_filename()))
1763	continue
1764
1765	## Catch some mulitpart execptions
1766	#
1767	if part.get_content_type() == 'multipart/alternative':
1768	ALTERNATIVE_MULTIPART = True
1769	continue
1770
1771	## Skip multipart containers
1772	#
1773	if part.get_content_maintype() == 'multipart':
1774	self.logger.debug("Skipping multipart container")
1775
1776	continue
1777
1778	## Check if this is an inline part. It's inline if there is co Cont-Disp header, or if there is one and it says "inline"
1779	#
1780	inline = self.inline_part(part)
1781
1782	## Drop HTML message
1783	#
1784	if ALTERNATIVE_MULTIPART and self.DROP_ALTERNATIVE_HTML_VERSION:
1785	if part.get_content_type() == 'text/html':
1786	self.logger.debug('Skipping alternative HTML message')
1787	ALTERNATIVE_MULTIPART = False
1788	continue
1789
1790	## Inline text parts are where the body is
1791	#
1792	if part.get_content_type() == 'text/plain' and inline:
1793	self.logger.debug(' Inline body part')
1794
1795	# Try to decode, if fails then do not decode
1796	#
1797	body_text = part.get_payload(decode=1)
1798	if not body_text:
1799	body_text = part.get_payload(decode=0)
1800
1801	format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
1802	delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
1803
1804	if self.REFLOW and not self.VERBATIM_FORMAT and format == 'flowed':
1805	body_text = self.reflow(body_text, delsp == 'yes')
1806
1807	if self.STRIP_SIGNATURE:
1808	body_text = self.strip_signature(body_text)
1809
1810	if self.STRIP_QUOTES:
1811	body_text = self.strip_quotes(body_text)
1812
1813	if self.INLINE_PROPERTIES:
1814	body_text = self.inline_properties(body_text)
1815
1816	if self.USE_TEXTWRAP:
1817	body_text = self.wrap_text(body_text)
1818
1819	## Get contents charset (iso-8859-15 if not defined in mail headers)
1820	#
1821	charset = part.get_content_charset()
1822	if not charset:
1823	charset = 'iso-8859-15'
1824
1825	try:
1826	ubody_text = unicode(body_text, charset)
1827
1828	except UnicodeError, detail:
1829	ubody_text = unicode(body_text, 'iso-8859-15')
1830
1831	except LookupError, detail:
1832	ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
1833
1834	if self.VERBATIM_FORMAT:
1835	message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
1836	else:
1837	message_parts.append('%s' %ubody_text)
1838	else:
1839	if self.parameters.debug:
1840	s = ' Filename: %s' % part.get_filename()
1841	self.print_unicode(s)
1842
1843	##
1844	# First try to use email header function to convert filename.
1845	# If this fails the use the plan filename
1846	try:
1847	filename = self.email_to_unicode(part.get_filename())
1848	except UnicodeEncodeError, detail:
1849	filename = part.get_filename()
1850
1851	message_parts.append((filename, part))
1852
1853	return message_parts
1854
1855	def unique_attachment_names(self, message_parts):
1856	"""
1857	"""
1858	renamed_parts = []
1859	attachment_names = set()
1860
1861	for item in message_parts:
1862
1863	## If not an attachment, leave it alone
1864	#
1865	if not isinstance(item, tuple):
1866	renamed_parts.append(item)
1867	continue
1868
1869	(filename, part) = item
1870
1871	## If no filename, use a default one
1872	#
1873	if not filename:
1874	filename = 'untitled-part'
1875
1876	# Guess the extension from the content type, use non strict mode
1877	# some additional non-standard but commonly used MIME types
1878	# are also recognized
1879	#
1880	ext = mimetypes.guess_extension(part.get_content_type(), False)
1881	if not ext:
1882	ext = '.bin'
1883
1884	filename = '%s%s' % (filename, ext)
1885
1886	## Discard relative paths for windows/unix in attachment names
1887	#
1888	#filename = filename.replace('\\', '/').replace(':', '/')
1889	filename = filename.replace('\\', '_')
1890	filename = filename.replace('/', '_')
1891
1892	#
1893	# We try to normalize the filename to utf-8 NFC if we can.
1894	# Files uploaded from OS X might be in NFD.
1895	# Check python version and then try it
1896	#
1897	#if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
1898	# try:
1899	# filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
1900	# except TypeError:
1901	# pass
1902
1903	# Make the filename unique for this ticket
1904	num = 0
1905	unique_filename = filename
1906	dummy_filename, ext = os.path.splitext(filename)
1907
1908	while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
1909	num += 1
1910	unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
1911
1912	if self.parameters.debug:
1913	s = 'Attachment with filename %s will be saved as %s' % (filename, unique_filename)
1914	self.print_unicode(s)
1915
1916	attachment_names.add(unique_filename)
1917
1918	renamed_parts.append((filename, unique_filename, part))
1919
1920	return renamed_parts
1921
1922
1923	def attachment_exists(self, filename):
1924
1925	if self.parameters.debug:
1926	s = 'attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
1927	self.print_unicode(s)
1928
1929	# We have no valid ticket id
1930	#
1931	if not self.id:
1932	return False
1933
1934	try:
1935	if self.system == 'discussion':
1936	att = attachment.Attachment(self.env, 'discussion', 'ticket/%s'
1937	% (self.id,), filename)
1938	else:
1939	att = attachment.Attachment(self.env, 'ticket', self.id,
1940	filename)
1941	return True
1942	except attachment.ResourceNotFound:
1943	return False
1944
1945	########## TRAC Ticket Text ###########################################################
1946
1947	def body_text(self, message_parts):
1948	body_text = []
1949
1950	for part in message_parts:
1951	# Plain text part, append it
1952	if not isinstance(part, tuple):
1953	body_text.extend(part.strip().splitlines())
1954	body_text.append("")
1955	continue
1956
1957	(original, filename, part) = part
1958	inline = self.inline_part(part)
1959
1960	if part.get_content_maintype() == 'image' and inline:
1961	if self.system != 'discussion':
1962	body_text.append('[[Image(%s)]]' % filename)
1963	body_text.append("")
1964	else:
1965	if self.system != 'discussion':
1966	body_text.append('[attachment:"%s"]' % filename)
1967	body_text.append("")
1968
1969	body_text = '\r\n'.join(body_text)
1970	return body_text
1971
1972	def html_mailto_link(self, subject):
1973	"""
1974	This function returns a HTML mailto tag with the ticket id and author email address
1975	"""
1976	if not self.author:
1977	author = self.email_addr
1978	else:
1979	author = self.author
1980
1981	if not self.parameters.mailto_cc:
1982	self.parameters.mailto_cc = ''
1983
1984	# use urllib to escape the chars
1985	#
1986	s = 'mailto:%s?Subject=%s&Cc=%s' %(
1987	urllib.quote(self.email_addr),
1988	urllib.quote('Re: #%s: %s' %(self.id, subject)),
1989	urllib.quote(self.parameters.mailto_cc)
1990	)
1991
1992	s = '\r\n{{{\r\n#!html\r\n<a\r\n href="%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
1993	return s
1994
1995	########## TRAC notify section ###########################################################
1996
1997	def notify(self, tkt, new=True, modtime=0):
1998	"""
1999	A wrapper for the TRAC notify function. So we can use templates
2000	"""
2001	self.logger.debug('function notify()')
2002
2003	if self.parameters.dry_run:
2004	print 'DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter']
2005	return
2006	try:
2007
2008	#from trac.ticket.web_ui import TicketModule
2009	#from trac.ticket.notification import TicketNotificationSystem
2010	#ticket_sys = TicketNotificationSystem(self.env)
2011	#a = TicketModule(self.env)
2012	#print a.__dict__
2013	#tn_sys = TicketNotificationSystem(self.env)
2014	#print tn_sys
2015	#print tn_sys.__dict__
2016	#sys.exit(0)
2017
2018	# create false {abs_}href properties, to trick Notify()
2019	#
2020	if not (self.VERSION in [0.11, 0.12]):
2021	self.env.abs_href = Href(self.get_config('project', 'url'))
2022	self.env.href = Href(self.get_config('project', 'url'))
2023
2024
2025	tn = TicketNotifyEmail(self.env)
2026
2027	if self.notify_template:
2028
2029	if self.VERSION >= 0.11:
2030
2031	from trac.web.chrome import Chrome
2032
2033	if self.notify_template_update and not new:
2034	tn.template_name = self.notify_template_update
2035	else:
2036	tn.template_name = self.notify_template
2037
2038	tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
2039
2040	else:
2041
2042	tn.template_name = self.notify_template;
2043
2044	tn.notify(tkt, new, modtime)
2045
2046	except Exception, e:
2047	self.logger.error('Failure sending notification on creation of ticket #%s: %s' %(self.id, e))
2048
2049
2050
2051	########## Parse Config File ###########################################################
2052
2053	def ReadConfig(file, name):
2054	"""
2055	Parse the config file
2056	"""
2057	if not os.path.isfile(file):
2058	print 'File %s does not exist' %file
2059	sys.exit(1)
2060
2061	config = trac_config.Configuration(file)
2062
2063	# Use given project name else use defaults
2064	#
2065	if name:
2066	sections = config.sections()
2067	if not name in sections:
2068	print "Not a valid project name: %s" %name
2069	print "Valid names: %s" %sections
2070	sys.exit(1)
2071
2072	project = SaraDict()
2073	for option, value in config.options(name):
2074	project[option] = value
2075
2076	else:
2077	# use some trac internals to get the defaults
2078	#
2079	tmp = config.parser.defaults()
2080	project = SaraDict()
2081
2082	for option,value in tmp.items():
2083	project[option] = value
2084
2085	## Convert debug value to int
2086	#
2087	if project.debug:
2088	project.debug = int(project.debug)
2089	else:
2090	project.debug = 0
2091
2092	return project
2093
2094	########## Setup Logging ###############################################################
2095
2096	def setup_log(parameters, project_name, interactive=None):
2097	"""
2098	Setup loging
2099
2100	Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
2101	would be interpreted by the ConfigParser itself.
2102	"""
2103	logger = logging.getLogger('email2trac %s' %project_name)
2104
2105	if interactive:
2106	parameters.log_type = 'stderr'
2107
2108	if not parameters.log_type:
2109	parameters.log_type = 'syslog'
2110
2111	if parameters.log_type == 'file':
2112
2113	if not parameters.log_file:
2114	parameters.log_file = 'email2trac.log'
2115
2116	if not os.path.isabs(parameters.log_file):
2117	import tempfile
2118	parameters.log_file = os.path.join(tempfile.gettempdir(), parameters.log_file)
2119
2120	log_handler = logging.FileHandler(parameters.log_file)
2121
2122	elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
2123	# Requires win32 extensions
2124	log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
2125
2126	elif parameters.log_type in ('syslog', 'unix'):
2127	log_handler = logging.handlers.SysLogHandler('/dev/log')
2128
2129	elif parameters.log_type in ('stderr'):
2130	log_handler = logging.StreamHandler(sys.stderr)
2131
2132	else:
2133	log_handler = logging.handlers.BufferingHandler(0)
2134
2135	if parameters.log_format:
2136	parameters.log_format = parameters.log_format.replace('$(', '%(')
2137	else:
2138	parameters.log_format = '%(name)s: %(message)s'
2139
2140	log_formatter = logging.Formatter(parameters.log_format)
2141	log_handler.setFormatter(log_formatter)
2142	logger.addHandler(log_handler)
2143
2144	if (parameters.log_level in ['DEBUG', 'ALL']) or (parameters.debug > 0):
2145	logger.setLevel(logging.DEBUG)
2146
2147	elif parameters.log_level in ['INFO'] or parameters.verbose:
2148	logger.setLevel(logging.INFO)
2149
2150	elif parameters.log_level in ['WARNING']:
2151	logger.setLevel(logging.WARNING)
2152
2153	elif parameters.log_level in ['ERROR']:
2154	logger.setLevel(logging.ERROR)
2155
2156	elif parameters.log_level in ['CRITICAL']:
2157	logger.setLevel(logging.CRITICAL)
2158
2159	else:
2160	logger.setLevel(logging.INFO)
2161
2162	return logger
2163
2164
2165	if __name__ == '__main__':
2166	# Default config file
2167	#
2168	configfile = '@email2trac_conf@'
2169	project = ''
2170	component = ''
2171	ticket_prefix = 'default'
2172	dry_run = None
2173	verbose = None
2174	debug_interactive = None
2175
2176	SHORT_OPT = 'cdhf:np:t:v'
2177	LONG_OPT = ['component=', 'debug', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=', 'verbose']
2178
2179	try:
2180	opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
2181	except getopt.error,detail:
2182	print __doc__
2183	print detail
2184	sys.exit(1)
2185
2186	project_name = None
2187	for opt,value in opts:
2188	if opt in [ '-h', '--help']:
2189	print __doc__
2190	sys.exit(0)
2191	elif opt in ['-c', '--component']:
2192	component = value
2193	elif opt in ['-d', '--debug']:
2194	debug_interactive = 1
2195	elif opt in ['-f', '--file']:
2196	configfile = value
2197	elif opt in ['-n', '--dry-run']:
2198	dry_run = True
2199	elif opt in ['-p', '--project']:
2200	project_name = value
2201	elif opt in ['-t', '--ticket_prefix']:
2202	ticket_prefix = value
2203	elif opt in ['-v', '--verbose']:
2204	verbose = True
2205
2206	settings = ReadConfig(configfile, project_name)
2207
2208	# The default prefix for ticket values in email2trac.conf
2209	#
2210	settings.ticket_prefix = ticket_prefix
2211	settings.dry_run = dry_run
2212	settings.verbose = verbose
2213
2214	if not settings.debug and debug_interactive:
2215	settings.debug = debug_interactive
2216
2217
2218	if not settings.project:
2219	print __doc__
2220	print 'No Trac project is defined in the email2trac config file.'
2221	sys.exit(1)
2222
2223	logger = setup_log(settings, os.path.basename(settings.project), debug_interactive)
2224
2225	if component:
2226	settings['component'] = component
2227
2228	# Determine major trac version used to be in email2trac.conf
2229	# Quick hack for 0.12
2230	#
2231	version = '0.%s' %(trac_version.split('.')[1])
2232	if version.startswith('0.12'):
2233	version = '0.12'
2234
2235	logger.debug("Found trac version: %s" %(version))
2236
2237	#debug HvB
2238	#print settings
2239
2240	try:
2241	if version == '0.10':
2242	from trac import attachment
2243	from trac.env import Environment
2244	from trac.ticket import Ticket
2245	from trac.web.href import Href
2246	from trac import util
2247	#
2248	# return util.text.to_unicode(str)
2249	#
2250	# see http://projects.edgewall.com/trac/changeset/2799
2251	from trac.ticket.notification import TicketNotifyEmail
2252	from trac import config as trac_config
2253	from trac.core import TracError
2254
2255	elif version == '0.11':
2256	from trac import attachment
2257	from trac.env import Environment
2258	from trac.ticket import Ticket
2259	from trac.web.href import Href
2260	from trac import config as trac_config
2261	from trac import util
2262	from trac.core import TracError
2263	from trac.perm import PermissionSystem
2264
2265	#
2266	# return util.text.to_unicode(str)
2267	#
2268	# see http://projects.edgewall.com/trac/changeset/2799
2269	from trac.ticket.notification import TicketNotifyEmail
2270
2271	elif version == '0.12':
2272	from trac import attachment
2273	from trac.env import Environment
2274	from trac.ticket import Ticket
2275	from trac.web.href import Href
2276	from trac import config as trac_config
2277	from trac import util
2278	from trac.core import TracError
2279	from trac.perm import PermissionSystem
2280
2281	#
2282	# return util.text.to_unicode(str)
2283	#
2284	# see http://projects.edgewall.com/trac/changeset/2799
2285	from trac.ticket.notification import TicketNotifyEmail
2286
2287
2288	else:
2289	logger.error('TRAC version %s is not supported' %version)
2290	sys.exit(1)
2291
2292	# Must be set before environment is created
2293	#
2294	if settings.has_key('python_egg_cache'):
2295	python_egg_cache = str(settings['python_egg_cache'])
2296	os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
2297
2298	if settings.debug > 0:
2299	logger.debug('Loading environment %s', settings.project)
2300
2301	env = Environment(settings['project'], create=0)
2302
2303	tktparser = TicketEmailParser(env, settings, logger, float(version))
2304	tktparser.parse(sys.stdin)
2305
2306	# Catch all errors ans log to SYSLOG if we have enabled this
2307	# else stdout
2308	#
2309	except Exception, error:
2310
2311	etype, evalue, etb = sys.exc_info()
2312	for e in traceback.format_exception(etype, evalue, etb):
2313	logger.critical(e)
2314
2315	if m:
2316	tktparser.save_email_for_debug(m, True)
2317
2318	sys.exit(1)
2319	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format