Context Navigation

source: trunk/email2trac.py.in @ 435

Last change on this file since 435 was 435, checked in by bas, 14 years ago
converterd SPAM_LEVEL and EMAIL_QUOTE to new UserDict?
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 59.7 KB

Line
1	#!@PYTHON@
2	# Copyright (C) 2002
3	#
4	# This file is part of the email2trac utils
5	#
6	# This program is free software; you can redistribute it and/or modify it
7	# under the terms of the GNU General Public License as published by the
8	# Free Software Foundation; either version 2, or (at your option) any
9	# later version.
10	#
11	# This program is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with this program; if not, write to the Free Software
18	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
19	#
20	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
21	#
22	"""
23	email2trac.py -- Email tickets to Trac.
24
25	A simple MTA filter to create Trac tickets from inbound emails.
26
27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28	Copyright 2005, Edgewall Software
29
30	Authors:
31	Bas van der Vlies <basv@sara.nl>
32	Walter de Jong <walter@sara.nl>
33
34	The scripts reads emails from stdin and inserts directly into a Trac database.
35
36	How to use
37	----------
38	* See https://subtrac.sara.nl/oss/email2trac/
39
40	* Create an config file:
41	[DEFAULT] # REQUIRED
42	project : /data/trac/test # REQUIRED
43	debug : 1 # OPTIONAL, if set print some DEBUG info
44
45	[jouvin] # OPTIONAL project declaration, if set both fields necessary
46	project : /data/trac/jouvin # use -p\|--project jouvin.
47
48	* default config file is : /etc/email2trac.conf
49
50	* Commandline opions:
51	-h,--help
52	-d, --debug
53	-f,--file <configuration file>
54	-n,--dry-run
55	-p, --project <project name>
56	-t, --ticket_prefix <name>
57
58	SVN Info:
59	$Id: email2trac.py.in 435 2010-07-22 12:26:29Z bas $
60	"""
61	import os
62	import sys
63	import string
64	import getopt
65	import time
66	import email
67	import email.Iterators
68	import email.Header
69	import re
70	import urllib
71	import unicodedata
72	import mimetypes
73	import traceback
74	import logging
75	import logging.handlers
76	import UserDict
77
78	from datetime import tzinfo, timedelta, datetime
79	from stat import *
80
81
82	from trac import __version__ as trac_version
83	from trac import config as trac_config
84
85	# Some global variables
86	#
87	trac_default_version = '0.11'
88	m = None
89
90	class SaraDict(UserDict.UserDict):
91	def __init__(self, dictin = None):
92	UserDict.UserDict.__init__(self)
93	self.name = None
94
95	if dictin:
96	if dictin.has_key('name'):
97	self.name = dictin['name']
98	del dictin['name']
99	self.data = dictin
100
101	def get_value(self, key):
102	if self.has_key(key):
103	return self[key]
104	else:
105	return None
106
107	def __repr__(self):
108	return repr(self.data)
109
110	def __str__(self):
111	return str(self.data)
112
113	def __getattr__(self, name):
114	"""
115	override the class attribute get method. Return the value
116	from the dictionary
117	"""
118	if self.data.has_key(name):
119	return self.data[name]
120	else:
121	return None
122
123	def __setattr__(self, name, value):
124	"""
125	override the class attribute set method only when the UserDict
126	has set its class attribute
127	"""
128	if self.__dict__.has_key('data'):
129	self.data[name] = value
130	else:
131	self.__dict__[name] = value
132
133	def __iter__(self):
134	return iter(self.data.keys())
135
136	class TicketEmailParser(object):
137	env = None
138	comment = '> '
139
140	def __init__(self, env, parameters, logger, version):
141	self.env = env
142
143	# Database connection
144	#
145	self.db = None
146
147	# Save parameters
148	#
149	self.parameters = parameters
150	self.logger = logger
151
152	# Some useful mail constants
153	#
154	self.email_name = None
155	self.email_addr = None
156	self.email_from = None
157	self.author = None
158	self.id = None
159
160	self.STRIP_CONTENT_TYPES = list()
161
162	self.VERSION = version
163
164	self.get_config = self.env.config.get
165
166	## init function ##
167	#
168	self.setup_parameters()
169
170	def setup_parameters(self):
171	if self.parameters.umask:
172	os.umask(int(self.parameters.umask, 8))
173
174	if self.parameters.spam_level:
175	self.parameters.spam_level = int(self.parameters.spam_level)
176	else:
177	self.parameters.spam_level = 0
178
179	if not self.parameters.spam_header:
180	self.parameters.spam_header = 'X-Spam-Score'
181
182	if not self.parameters.email_quote):
183	self.parameters.email_quote = '> '
184
185	if self.parameters.has_key('email_header'):
186	self.EMAIL_HEADER = int(self.parameters['email_header'])
187	else:
188	self.EMAIL_HEADER = 0
189
190	if self.parameters.has_key('alternate_notify_template'):
191	self.notify_template = str(self.parameters['alternate_notify_template'])
192	else:
193	self.notify_template = None
194
195	if self.parameters.has_key('alternate_notify_template_update'):
196	self.notify_template_update = str(self.parameters['alternate_notify_template_update'])
197	else:
198	self.notify_template_update = None
199
200	if self.parameters.has_key('reply_all'):
201	self.REPLY_ALL = int(self.parameters['reply_all'])
202	else:
203	self.REPLY_ALL = 0
204
205	if self.parameters.has_key('ticket_permission_system'):
206	self.TICKET_PERMISSION_SYSTEM = str(self.parameters['ticket_permission_system'])
207	else:
208	self.TICKET_PERMISSION_SYSTEM = None
209
210	if self.parameters.has_key('ticket_update'):
211	self.TICKET_UPDATE = int(self.parameters['ticket_update'])
212	else:
213	self.TICKET_UPDATE = 0
214
215	if self.parameters.has_key('ticket_update_by_subject'):
216	self.TICKET_UPDATE_BY_SUBJECT = int(self.parameters['ticket_update_by_subject'])
217	else:
218	self.TICKET_UPDATE_BY_SUBJECT = 0
219
220	if self.parameters.has_key('ticket_update_by_subject_lookback'):
221	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK = int(self.parameters['ticket_update_by_subject_lookback'])
222	else:
223	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK = 30
224
225	if self.parameters.has_key('drop_spam'):
226	self.DROP_SPAM = int(self.parameters['drop_spam'])
227	else:
228	self.DROP_SPAM = 0
229
230	if self.parameters.has_key('verbatim_format'):
231	self.VERBATIM_FORMAT = int(self.parameters['verbatim_format'])
232	else:
233	self.VERBATIM_FORMAT = 1
234
235	if self.parameters.has_key('reflow'):
236	self.REFLOW = int(self.parameters['reflow'])
237	else:
238	self.REFLOW = 1
239
240	if self.parameters.has_key('drop_alternative_html_version'):
241	self.DROP_ALTERNATIVE_HTML_VERSION = int(self.parameters['drop_alternative_html_version'])
242	else:
243	self.DROP_ALTERNATIVE_HTML_VERSION = 0
244
245	if self.parameters.has_key('strip_signature'):
246	self.STRIP_SIGNATURE = int(self.parameters['strip_signature'])
247	else:
248	self.STRIP_SIGNATURE = 0
249
250	if self.parameters.has_key('strip_quotes'):
251	self.STRIP_QUOTES = int(self.parameters['strip_quotes'])
252	else:
253	self.STRIP_QUOTES = 0
254
255	self.properties = dict()
256	if self.parameters.has_key('inline_properties'):
257	self.INLINE_PROPERTIES = int(self.parameters['inline_properties'])
258	else:
259	self.INLINE_PROPERTIES = 0
260
261	if self.parameters.has_key('use_textwrap'):
262	self.USE_TEXTWRAP = int(self.parameters['use_textwrap'])
263	else:
264	self.USE_TEXTWRAP = 0
265
266	if self.parameters.has_key('binhex'):
267	self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
268
269	if self.parameters.has_key('applesingle'):
270	self.STRIP_CONTENT_TYPES.append('application/applefile')
271
272	if self.parameters.has_key('appledouble'):
273	self.STRIP_CONTENT_TYPES.append('application/applefile')
274
275	if self.parameters.has_key('strip_content_types'):
276	items = self.parameters['strip_content_types'].split(',')
277	for item in items:
278	self.STRIP_CONTENT_TYPES.append(item.strip())
279
280	self.WORKFLOW = None
281	if self.parameters.has_key('workflow'):
282	self.WORKFLOW = self.parameters['workflow']
283
284	# Use OS independend functions
285	#
286	self.TMPDIR = os.path.normcase('/tmp')
287	if self.parameters.has_key('tmpdir'):
288	self.TMPDIR = os.path.normcase(str(self.parameters['tmpdir']))
289
290	if self.parameters.has_key('ignore_trac_user_settings'):
291	self.IGNORE_TRAC_USER_SETTINGS = int(self.parameters['ignore_trac_user_settings'])
292	else:
293	self.IGNORE_TRAC_USER_SETTINGS = 0
294
295	if self.parameters.has_key('email_triggers_workflow'):
296	self.EMAIL_TRIGGERS_WORKFLOW = int(self.parameters['email_triggers_workflow'])
297	else:
298	self.EMAIL_TRIGGERS_WORKFLOW = 1
299
300	if self.parameters.has_key('subject_field_separator'):
301	self.SUBJECT_FIELD_SEPARATOR = self.parameters['subject_field_separator'].strip()
302	else:
303	self.SUBJECT_FIELD_SEPARATOR = '&'
304
305	self.trac_smtp_from = self.get_config('notification', 'smtp_from')
306
307	self.system = None
308
309	########## Email Header Functions ###########################################################
310
311	def spam(self, message):
312	"""
313	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
314	# Note if Spam_level then '*' are included
315	"""
316	spam = False
317	if message.has_key(self.parameters.spam_header):
318	spam_l = string.split(message[self.parameters.spam_header])
319
320	try:
321	number = spam_l[0].count('*')
322	except IndexError, detail:
323	number = 0
324
325	if number >= self.parameters.spam_level:
326	spam = True
327
328	# treat virus mails as spam
329	#
330	elif message.has_key('X-Virus-found'):
331	spam = True
332
333	# How to handle SPAM messages
334	#
335	if self.DROP_SPAM and spam:
336
337	self.logger.info('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' %self.parameters.spam_level)
338	return 'drop'
339
340	elif spam:
341
342	return 'Spam'
343	else:
344
345	return False
346
347	def email_header_acl(self, keyword, header_field, default):
348	"""
349	This function wil check if the email address is allowed or denied
350	to send mail to the ticket list
351	"""
352	self.logger.debug('function email_header_acl: %s' %keyword)
353
354	try:
355	mail_addresses = self.parameters[keyword]
356
357	# Check if we have an empty string
358	#
359	if not mail_addresses:
360	return default
361
362	except KeyError, detail:
363	self.logger.debug('%s not defined, all messages are allowed.' %(keyword))
364
365	return default
366
367	mail_addresses = string.split(mail_addresses, ',')
368
369	for entry in mail_addresses:
370	entry = entry.strip()
371	TO_RE = re.compile(entry, re.VERBOSE\|re.IGNORECASE)
372	result = TO_RE.search(header_field)
373	if result:
374	return True
375
376	return False
377
378	def email_header_txt(self, m):
379	"""
380	Display To and CC addresses in description field
381	"""
382	s = ''
383
384	if m['To'] and len(m['To']) > 0:
385	s = "'''To:''' %s\r\n" %(m['To'])
386	if m['Cc'] and len(m['Cc']) > 0:
387	s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
388
389	return self.email_to_unicode(s)
390
391
392	def get_sender_info(self, message):
393	"""
394	Get the default author name and email address from the message
395	"""
396
397	self.email_to = self.email_to_unicode(message['to'])
398	self.to_name, self.to_email_addr = email.Utils.parseaddr (self.email_to)
399
400	self.email_from = self.email_to_unicode(message['from'])
401	self.email_name, self.email_addr = email.Utils.parseaddr(self.email_from)
402
403	## Trac can not handle author's name that contains spaces
404	# and forbid the ticket email address as author field
405
406	if self.email_addr == self.trac_smtp_from:
407	if self.email_name:
408	self.author = self.email_name
409	else:
410	self.author = "email2trac"
411	else:
412	self.author = self.email_addr
413
414	if self.IGNORE_TRAC_USER_SETTINGS:
415	return
416
417	# Is this a registered user, use email address as search key:
418	# result:
419	# u : login name
420	# n : Name that the user has set in the settings tab
421	# e : email address that the user has set in the settings tab
422	#
423	users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
424	if e and (e.lower() == self.email_addr.lower()) ]
425
426	if len(users) == 1:
427	self.email_from = users[0][0]
428	self.author = users[0][0]
429
430	def set_reply_fields(self, ticket, message):
431	"""
432	Set all the right fields for a new ticket
433	"""
434	self.logger.debug('function set_reply_fields')
435
436	## Only use name or email adress
437	#ticket['reporter'] = self.email_from
438	ticket['reporter'] = self.author
439
440
441	# Put all CC-addresses in ticket CC field
442	#
443	if self.REPLY_ALL:
444
445	email_cc = ''
446
447	cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
448
449	if not cc_addrs:
450	return
451
452	## Build a list of forbidden CC addresses
453	#
454	#to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
455	#to_list = list()
456	#for n,e in to_addrs:
457	# to_list.append(e)
458
459	# Always Remove reporter email address from cc-list
460	#
461	try:
462	cc_addrs.remove((self.author, self.email_addr))
463	except ValueError, detail:
464	pass
465
466	for name,addr in cc_addrs:
467
468	## Prevent mail loop
469	#
470	#if addr in to_list:
471
472	if addr == self.trac_smtp_from:
473	self.logger.debug("Skipping %s mail address for CC-field" %(addr))
474	continue
475
476	if email_cc:
477	email_cc = '%s, %s' %(email_cc, addr)
478	else:
479	email_cc = addr
480
481	if email_cc:
482	self.logger.debug('set_reply_fields: %s' %email_cc)
483
484	ticket['cc'] = self.email_to_unicode(email_cc)
485
486
487	########## DEBUG functions ###########################################################
488
489	def debug_body(self, message_body, tempfile=False):
490	if tempfile:
491	import tempfile
492	body_file = tempfile.mktemp('.email2trac')
493	else:
494	body_file = os.path.join(self.TMPDIR, 'body.txt')
495
496	if self.parameters.dry_run:
497	print 'DRY-RUN: not saving body to %s' %(body_file)
498	return
499
500	print 'writing body to %s' %(body_file)
501	fx = open(body_file, 'wb')
502	if not message_body:
503	message_body = '(None)'
504
505	message_body = message_body.encode('utf-8')
506	#message_body = unicode(message_body, 'iso-8859-15')
507
508	fx.write(message_body)
509	fx.close()
510	try:
511	os.chmod(body_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
512	except OSError:
513	pass
514
515	def debug_attachments(self, message_parts):
516	"""
517	"""
518	self.logger.debug('function debug_attachments')
519
520	n = 0
521	for item in message_parts:
522	# Skip inline text parts
523	if not isinstance(item, tuple):
524	continue
525
526	(original, filename, part) = item
527
528	n = n + 1
529	print 'part%d: Content-Type: %s' % (n, part.get_content_type())
530
531	s = 'part%d: filename: %s' %(n, filename)
532	self.print_unicode(s)
533
534	## Forbidden chars
535	#
536	filename = filename.replace('\\', '_')
537	filename = filename.replace('/', '_')
538
539
540	part_file = os.path.join(self.TMPDIR, filename)
541	s = 'writing part%d (%s)' % (n,part_file)
542	self.print_unicode(s)
543
544	if self.parameters.dry_run:
545	print 'DRY_RUN: NOT saving attachments'
546	continue
547
548	part_file = util.text.unicode_quote(part_file)
549
550	fx = open(part_file, 'wb')
551	text = part.get_payload(decode=1)
552
553	if not text:
554	text = '(None)'
555
556	fx.write(text)
557	fx.close()
558
559	try:
560	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
561	except OSError:
562	pass
563
564	def save_email_for_debug(self, message, tempfile=False):
565
566	if tempfile:
567	import tempfile
568	msg_file = tempfile.mktemp('.email2trac')
569	else:
570	#msg_file = '/var/tmp/msg.txt'
571	msg_file = os.path.join(self.TMPDIR, 'msg.txt')
572
573	if self.parameters.dry_run:
574	print 'DRY_RUN: NOT saving email message to %s' %(msg_file)
575	else:
576	print 'saving email to %s' %(msg_file)
577
578	fx = open(msg_file, 'wb')
579	fx.write('%s' % message)
580	fx.close()
581
582	try:
583	os.chmod(msg_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
584	except OSError:
585	pass
586
587	message_parts = self.get_message_parts(message)
588	message_parts = self.unique_attachment_names(message_parts)
589	body_text = self.body_text(message_parts)
590	self.debug_body(body_text, True)
591	self.debug_attachments(message_parts)
592
593	########## Conversion functions ###########################################################
594
595	def email_to_unicode(self, message_str):
596	"""
597	Email has 7 bit ASCII code, convert it to unicode with the charset
598	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
599	understands it.
600	"""
601	self.logger.debug("function email_to_unicode")
602
603	results = email.Header.decode_header(message_str)
604
605	s = None
606	for text,format in results:
607	if format:
608	try:
609	temp = unicode(text, format)
610	except UnicodeError, detail:
611	# This always works
612	#
613	temp = unicode(text, 'iso-8859-15')
614	except LookupError, detail:
615	#text = 'ERROR: Could not find charset: %s, please install' %format
616	#temp = unicode(text, 'iso-8859-15')
617	temp = message_str
618
619	else:
620	temp = string.strip(text)
621	temp = unicode(text, 'iso-8859-15')
622
623	if s:
624	s = '%s %s' %(s, temp)
625	else:
626	s = '%s' %temp
627
628	#s = s.encode('utf-8')
629	return s
630
631	def str_to_dict(self, s):
632	"""
633	Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
634	"""
635	self.logger.debug("function str_to_dict")
636
637	fields = string.split(s, self.SUBJECT_FIELD_SEPARATOR)
638
639	result = dict()
640	for field in fields:
641	try:
642	index, value = string.split(field, '=')
643
644	# We can not change the description of a ticket via the subject
645	# line. The description is the body of the email
646	#
647	if index.lower() in ['description']:
648	continue
649
650	if value:
651	result[index.lower()] = value
652
653	except ValueError:
654	pass
655	return result
656
657	def print_unicode(self,s):
658	"""
659	This function prints unicode strings uif possible else it will quote it
660	"""
661	try:
662	self.logger.debug(s)
663	except UnicodeEncodeError, detail:
664	self.logger.debug(util.text.unicode_quote(s))
665
666	########## TRAC ticket functions ###########################################################
667
668	def check_permission_participants(self, tkt):
669	"""
670	Check if the mailer is allowed to update the ticket
671	"""
672	self.logger.debug('function check_permission_participants')
673
674	if tkt['reporter'].lower() in [self.author, self.email_addr]:
675	self.logger.debug('ALLOW, %s is the ticket reporter' %(self.email_addr))
676
677	return True
678
679	perm = PermissionSystem(self.env)
680	if perm.check_permission('TICKET_MODIFY', self.author):
681	self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
682
683	return True
684
685	else:
686	return False
687
688
689	# Is the updater in the CC?
690	try:
691	cc_list = tkt['cc'].split(',')
692	for cc in cc_list:
693	if self.email_addr.lower() in cc.strip():
694	self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
695
696	return True
697
698	except KeyError:
699	return False
700
701	def check_permission(self, tkt, action):
702	"""
703	check if the reporter has the right permission for the action:
704	- TICKET_CREATE
705	- TICKET_MODIFY
706
707	There are three models:
708	- None : no checking at all
709	- trac : check the permission via trac permission model
710	- email2trac: ....
711	"""
712	self.logger.debug("function check_permission")
713
714	if self.TICKET_PERMISSION_SYSTEM in ['trac']:
715
716	perm = PermissionSystem(self.env)
717	if perm.check_permission(action, self.author):
718	return True
719	else:
720	return False
721
722	elif self.TICKET_PERMISSION_SYSTEM in ['update_restricted_to_participants']:
723	if action in ['TICKET_MODIFY']:
724	return (self.check_permission_participants(tkt))
725	else:
726	return True
727
728	# Default is to allow everybody ticket updates and ticket creation
729	else:
730	return True
731
732
733	def update_ticket_fields(self, ticket, user_dict, use_default=None):
734	"""
735	This will update the ticket fields. It will check if the
736	given fields are known and if the right values are specified
737	It will only update the ticket field value:
738	- If the field is known
739	- If the value supplied is valid for the ticket field.
740	If not then there are two options:
741	1) Skip the value (use_default=None)
742	2) Set default value for field (use_default=1)
743	"""
744	self.logger.debug("function update_ticket_fields")
745
746	# Build a system dictionary from the ticket fields
747	# with field as index and option as value
748	#
749	sys_dict = dict()
750	for field in ticket.fields:
751	try:
752	sys_dict[field['name']] = field['options']
753
754	except KeyError:
755	sys_dict[field['name']] = None
756	pass
757
758	## Check user supplied fields an compare them with the
759	# system one's
760	#
761	for field,value in user_dict.items():
762	if self.parameters.debug:
763	s = 'user_field\t %s = %s' %(field,value)
764	self.print_unicode(s)
765
766	## To prevent mail loop
767	#
768	if field == 'cc':
769
770	cc_list = user_dict['cc'].split(',')
771
772	if self.trac_smtp_from in cc_list:
773	self.logger.debug('MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
774
775	cc_list.remove(self.trac_smtp_from)
776
777	value = ','.join(cc_list)
778
779
780	if sys_dict.has_key(field):
781
782	# Check if value is an allowed system option, if TypeError then
783	# every value is allowed
784	#
785	try:
786	if value in sys_dict[field]:
787	ticket[field] = value
788	else:
789	# Must we set a default if value is not allowed
790	#
791	if use_default:
792	value = self.get_config('ticket', 'default_%s' %(field) )
793
794	except TypeError:
795	pass
796
797	## Only set if we have a value
798	#
799	if value:
800	ticket[field] = value
801
802	if self.parameters.debug:
803	s = 'ticket_field\t %s = %s' %(field, ticket[field])
804	self.print_unicode(s)
805
806	def ticket_update(self, m, id, spam):
807	"""
808	If the current email is a reply to an existing ticket, this function
809	will append the contents of this email to that ticket, instead of
810	creating a new one.
811	"""
812	self.logger.debug("function ticket_update")
813
814	# Must we update ticket fields
815	#
816	update_fields = dict()
817	try:
818	id, keywords = string.split(id, '?')
819
820	update_fields = self.str_to_dict(keywords)
821
822	# Strip '#'
823	#
824	self.id = int(id[1:])
825
826	except ValueError:
827
828	# Strip '#'
829	#
830	self.id = int(id[1:])
831
832	self.logger.debug("function ticket_update id %s" %id)
833
834	# When is the change committed
835	#
836	if self.VERSION < 0.11:
837	when = int(time.time())
838	else:
839	when = datetime.now(util.datefmt.utc)
840
841	try:
842	tkt = Ticket(self.env, self.id, self.db)
843
844	except util.TracError, detail:
845
846	# Not a valid ticket
847
848	self.id = None
849	return False
850
851	# Check the permission of the reporter
852	#
853	if self.TICKET_PERMISSION_SYSTEM:
854	if not self.check_permission(tkt, 'TICKET_MODIFY'):
855	self.logger.info('Reporter: %s has no permission to modify tickets' %self.author)
856	return False
857
858	# How many changes has this ticket
859	cnum = len(tkt.get_changelog())
860
861
862	# reopen the ticket if it is was closed
863	# We must use the ticket workflow framework
864	#
865	if tkt['status'] in ['closed'] and self.EMAIL_TRIGGERS_WORKFLOW:
866
867	#print controller.actions['reopen']
868	#
869	# As reference
870	# req = Mock(href=Href('/'), abs_href=Href('http://www.example.com/'), authname='anonymous', perm=MockPerm(), args={})
871	#
872	#a = controller.render_ticket_action_control(req, tkt, 'reopen')
873	#print 'controller : ', a
874	#
875	#b = controller.get_all_status()
876	#print 'get all status: ', b
877	#
878	#b = controller.get_ticket_changes(req, tkt, 'reopen')
879	#print 'get_ticket_changes :', b
880
881	if self.WORKFLOW and (self.VERSION >= 0.11 ) :
882	from trac.ticket.default_workflow import ConfigurableTicketWorkflow
883	from trac.test import Mock, MockPerm
884
885	req = Mock(authname='anonymous', perm=MockPerm(), args={})
886
887	controller = ConfigurableTicketWorkflow(self.env)
888	fields = controller.get_ticket_changes(req, tkt, self.WORKFLOW)
889
890	self.logger.debug('Workflow ticket update fields: ')
891
892	for key in fields.keys():
893	self.logger.debug('\t %s : %s' %(key, fields[key]))
894	tkt[key] = fields[key]
895
896	else:
897	tkt['status'] = 'reopened'
898	tkt['resolution'] = ''
899
900	# Must we update some ticket fields properties via subjectline
901	#
902	if update_fields:
903	self.update_ticket_fields(tkt, update_fields)
904
905	message_parts = self.get_message_parts(m)
906	message_parts = self.unique_attachment_names(message_parts)
907
908	# Must we update some ticket fields properties via body_text
909	#
910	if self.properties:
911	self.update_ticket_fields(tkt, self.properties)
912
913	if self.EMAIL_HEADER:
914	message_parts.insert(0, self.email_header_txt(m))
915
916	body_text = self.body_text(message_parts)
917
918	error_with_attachments = self.attach_attachments(message_parts)
919
920	if body_text.strip() or update_fields or self.properties:
921	if self.parameters.dry_run:
922	print 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) ', self.author, cnum
923	else:
924	if error_with_attachments:
925	body_text = '%s\\%s' %(error_with_attachments, body_text)
926	self.logger.debug('tkt.save_changes(%s, %d)' %(self.author, cnum))
927	tkt.save_changes(self.author, body_text, when, None, str(cnum))
928
929
930	if not spam:
931	self.notify(tkt, False, when)
932
933	return True
934
935	def set_ticket_fields(self, ticket):
936	"""
937	set the ticket fields to value specified
938	- /etc/email2trac.conf with <prefix>_<field>
939	- trac default values, trac.ini
940	"""
941	self.logger.debug('function set_ticket_fields')
942
943	user_dict = dict()
944
945	for field in ticket.fields:
946
947	name = field['name']
948
949	## default trac value
950	#
951	if not field.get('custom'):
952	value = self.get_config('ticket', 'default_%s' %(name) )
953	else:
954	## Else we get the default value for reporter
955	#
956	value = field.get('value')
957	options = field.get('options')
958
959	if value and options and (value not in options):
960	value = options[int(value)]
961
962	if self.parameters.debug:
963	s = 'trac[%s] = %s' %(name, value)
964	self.print_unicode(s)
965
966	## email2trac.conf settings
967	#
968	prefix = self.parameters['ticket_prefix']
969	try:
970	value = self.parameters['%s_%s' %(prefix, name)]
971	if self.parameters.debug > 10:
972	s = 'email2trac[%s] = %s ' %(name, value)
973	self.print_unicode(s)
974
975	except KeyError, detail:
976	pass
977
978	if self.parameters.debug:
979	s = 'used %s = %s' %(name, value)
980	self.print_unicode(s)
981
982	if value:
983	user_dict[name] = value
984
985	self.update_ticket_fields(ticket, user_dict, use_default=1)
986
987	if 'status' not in user_dict.keys():
988	ticket['status'] = 'new'
989
990
991	def ticket_update_by_subject(self, subject):
992	"""
993	This list of Re: prefixes is probably incomplete. Taken from
994	wikipedia. Here is how the subject is matched
995	- Re: <subject>
996	- Re: (<Mail list label>:)+ <subject>
997
998	So we must have the last column
999	"""
1000	self.logger.debug('function ticket_update_by_subject')
1001
1002	matched_id = None
1003	if self.TICKET_UPDATE and self.TICKET_UPDATE_BY_SUBJECT:
1004
1005	SUBJECT_RE = re.compile(r'^(RE\|AW\|VS\|SV):(.:)\s(.)', re.IGNORECASE)
1006	result = SUBJECT_RE.search(subject)
1007
1008	if result:
1009	# This is a reply
1010	orig_subject = result.group(3)
1011
1012	self.logger.debug('subject search string: %s' %(orig_subject))
1013
1014	cursor = self.db.cursor()
1015	summaries = [orig_subject, '%%: %s' % orig_subject]
1016
1017	##
1018	# Convert days to seconds
1019	lookback = int(time.mktime(time.gmtime())) - \
1020	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK * 24 * 3600
1021
1022
1023	for summary in summaries:
1024	self.logger.debug('Looking for summary matching: "%s"' % summary)
1025
1026	sql = """SELECT id FROM ticket
1027	WHERE changetime >= %s AND summary LIKE %s
1028	ORDER BY changetime DESC"""
1029	cursor.execute(sql, [lookback, summary.strip()])
1030
1031	for row in cursor:
1032	(matched_id,) = row
1033
1034	self.logger.debug('Found matching ticket id: %d' % matched_id)
1035
1036	break
1037
1038	if matched_id:
1039	matched_id = '#%d' % matched_id
1040	return matched_id
1041
1042	return matched_id
1043
1044
1045	def new_ticket(self, msg, subject, spam, set_fields = None):
1046	"""
1047	Create a new ticket
1048	"""
1049	self.logger.debug('function new_ticket')
1050
1051	tkt = Ticket(self.env)
1052
1053	self.set_reply_fields(tkt, msg)
1054
1055	self.set_ticket_fields(tkt)
1056
1057	# Check the permission of the reporter
1058	#
1059	if self.TICKET_PERMISSION_SYSTEM:
1060	if not self.check_permission(tkt, 'TICKET_CREATE'):
1061	self.logger.info('Reporter: %s has no permission to create tickets' %self.author)
1062	return False
1063
1064	# Old style setting for component, will be removed
1065	#
1066	if spam:
1067	tkt['component'] = 'Spam'
1068
1069	elif self.parameters.has_key('component'):
1070	tkt['component'] = self.parameters['component']
1071
1072	if not msg['Subject']:
1073	tkt['summary'] = u'(No subject)'
1074	else:
1075	tkt['summary'] = subject
1076
1077
1078	if set_fields:
1079	rest, keywords = string.split(set_fields, '?')
1080
1081	if keywords:
1082	update_fields = self.str_to_dict(keywords)
1083	self.update_ticket_fields(tkt, update_fields)
1084
1085	# produce e-mail like header
1086	#
1087	head = ''
1088	if self.EMAIL_HEADER > 0:
1089	head = self.email_header_txt(msg)
1090
1091	message_parts = self.get_message_parts(msg)
1092
1093	# Must we update some ticket fields properties via body_text
1094	#
1095	if self.properties:
1096	self.update_ticket_fields(tkt, self.properties)
1097
1098	message_parts = self.unique_attachment_names(message_parts)
1099
1100	if self.EMAIL_HEADER > 0:
1101	message_parts.insert(0, self.email_header_txt(msg))
1102
1103	body_text = self.body_text(message_parts)
1104
1105	tkt['description'] = body_text
1106
1107	# When is the change committed
1108	#
1109	if self.VERSION < 0.11:
1110	when = int(time.time())
1111	else:
1112	when = datetime.now(util.datefmt.utc)
1113
1114	if self.parameters.dry_run:
1115	print 'DRY_RUN: tkt.insert()'
1116	else:
1117	self.id = tkt.insert()
1118
1119	changed = False
1120	comment = ''
1121
1122	# some routines in trac are dependend on ticket id
1123	# like alternate notify template
1124	#
1125	if self.notify_template:
1126	tkt['id'] = self.id
1127	changed = True
1128
1129	## Rewrite the description if we have mailto enabled
1130	#
1131	if self.parameters.mailto_link:
1132	changed = True
1133	comment = u'\nadded mailto line\n'
1134	mailto = self.html_mailto_link( m['Subject'])
1135
1136	tkt['description'] = u'%s\r\n%s%s\r\n' \
1137	%(head, mailto, body_text)
1138
1139	## Save the attachments to the ticket
1140	#
1141	error_with_attachments = self.attach_attachments(message_parts)
1142
1143	if error_with_attachments:
1144	changed = True
1145	comment = '%s\n%s\n' %(comment, error_with_attachments)
1146
1147	if changed:
1148	if self.parameters.dry_run:
1149	print 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
1150	else:
1151	tkt.save_changes(tkt['reporter'], comment)
1152	#print tkt.get_changelog(self.db, when)
1153
1154	if not spam:
1155	self.notify(tkt, True)
1156
1157
1158	def attach_attachments(self, message_parts, update=False):
1159	'''
1160	save any attachments as files in the ticket's directory
1161	'''
1162	self.logger.debug('function attach_attachments()')
1163
1164	if self.parameters.dry_run:
1165	print "DRY_RUN: no attachments attached to tickets"
1166	return ''
1167
1168	count = 0
1169
1170	# Get Maxium attachment size
1171	#
1172	max_size = int(self.get_config('attachment', 'max_size'))
1173	status = None
1174
1175	for item in message_parts:
1176	# Skip body parts
1177	if not isinstance(item, tuple):
1178	continue
1179
1180	(original, filename, part) = item
1181	#
1182	# We have to determine the size so we use this temporary solution. we must escape it
1183	# else we get UnicodeErrors.
1184	#
1185	path, fd = util.create_unique_file(os.path.join(self.TMPDIR, util.text.unicode_quote(filename)))
1186	text = part.get_payload(decode=1)
1187	if not text:
1188	text = '(None)'
1189	fd.write(text)
1190	fd.close()
1191
1192	# get the file_size
1193	#
1194	stats = os.lstat(path)
1195	file_size = stats[ST_SIZE]
1196
1197	# Check if the attachment size is allowed
1198	#
1199	if (max_size != -1) and (file_size > max_size):
1200	status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1201	%(status, original, file_size, max_size)
1202
1203	os.unlink(path)
1204	continue
1205	else:
1206	count = count + 1
1207
1208	# Insert the attachment
1209	#
1210	fd = open(path, 'rb')
1211	if self.system == 'discussion':
1212	att = attachment.Attachment(self.env, 'discussion', 'topic/%s'
1213	% (self.id,))
1214	else:
1215	self.logger.debug('Attach %s to ticket %d' %(util.text.unicode_quote(filename), self.id))
1216	att = attachment.Attachment(self.env, 'ticket', self.id)
1217
1218	# This will break the ticket_update system, the body_text is vaporized
1219	# ;-(
1220	#
1221	if not update:
1222	att.author = self.author
1223	att.description = self.email_to_unicode('Added by email2trac')
1224
1225	try:
1226	self.logger.debug('Insert atachment')
1227	att.insert(filename, fd, file_size)
1228	except OSError, detail:
1229	self.logger.info('%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail))
1230	status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
1231
1232	# Remove the created temporary filename
1233	#
1234	fd.close()
1235	os.unlink(path)
1236
1237	## return error
1238	#
1239	return status
1240
1241	########## Fullblog functions #################################################
1242
1243	def blog(self, id):
1244	"""
1245	The blog create/update function
1246	"""
1247	# import the modules
1248	#
1249	from tracfullblog.core import FullBlogCore
1250	from tracfullblog.model import BlogPost, BlogComment
1251	from trac.test import Mock, MockPerm
1252
1253	# instantiate blog core
1254	blog = FullBlogCore(self.env)
1255	req = Mock(authname='anonymous', perm=MockPerm(), args={})
1256
1257	if id:
1258
1259	# update blog
1260	#
1261	comment = BlogComment(self.env, id)
1262	comment.author = self.author
1263
1264	message_parts = self.get_message_parts(m)
1265	comment.comment = self.body_text(message_parts)
1266
1267	blog.create_comment(req, comment)
1268
1269	else:
1270	# create blog
1271	#
1272	import time
1273	post = BlogPost(self.env, 'blog_'+time.strftime("%Y%m%d%H%M%S", time.gmtime()))
1274
1275	#post = BlogPost(self.env, blog._get_default_postname(self.env))
1276
1277	post.author = self.author
1278	post.title = self.email_to_unicode(m['Subject'])
1279
1280	message_parts = self.get_message_parts(m)
1281	post.body = self.body_text(message_parts)
1282
1283	blog.create_post(req, post, self.author, u'Created by email2trac', False)
1284
1285
1286	########## Discussion functions ##############################################
1287
1288	def discussion_topic(self, content, subject):
1289
1290	# Import modules.
1291	from tracdiscussion.api import DiscussionApi
1292	from trac.util.datefmt import to_timestamp, utc
1293
1294	self.logger.debug('Creating a new topic in forum:', self.id)
1295
1296	# Get dissussion API component.
1297	api = self.env[DiscussionApi]
1298	context = self._create_context(content, subject)
1299
1300	# Get forum for new topic.
1301	forum = api.get_forum(context, self.id)
1302
1303	if not forum:
1304	self.logger.error("ERROR: Replied forum doesn't exist")
1305
1306	# Prepare topic.
1307	topic = {'forum' : forum['id'],
1308	'subject' : context.subject,
1309	'time': to_timestamp(datetime.now(utc)),
1310	'author' : self.author,
1311	'subscribers' : [self.email_addr],
1312	'body' : self.body_text(context.content_parts)}
1313
1314	# Add topic to DB and commit it.
1315	self._add_topic(api, context, topic)
1316	self.db.commit()
1317
1318	def discussion_topic_reply(self, content, subject):
1319
1320	# Import modules.
1321	from tracdiscussion.api import DiscussionApi
1322	from trac.util.datefmt import to_timestamp, utc
1323
1324	self.logger.debug('Replying to discussion topic', self.id)
1325
1326	# Get dissussion API component.
1327	api = self.env[DiscussionApi]
1328	context = self._create_context(content, subject)
1329
1330	# Get replied topic.
1331	topic = api.get_topic(context, self.id)
1332
1333	if not topic:
1334	self.logger.error("ERROR: Replied topic doesn't exist")
1335
1336	# Prepare message.
1337	message = {'forum' : topic['forum'],
1338	'topic' : topic['id'],
1339	'replyto' : -1,
1340	'time' : to_timestamp(datetime.now(utc)),
1341	'author' : self.author,
1342	'body' : self.body_text(context.content_parts)}
1343
1344	# Add message to DB and commit it.
1345	self._add_message(api, context, message)
1346	self.db.commit()
1347
1348	def discussion_message_reply(self, content, subject):
1349
1350	# Import modules.
1351	from tracdiscussion.api import DiscussionApi
1352	from trac.util.datefmt import to_timestamp, utc
1353
1354	self.loggger.debug('Replying to discussion message', self.id)
1355
1356	# Get dissussion API component.
1357	api = self.env[DiscussionApi]
1358	context = self._create_context(content, subject)
1359
1360	# Get replied message.
1361	message = api.get_message(context, self.id)
1362
1363	if not message:
1364	self.logger.error("ERROR: Replied message doesn't exist")
1365
1366	# Prepare message.
1367	message = {'forum' : message['forum'],
1368	'topic' : message['topic'],
1369	'replyto' : message['id'],
1370	'time' : to_timestamp(datetime.now(utc)),
1371	'author' : self.author,
1372	'body' : self.body_text(context.content_parts)}
1373
1374	# Add message to DB and commit it.
1375	self._add_message(api, context, message)
1376	self.db.commit()
1377
1378	def _create_context(self, content, subject):
1379
1380	# Import modules.
1381	from trac.mimeview import Context
1382	from trac.web.api import Request
1383	from trac.perm import PermissionCache
1384
1385	# TODO: Read server base URL from config.
1386	# Create request object to mockup context creation.
1387	#
1388	environ = {'SERVER_PORT' : 80,
1389	'SERVER_NAME' : 'test',
1390	'REQUEST_METHOD' : 'POST',
1391	'wsgi.url_scheme' : 'http',
1392	'wsgi.input' : sys.stdin}
1393	chrome = {'links': {},
1394	'scripts': [],
1395	'ctxtnav': [],
1396	'warnings': [],
1397	'notices': []}
1398
1399	if self.env.base_url_for_redirect:
1400	environ['trac.base_url'] = self.env.base_url
1401
1402	req = Request(environ, None)
1403	req.chrome = chrome
1404	req.tz = 'missing'
1405	req.authname = self.author
1406	req.perm = PermissionCache(self.env, self.author)
1407
1408	# Create and return context.
1409	context = Context.from_request(req)
1410	context.realm = 'discussion-email2trac'
1411	context.cursor = self.db.cursor()
1412	context.content = content
1413	context.subject = subject
1414
1415	# Read content parts from content.
1416	context.content_parts = self.get_message_parts(content)
1417	context.content_parts = self.unique_attachment_names(
1418	context.content_parts)
1419
1420	return context
1421
1422	def _add_topic(self, api, context, topic):
1423	context.req.perm.assert_permission('DISCUSSION_APPEND')
1424
1425	# Filter topic.
1426	for discussion_filter in api.discussion_filters:
1427	accept, topic_or_error = discussion_filter.filter_topic(
1428	context, topic)
1429	if accept:
1430	topic = topic_or_error
1431	else:
1432	raise TracError(topic_or_error)
1433
1434	# Add a new topic.
1435	api.add_topic(context, topic)
1436
1437	# Get inserted topic with new ID.
1438	topic = api.get_topic_by_time(context, topic['time'])
1439
1440	# Attach attachments.
1441	self.id = topic['id']
1442	self.attach_attachments(context.content_parts, True)
1443
1444	# Notify change listeners.
1445	for listener in api.topic_change_listeners:
1446	listener.topic_created(context, topic)
1447
1448	def _add_message(self, api, context, message):
1449	context.req.perm.assert_permission('DISCUSSION_APPEND')
1450
1451	# Filter message.
1452	for discussion_filter in api.discussion_filters:
1453	accept, message_or_error = discussion_filter.filter_message(
1454	context, message)
1455	if accept:
1456	message = message_or_error
1457	else:
1458	raise TracError(message_or_error)
1459
1460	# Add message.
1461	api.add_message(context, message)
1462
1463	# Get inserted message with new ID.
1464	message = api.get_message_by_time(context, message['time'])
1465
1466	# Attach attachments.
1467	self.id = message['topic']
1468	self.attach_attachments(context.content_parts, True)
1469
1470	# Notify change listeners.
1471	for listener in api.message_change_listeners:
1472	listener.message_created(context, message)
1473
1474	########## MAIN function ######################################################
1475
1476	def parse(self, fp):
1477	"""
1478	"""
1479	self.logger.debug('Main function parse')
1480	global m
1481
1482	m = email.message_from_file(fp)
1483
1484	if not m:
1485	self.logger.debug('This is not a valid email message format')
1486	return
1487
1488	# Work around lack of header folding in Python; see http://bugs.python.org/issue4696
1489	try:
1490	m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
1491	except AttributeError, detail:
1492	pass
1493
1494	if self.parameters.debug: # save the entire e-mail message text
1495	self.save_email_for_debug(m, True)
1496
1497	self.db = self.env.get_db_cnx()
1498	self.get_sender_info(m)
1499
1500	if not self.email_header_acl('white_list', self.email_addr, True):
1501	self.logger.info('Message rejected : %s not in white list' %(self.email_addr))
1502	return False
1503
1504	if self.email_header_acl('black_list', self.email_addr, False):
1505	self.logger.info('Message rejected : %s in black list' %(self.email_addr))
1506	return False
1507
1508	if not self.email_header_acl('recipient_list', self.to_email_addr, True):
1509	self.logger.info('Message rejected : %s not in recipient list' %(self.to_email_addr))
1510	return False
1511
1512	# If spam drop the message
1513	#
1514	if self.spam(m) == 'drop':
1515	return False
1516
1517	elif self.spam(m) == 'spam':
1518	spam_msg = True
1519	else:
1520	spam_msg = False
1521
1522	if not m['Subject']:
1523	subject = 'No Subject'
1524	else:
1525	subject = self.email_to_unicode(m['Subject'])
1526
1527	self.logger.debug('subject: %s' %subject)
1528
1529	#
1530	# [hic] #1529: Re: LRZ
1531	# [hic] #1529?owner=bas,priority=medium: Re: LRZ
1532	#
1533	ticket_regex = r'''
1534	(?P<new_fields>[#][?].*)
1535	\|(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?.*)?:)
1536	'''
1537	# Check if FullBlogPlugin is installed
1538	#
1539	blog_enabled = None
1540	blog_regex = ''
1541	if self.get_config('components', 'tracfullblog.*') in ['enabled']:
1542	blog_enabled = True
1543	blog_regex = '''\|(?P<blog>blog:(?P<blog_id>\w*))'''
1544
1545
1546	# Check if DiscussionPlugin is installed
1547	#
1548	discussion_enabled = None
1549	discussion_regex = ''
1550	if self.get_config('components', 'tracdiscussion.api.*') in ['enabled']:
1551	discussion_enabled = True
1552	discussion_regex = r'''
1553	\|(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
1554	\|(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
1555	\|(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
1556	'''
1557
1558
1559	regex_str = ticket_regex + blog_regex + discussion_regex
1560	SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
1561
1562	# Find out if this is a ticket, a blog or a discussion
1563	#
1564	result = SYSTEM_RE.search(subject)
1565
1566	if result:
1567	# update ticket + fields
1568	#
1569	if result.group('reply') and self.TICKET_UPDATE:
1570	self.system = 'ticket'
1571
1572	# Skip the last ':' character
1573	#
1574	if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
1575	self.new_ticket(m, subject, spam_msg)
1576
1577	# New ticket + fields
1578	#
1579	elif result.group('new_fields'):
1580	self.system = 'ticket'
1581	self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
1582
1583	if blog_enabled:
1584	if result.group('blog'):
1585	self.system = 'blog'
1586	self.blog(result.group('blog_id'))
1587
1588	if discussion_enabled:
1589	# New topic.
1590	#
1591	if result.group('forum'):
1592	self.system = 'discussion'
1593	self.id = int(result.group('forum_id'))
1594	self.discussion_topic(m, subject[result.end('forum'):])
1595
1596	# Reply to topic.
1597	#
1598	elif result.group('topic'):
1599	self.system = 'discussion'
1600	self.id = int(result.group('topic_id'))
1601	self.discussion_topic_reply(m, subject[result.end('topic'):])
1602
1603	# Reply to topic message.
1604	#
1605	elif result.group('message'):
1606	self.system = 'discussion'
1607	self.id = int(result.group('message_id'))
1608	self.discussion_message_reply(m, subject[result.end('message'):])
1609
1610	else:
1611	self.system = 'ticket'
1612	result = self.ticket_update_by_subject(subject)
1613	if result:
1614	if not self.ticket_update(m, result, spam_msg):
1615	self.new_ticket(m, subject, spam_msg)
1616	else:
1617	# No update by subject, so just create a new ticket
1618	self.new_ticket(m, subject, spam_msg)
1619
1620
1621	########## BODY TEXT functions ###########################################################
1622
1623	def strip_signature(self, text):
1624	"""
1625	Strip signature from message, inspired by Mailman software
1626	"""
1627	body = []
1628	for line in text.splitlines():
1629	if line == '-- ':
1630	break
1631	body.append(line)
1632
1633	return ('\n'.join(body))
1634
1635	def reflow(self, text, delsp = 0):
1636	"""
1637	Reflow the message based on the format="flowed" specification (RFC 3676)
1638	"""
1639	flowedlines = []
1640	quotelevel = 0
1641	prevflowed = 0
1642
1643	for line in text.splitlines():
1644	from re import match
1645
1646	# Figure out the quote level and the content of the current line
1647	m = match('(>)( ?)(.)', line)
1648	linequotelevel = len(m.group(1))
1649	line = m.group(3)
1650
1651	# Determine whether this line is flowed
1652	if line and line != '-- ' and line[-1] == ' ':
1653	flowed = 1
1654	else:
1655	flowed = 0
1656
1657	if flowed and delsp and line and line[-1] == ' ':
1658	line = line[:-1]
1659
1660	# If the previous line is flowed, append this line to it
1661	if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1662	flowedlines[-1] += line
1663	# Otherwise, start a new line
1664	else:
1665	flowedlines.append('>' * linequotelevel + line)
1666
1667	prevflowed = flowed
1668
1669
1670	return '\n'.join(flowedlines)
1671
1672	def strip_quotes(self, text):
1673	"""
1674	Strip quotes from message by Nicolas Mendoza
1675	"""
1676	body = []
1677	for line in text.splitlines():
1678	if line.startswith(self.parameters.email_quote):
1679	continue
1680	body.append(line)
1681
1682	return ('\n'.join(body))
1683
1684	def inline_properties(self, text):
1685	"""
1686	Parse text if we use inline keywords to set ticket fields
1687	"""
1688	self.logger.debug('function inline_properties')
1689
1690	properties = dict()
1691	body = list()
1692
1693	INLINE_EXP = re.compile('\s[@]\s([a-zA-Z]+)\s:(.)$')
1694
1695	for line in text.splitlines():
1696	match = INLINE_EXP.match(line)
1697	if match:
1698	keyword, value = match.groups()
1699	self.properties[keyword] = value.strip()
1700
1701	self.logger.debug('inline properties: %s : %s' %(keyword,value))
1702
1703	else:
1704	body.append(line)
1705
1706	return '\n'.join(body)
1707
1708
1709	def wrap_text(self, text, replace_whitespace = False):
1710	"""
1711	Will break a lines longer then given length into several small
1712	lines of size given length
1713	"""
1714	import textwrap
1715
1716	LINESEPARATOR = '\n'
1717	reformat = ''
1718
1719	for s in text.split(LINESEPARATOR):
1720	tmp = textwrap.fill(s,self.USE_TEXTWRAP)
1721	if tmp:
1722	reformat = '%s\n%s' %(reformat,tmp)
1723	else:
1724	reformat = '%s\n' %reformat
1725
1726	return reformat
1727
1728	# Python2.4 and higher
1729	#
1730	#return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
1731	#
1732
1733	########## EMAIL attachements functions ###########################################################
1734
1735	def inline_part(self, part):
1736	"""
1737	"""
1738	self.logger.debug('function inline_part()')
1739
1740	return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
1741
1742	def get_message_parts(self, msg):
1743	"""
1744	parses the email message and returns a list of body parts and attachments
1745	body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
1746	"""
1747	self.logger.debug('function get_message_parts()')
1748
1749	message_parts = list()
1750
1751	ALTERNATIVE_MULTIPART = False
1752
1753	for part in msg.walk():
1754	self.logger.debug('Message part: Main-Type: %s' % part.get_content_maintype())
1755	self.logger.debug('Message part: Content-Type: %s' % part.get_content_type())
1756
1757	## Check content type
1758	#
1759	if part.get_content_type() in self.STRIP_CONTENT_TYPES:
1760	self.logger.debug("A %s attachment named '%s' was skipped" %(part.get_content_type(), part.get_filename()))
1761	continue
1762
1763	## Catch some mulitpart execptions
1764	#
1765	if part.get_content_type() == 'multipart/alternative':
1766	ALTERNATIVE_MULTIPART = True
1767	continue
1768
1769	## Skip multipart containers
1770	#
1771	if part.get_content_maintype() == 'multipart':
1772	self.logger.debug("Skipping multipart container")
1773
1774	continue
1775
1776	## Check if this is an inline part. It's inline if there is co Cont-Disp header, or if there is one and it says "inline"
1777	#
1778	inline = self.inline_part(part)
1779
1780	## Drop HTML message
1781	#
1782	if ALTERNATIVE_MULTIPART and self.DROP_ALTERNATIVE_HTML_VERSION:
1783	if part.get_content_type() == 'text/html':
1784	self.logger.debug('Skipping alternative HTML message')
1785	ALTERNATIVE_MULTIPART = False
1786	continue
1787
1788	## Inline text parts are where the body is
1789	#
1790	if part.get_content_type() == 'text/plain' and inline:
1791	self.logger.debug(' Inline body part')
1792
1793	# Try to decode, if fails then do not decode
1794	#
1795	body_text = part.get_payload(decode=1)
1796	if not body_text:
1797	body_text = part.get_payload(decode=0)
1798
1799	format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
1800	delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
1801
1802	if self.REFLOW and not self.VERBATIM_FORMAT and format == 'flowed':
1803	body_text = self.reflow(body_text, delsp == 'yes')
1804
1805	if self.STRIP_SIGNATURE:
1806	body_text = self.strip_signature(body_text)
1807
1808	if self.STRIP_QUOTES:
1809	body_text = self.strip_quotes(body_text)
1810
1811	if self.INLINE_PROPERTIES:
1812	body_text = self.inline_properties(body_text)
1813
1814	if self.USE_TEXTWRAP:
1815	body_text = self.wrap_text(body_text)
1816
1817	## Get contents charset (iso-8859-15 if not defined in mail headers)
1818	#
1819	charset = part.get_content_charset()
1820	if not charset:
1821	charset = 'iso-8859-15'
1822
1823	try:
1824	ubody_text = unicode(body_text, charset)
1825
1826	except UnicodeError, detail:
1827	ubody_text = unicode(body_text, 'iso-8859-15')
1828
1829	except LookupError, detail:
1830	ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
1831
1832	if self.VERBATIM_FORMAT:
1833	message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
1834	else:
1835	message_parts.append('%s' %ubody_text)
1836	else:
1837	if self.parameters.debug:
1838	s = ' Filename: %s' % part.get_filename()
1839	self.print_unicode(s)
1840
1841	##
1842	# First try to use email header function to convert filename.
1843	# If this fails the use the plan filename
1844	try:
1845	filename = self.email_to_unicode(part.get_filename())
1846	except UnicodeEncodeError, detail:
1847	filename = part.get_filename()
1848
1849	message_parts.append((filename, part))
1850
1851	return message_parts
1852
1853	def unique_attachment_names(self, message_parts):
1854	"""
1855	"""
1856	renamed_parts = []
1857	attachment_names = set()
1858
1859	for item in message_parts:
1860
1861	## If not an attachment, leave it alone
1862	#
1863	if not isinstance(item, tuple):
1864	renamed_parts.append(item)
1865	continue
1866
1867	(filename, part) = item
1868
1869	## If no filename, use a default one
1870	#
1871	if not filename:
1872	filename = 'untitled-part'
1873
1874	# Guess the extension from the content type, use non strict mode
1875	# some additional non-standard but commonly used MIME types
1876	# are also recognized
1877	#
1878	ext = mimetypes.guess_extension(part.get_content_type(), False)
1879	if not ext:
1880	ext = '.bin'
1881
1882	filename = '%s%s' % (filename, ext)
1883
1884	## Discard relative paths for windows/unix in attachment names
1885	#
1886	#filename = filename.replace('\\', '/').replace(':', '/')
1887	filename = filename.replace('\\', '_')
1888	filename = filename.replace('/', '_')
1889
1890	#
1891	# We try to normalize the filename to utf-8 NFC if we can.
1892	# Files uploaded from OS X might be in NFD.
1893	# Check python version and then try it
1894	#
1895	#if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
1896	# try:
1897	# filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
1898	# except TypeError:
1899	# pass
1900
1901	# Make the filename unique for this ticket
1902	num = 0
1903	unique_filename = filename
1904	dummy_filename, ext = os.path.splitext(filename)
1905
1906	while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
1907	num += 1
1908	unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
1909
1910	if self.parameters.debug:
1911	s = 'Attachment with filename %s will be saved as %s' % (filename, unique_filename)
1912	self.print_unicode(s)
1913
1914	attachment_names.add(unique_filename)
1915
1916	renamed_parts.append((filename, unique_filename, part))
1917
1918	return renamed_parts
1919
1920
1921	def attachment_exists(self, filename):
1922
1923	if self.parameters.debug:
1924	s = 'attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
1925	self.print_unicode(s)
1926
1927	# We have no valid ticket id
1928	#
1929	if not self.id:
1930	return False
1931
1932	try:
1933	if self.system == 'discussion':
1934	att = attachment.Attachment(self.env, 'discussion', 'ticket/%s'
1935	% (self.id,), filename)
1936	else:
1937	att = attachment.Attachment(self.env, 'ticket', self.id,
1938	filename)
1939	return True
1940	except attachment.ResourceNotFound:
1941	return False
1942
1943	########## TRAC Ticket Text ###########################################################
1944
1945	def body_text(self, message_parts):
1946	body_text = []
1947
1948	for part in message_parts:
1949	# Plain text part, append it
1950	if not isinstance(part, tuple):
1951	body_text.extend(part.strip().splitlines())
1952	body_text.append("")
1953	continue
1954
1955	(original, filename, part) = part
1956	inline = self.inline_part(part)
1957
1958	if part.get_content_maintype() == 'image' and inline:
1959	if self.system != 'discussion':
1960	body_text.append('[[Image(%s)]]' % filename)
1961	body_text.append("")
1962	else:
1963	if self.system != 'discussion':
1964	body_text.append('[attachment:"%s"]' % filename)
1965	body_text.append("")
1966
1967	body_text = '\r\n'.join(body_text)
1968	return body_text
1969
1970	def html_mailto_link(self, subject):
1971	"""
1972	This function returns a HTML mailto tag with the ticket id and author email address
1973	"""
1974	if not self.author:
1975	author = self.email_addr
1976	else:
1977	author = self.author
1978
1979	if not self.parameters.mailto_cc:
1980	self.parameters.mailto_cc = ''
1981
1982	# use urllib to escape the chars
1983	#
1984	s = 'mailto:%s?Subject=%s&Cc=%s' %(
1985	urllib.quote(self.email_addr),
1986	urllib.quote('Re: #%s: %s' %(self.id, subject)),
1987	urllib.quote(self.parameters.mailto_cc)
1988	)
1989
1990	s = '\r\n{{{\r\n#!html\r\n<a\r\n href="%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
1991	return s
1992
1993	########## TRAC notify section ###########################################################
1994
1995	def notify(self, tkt, new=True, modtime=0):
1996	"""
1997	A wrapper for the TRAC notify function. So we can use templates
1998	"""
1999	self.logger.debug('function notify()')
2000
2001	if self.parameters.dry_run:
2002	print 'DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter']
2003	return
2004	try:
2005
2006	#from trac.ticket.web_ui import TicketModule
2007	#from trac.ticket.notification import TicketNotificationSystem
2008	#ticket_sys = TicketNotificationSystem(self.env)
2009	#a = TicketModule(self.env)
2010	#print a.__dict__
2011	#tn_sys = TicketNotificationSystem(self.env)
2012	#print tn_sys
2013	#print tn_sys.__dict__
2014	#sys.exit(0)
2015
2016	# create false {abs_}href properties, to trick Notify()
2017	#
2018	if not (self.VERSION in [0.11, 0.12]):
2019	self.env.abs_href = Href(self.get_config('project', 'url'))
2020	self.env.href = Href(self.get_config('project', 'url'))
2021
2022
2023	tn = TicketNotifyEmail(self.env)
2024
2025	if self.notify_template:
2026
2027	if self.VERSION >= 0.11:
2028
2029	from trac.web.chrome import Chrome
2030
2031	if self.notify_template_update and not new:
2032	tn.template_name = self.notify_template_update
2033	else:
2034	tn.template_name = self.notify_template
2035
2036	tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
2037
2038	else:
2039
2040	tn.template_name = self.notify_template;
2041
2042	tn.notify(tkt, new, modtime)
2043
2044	except Exception, e:
2045	self.logger.error('Failure sending notification on creation of ticket #%s: %s' %(self.id, e))
2046
2047
2048
2049	########## Parse Config File ###########################################################
2050
2051	def ReadConfig(file, name):
2052	"""
2053	Parse the config file
2054	"""
2055	if not os.path.isfile(file):
2056	print 'File %s does not exist' %file
2057	sys.exit(1)
2058
2059	config = trac_config.Configuration(file)
2060
2061	# Use given project name else use defaults
2062	#
2063	if name:
2064	sections = config.sections()
2065	if not name in sections:
2066	print "Not a valid project name: %s" %name
2067	print "Valid names: %s" %sections
2068	sys.exit(1)
2069
2070	project = SaraDict()
2071	for option, value in config.options(name):
2072	project[option] = value
2073
2074	else:
2075	# use some trac internals to get the defaults
2076	#
2077	tmp = config.parser.defaults()
2078	project = SaraDict()
2079
2080	for option,value in tmp.items():
2081	project[option] = value
2082
2083	## Convert debug value to int
2084	#
2085	if project.debug:
2086	project.debug = int(project.debug)
2087	else:
2088	project.debug = 0
2089
2090	return project
2091
2092	########## Setup Logging ###############################################################
2093
2094	def setup_log(parameters, project_name, interactive=None):
2095	"""
2096	Setup loging
2097
2098	Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
2099	would be interpreted by the ConfigParser itself.
2100	"""
2101	logger = logging.getLogger('email2trac %s' %project_name)
2102
2103	if interactive:
2104	parameters.log_type = 'stderr'
2105
2106	if not parameters.log_type:
2107	parameters.log_type = 'syslog'
2108
2109	if parameters.log_type == 'file':
2110
2111	if not parameters.log_file:
2112	parameters.log_file = 'email2trac.log'
2113
2114	if not os.path.isabs(parameters.log_file):
2115	import tempfile
2116	parameters.log_file = os.path.join(tempfile.gettempdir(), parameters.log_file)
2117
2118	log_handler = logging.FileHandler(parameters.log_file)
2119
2120	elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
2121	# Requires win32 extensions
2122	log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
2123
2124	elif parameters.log_type in ('syslog', 'unix'):
2125	log_handler = logging.handlers.SysLogHandler('/dev/log')
2126
2127	elif parameters.log_type in ('stderr'):
2128	log_handler = logging.StreamHandler(sys.stderr)
2129
2130	else:
2131	log_handler = logging.handlers.BufferingHandler(0)
2132
2133	if parameters.log_format:
2134	parameters.log_format = parameters.log_format.replace('$(', '%(')
2135	else:
2136	parameters.log_format = '%(name)s: %(message)s'
2137
2138	log_formatter = logging.Formatter(parameters.log_format)
2139	log_handler.setFormatter(log_formatter)
2140	logger.addHandler(log_handler)
2141
2142	if (parameters.log_level in ['DEBUG', 'ALL']) or (parameters.debug > 0):
2143	logger.setLevel(logging.DEBUG)
2144
2145	elif parameters.log_level in ['INFO'] or parameters.verbose:
2146	logger.setLevel(logging.INFO)
2147
2148	elif parameters.log_level in ['WARNING']:
2149	logger.setLevel(logging.WARNING)
2150
2151	elif parameters.log_level in ['ERROR']:
2152	logger.setLevel(logging.ERROR)
2153
2154	elif parameters.log_level in ['CRITICAL']:
2155	logger.setLevel(logging.CRITICAL)
2156
2157	else:
2158	logger.setLevel(logging.INFO)
2159
2160	return logger
2161
2162
2163	if __name__ == '__main__':
2164	# Default config file
2165	#
2166	configfile = '@email2trac_conf@'
2167	project = ''
2168	component = ''
2169	ticket_prefix = 'default'
2170	dry_run = None
2171	verbose = None
2172	debug_interactive = None
2173
2174	SHORT_OPT = 'cdhf:np:t:v'
2175	LONG_OPT = ['component=', 'debug', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=', 'verbose']
2176
2177	try:
2178	opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
2179	except getopt.error,detail:
2180	print __doc__
2181	print detail
2182	sys.exit(1)
2183
2184	project_name = None
2185	for opt,value in opts:
2186	if opt in [ '-h', '--help']:
2187	print __doc__
2188	sys.exit(0)
2189	elif opt in ['-c', '--component']:
2190	component = value
2191	elif opt in ['-d', '--debug']:
2192	debug_interactive = 1
2193	elif opt in ['-f', '--file']:
2194	configfile = value
2195	elif opt in ['-n', '--dry-run']:
2196	dry_run = True
2197	elif opt in ['-p', '--project']:
2198	project_name = value
2199	elif opt in ['-t', '--ticket_prefix']:
2200	ticket_prefix = value
2201	elif opt in ['-v', '--verbose']:
2202	verbose = True
2203
2204	settings = ReadConfig(configfile, project_name)
2205
2206	# The default prefix for ticket values in email2trac.conf
2207	#
2208	settings.ticket_prefix = ticket_prefix
2209	settings.dry_run = dry_run
2210	settings.verbose = verbose
2211
2212	if not settings.debug and debug_interactive:
2213	settings.debug = debug_interactive
2214
2215
2216	if not settings.project:
2217	print __doc__
2218	print 'No Trac project is defined in the email2trac config file.'
2219	sys.exit(1)
2220
2221	logger = setup_log(settings, os.path.basename(settings.project), debug_interactive)
2222
2223	if component:
2224	settings['component'] = component
2225
2226	# Determine major trac version used to be in email2trac.conf
2227	# Quick hack for 0.12
2228	#
2229	version = '0.%s' %(trac_version.split('.')[1])
2230	if version.startswith('0.12'):
2231	version = '0.12'
2232
2233	logger.debug("Found trac version: %s" %(version))
2234
2235	#debug HvB
2236	#print settings
2237
2238	try:
2239	if version == '0.10':
2240	from trac import attachment
2241	from trac.env import Environment
2242	from trac.ticket import Ticket
2243	from trac.web.href import Href
2244	from trac import util
2245	#
2246	# return util.text.to_unicode(str)
2247	#
2248	# see http://projects.edgewall.com/trac/changeset/2799
2249	from trac.ticket.notification import TicketNotifyEmail
2250	from trac import config as trac_config
2251	from trac.core import TracError
2252
2253	elif version == '0.11':
2254	from trac import attachment
2255	from trac.env import Environment
2256	from trac.ticket import Ticket
2257	from trac.web.href import Href
2258	from trac import config as trac_config
2259	from trac import util
2260	from trac.core import TracError
2261	from trac.perm import PermissionSystem
2262
2263	#
2264	# return util.text.to_unicode(str)
2265	#
2266	# see http://projects.edgewall.com/trac/changeset/2799
2267	from trac.ticket.notification import TicketNotifyEmail
2268
2269	elif version == '0.12':
2270	from trac import attachment
2271	from trac.env import Environment
2272	from trac.ticket import Ticket
2273	from trac.web.href import Href
2274	from trac import config as trac_config
2275	from trac import util
2276	from trac.core import TracError
2277	from trac.perm import PermissionSystem
2278
2279	#
2280	# return util.text.to_unicode(str)
2281	#
2282	# see http://projects.edgewall.com/trac/changeset/2799
2283	from trac.ticket.notification import TicketNotifyEmail
2284
2285
2286	else:
2287	logger.error('TRAC version %s is not supported' %version)
2288	sys.exit(1)
2289
2290	# Must be set before environment is created
2291	#
2292	if settings.has_key('python_egg_cache'):
2293	python_egg_cache = str(settings['python_egg_cache'])
2294	os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
2295
2296	if settings.debug > 0:
2297	logger.debug('Loading environment %s', settings.project)
2298
2299	env = Environment(settings['project'], create=0)
2300
2301	tktparser = TicketEmailParser(env, settings, logger, float(version))
2302	tktparser.parse(sys.stdin)
2303
2304	# Catch all errors ans log to SYSLOG if we have enabled this
2305	# else stdout
2306	#
2307	except Exception, error:
2308
2309	etype, evalue, etb = sys.exc_info()
2310	for e in traceback.format_exception(etype, evalue, etb):
2311	logger.critical(e)
2312
2313	if m:
2314	tktparser.save_email_for_debug(m, True)
2315
2316	sys.exit(1)
2317	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format