Context Navigation

source: trunk/email2trac.py.in @ 410

Last change on this file since 410 was 410, checked in by bas, 14 years ago
Moved logging out class TicketParser? so we can use outside this class
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 60.8 KB

Line
1	#!@PYTHON@
2	# Copyright (C) 2002
3	#
4	# This file is part of the email2trac utils
5	#
6	# This program is free software; you can redistribute it and/or modify it
7	# under the terms of the GNU General Public License as published by the
8	# Free Software Foundation; either version 2, or (at your option) any
9	# later version.
10	#
11	# This program is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with this program; if not, write to the Free Software
18	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
19	#
20	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
21	#
22	"""
23	email2trac.py -- Email tickets to Trac.
24
25	A simple MTA filter to create Trac tickets from inbound emails.
26
27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28	Copyright 2005, Edgewall Software
29
30	Authors:
31	Bas van der Vlies <basv@sara.nl>
32	Walter de Jong <walter@sara.nl>
33
34	The scripts reads emails from stdin and inserts directly into a Trac database.
35
36	How to use
37	----------
38	* See https://subtrac.sara.nl/oss/email2trac/
39
40	* Create an config file:
41	[DEFAULT] # REQUIRED
42	project : /data/trac/test # REQUIRED
43	debug : 1 # OPTIONAL, if set print some DEBUG info
44
45	[jouvin] # OPTIONAL project declaration, if set both fields necessary
46	project : /data/trac/jouvin # use -p\|--project jouvin.
47
48	* default config file is : /etc/email2trac.conf
49
50	* Commandline opions:
51	-h,--help
52	-f,--file <configuration file>
53	-n,--dry-run
54	-p, --project <project name>
55	-t, --ticket_prefix <name>
56
57	SVN Info:
58	$Id: email2trac.py.in 410 2010-07-20 08:44:34Z bas $
59	"""
60	import os
61	import sys
62	import string
63	import getopt
64	import stat
65	import time
66	import email
67	import email.Iterators
68	import email.Header
69	import re
70	import urllib
71	import unicodedata
72	from stat import *
73	import mimetypes
74	import traceback
75	import logging
76	import logging.handlers
77	import UserDict
78
79
80	from trac import __version__ as trac_version
81
82	# Will fail where unavailable, e.g. Windows
83	#
84	try:
85	import syslog
86	SYSLOG_AVAILABLE = True
87	except ImportError:
88	SYSLOG_AVAILABLE = False
89
90	from datetime import tzinfo, timedelta, datetime
91	from trac import config as trac_config
92
93	# Some global variables
94	#
95	trac_default_version = '0.11'
96	m = None
97
98	# A UTC class needed for trac version 0.11, added by
99	# tbaschak at ktc dot mb dot ca
100	#
101	class UTC(tzinfo):
102	"""UTC"""
103	ZERO = timedelta(0)
104	HOUR = timedelta(hours=1)
105
106	def utcoffset(self, dt):
107	return self.ZERO
108
109	def tzname(self, dt):
110	return "UTC"
111
112	def dst(self, dt):
113	return self.ZERO
114
115	class SaraDict(UserDict.UserDict):
116	def __init__(self, dictin = None):
117	UserDict.UserDict.__init__(self)
118	self.name = None
119
120	if dictin:
121	if dictin.has_key('name'):
122	self.name = dictin['name']
123	del dictin['name']
124	self.data = dictin
125
126	def get_value(self, key):
127	if self.has_key(key):
128	return self[key]
129	else:
130	return None
131
132	def __repr__(self):
133	return repr(self.data)
134
135	def __str__(self):
136	return str(self.data)
137
138	def __getattr__(self, name):
139	"""
140	override the class attribute get method. Return the value
141	from the dictionary
142	"""
143	if self.data.has_key(name):
144	return self.data[name]
145	else:
146	return None
147
148	def __setattr__(self, name, value):
149	"""
150	override the class attribute set method only when the UserDict
151	has set its class attribute
152	"""
153	if self.__dict__.has_key('data'):
154	self.data[name] = value
155	else:
156	self.__dict__[name] = value
157
158	def __iter__(self):
159	return iter(self.data.keys())
160
161	class TicketEmailParser(object):
162	env = None
163	comment = '> '
164
165	def __init__(self, env, parameters, logger, version):
166	self.env = env
167
168	# Database connection
169	#
170	self.db = None
171
172	# Save parameters
173	#
174	self.parameters = parameters
175	self.logger = logger
176
177	# Some useful mail constants
178	#
179	self.email_name = None
180	self.email_addr = None
181	self.email_from = None
182	self.author = None
183	self.id = None
184
185	self.STRIP_CONTENT_TYPES = list()
186
187	self.VERSION = version
188
189	self.get_config = self.env.config.get
190
191	## init function ##
192	#
193	self.setup_parameters()
194
195	def setup_parameters(self):
196	if self.parameters.has_key('umask'):
197	os.umask(int(self.parameters['umask'], 8))
198
199
200	if self.parameters.has_key('mailto_link'):
201	self.MAILTO = int(self.parameters['mailto_link'])
202	if self.parameters.has_key('mailto_cc'):
203	self.MAILTO_CC = self.parameters['mailto_cc']
204	else:
205	self.MAILTO_CC = ''
206	else:
207	self.MAILTO = 0
208
209	if self.parameters.has_key('spam_level'):
210	self.SPAM_LEVEL = int(self.parameters['spam_level'])
211	else:
212	self.SPAM_LEVEL = 0
213
214	if self.parameters.has_key('spam_header'):
215	self.SPAM_HEADER = self.parameters['spam_header']
216	else:
217	self.SPAM_HEADER = 'X-Spam-Score'
218
219	if self.parameters.has_key('email_quote'):
220	self.EMAIL_QUOTE = str(self.parameters['email_quote'])
221	else:
222	self.EMAIL_QUOTE = '> '
223
224	if self.parameters.has_key('email_header'):
225	self.EMAIL_HEADER = int(self.parameters['email_header'])
226	else:
227	self.EMAIL_HEADER = 0
228
229	if self.parameters.has_key('alternate_notify_template'):
230	self.notify_template = str(self.parameters['alternate_notify_template'])
231	else:
232	self.notify_template = None
233
234	if self.parameters.has_key('alternate_notify_template_update'):
235	self.notify_template_update = str(self.parameters['alternate_notify_template_update'])
236	else:
237	self.notify_template_update = None
238
239	if self.parameters.has_key('reply_all'):
240	self.REPLY_ALL = int(self.parameters['reply_all'])
241	else:
242	self.REPLY_ALL = 0
243
244	if self.parameters.has_key('ticket_permission_system'):
245	self.TICKET_PERMISSION_SYSTEM = str(self.parameters['ticket_permission_system'])
246	else:
247	self.TICKET_PERMISSION_SYSTEM = None
248
249	if self.parameters.has_key('ticket_update'):
250	self.TICKET_UPDATE = int(self.parameters['ticket_update'])
251	else:
252	self.TICKET_UPDATE = 0
253
254	if self.parameters.has_key('ticket_update_by_subject'):
255	self.TICKET_UPDATE_BY_SUBJECT = int(self.parameters['ticket_update_by_subject'])
256	else:
257	self.TICKET_UPDATE_BY_SUBJECT = 0
258
259	if self.parameters.has_key('ticket_update_by_subject_lookback'):
260	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK = int(self.parameters['ticket_update_by_subject_lookback'])
261	else:
262	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK = 30
263
264	if self.parameters.has_key('drop_spam'):
265	self.DROP_SPAM = int(self.parameters['drop_spam'])
266	else:
267	self.DROP_SPAM = 0
268
269	if self.parameters.has_key('verbatim_format'):
270	self.VERBATIM_FORMAT = int(self.parameters['verbatim_format'])
271	else:
272	self.VERBATIM_FORMAT = 1
273
274	if self.parameters.has_key('reflow'):
275	self.REFLOW = int(self.parameters['reflow'])
276	else:
277	self.REFLOW = 1
278
279	if self.parameters.has_key('drop_alternative_html_version'):
280	self.DROP_ALTERNATIVE_HTML_VERSION = int(self.parameters['drop_alternative_html_version'])
281	else:
282	self.DROP_ALTERNATIVE_HTML_VERSION = 0
283
284	if self.parameters.has_key('strip_signature'):
285	self.STRIP_SIGNATURE = int(self.parameters['strip_signature'])
286	else:
287	self.STRIP_SIGNATURE = 0
288
289	if self.parameters.has_key('strip_quotes'):
290	self.STRIP_QUOTES = int(self.parameters['strip_quotes'])
291	else:
292	self.STRIP_QUOTES = 0
293
294	self.properties = dict()
295	if self.parameters.has_key('inline_properties'):
296	self.INLINE_PROPERTIES = int(self.parameters['inline_properties'])
297	else:
298	self.INLINE_PROPERTIES = 0
299
300	if self.parameters.has_key('use_textwrap'):
301	self.USE_TEXTWRAP = int(self.parameters['use_textwrap'])
302	else:
303	self.USE_TEXTWRAP = 0
304
305	if self.parameters.has_key('binhex'):
306	self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
307
308	if self.parameters.has_key('applesingle'):
309	self.STRIP_CONTENT_TYPES.append('application/applefile')
310
311	if self.parameters.has_key('appledouble'):
312	self.STRIP_CONTENT_TYPES.append('application/applefile')
313
314	if self.parameters.has_key('strip_content_types'):
315	items = self.parameters['strip_content_types'].split(',')
316	for item in items:
317	self.STRIP_CONTENT_TYPES.append(item.strip())
318
319	self.WORKFLOW = None
320	if self.parameters.has_key('workflow'):
321	self.WORKFLOW = self.parameters['workflow']
322
323	# Use OS independend functions
324	#
325	self.TMPDIR = os.path.normcase('/tmp')
326	if self.parameters.has_key('tmpdir'):
327	self.TMPDIR = os.path.normcase(str(self.parameters['tmpdir']))
328
329	if self.parameters.has_key('ignore_trac_user_settings'):
330	self.IGNORE_TRAC_USER_SETTINGS = int(self.parameters['ignore_trac_user_settings'])
331	else:
332	self.IGNORE_TRAC_USER_SETTINGS = 0
333
334	if self.parameters.has_key('email_triggers_workflow'):
335	self.EMAIL_TRIGGERS_WORKFLOW = int(self.parameters['email_triggers_workflow'])
336	else:
337	self.EMAIL_TRIGGERS_WORKFLOW = 1
338
339	if self.parameters.has_key('subject_field_separator'):
340	self.SUBJECT_FIELD_SEPARATOR = self.parameters['subject_field_separator'].strip()
341	else:
342	self.SUBJECT_FIELD_SEPARATOR = '&'
343
344	self.trac_smtp_from = self.get_config('notification', 'smtp_from')
345
346	self.system = None
347
348	########## Email Header Functions ###########################################################
349
350	def spam(self, message):
351	"""
352	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
353	# Note if Spam_level then '*' are included
354	"""
355	spam = False
356	if message.has_key(self.SPAM_HEADER):
357	spam_l = string.split(message[self.SPAM_HEADER])
358
359	try:
360	number = spam_l[0].count('*')
361	except IndexError, detail:
362	number = 0
363
364	if number >= self.SPAM_LEVEL:
365	spam = True
366
367	# treat virus mails as spam
368	#
369	elif message.has_key('X-Virus-found'):
370	spam = True
371
372	# How to handle SPAM messages
373	#
374	if self.DROP_SPAM and spam:
375	if self.parameters.debug > 2 :
376	set.logger.debug('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' % self.SPAM_LEVEL)
377
378	return 'drop'
379
380	elif spam:
381
382	return 'Spam'
383
384	else:
385
386	return False
387
388	def email_header_acl(self, keyword, header_field, default):
389	"""
390	This function wil check if the email address is allowed or denied
391	to send mail to the ticket list
392	"""
393	self.logger.debug('function email_header_acl: %s' %keyword)
394
395	try:
396	mail_addresses = self.parameters[keyword]
397
398	# Check if we have an empty string
399	#
400	if not mail_addresses:
401	return default
402
403	except KeyError, detail:
404	if self.parameters.debug > 2 :
405	self.logger.debug('%s not defined, all messages are allowed.' %(keyword))
406
407	return default
408
409	mail_addresses = string.split(mail_addresses, ',')
410
411	for entry in mail_addresses:
412	entry = entry.strip()
413	TO_RE = re.compile(entry, re.VERBOSE\|re.IGNORECASE)
414	result = TO_RE.search(header_field)
415	if result:
416	return True
417
418	return False
419
420	def email_header_txt(self, m):
421	"""
422	Display To and CC addresses in description field
423	"""
424	s = ''
425
426	if m['To'] and len(m['To']) > 0:
427	s = "'''To:''' %s\r\n" %(m['To'])
428	if m['Cc'] and len(m['Cc']) > 0:
429	s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
430
431	return self.email_to_unicode(s)
432
433
434	def get_sender_info(self, message):
435	"""
436	Get the default author name and email address from the message
437	"""
438
439	self.email_to = self.email_to_unicode(message['to'])
440	self.to_name, self.to_email_addr = email.Utils.parseaddr (self.email_to)
441
442	self.email_from = self.email_to_unicode(message['from'])
443	self.email_name, self.email_addr = email.Utils.parseaddr(self.email_from)
444
445	## Trac can not handle author's name that contains spaces
446	# and forbid the ticket email address as author field
447
448	if self.email_addr == self.trac_smtp_from:
449	if self.email_name:
450	self.author = self.email_name
451	else:
452	self.author = "email2trac"
453	else:
454	self.author = self.email_addr
455
456	if self.IGNORE_TRAC_USER_SETTINGS:
457	return
458
459	# Is this a registered user, use email address as search key:
460	# result:
461	# u : login name
462	# n : Name that the user has set in the settings tab
463	# e : email address that the user has set in the settings tab
464	#
465	users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
466	if e and (e.lower() == self.email_addr.lower()) ]
467
468	if len(users) == 1:
469	self.email_from = users[0][0]
470	self.author = users[0][0]
471
472	def set_reply_fields(self, ticket, message):
473	"""
474	Set all the right fields for a new ticket
475	"""
476	self.logger.debug('function set_reply_fields')
477
478	## Only use name or email adress
479	#ticket['reporter'] = self.email_from
480	ticket['reporter'] = self.author
481
482
483	# Put all CC-addresses in ticket CC field
484	#
485	if self.REPLY_ALL:
486
487	email_cc = ''
488
489	cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
490
491	if not cc_addrs:
492	return
493
494	## Build a list of forbidden CC addresses
495	#
496	#to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
497	#to_list = list()
498	#for n,e in to_addrs:
499	# to_list.append(e)
500
501	# Always Remove reporter email address from cc-list
502	#
503	try:
504	cc_addrs.remove((self.author, self.email_addr))
505	except ValueError, detail:
506	pass
507
508	for name,addr in cc_addrs:
509
510	## Prevent mail loop
511	#
512	#if addr in to_list:
513
514	if addr == self.trac_smtp_from:
515	if self.parameters.debug:
516	self.logger.debug("Skipping %s mail address for CC-field" %(addr))
517	continue
518
519	if email_cc:
520	email_cc = '%s, %s' %(email_cc, addr)
521	else:
522	email_cc = addr
523
524	if email_cc:
525	if self.parameters.debug:
526	self.logger.debug('set_reply_fields: %s' %email_cc)
527
528	ticket['cc'] = self.email_to_unicode(email_cc)
529
530
531	########## DEBUG functions ###########################################################
532
533	def debug_body(self, message_body, tempfile=False):
534	if tempfile:
535	import tempfile
536	body_file = tempfile.mktemp('.email2trac')
537	else:
538	body_file = os.path.join(self.TMPDIR, 'body.txt')
539
540	if self.parameters.dry_run:
541	print 'DRY-RUN: not saving body to %s' %(body_file)
542	return
543
544	print 'TD: writing body to %s' %(body_file)
545	fx = open(body_file, 'wb')
546	if not message_body:
547	message_body = '(None)'
548
549	message_body = message_body.encode('utf-8')
550	#message_body = unicode(message_body, 'iso-8859-15')
551
552	fx.write(message_body)
553	fx.close()
554	try:
555	os.chmod(body_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
556	except OSError:
557	pass
558
559	def debug_attachments(self, message_parts):
560	"""
561	"""
562	self.logger.debug('function debug_attachments')
563
564	n = 0
565	for item in message_parts:
566	# Skip inline text parts
567	if not isinstance(item, tuple):
568	continue
569
570	(original, filename, part) = item
571
572	n = n + 1
573	print 'TD: part%d: Content-Type: %s' % (n, part.get_content_type())
574
575	s = 'TD: part%d: filename: %s' %(n, filename)
576	self.print_unicode(s)
577
578	## Forbidden chars
579	#
580	filename = filename.replace('\\', '_')
581	filename = filename.replace('/', '_')
582
583
584	part_file = os.path.join(self.TMPDIR, filename)
585	s = 'TD: writing part%d (%s)' % (n,part_file)
586	self.print_unicode(s)
587
588	if self.parameters.dry_run:
589	print 'DRY_RUN: NOT saving attachments'
590	continue
591
592	part_file = util.text.unicode_quote(part_file)
593
594	fx = open(part_file, 'wb')
595	text = part.get_payload(decode=1)
596
597	if not text:
598	text = '(None)'
599
600	fx.write(text)
601	fx.close()
602
603	try:
604	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
605	except OSError:
606	pass
607
608	def save_email_for_debug(self, message, tempfile=False):
609
610	if tempfile:
611	import tempfile
612	msg_file = tempfile.mktemp('.email2trac')
613	else:
614	#msg_file = '/var/tmp/msg.txt'
615	msg_file = os.path.join(self.TMPDIR, 'msg.txt')
616
617	if self.parameters.dry_run:
618	print 'DRY_RUN: NOT saving email message to %s' %(msg_file)
619	else:
620	print 'TD: saving email to %s' %(msg_file)
621
622	fx = open(msg_file, 'wb')
623	fx.write('%s' % message)
624	fx.close()
625
626	try:
627	os.chmod(msg_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
628	except OSError:
629	pass
630
631	message_parts = self.get_message_parts(message)
632	message_parts = self.unique_attachment_names(message_parts)
633	body_text = self.body_text(message_parts)
634	self.debug_body(body_text, True)
635	self.debug_attachments(message_parts)
636
637	########## Conversion functions ###########################################################
638
639	def email_to_unicode(self, message_str):
640	"""
641	Email has 7 bit ASCII code, convert it to unicode with the charset
642	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
643	understands it.
644	"""
645	self.logger.debug("function email_to_unicode")
646
647	results = email.Header.decode_header(message_str)
648
649	s = None
650	for text,format in results:
651	if format:
652	try:
653	temp = unicode(text, format)
654	except UnicodeError, detail:
655	# This always works
656	#
657	temp = unicode(text, 'iso-8859-15')
658	except LookupError, detail:
659	#text = 'ERROR: Could not find charset: %s, please install' %format
660	#temp = unicode(text, 'iso-8859-15')
661	temp = message_str
662
663	else:
664	temp = string.strip(text)
665	temp = unicode(text, 'iso-8859-15')
666
667	if s:
668	s = '%s %s' %(s, temp)
669	else:
670	s = '%s' %temp
671
672	#s = s.encode('utf-8')
673	return s
674
675	def str_to_dict(self, s):
676	"""
677	Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
678	"""
679	self.logger.debug("function str_to_dict")
680
681	fields = string.split(s, self.SUBJECT_FIELD_SEPARATOR)
682
683	result = dict()
684	for field in fields:
685	try:
686	index, value = string.split(field, '=')
687
688	# We can not change the description of a ticket via the subject
689	# line. The description is the body of the email
690	#
691	if index.lower() in ['description']:
692	continue
693
694	if value:
695	result[index.lower()] = value
696
697	except ValueError:
698	pass
699	return result
700
701	def print_unicode(self,s):
702	"""
703	This function prints unicode strings uif possible else it will quote it
704	"""
705	try:
706	self.logger.debug(s)
707	except UnicodeEncodeError, detail:
708	self.logger.debug(util.text.unicode_quote(s))
709
710	########## TRAC ticket functions ###########################################################
711
712	def check_permission_participants(self, tkt):
713	"""
714	Check if the mailer is allowed to update the ticket
715	"""
716	self.logger.debug('function check_permission_participants')
717
718	if tkt['reporter'].lower() in [self.author, self.email_addr]:
719	if self.parameters.debug:
720	self.logger.info('ALLOW, %s is the ticket reporter' %(self.email_addr))
721
722	return True
723
724	perm = PermissionSystem(self.env)
725	if perm.check_permission('TICKET_MODIFY', self.author):
726	if self.parameters.debug:
727	self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
728
729	return True
730
731	else:
732	return False
733
734
735	# Is the updater in the CC?
736	try:
737	cc_list = tkt['cc'].split(',')
738	for cc in cc_list:
739	if self.email_addr.lower() in cc.strip():
740
741	if self.parameters.debug:
742	self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
743
744	return True
745
746	except KeyError:
747	return False
748
749	def check_permission(self, tkt, action):
750	"""
751	check if the reporter has the right permission for the action:
752	- TICKET_CREATE
753	- TICKET_MODIFY
754
755	There are three models:
756	- None : no checking at all
757	- trac : check the permission via trac permission model
758	- email2trac: ....
759	"""
760	self.logger.debug("function check_permission")
761
762	if self.TICKET_PERMISSION_SYSTEM in ['trac']:
763
764	perm = PermissionSystem(self.env)
765	if perm.check_permission(action, self.author):
766	return True
767	else:
768	return False
769
770	elif self.TICKET_PERMISSION_SYSTEM in ['update_restricted_to_participants']:
771	if action in ['TICKET_MODIFY']:
772	return (self.check_permission_participants(tkt))
773	else:
774	return True
775
776	# Default is to allow everybody ticket updates and ticket creation
777	else:
778	return True
779
780
781	def update_ticket_fields(self, ticket, user_dict, use_default=None):
782	"""
783	This will update the ticket fields. It will check if the
784	given fields are known and if the right values are specified
785	It will only update the ticket field value:
786	- If the field is known
787	- If the value supplied is valid for the ticket field.
788	If not then there are two options:
789	1) Skip the value (use_default=None)
790	2) Set default value for field (use_default=1)
791	"""
792	self.logger.debug("function update_ticket_fields")
793
794	# Build a system dictionary from the ticket fields
795	# with field as index and option as value
796	#
797	sys_dict = dict()
798	for field in ticket.fields:
799	try:
800	sys_dict[field['name']] = field['options']
801
802	except KeyError:
803	sys_dict[field['name']] = None
804	pass
805
806	## Check user supplied fields an compare them with the
807	# system one's
808	#
809	for field,value in user_dict.items():
810	if self.parameters.debug:
811	s = 'TD: user_field\t %s = %s' %(field,value)
812	self.print_unicode(s)
813
814	## To prevent mail loop
815	#
816	if field == 'cc':
817
818	cc_list = user_dict['cc'].split(',')
819
820	if self.trac_smtp_from in cc_list:
821	if self.parameters.debug:
822	self.logger.debug('TD: MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
823
824	cc_list.remove(self.trac_smtp_from)
825
826	value = ','.join(cc_list)
827
828
829	if sys_dict.has_key(field):
830
831	# Check if value is an allowed system option, if TypeError then
832	# every value is allowed
833	#
834	try:
835	if value in sys_dict[field]:
836	ticket[field] = value
837	else:
838	# Must we set a default if value is not allowed
839	#
840	if use_default:
841	value = self.get_config('ticket', 'default_%s' %(field) )
842
843	except TypeError:
844	pass
845
846	## Only set if we have a value
847	#
848	if value:
849	ticket[field] = value
850
851	if self.parameters.debug:
852	s = 'ticket_field\t %s = %s' %(field, ticket[field])
853	self.print_unicode(s)
854
855	def ticket_update(self, m, id, spam):
856	"""
857	If the current email is a reply to an existing ticket, this function
858	will append the contents of this email to that ticket, instead of
859	creating a new one.
860	"""
861	self.logger.debug("function ticket_update")
862
863	# Must we update ticket fields
864	#
865	update_fields = dict()
866	try:
867	id, keywords = string.split(id, '?')
868
869	update_fields = self.str_to_dict(keywords)
870
871	# Strip '#'
872	#
873	self.id = int(id[1:])
874
875	except ValueError:
876
877	# Strip '#'
878	#
879	self.id = int(id[1:])
880
881	self.logger.debug("function ticket_update id %s" %id)
882
883	# When is the change committed
884	#
885	if self.VERSION < 0.11:
886	when = int(time.time())
887	else:
888	utc = UTC()
889	when = datetime.now(utc)
890
891	try:
892	tkt = Ticket(self.env, self.id, self.db)
893
894	except util.TracError, detail:
895
896	# Not a valid ticket
897
898	self.id = None
899	return False
900
901	# Check the permission of the reporter
902	#
903	if self.TICKET_PERMISSION_SYSTEM:
904	if not self.check_permission(tkt, 'TICKET_MODIFY'):
905	self.logger.info('Reporter: %s has no permission to modify tickets' %self.author)
906	return False
907
908	# How many changes has this ticket
909	cnum = len(tkt.get_changelog())
910
911
912	# reopen the ticket if it is was closed
913	# We must use the ticket workflow framework
914	#
915	if tkt['status'] in ['closed'] and self.EMAIL_TRIGGERS_WORKFLOW:
916
917	#print controller.actions['reopen']
918	#
919	# As reference
920	# req = Mock(href=Href('/'), abs_href=Href('http://www.example.com/'), authname='anonymous', perm=MockPerm(), args={})
921	#
922	#a = controller.render_ticket_action_control(req, tkt, 'reopen')
923	#print 'controller : ', a
924	#
925	#b = controller.get_all_status()
926	#print 'get all status: ', b
927	#
928	#b = controller.get_ticket_changes(req, tkt, 'reopen')
929	#print 'get_ticket_changes :', b
930
931	if self.WORKFLOW and (self.VERSION >= 0.11 ) :
932	from trac.ticket.default_workflow import ConfigurableTicketWorkflow
933	from trac.test import Mock, MockPerm
934
935	req = Mock(authname='anonymous', perm=MockPerm(), args={})
936
937	controller = ConfigurableTicketWorkflow(self.env)
938	fields = controller.get_ticket_changes(req, tkt, self.WORKFLOW)
939
940	if self.parameters.debug:
941	self.logger.debug('Workflow ticket update fields: ')
942
943	for key in fields.keys():
944	if self.parameters.debug:
945	self.logger.debug('\t %s : %s' %(key, fields[key]))
946
947	tkt[key] = fields[key]
948
949	else:
950	tkt['status'] = 'reopened'
951	tkt['resolution'] = ''
952
953	# Must we update some ticket fields properties via subjectline
954	#
955	if update_fields:
956	self.update_ticket_fields(tkt, update_fields)
957
958	message_parts = self.get_message_parts(m)
959	message_parts = self.unique_attachment_names(message_parts)
960
961	# Must we update some ticket fields properties via body_text
962	#
963	if self.properties:
964	self.update_ticket_fields(tkt, self.properties)
965
966	if self.EMAIL_HEADER:
967	message_parts.insert(0, self.email_header_txt(m))
968
969	body_text = self.body_text(message_parts)
970
971	error_with_attachments = self.attach_attachments(message_parts)
972
973	if body_text.strip() or update_fields or self.properties:
974	if self.parameters.dry_run:
975	print 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) ', self.author, cnum
976	else:
977	if error_with_attachments:
978	body_text = '%s\\%s' %(error_with_attachments, body_text)
979
980	tkt.save_changes(self.author, body_text, when, None, str(cnum))
981
982
983	if not spam:
984	self.notify(tkt, False, when)
985
986	return True
987
988	def set_ticket_fields(self, ticket):
989	"""
990	set the ticket fields to value specified
991	- /etc/email2trac.conf with <prefix>_<field>
992	- trac default values, trac.ini
993	"""
994	self.logger.debug('function set_ticket_fields')
995
996	user_dict = dict()
997
998	for field in ticket.fields:
999
1000	name = field['name']
1001
1002	## default trac value
1003	#
1004	if not field.get('custom'):
1005	value = self.get_config('ticket', 'default_%s' %(name) )
1006	else:
1007	## Else we get the default value for reporter
1008	#
1009	value = field.get('value')
1010	options = field.get('options')
1011
1012	if value and options and (value not in options):
1013	value = options[int(value)]
1014
1015	if self.parameters.debug:
1016	s = 'TD: trac.ini name %s = %s' %(name, value)
1017	self.print_unicode(s)
1018
1019	## email2trac.conf settings
1020	#
1021	prefix = self.parameters['ticket_prefix']
1022	try:
1023	value = self.parameters['%s_%s' %(prefix, name)]
1024	if self.parameters.debug > 10:
1025	s = 'TD: email2trac.conf %s = %s ' %(name, value)
1026	self.print_unicode(s)
1027
1028	except KeyError, detail:
1029	pass
1030
1031	if self.parameters.debug:
1032	s = 'TD: user_dict[%s] = %s' %(name, value)
1033	self.print_unicode(s)
1034
1035	if value:
1036	user_dict[name] = value
1037
1038	self.update_ticket_fields(ticket, user_dict, use_default=1)
1039
1040	if 'status' not in user_dict.keys():
1041	ticket['status'] = 'new'
1042
1043
1044	def ticket_update_by_subject(self, subject):
1045	"""
1046	This list of Re: prefixes is probably incomplete. Taken from
1047	wikipedia. Here is how the subject is matched
1048	- Re: <subject>
1049	- Re: (<Mail list label>:)+ <subject>
1050
1051	So we must have the last column
1052	"""
1053	self.logger.debug('function ticket_update_by_subject')
1054
1055	matched_id = None
1056	if self.TICKET_UPDATE and self.TICKET_UPDATE_BY_SUBJECT:
1057
1058	SUBJECT_RE = re.compile(r'^(RE\|AW\|VS\|SV):(.:)\s(.)', re.IGNORECASE)
1059	result = SUBJECT_RE.search(subject)
1060
1061	if result:
1062	# This is a reply
1063	orig_subject = result.group(3)
1064
1065	if self.parameters.debug:
1066	self.logger.debug('subject search string: %s' %(orig_subject))
1067
1068	cursor = self.db.cursor()
1069	summaries = [orig_subject, '%%: %s' % orig_subject]
1070
1071	##
1072	# Convert days to seconds
1073	lookback = int(time.mktime(time.gmtime())) - \
1074	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK * 24 * 3600
1075
1076
1077	for summary in summaries:
1078	if self.parameters.debug:
1079	self.logger.debug('Looking for summary matching: "%s"' % summary)
1080
1081	sql = """SELECT id FROM ticket
1082	WHERE changetime >= %s AND summary LIKE %s
1083	ORDER BY changetime DESC"""
1084	cursor.execute(sql, [lookback, summary.strip()])
1085
1086	for row in cursor:
1087	(matched_id,) = row
1088
1089	if self.parameters.debug:
1090	self.logger.debug('Found matching ticket id: %d' % matched_id)
1091
1092	break
1093
1094	if matched_id:
1095	matched_id = '#%d' % matched_id
1096	return matched_id
1097
1098	return matched_id
1099
1100
1101	def new_ticket(self, msg, subject, spam, set_fields = None):
1102	"""
1103	Create a new ticket
1104	"""
1105	self.logger.debug('function new_ticket')
1106
1107	tkt = Ticket(self.env)
1108
1109	self.set_reply_fields(tkt, msg)
1110
1111	self.set_ticket_fields(tkt)
1112
1113	# Check the permission of the reporter
1114	#
1115	if self.TICKET_PERMISSION_SYSTEM:
1116	if not self.check_permission(tkt, 'TICKET_CREATE'):
1117	self.logger.info('Reporter: %s has no permission to create tickets' %self.author)
1118	return False
1119
1120	# Old style setting for component, will be removed
1121	#
1122	if spam:
1123	tkt['component'] = 'Spam'
1124
1125	elif self.parameters.has_key('component'):
1126	tkt['component'] = self.parameters['component']
1127
1128	if not msg['Subject']:
1129	tkt['summary'] = u'(No subject)'
1130	else:
1131	tkt['summary'] = subject
1132
1133
1134	if set_fields:
1135	rest, keywords = string.split(set_fields, '?')
1136
1137	if keywords:
1138	update_fields = self.str_to_dict(keywords)
1139	self.update_ticket_fields(tkt, update_fields)
1140
1141	# produce e-mail like header
1142	#
1143	head = ''
1144	if self.EMAIL_HEADER > 0:
1145	head = self.email_header_txt(msg)
1146
1147	message_parts = self.get_message_parts(msg)
1148
1149	# Must we update some ticket fields properties via body_text
1150	#
1151	if self.properties:
1152	self.update_ticket_fields(tkt, self.properties)
1153
1154	message_parts = self.unique_attachment_names(message_parts)
1155
1156	if self.EMAIL_HEADER > 0:
1157	message_parts.insert(0, self.email_header_txt(msg))
1158
1159	body_text = self.body_text(message_parts)
1160
1161	tkt['description'] = body_text
1162
1163	#when = int(time.time())
1164	#
1165	utc = UTC()
1166	when = datetime.now(utc)
1167
1168	if self.parameters.dry_run:
1169	print 'DRY_RUN: tkt.insert()'
1170	else:
1171	self.id = tkt.insert()
1172
1173	changed = False
1174	comment = ''
1175
1176	# some routines in trac are dependend on ticket id
1177	# like alternate notify template
1178	#
1179	if self.notify_template:
1180	tkt['id'] = self.id
1181	changed = True
1182
1183	## Rewrite the description if we have mailto enabled
1184	#
1185	if self.MAILTO:
1186	changed = True
1187	comment = u'\nadded mailto line\n'
1188	mailto = self.html_mailto_link( m['Subject'])
1189
1190	tkt['description'] = u'%s\r\n%s%s\r\n' \
1191	%(head, mailto, body_text)
1192
1193	## Save the attachments to the ticket
1194	#
1195	error_with_attachments = self.attach_attachments(message_parts)
1196
1197	if error_with_attachments:
1198	changed = True
1199	comment = '%s\n%s\n' %(comment, error_with_attachments)
1200
1201	if changed:
1202	if self.parameters.dry_run:
1203	print 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
1204	else:
1205	tkt.save_changes(tkt['reporter'], comment)
1206	#print tkt.get_changelog(self.db, when)
1207
1208	if not spam:
1209	self.notify(tkt, True)
1210
1211
1212	def attach_attachments(self, message_parts, update=False):
1213	'''
1214	save any attachments as files in the ticket's directory
1215	'''
1216	self.logger.debug('function attach_attachments()')
1217
1218	if self.parameters.dry_run:
1219	print "DRY_RUN: no attachments attached to tickets"
1220	return ''
1221
1222	count = 0
1223
1224	# Get Maxium attachment size
1225	#
1226	max_size = int(self.get_config('attachment', 'max_size'))
1227	status = None
1228
1229	for item in message_parts:
1230	# Skip body parts
1231	if not isinstance(item, tuple):
1232	continue
1233
1234	(original, filename, part) = item
1235	#
1236	# We have to determine the size so we use this temporary solution. we must escape it
1237	# else we get UnicodeErrors.
1238	#
1239	path, fd = util.create_unique_file(os.path.join(self.TMPDIR, util.text.unicode_quote(filename)))
1240	text = part.get_payload(decode=1)
1241	if not text:
1242	text = '(None)'
1243	fd.write(text)
1244	fd.close()
1245
1246	# get the file_size
1247	#
1248	stats = os.lstat(path)
1249	file_size = stats[stat.ST_SIZE]
1250
1251	# Check if the attachment size is allowed
1252	#
1253	if (max_size != -1) and (file_size > max_size):
1254	status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1255	%(status, original, file_size, max_size)
1256
1257	os.unlink(path)
1258	continue
1259	else:
1260	count = count + 1
1261
1262	# Insert the attachment
1263	#
1264	fd = open(path, 'rb')
1265	if self.system == 'discussion':
1266	att = attachment.Attachment(self.env, 'discussion', 'topic/%s'
1267	% (self.id,))
1268	else:
1269	att = attachment.Attachment(self.env, 'ticket', self.id)
1270
1271	# This will break the ticket_update system, the body_text is vaporized
1272	# ;-(
1273	#
1274	if not update:
1275	att.author = self.author
1276	att.description = self.email_to_unicode('Added by email2trac')
1277
1278	try:
1279	att.insert(filename, fd, file_size)
1280	except OSError, detail:
1281	status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
1282
1283	# Remove the created temporary filename
1284	#
1285	fd.close()
1286	os.unlink(path)
1287
1288	## return error
1289	#
1290	return status
1291
1292	########## Fullblog functions #################################################
1293
1294	def blog(self, id):
1295	"""
1296	The blog create/update function
1297	"""
1298	# import the modules
1299	#
1300	from tracfullblog.core import FullBlogCore
1301	from tracfullblog.model import BlogPost, BlogComment
1302	from trac.test import Mock, MockPerm
1303
1304	# instantiate blog core
1305	blog = FullBlogCore(self.env)
1306	req = Mock(authname='anonymous', perm=MockPerm(), args={})
1307
1308	if id:
1309
1310	# update blog
1311	#
1312	comment = BlogComment(self.env, id)
1313	comment.author = self.author
1314
1315	message_parts = self.get_message_parts(m)
1316	comment.comment = self.body_text(message_parts)
1317
1318	blog.create_comment(req, comment)
1319
1320	else:
1321	# create blog
1322	#
1323	import time
1324	post = BlogPost(self.env, 'blog_'+time.strftime("%Y%m%d%H%M%S", time.gmtime()))
1325
1326	#post = BlogPost(self.env, blog._get_default_postname(self.env))
1327
1328	post.author = self.author
1329	post.title = self.email_to_unicode(m['Subject'])
1330
1331	message_parts = self.get_message_parts(m)
1332	post.body = self.body_text(message_parts)
1333
1334	blog.create_post(req, post, self.author, u'Created by email2trac', False)
1335
1336
1337	########## Discussion functions ##############################################
1338
1339	def discussion_topic(self, content, subject):
1340
1341	# Import modules.
1342	from tracdiscussion.api import DiscussionApi
1343	from trac.util.datefmt import to_timestamp, utc
1344
1345	if self.parameters.debug:
1346	self.logger.debug('Creating a new topic in forum:', self.id)
1347
1348	# Get dissussion API component.
1349	api = self.env[DiscussionApi]
1350	context = self._create_context(content, subject)
1351
1352	# Get forum for new topic.
1353	forum = api.get_forum(context, self.id)
1354
1355	if not forum and self.parameters.debug:
1356	self.logger.debug("ERROR: Replied forum doesn't exist")
1357
1358	# Prepare topic.
1359	topic = {'forum' : forum['id'],
1360	'subject' : context.subject,
1361	'time': to_timestamp(datetime.now(utc)),
1362	'author' : self.author,
1363	'subscribers' : [self.email_addr],
1364	'body' : self.body_text(context.content_parts)}
1365
1366	# Add topic to DB and commit it.
1367	self._add_topic(api, context, topic)
1368	self.db.commit()
1369
1370	def discussion_topic_reply(self, content, subject):
1371
1372	# Import modules.
1373	from tracdiscussion.api import DiscussionApi
1374	from trac.util.datefmt import to_timestamp, utc
1375
1376	if self.parameters.debug:
1377	self.logger.debug('Replying to discussion topic', self.id)
1378
1379	# Get dissussion API component.
1380	api = self.env[DiscussionApi]
1381	context = self._create_context(content, subject)
1382
1383	# Get replied topic.
1384	topic = api.get_topic(context, self.id)
1385
1386	if not topic and self.parameters.debug:
1387	self.logger.debug("ERROR: Replied topic doesn't exist")
1388
1389	# Prepare message.
1390	message = {'forum' : topic['forum'],
1391	'topic' : topic['id'],
1392	'replyto' : -1,
1393	'time' : to_timestamp(datetime.now(utc)),
1394	'author' : self.author,
1395	'body' : self.body_text(context.content_parts)}
1396
1397	# Add message to DB and commit it.
1398	self._add_message(api, context, message)
1399	self.db.commit()
1400
1401	def discussion_message_reply(self, content, subject):
1402
1403	# Import modules.
1404	from tracdiscussion.api import DiscussionApi
1405	from trac.util.datefmt import to_timestamp, utc
1406
1407	if self.parameters.debug:
1408	self.loggger.debug('TD: Replying to discussion message', self.id)
1409
1410	# Get dissussion API component.
1411	api = self.env[DiscussionApi]
1412	context = self._create_context(content, subject)
1413
1414	# Get replied message.
1415	message = api.get_message(context, self.id)
1416
1417	if not message and self.parameters.debug:
1418	self.logger.debug("ERROR: Replied message doesn't exist")
1419
1420	# Prepare message.
1421	message = {'forum' : message['forum'],
1422	'topic' : message['topic'],
1423	'replyto' : message['id'],
1424	'time' : to_timestamp(datetime.now(utc)),
1425	'author' : self.author,
1426	'body' : self.body_text(context.content_parts)}
1427
1428	# Add message to DB and commit it.
1429	self._add_message(api, context, message)
1430	self.db.commit()
1431
1432	def _create_context(self, content, subject):
1433
1434	# Import modules.
1435	from trac.mimeview import Context
1436	from trac.web.api import Request
1437	from trac.perm import PermissionCache
1438
1439	# TODO: Read server base URL from config.
1440	# Create request object to mockup context creation.
1441	#
1442	environ = {'SERVER_PORT' : 80,
1443	'SERVER_NAME' : 'test',
1444	'REQUEST_METHOD' : 'POST',
1445	'wsgi.url_scheme' : 'http',
1446	'wsgi.input' : sys.stdin}
1447	chrome = {'links': {},
1448	'scripts': [],
1449	'ctxtnav': [],
1450	'warnings': [],
1451	'notices': []}
1452
1453	if self.env.base_url_for_redirect:
1454	environ['trac.base_url'] = self.env.base_url
1455
1456	req = Request(environ, None)
1457	req.chrome = chrome
1458	req.tz = 'missing'
1459	req.authname = self.author
1460	req.perm = PermissionCache(self.env, self.author)
1461
1462	# Create and return context.
1463	context = Context.from_request(req)
1464	context.realm = 'discussion-email2trac'
1465	context.cursor = self.db.cursor()
1466	context.content = content
1467	context.subject = subject
1468
1469	# Read content parts from content.
1470	context.content_parts = self.get_message_parts(content)
1471	context.content_parts = self.unique_attachment_names(
1472	context.content_parts)
1473
1474	return context
1475
1476	def _add_topic(self, api, context, topic):
1477	context.req.perm.assert_permission('DISCUSSION_APPEND')
1478
1479	# Filter topic.
1480	for discussion_filter in api.discussion_filters:
1481	accept, topic_or_error = discussion_filter.filter_topic(
1482	context, topic)
1483	if accept:
1484	topic = topic_or_error
1485	else:
1486	raise TracError(topic_or_error)
1487
1488	# Add a new topic.
1489	api.add_topic(context, topic)
1490
1491	# Get inserted topic with new ID.
1492	topic = api.get_topic_by_time(context, topic['time'])
1493
1494	# Attach attachments.
1495	self.id = topic['id']
1496	self.attach_attachments(context.content_parts, True)
1497
1498	# Notify change listeners.
1499	for listener in api.topic_change_listeners:
1500	listener.topic_created(context, topic)
1501
1502	def _add_message(self, api, context, message):
1503	context.req.perm.assert_permission('DISCUSSION_APPEND')
1504
1505	# Filter message.
1506	for discussion_filter in api.discussion_filters:
1507	accept, message_or_error = discussion_filter.filter_message(
1508	context, message)
1509	if accept:
1510	message = message_or_error
1511	else:
1512	raise TracError(message_or_error)
1513
1514	# Add message.
1515	api.add_message(context, message)
1516
1517	# Get inserted message with new ID.
1518	message = api.get_message_by_time(context, message['time'])
1519
1520	# Attach attachments.
1521	self.id = message['topic']
1522	self.attach_attachments(context.content_parts, True)
1523
1524	# Notify change listeners.
1525	for listener in api.message_change_listeners:
1526	listener.message_created(context, message)
1527
1528	########## MAIN function ######################################################
1529
1530	def parse(self, fp):
1531	"""
1532	"""
1533	self.logger.debug('Main function parse')
1534	global m
1535
1536	m = email.message_from_file(fp)
1537
1538	if not m:
1539	if self.parameters.debug:
1540	self.logger.debug('This is not a valid email message format')
1541
1542	return
1543
1544	# Work around lack of header folding in Python; see http://bugs.python.org/issue4696
1545	try:
1546	m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
1547	except AttributeError, detail:
1548	pass
1549
1550	if self.parameters.debug: # save the entire e-mail message text
1551	self.save_email_for_debug(m, True)
1552
1553	self.db = self.env.get_db_cnx()
1554	self.get_sender_info(m)
1555
1556	if not self.email_header_acl('white_list', self.email_addr, True):
1557	if self.parameters.debug:
1558	self.logger.debug('Message rejected : %s not in white list' %(self.email_addr))
1559
1560	return False
1561
1562	if self.email_header_acl('black_list', self.email_addr, False):
1563	if self.parameters.debug:
1564	self.logger.debug('Message rejected : %s in black list' %(self.email_addr))
1565
1566	return False
1567
1568	if not self.email_header_acl('recipient_list', self.to_email_addr, True):
1569	if self.parameters.debug:
1570	self.logger.debug('Message rejected : %s not in recipient list' %(self.to_email_addr))
1571
1572	return False
1573
1574	# If drop the message
1575	#
1576	if self.spam(m) == 'drop':
1577	return False
1578
1579	elif self.spam(m) == 'spam':
1580	spam_msg = True
1581	else:
1582	spam_msg = False
1583
1584	if not m['Subject']:
1585	subject = 'No Subject'
1586	else:
1587	subject = self.email_to_unicode(m['Subject'])
1588
1589	if self.parameters.debug:
1590	self.logger.debug('subject: %s' %subject)
1591
1592	#
1593	# [hic] #1529: Re: LRZ
1594	# [hic] #1529?owner=bas,priority=medium: Re: LRZ
1595	#
1596	ticket_regex = r'''
1597	(?P<new_fields>[#][?].*)
1598	\|(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?.*)?:)
1599	'''
1600	# Check if FullBlogPlugin is installed
1601	#
1602	blog_enabled = None
1603	blog_regex = ''
1604	if self.get_config('components', 'tracfullblog.*') in ['enabled']:
1605	blog_enabled = True
1606	blog_regex = '''\|(?P<blog>blog:(?P<blog_id>\w*))'''
1607
1608
1609	# Check if DiscussionPlugin is installed
1610	#
1611	discussion_enabled = None
1612	discussion_regex = ''
1613	if self.get_config('components', 'tracdiscussion.api.*') in ['enabled']:
1614	discussion_enabled = True
1615	discussion_regex = r'''
1616	\|(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
1617	\|(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
1618	\|(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
1619	'''
1620
1621
1622	regex_str = ticket_regex + blog_regex + discussion_regex
1623	SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
1624
1625	# Find out if this is a ticket, a blog or a discussion
1626	#
1627	result = SYSTEM_RE.search(subject)
1628
1629	if result:
1630	# update ticket + fields
1631	#
1632	if result.group('reply') and self.TICKET_UPDATE:
1633	self.system = 'ticket'
1634
1635	# Skip the last ':' character
1636	#
1637	if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
1638	self.new_ticket(m, subject, spam_msg)
1639
1640	# New ticket + fields
1641	#
1642	elif result.group('new_fields'):
1643	self.system = 'ticket'
1644	self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
1645
1646	if blog_enabled:
1647	if result.group('blog'):
1648	self.system = 'blog'
1649	self.blog(result.group('blog_id'))
1650
1651	if discussion_enabled:
1652	# New topic.
1653	#
1654	if result.group('forum'):
1655	self.system = 'discussion'
1656	self.id = int(result.group('forum_id'))
1657	self.discussion_topic(m, subject[result.end('forum'):])
1658
1659	# Reply to topic.
1660	#
1661	elif result.group('topic'):
1662	self.system = 'discussion'
1663	self.id = int(result.group('topic_id'))
1664	self.discussion_topic_reply(m, subject[result.end('topic'):])
1665
1666	# Reply to topic message.
1667	#
1668	elif result.group('message'):
1669	self.system = 'discussion'
1670	self.id = int(result.group('message_id'))
1671	self.discussion_message_reply(m, subject[result.end('message'):])
1672
1673	else:
1674	self.system = 'ticket'
1675	result = self.ticket_update_by_subject(subject)
1676	if result:
1677	if not self.ticket_update(m, result, spam_msg):
1678	self.new_ticket(m, subject, spam_msg)
1679	else:
1680	# No update by subject, so just create a new ticket
1681	self.new_ticket(m, subject, spam_msg)
1682
1683
1684	########## BODY TEXT functions ###########################################################
1685
1686	def strip_signature(self, text):
1687	"""
1688	Strip signature from message, inspired by Mailman software
1689	"""
1690	body = []
1691	for line in text.splitlines():
1692	if line == '-- ':
1693	break
1694	body.append(line)
1695
1696	return ('\n'.join(body))
1697
1698	def reflow(self, text, delsp = 0):
1699	"""
1700	Reflow the message based on the format="flowed" specification (RFC 3676)
1701	"""
1702	flowedlines = []
1703	quotelevel = 0
1704	prevflowed = 0
1705
1706	for line in text.splitlines():
1707	from re import match
1708
1709	# Figure out the quote level and the content of the current line
1710	m = match('(>)( ?)(.)', line)
1711	linequotelevel = len(m.group(1))
1712	line = m.group(3)
1713
1714	# Determine whether this line is flowed
1715	if line and line != '-- ' and line[-1] == ' ':
1716	flowed = 1
1717	else:
1718	flowed = 0
1719
1720	if flowed and delsp and line and line[-1] == ' ':
1721	line = line[:-1]
1722
1723	# If the previous line is flowed, append this line to it
1724	if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1725	flowedlines[-1] += line
1726	# Otherwise, start a new line
1727	else:
1728	flowedlines.append('>' * linequotelevel + line)
1729
1730	prevflowed = flowed
1731
1732
1733	return '\n'.join(flowedlines)
1734
1735	def strip_quotes(self, text):
1736	"""
1737	Strip quotes from message by Nicolas Mendoza
1738	"""
1739	body = []
1740	for line in text.splitlines():
1741	if line.startswith(self.EMAIL_QUOTE):
1742	continue
1743	body.append(line)
1744
1745	return ('\n'.join(body))
1746
1747	def inline_properties(self, text):
1748	"""
1749	Parse text if we use inline keywords to set ticket fields
1750	"""
1751	self.logger.debug('function inline_properties')
1752
1753	properties = dict()
1754	body = list()
1755
1756	INLINE_EXP = re.compile('\s[@]\s([a-zA-Z]+)\s:(.)$')
1757
1758	for line in text.splitlines():
1759	match = INLINE_EXP.match(line)
1760	if match:
1761	keyword, value = match.groups()
1762	self.properties[keyword] = value.strip()
1763
1764	if self.parameters.debug:
1765	self.logger.debug('inline properties: %s : %s' %(keyword,value))
1766	else:
1767	body.append(line)
1768
1769	return '\n'.join(body)
1770
1771
1772	def wrap_text(self, text, replace_whitespace = False):
1773	"""
1774	Will break a lines longer then given length into several small
1775	lines of size given length
1776	"""
1777	import textwrap
1778
1779	LINESEPARATOR = '\n'
1780	reformat = ''
1781
1782	for s in text.split(LINESEPARATOR):
1783	tmp = textwrap.fill(s,self.USE_TEXTWRAP)
1784	if tmp:
1785	reformat = '%s\n%s' %(reformat,tmp)
1786	else:
1787	reformat = '%s\n' %reformat
1788
1789	return reformat
1790
1791	# Python2.4 and higher
1792	#
1793	#return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
1794	#
1795
1796	########## EMAIL attachements functions ###########################################################
1797
1798	def inline_part(self, part):
1799	"""
1800	"""
1801	self.logger.debug('function inline_part()')
1802
1803	return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
1804
1805	def get_message_parts(self, msg):
1806	"""
1807	parses the email message and returns a list of body parts and attachments
1808	body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
1809	"""
1810	self.logger.debug('function get_message_parts()')
1811
1812	message_parts = list()
1813
1814	ALTERNATIVE_MULTIPART = False
1815
1816	for part in msg.walk():
1817	if self.parameters.debug:
1818	self.logger.debug('Message part: Main-Type: %s' % part.get_content_maintype())
1819	self.logger.debug('Message part: Content-Type: %s' % part.get_content_type())
1820
1821	## Check content type
1822	#
1823	if part.get_content_type() in self.STRIP_CONTENT_TYPES:
1824
1825	if self.parameters.debug:
1826	self.logger.debug("A %s attachment named '%s' was skipped" %(part.get_content_type(), part.get_filename()))
1827
1828	continue
1829
1830	## Catch some mulitpart execptions
1831	#
1832	if part.get_content_type() == 'multipart/alternative':
1833	ALTERNATIVE_MULTIPART = True
1834	continue
1835
1836	## Skip multipart containers
1837	#
1838	if part.get_content_maintype() == 'multipart':
1839	if self.parameters.debug:
1840	self.logger.debug("Skipping multipart container")
1841	continue
1842
1843	## Check if this is an inline part. It's inline if there is co Cont-Disp header, or if there is one and it says "inline"
1844	#
1845	inline = self.inline_part(part)
1846
1847	## Drop HTML message
1848	#
1849	if ALTERNATIVE_MULTIPART and self.DROP_ALTERNATIVE_HTML_VERSION:
1850	if part.get_content_type() == 'text/html':
1851	if self.parameters.debug:
1852	self.logger.debug('Skipping alternative HTML message')
1853
1854	ALTERNATIVE_MULTIPART = False
1855	continue
1856
1857	## Inline text parts are where the body is
1858	#
1859	if part.get_content_type() == 'text/plain' and inline:
1860	if self.parameters.debug:
1861	self.logger.debug(' Inline body part')
1862
1863	# Try to decode, if fails then do not decode
1864	#
1865	body_text = part.get_payload(decode=1)
1866	if not body_text:
1867	body_text = part.get_payload(decode=0)
1868
1869	format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
1870	delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
1871
1872	if self.REFLOW and not self.VERBATIM_FORMAT and format == 'flowed':
1873	body_text = self.reflow(body_text, delsp == 'yes')
1874
1875	if self.STRIP_SIGNATURE:
1876	body_text = self.strip_signature(body_text)
1877
1878	if self.STRIP_QUOTES:
1879	body_text = self.strip_quotes(body_text)
1880
1881	if self.INLINE_PROPERTIES:
1882	body_text = self.inline_properties(body_text)
1883
1884	if self.USE_TEXTWRAP:
1885	body_text = self.wrap_text(body_text)
1886
1887	## Get contents charset (iso-8859-15 if not defined in mail headers)
1888	#
1889	charset = part.get_content_charset()
1890	if not charset:
1891	charset = 'iso-8859-15'
1892
1893	try:
1894	ubody_text = unicode(body_text, charset)
1895
1896	except UnicodeError, detail:
1897	ubody_text = unicode(body_text, 'iso-8859-15')
1898
1899	except LookupError, detail:
1900	ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
1901
1902	if self.VERBATIM_FORMAT:
1903	message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
1904	else:
1905	message_parts.append('%s' %ubody_text)
1906	else:
1907	if self.parameters.debug:
1908	s = 'TD: Filename: %s' % part.get_filename()
1909	self.print_unicode(s)
1910
1911	##
1912	# First try to use email header function to convert filename.
1913	# If this fails the use the plan filename
1914	try:
1915	filename = self.email_to_unicode(part.get_filename())
1916	except UnicodeEncodeError, detail:
1917	filename = part.get_filename()
1918
1919	message_parts.append((filename, part))
1920
1921	return message_parts
1922
1923	def unique_attachment_names(self, message_parts):
1924	"""
1925	"""
1926	renamed_parts = []
1927	attachment_names = set()
1928
1929	for item in message_parts:
1930
1931	## If not an attachment, leave it alone
1932	#
1933	if not isinstance(item, tuple):
1934	renamed_parts.append(item)
1935	continue
1936
1937	(filename, part) = item
1938
1939	## If no filename, use a default one
1940	#
1941	if not filename:
1942	filename = 'untitled-part'
1943
1944	# Guess the extension from the content type, use non strict mode
1945	# some additional non-standard but commonly used MIME types
1946	# are also recognized
1947	#
1948	ext = mimetypes.guess_extension(part.get_content_type(), False)
1949	if not ext:
1950	ext = '.bin'
1951
1952	filename = '%s%s' % (filename, ext)
1953
1954	## Discard relative paths for windows/unix in attachment names
1955	#
1956	#filename = filename.replace('\\', '/').replace(':', '/')
1957	filename = filename.replace('\\', '_')
1958	filename = filename.replace('/', '_')
1959
1960	#
1961	# We try to normalize the filename to utf-8 NFC if we can.
1962	# Files uploaded from OS X might be in NFD.
1963	# Check python version and then try it
1964	#
1965	#if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
1966	# try:
1967	# filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
1968	# except TypeError:
1969	# pass
1970
1971	# Make the filename unique for this ticket
1972	num = 0
1973	unique_filename = filename
1974	dummy_filename, ext = os.path.splitext(filename)
1975
1976	while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
1977	num += 1
1978	unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
1979
1980	if self.parameters.debug:
1981	s = 'TD: Attachment with filename %s will be saved as %s' % (filename, unique_filename)
1982	self.print_unicode(s)
1983
1984	attachment_names.add(unique_filename)
1985
1986	renamed_parts.append((filename, unique_filename, part))
1987
1988	return renamed_parts
1989
1990
1991	def attachment_exists(self, filename):
1992
1993	if self.parameters.debug:
1994	s = 'TD: attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
1995	self.print_unicode(s)
1996
1997	# We have no valid ticket id
1998	#
1999	if not self.id:
2000	return False
2001
2002	try:
2003	if self.system == 'discussion':
2004	att = attachment.Attachment(self.env, 'discussion', 'ticket/%s'
2005	% (self.id,), filename)
2006	else:
2007	att = attachment.Attachment(self.env, 'ticket', self.id,
2008	filename)
2009	return True
2010	except attachment.ResourceNotFound:
2011	return False
2012
2013	########## TRAC Ticket Text ###########################################################
2014
2015	def body_text(self, message_parts):
2016	body_text = []
2017
2018	for part in message_parts:
2019	# Plain text part, append it
2020	if not isinstance(part, tuple):
2021	body_text.extend(part.strip().splitlines())
2022	body_text.append("")
2023	continue
2024
2025	(original, filename, part) = part
2026	inline = self.inline_part(part)
2027
2028	if part.get_content_maintype() == 'image' and inline:
2029	if self.system != 'discussion':
2030	body_text.append('[[Image(%s)]]' % filename)
2031	body_text.append("")
2032	else:
2033	if self.system != 'discussion':
2034	body_text.append('[attachment:"%s"]' % filename)
2035	body_text.append("")
2036
2037	body_text = '\r\n'.join(body_text)
2038	return body_text
2039
2040	def html_mailto_link(self, subject):
2041	"""
2042	This function returns a HTML mailto tag with the ticket id and author email address
2043	"""
2044	if not self.author:
2045	author = self.email_addr
2046	else:
2047	author = self.author
2048
2049	# use urllib to escape the chars
2050	#
2051	s = 'mailto:%s?Subject=%s&Cc=%s' %(
2052	urllib.quote(self.email_addr),
2053	urllib.quote('Re: #%s: %s' %(self.id, subject)),
2054	urllib.quote(self.MAILTO_CC)
2055	)
2056
2057	s = '\r\n{{{\r\n#!html\r\n<a\r\n href="%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
2058	return s
2059
2060	########## TRAC notify section ###########################################################
2061
2062	def notify(self, tkt, new=True, modtime=0):
2063	"""
2064	A wrapper for the TRAC notify function. So we can use templates
2065	"""
2066	self.logger.debug('function notify()')
2067
2068	if self.parameters.dry_run:
2069	print 'DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter']
2070	return
2071	try:
2072
2073	#from trac.ticket.web_ui import TicketModule
2074	#from trac.ticket.notification import TicketNotificationSystem
2075	#ticket_sys = TicketNotificationSystem(self.env)
2076	#a = TicketModule(self.env)
2077	#print a.__dict__
2078	#tn_sys = TicketNotificationSystem(self.env)
2079	#print tn_sys
2080	#print tn_sys.__dict__
2081	#sys.exit(0)
2082
2083	# create false {abs_}href properties, to trick Notify()
2084	#
2085	if not (self.VERSION in [0.11, 0.12]):
2086	self.env.abs_href = Href(self.get_config('project', 'url'))
2087	self.env.href = Href(self.get_config('project', 'url'))
2088
2089
2090	tn = TicketNotifyEmail(self.env)
2091
2092	if self.notify_template:
2093
2094	if self.VERSION >= 0.11:
2095
2096	from trac.web.chrome import Chrome
2097
2098	if self.notify_template_update and not new:
2099	tn.template_name = self.notify_template_update
2100	else:
2101	tn.template_name = self.notify_template
2102
2103	tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
2104
2105	else:
2106
2107	tn.template_name = self.notify_template;
2108
2109	tn.notify(tkt, new, modtime)
2110
2111	except Exception, e:
2112	set.logger.error('Failure sending notification on creation of ticket #%s: %s' %(self.id, e))
2113
2114
2115
2116	########## Parse Config File ###########################################################
2117
2118	def ReadConfig(file, name):
2119	"""
2120	Parse the config file
2121	"""
2122	if not os.path.isfile(file):
2123	print 'File %s does not exist' %file
2124	sys.exit(1)
2125
2126	config = trac_config.Configuration(file)
2127
2128	# Use given project name else use defaults
2129	#
2130	if name:
2131	sections = config.sections()
2132	if not name in sections:
2133	print "Not a valid project name: %s" %name
2134	print "Valid names: %s" %sections
2135	sys.exit(1)
2136
2137	project = SaraDict()
2138	for option, value in config.options(name):
2139	project[option] = value
2140
2141	else:
2142	# use some trac internals to get the defaults
2143	#
2144	tmp = config.parser.defaults()
2145	project = SaraDict()
2146
2147	for option,value in tmp.items():
2148	project[option] = value
2149
2150	## Convert debug value to int
2151	#
2152	if project.debug:
2153	project.debug = int(project.debug)
2154	else:
2155	project.debug = 0
2156
2157	return project
2158
2159	########## Setup Logging ###############################################################
2160
2161	def setup_log(parameters):
2162	"""
2163	Setup loging
2164
2165	Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
2166	would be interpreted by the ConfigParser itself.
2167
2168	"""
2169	logger = logging.getLogger('email2trac')
2170
2171	if not parameters.log_type:
2172	parameters.log_type = 'syslog'
2173
2174	if parameters.log_type == 'file':
2175	if not os.path.isabs(parameters.log_file):
2176	parameters.log_file = os.path.join('log')
2177	log_handler = logging.FileHandler()
2178
2179	elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
2180	# Requires win32 extensions
2181	log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
2182
2183	elif parameters.log_type in ('syslog', 'unix'):
2184	log_handler = logging.handlers.SysLogHandler('/dev/log')
2185
2186	elif parameters.log_type in ('stderr'):
2187	log_handler = logging.StreamHandler(sys.stderr)
2188
2189	else:
2190	log_handler = logging.handlers.BufferingHandler(0)
2191
2192	if parameters.log_format:
2193	parameters.log_format = parameters.log_format.replace('$(', '%(')
2194	else:
2195	parameters.log_format = 'Email2trac: %(message)s'
2196
2197	log_formatter = logging.Formatter(parameters.log_format)
2198	log_handler.setFormatter(log_formatter)
2199	logger.addHandler(log_handler)
2200
2201	if (parameters.log_level in ['DEBUG', 'ALL']) or (parameters.debug > 0):
2202	logger.setLevel(logging.DEBUG)
2203
2204	elif parameters.log_level in ['INFO'] or parameters.verbose:
2205	logger.setLevel(logging.INFO)
2206
2207	elif parameters.log_level in ['WARNING']:
2208	logger.setLevel(logging.WARNING)
2209
2210	elif parameters.log_level in ['ERROR']:
2211	logger.setLevel(logging.ERROR)
2212
2213	elif parameters.log_level in ['CRITICAL']:
2214	logger.setLevel(logging.CRITICAL)
2215
2216	else:
2217	logger.setLevel(logging.INFO)
2218
2219	return logger
2220
2221
2222	if __name__ == '__main__':
2223	# Default config file
2224	#
2225	configfile = '@email2trac_conf@'
2226	project = ''
2227	component = ''
2228	ticket_prefix = 'default'
2229	dry_run = None
2230	verbose = None
2231
2232	ENABLE_SYSLOG = 0
2233
2234	SHORT_OPT = 'chf:np:t:v'
2235	LONG_OPT = ['component=', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=', 'verbose']
2236
2237	try:
2238	opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
2239	except getopt.error,detail:
2240	print __doc__
2241	print detail
2242	sys.exit(1)
2243
2244	project_name = None
2245	for opt,value in opts:
2246	if opt in [ '-h', '--help']:
2247	print __doc__
2248	sys.exit(0)
2249	elif opt in ['-c', '--component']:
2250	component = value
2251	elif opt in ['-f', '--file']:
2252	configfile = value
2253	elif opt in ['-n', '--dry-run']:
2254	dry_run = True
2255	elif opt in ['-p', '--project']:
2256	project_name = value
2257	elif opt in ['-t', '--ticket_prefix']:
2258	ticket_prefix = value
2259	elif opt in ['-v', '--verbose']:
2260	verbose = True
2261
2262	settings = ReadConfig(configfile, project_name)
2263
2264	# The default prefix for ticket values in email2trac.conf
2265	#
2266	settings['ticket_prefix'] = ticket_prefix
2267	settings['dry_run'] = dry_run
2268	settings['verbose'] = verbose
2269
2270	logger = setup_log(settings)
2271
2272	if not settings.has_key('project'):
2273	print __doc__
2274	print 'No Trac project is defined in the email2trac config file.'
2275	sys.exit(1)
2276
2277	if component:
2278	settings['component'] = component
2279
2280	# Determine major trac version used to be in email2trac.conf
2281	# Quick hack for 0.12
2282	#
2283	version = '0.%s' %(trac_version.split('.')[1])
2284	if version.startswith('0.12'):
2285	version = '0.12'
2286
2287	logger.debug("Found trac version: %s" %(version))
2288
2289	#debug HvB
2290	#print settings
2291
2292	try:
2293	if version == '0.10':
2294	from trac import attachment
2295	from trac.env import Environment
2296	from trac.ticket import Ticket
2297	from trac.web.href import Href
2298	from trac import util
2299	#
2300	# return util.text.to_unicode(str)
2301	#
2302	# see http://projects.edgewall.com/trac/changeset/2799
2303	from trac.ticket.notification import TicketNotifyEmail
2304	from trac import config as trac_config
2305	from trac.core import TracError
2306
2307	elif version == '0.11':
2308	from trac import attachment
2309	from trac.env import Environment
2310	from trac.ticket import Ticket
2311	from trac.web.href import Href
2312	from trac import config as trac_config
2313	from trac import util
2314	from trac.core import TracError
2315	from trac.perm import PermissionSystem
2316
2317	#
2318	# return util.text.to_unicode(str)
2319	#
2320	# see http://projects.edgewall.com/trac/changeset/2799
2321	from trac.ticket.notification import TicketNotifyEmail
2322
2323	elif version == '0.12':
2324	from trac import attachment
2325	from trac.env import Environment
2326	from trac.ticket import Ticket
2327	from trac.web.href import Href
2328	from trac import config as trac_config
2329	from trac import util
2330	from trac.core import TracError
2331	from trac.perm import PermissionSystem
2332
2333	#
2334	# return util.text.to_unicode(str)
2335	#
2336	# see http://projects.edgewall.com/trac/changeset/2799
2337	from trac.ticket.notification import TicketNotifyEmail
2338
2339
2340	else:
2341	logger.error('TRAC version %s is not supported' %version)
2342	sys.exit(1)
2343
2344	if settings.has_key('enable_syslog'):
2345	if SYSLOG_AVAILABLE:
2346	ENABLE_SYSLOG = float(settings['enable_syslog'])
2347
2348
2349	# Must be set before environment is created
2350	#
2351	if settings.has_key('python_egg_cache'):
2352	python_egg_cache = str(settings['python_egg_cache'])
2353	os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
2354
2355
2356	if settings.debug > 0:
2357	logger.debug('Loading environment %s', settings.project)
2358
2359	env = Environment(settings['project'], create=0)
2360
2361	tktparser = TicketEmailParser(env, settings, logger, float(version))
2362	tktparser.parse(sys.stdin)
2363
2364	# Catch all errors ans log to SYSLOG if we have enabled this
2365	# else stdout
2366	#
2367	except Exception, error:
2368	if ENABLE_SYSLOG:
2369	syslog.openlog('email2trac', syslog.LOG_NOWAIT)
2370
2371	etype, evalue, etb = sys.exc_info()
2372	for e in traceback.format_exception(etype, evalue, etb):
2373	syslog.syslog(e)
2374
2375	syslog.closelog()
2376	else:
2377	traceback.print_exc()
2378
2379	if m:
2380	tktparser.save_email_for_debug(m, True)
2381
2382	sys.exit(1)
2383	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format