Context Navigation

source: trunk/email2trac.py.in @ 403

Last change on this file since 403 was 403, checked in by bas, 14 years ago
Switching to the python logging module inspired by the Trac source code
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 58.2 KB

Line
1	#!@PYTHON@
2	# Copyright (C) 2002
3	#
4	# This file is part of the email2trac utils
5	#
6	# This program is free software; you can redistribute it and/or modify it
7	# under the terms of the GNU General Public License as published by the
8	# Free Software Foundation; either version 2, or (at your option) any
9	# later version.
10	#
11	# This program is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with this program; if not, write to the Free Software
18	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
19	#
20	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
21	#
22	"""
23	email2trac.py -- Email tickets to Trac.
24
25	A simple MTA filter to create Trac tickets from inbound emails.
26
27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28	Copyright 2005, Edgewall Software
29
30	Authors:
31	Bas van der Vlies <basv@sara.nl>
32	Walter de Jong <walter@sara.nl>
33
34	The scripts reads emails from stdin and inserts directly into a Trac database.
35
36	How to use
37	----------
38	* See https://subtrac.sara.nl/oss/email2trac/
39
40	* Create an config file:
41	[DEFAULT] # REQUIRED
42	project : /data/trac/test # REQUIRED
43	debug : 1 # OPTIONAL, if set print some DEBUG info
44
45	[jouvin] # OPTIONAL project declaration, if set both fields necessary
46	project : /data/trac/jouvin # use -p\|--project jouvin.
47
48	* default config file is : /etc/email2trac.conf
49
50	* Commandline opions:
51	-h,--help
52	-f,--file <configuration file>
53	-n,--dry-run
54	-p, --project <project name>
55	-t, --ticket_prefix <name>
56
57	SVN Info:
58	$Id: email2trac.py.in 403 2010-07-16 09:41:11Z bas $
59	"""
60	import os
61	import sys
62	import string
63	import getopt
64	import stat
65	import time
66	import email
67	import email.Iterators
68	import email.Header
69	import re
70	import urllib
71	import unicodedata
72	from stat import *
73	import mimetypes
74	import traceback
75	import logging
76	import logging.handlers
77
78
79	from trac import __version__ as trac_version
80
81	# Will fail where unavailable, e.g. Windows
82	#
83	try:
84	import syslog
85	SYSLOG_AVAILABLE = True
86	except ImportError:
87	SYSLOG_AVAILABLE = False
88
89	from datetime import tzinfo, timedelta, datetime
90	from trac import config as trac_config
91
92	# Some global variables
93	#
94	trac_default_version = '0.11'
95	m = None
96
97	# A UTC class needed for trac version 0.11, added by
98	# tbaschak at ktc dot mb dot ca
99	#
100	class UTC(tzinfo):
101	"""UTC"""
102	ZERO = timedelta(0)
103	HOUR = timedelta(hours=1)
104
105	def utcoffset(self, dt):
106	return self.ZERO
107
108	def tzname(self, dt):
109	return "UTC"
110
111	def dst(self, dt):
112	return self.ZERO
113
114
115	class TicketEmailParser(object):
116	env = None
117	comment = '> '
118
119	def __init__(self, env, parameters, version):
120	self.env = env
121
122	# Database connection
123	#
124	self.db = None
125
126	# Save parameters
127	#
128	self.parameters = parameters
129
130	# Some useful mail constants
131	#
132	self.email_name = None
133	self.email_addr = None
134	self.email_from = None
135	self.author = None
136	self.id = None
137
138	self.STRIP_CONTENT_TYPES = list()
139
140	self.VERSION = version
141
142	self.setup_log()
143	self.setup_parameters()
144
145	self.DRY_RUN = parameters['dry_run']
146	self.VERBOSE = parameters['verbose']
147
148	self.get_config = self.env.config.get
149
150
151	def setup_log():
152	"""
153	Setup loging
154
155	Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
156	would be interpreted by the ConfigParser itself.
157
158	"""
159	if not parameters.log_type:
160	paramters.log_type = 'syslog'
161
162	if parameters.log_type = 'file'
163	if not os.path.isabs(parameters.log_file):
164	parameters.log_file = os.path.jpin(self.env.path. 'log')
165	hdlr = logging.FileHandler()
166
167	elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
168	# Requires win32 extensions
169	hdlr = logging.handlers.NTEventLogHandler(logid, logtype='Application')
170
171	elif parameters.log_type in ('syslog', 'unix'):
172	hdlr = logging.handlers.SysLogHandler('/dev/log')
173
174	elif parameters.log_type in ('stderr'):
175	hdlr = logging.StreamHandler(sys.stderr)
176
177	else:
178	hdlr = logging.handlers.BufferingHandler(0)
179
180	if parameters.format:
181	parameters.format = parameters.format.replace('$(', '%(')
182	else:
183	parameters.format = 'Email2trac: %(message)s'
184
185
186	self.logger = logging.getLogger('email2trac')
187
188	## Setup debug level
189
190
191	def setup_parameters():
192	if self.parameters.has_key('umask'):
193	os.umask(int(self.parameters['umask'], 8))
194
195	if self.parameters.has_key('debug'):
196	self.DEBUG = int(self.parameters['debug'])
197	else:
198	self.DEBUG = 0
199
200	if self.parameters.has_key('mailto_link'):
201	self.MAILTO = int(self.parameters['mailto_link'])
202	if self.parameters.has_key('mailto_cc'):
203	self.MAILTO_CC = self.parameters['mailto_cc']
204	else:
205	self.MAILTO_CC = ''
206	else:
207	self.MAILTO = 0
208
209	if self.parameters.has_key('spam_level'):
210	self.SPAM_LEVEL = int(self.parameters['spam_level'])
211	else:
212	self.SPAM_LEVEL = 0
213
214	if self.parameters.has_key('spam_header'):
215	self.SPAM_HEADER = self.parameters['spam_header']
216	else:
217	self.SPAM_HEADER = 'X-Spam-Score'
218
219	if self.parameters.has_key('email_quote'):
220	self.EMAIL_QUOTE = str(self.parameters['email_quote'])
221	else:
222	self.EMAIL_QUOTE = '> '
223
224	if self.parameters.has_key('email_header'):
225	self.EMAIL_HEADER = int(self.parameters['email_header'])
226	else:
227	self.EMAIL_HEADER = 0
228
229	if self.parameters.has_key('alternate_notify_template'):
230	self.notify_template = str(self.parameters['alternate_notify_template'])
231	else:
232	self.notify_template = None
233
234	if self.parameters.has_key('alternate_notify_template_update'):
235	self.notify_template_update = str(self.parameters['alternate_notify_template_update'])
236	else:
237	self.notify_template_update = None
238
239	if self.parameters.has_key('reply_all'):
240	self.REPLY_ALL = int(self.parameters['reply_all'])
241	else:
242	self.REPLY_ALL = 0
243
244	if self.parameters.has_key('ticket_permission_system'):
245	self.TICKET_PERMISSION_SYSTEM = str(self.parameters['ticket_permission_system'])
246	else:
247	self.TICKET_PERMISSION_SYSTEM = None
248
249	if self.parameters.has_key('ticket_update'):
250	self.TICKET_UPDATE = int(self.parameters['ticket_update'])
251	else:
252	self.TICKET_UPDATE = 0
253
254	if self.parameters.has_key('ticket_update_by_subject'):
255	self.TICKET_UPDATE_BY_SUBJECT = int(self.parameters['ticket_update_by_subject'])
256	else:
257	self.TICKET_UPDATE_BY_SUBJECT = 0
258
259	if self.parameters.has_key('ticket_update_by_subject_lookback'):
260	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK = int(self.parameters['ticket_update_by_subject_lookback'])
261	else:
262	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK = 30
263
264	if self.parameters.has_key('drop_spam'):
265	self.DROP_SPAM = int(self.parameters['drop_spam'])
266	else:
267	self.DROP_SPAM = 0
268
269	if self.parameters.has_key('verbatim_format'):
270	self.VERBATIM_FORMAT = int(self.parameters['verbatim_format'])
271	else:
272	self.VERBATIM_FORMAT = 1
273
274	if self.parameters.has_key('reflow'):
275	self.REFLOW = int(self.parameters['reflow'])
276	else:
277	self.REFLOW = 1
278
279	if self.parameters.has_key('drop_alternative_html_version'):
280	self.DROP_ALTERNATIVE_HTML_VERSION = int(self.parameters['drop_alternative_html_version'])
281	else:
282	self.DROP_ALTERNATIVE_HTML_VERSION = 0
283
284	if self.parameters.has_key('strip_signature'):
285	self.STRIP_SIGNATURE = int(self.parameters['strip_signature'])
286	else:
287	self.STRIP_SIGNATURE = 0
288
289	if self.parameters.has_key('strip_quotes'):
290	self.STRIP_QUOTES = int(self.parameters['strip_quotes'])
291	else:
292	self.STRIP_QUOTES = 0
293
294	self.properties = dict()
295	if self.parameters.has_key('inline_properties'):
296	self.INLINE_PROPERTIES = int(self.parameters['inline_properties'])
297	else:
298	self.INLINE_PROPERTIES = 0
299
300	if self.parameters.has_key('use_textwrap'):
301	self.USE_TEXTWRAP = int(self.parameters['use_textwrap'])
302	else:
303	self.USE_TEXTWRAP = 0
304
305	if self.parameters.has_key('binhex'):
306	self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
307
308	if self.parameters.has_key('applesingle'):
309	self.STRIP_CONTENT_TYPES.append('application/applefile')
310
311	if self.parameters.has_key('appledouble'):
312	self.STRIP_CONTENT_TYPES.append('application/applefile')
313
314	if self.parameters.has_key('strip_content_types'):
315	items = self.parameters['strip_content_types'].split(',')
316	for item in items:
317	self.STRIP_CONTENT_TYPES.append(item.strip())
318
319	self.WORKFLOW = None
320	if self.parameters.has_key('workflow'):
321	self.WORKFLOW = self.parameters['workflow']
322
323	# Use OS independend functions
324	#
325	self.TMPDIR = os.path.normcase('/tmp')
326	if self.parameters.has_key('tmpdir'):
327	self.TMPDIR = os.path.normcase(str(self.parameters['tmpdir']))
328
329	if self.parameters.has_key('ignore_trac_user_settings'):
330	self.IGNORE_TRAC_USER_SETTINGS = int(self.parameters['ignore_trac_user_settings'])
331	else:
332	self.IGNORE_TRAC_USER_SETTINGS = 0
333
334	if self.parameters.has_key('email_triggers_workflow'):
335	self.EMAIL_TRIGGERS_WORKFLOW = int(self.parameters['email_triggers_workflow'])
336	else:
337	self.EMAIL_TRIGGERS_WORKFLOW = 1
338
339	if self.parameters.has_key('subject_field_separator'):
340	self.SUBJECT_FIELD_SEPARATOR = self.parameters['subject_field_separator'].strip()
341	else:
342	self.SUBJECT_FIELD_SEPARATOR = '&'
343
344	self.trac_smtp_from = self.get_config('notification', 'smtp_from')
345
346	self.system = None
347
348	########## Email Header Functions ###########################################################
349
350	def spam(self, message):
351	"""
352	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
353	# Note if Spam_level then '*' are included
354	"""
355	spam = False
356	if message.has_key(self.SPAM_HEADER):
357	spam_l = string.split(message[self.SPAM_HEADER])
358
359	try:
360	number = spam_l[0].count('*')
361	except IndexError, detail:
362	number = 0
363
364	if number >= self.SPAM_LEVEL:
365	spam = True
366
367	# treat virus mails as spam
368	#
369	elif message.has_key('X-Virus-found'):
370	spam = True
371
372	# How to handle SPAM messages
373	#
374	if self.DROP_SPAM and spam:
375	if self.DEBUG > 2 :
376	print 'This message is a SPAM. Automatic ticket insertion refused (SPAM level > %d' % self.SPAM_LEVEL
377
378	return 'drop'
379
380	elif spam:
381
382	return 'Spam'
383
384	else:
385
386	return False
387
388	def email_header_acl(self, keyword, header_field, default):
389	"""
390	This function wil check if the email address is allowed or denied
391	to send mail to the ticket list
392	"""
393	try:
394	mail_addresses = self.parameters[keyword]
395
396	# Check if we have an empty string
397	#
398	if not mail_addresses:
399	return default
400
401	except KeyError, detail:
402	if self.DEBUG > 2 :
403	print 'TD: %s not defined, all messages are allowed.' %(keyword)
404
405	return default
406
407	mail_addresses = string.split(mail_addresses, ',')
408
409	for entry in mail_addresses:
410	entry = entry.strip()
411	TO_RE = re.compile(entry, re.VERBOSE\|re.IGNORECASE)
412	result = TO_RE.search(header_field)
413	if result:
414	return True
415
416	return False
417
418	def email_header_txt(self, m):
419	"""
420	Display To and CC addresses in description field
421	"""
422	s = ''
423
424	if m['To'] and len(m['To']) > 0:
425	s = "'''To:''' %s\r\n" %(m['To'])
426	if m['Cc'] and len(m['Cc']) > 0:
427	s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
428
429	return self.email_to_unicode(s)
430
431
432	def get_sender_info(self, message):
433	"""
434	Get the default author name and email address from the message
435	"""
436
437	self.email_to = self.email_to_unicode(message['to'])
438	self.to_name, self.to_email_addr = email.Utils.parseaddr (self.email_to)
439
440	self.email_from = self.email_to_unicode(message['from'])
441	self.email_name, self.email_addr = email.Utils.parseaddr(self.email_from)
442
443	## Trac can not handle author's name that contains spaces
444	# and forbid the ticket email address as author field
445
446	if self.email_addr == self.trac_smtp_from:
447	if self.email_name:
448	self.author = self.email_name
449	else:
450	self.author = "email2trac"
451	else:
452	self.author = self.email_addr
453
454	if self.IGNORE_TRAC_USER_SETTINGS:
455	return
456
457	# Is this a registered user, use email address as search key:
458	# result:
459	# u : login name
460	# n : Name that the user has set in the settings tab
461	# e : email address that the user has set in the settings tab
462	#
463	users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
464	if e and (e.lower() == self.email_addr.lower()) ]
465
466	if len(users) == 1:
467	self.email_from = users[0][0]
468	self.author = users[0][0]
469
470	def set_reply_fields(self, ticket, message):
471	"""
472	Set all the right fields for a new ticket
473	"""
474	if self.DEBUG:
475	print 'TD: set_reply_fields'
476
477	## Only use name or email adress
478	#ticket['reporter'] = self.email_from
479	ticket['reporter'] = self.author
480
481
482	# Put all CC-addresses in ticket CC field
483	#
484	if self.REPLY_ALL:
485
486	email_cc = ''
487
488	cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
489
490	if not cc_addrs:
491	return
492
493	## Build a list of forbidden CC addresses
494	#
495	#to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
496	#to_list = list()
497	#for n,e in to_addrs:
498	# to_list.append(e)
499
500	# Always Remove reporter email address from cc-list
501	#
502	try:
503	cc_addrs.remove((self.author, self.email_addr))
504	except ValueError, detail:
505	pass
506
507	for name,addr in cc_addrs:
508
509	## Prevent mail loop
510	#
511	#if addr in to_list:
512
513	if addr == self.trac_smtp_from:
514	if self.DEBUG:
515	print "Skipping %s mail address for CC-field" %(addr)
516	continue
517
518	if email_cc:
519	email_cc = '%s, %s' %(email_cc, addr)
520	else:
521	email_cc = addr
522
523	if email_cc:
524	if self.DEBUG:
525	print 'TD: set_reply_fields: %s' %email_cc
526
527	ticket['cc'] = self.email_to_unicode(email_cc)
528
529
530	########## DEBUG functions ###########################################################
531
532	def debug_body(self, message_body, tempfile=False):
533	if tempfile:
534	import tempfile
535	body_file = tempfile.mktemp('.email2trac')
536	else:
537	body_file = os.path.join(self.TMPDIR, 'body.txt')
538
539	if self.DRY_RUN:
540	print 'DRY-RUN: not saving body to %s' %(body_file)
541	return
542
543	print 'TD: writing body to %s' %(body_file)
544	fx = open(body_file, 'wb')
545	if not message_body:
546	message_body = '(None)'
547
548	message_body = message_body.encode('utf-8')
549	#message_body = unicode(message_body, 'iso-8859-15')
550
551	fx.write(message_body)
552	fx.close()
553	try:
554	os.chmod(body_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
555	except OSError:
556	pass
557
558	def debug_attachments(self, message_parts):
559	"""
560	"""
561	if self.VERBOSE:
562	print "VB: debug_attachments"
563
564	n = 0
565	for item in message_parts:
566	# Skip inline text parts
567	if not isinstance(item, tuple):
568	continue
569
570	(original, filename, part) = item
571
572	n = n + 1
573	print 'TD: part%d: Content-Type: %s' % (n, part.get_content_type())
574
575	s = 'TD: part%d: filename: %s' %(n, filename)
576	self.print_unicode(s)
577
578	## Forbidden chars
579	#
580	filename = filename.replace('\\', '_')
581	filename = filename.replace('/', '_')
582
583
584	part_file = os.path.join(self.TMPDIR, filename)
585	s = 'TD: writing part%d (%s)' % (n,part_file)
586	self.print_unicode(s)
587
588	if self.DRY_RUN:
589	print 'DRY_RUN: NOT saving attachments'
590	continue
591
592	part_file = util.text.unicode_quote(part_file)
593
594	fx = open(part_file, 'wb')
595	text = part.get_payload(decode=1)
596
597	if not text:
598	text = '(None)'
599
600	fx.write(text)
601	fx.close()
602
603	try:
604	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
605	except OSError:
606	pass
607
608	def save_email_for_debug(self, message, tempfile=False):
609
610	if tempfile:
611	import tempfile
612	msg_file = tempfile.mktemp('.email2trac')
613	else:
614	#msg_file = '/var/tmp/msg.txt'
615	msg_file = os.path.join(self.TMPDIR, 'msg.txt')
616
617	if self.DRY_RUN:
618	print 'DRY_RUN: NOT saving email message to %s' %(msg_file)
619	else:
620	print 'TD: saving email to %s' %(msg_file)
621
622	fx = open(msg_file, 'wb')
623	fx.write('%s' % message)
624	fx.close()
625
626	try:
627	os.chmod(msg_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
628	except OSError:
629	pass
630
631	message_parts = self.get_message_parts(message)
632	message_parts = self.unique_attachment_names(message_parts)
633	body_text = self.body_text(message_parts)
634	self.debug_body(body_text, True)
635	self.debug_attachments(message_parts)
636
637	########## Conversion functions ###########################################################
638
639	def email_to_unicode(self, message_str):
640	"""
641	Email has 7 bit ASCII code, convert it to unicode with the charset
642	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
643	understands it.
644	"""
645	if self.VERBOSE:
646	print "VB: email_to_unicode"
647
648	results = email.Header.decode_header(message_str)
649
650	s = None
651	for text,format in results:
652	if format:
653	try:
654	temp = unicode(text, format)
655	except UnicodeError, detail:
656	# This always works
657	#
658	temp = unicode(text, 'iso-8859-15')
659	except LookupError, detail:
660	#text = 'ERROR: Could not find charset: %s, please install' %format
661	#temp = unicode(text, 'iso-8859-15')
662	temp = message_str
663
664	else:
665	temp = string.strip(text)
666	temp = unicode(text, 'iso-8859-15')
667
668	if s:
669	s = '%s %s' %(s, temp)
670	else:
671	s = '%s' %temp
672
673	#s = s.encode('utf-8')
674	return s
675
676	def str_to_dict(self, s):
677	"""
678	Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
679	"""
680	if self.VERBOSE:
681	print "VB: str_to_dict"
682
683	fields = string.split(s, self.SUBJECT_FIELD_SEPARATOR)
684
685	result = dict()
686	for field in fields:
687	try:
688	index, value = string.split(field, '=')
689
690	# We can not change the description of a ticket via the subject
691	# line. The description is the body of the email
692	#
693	if index.lower() in ['description']:
694	continue
695
696	if value:
697	result[index.lower()] = value
698
699	except ValueError:
700	pass
701	return result
702
703	def print_unicode(self,s):
704	"""
705	This function prints unicode strings uif possible else it will quote it
706	"""
707	try:
708	print s
709	except UnicodeEncodeError, detail:
710	print util.text.unicode_quote(s)
711
712	########## TRAC ticket functions ###########################################################
713
714	def check_permission_participants(self, tkt):
715	"""
716	Check if the mailer is allowed to update the ticket
717	"""
718
719	if tkt['reporter'].lower() in [self.author, self.email_addr]:
720	if self.DEBUG:
721	print 'ALLOW, %s is the ticket reporter' %(self.email_addr)
722	return True
723
724	perm = PermissionSystem(self.env)
725	if perm.check_permission('TICKET_MODIFY', self.author):
726	if self.DEBUG:
727	print 'ALLOW, %s has trac permission to update the ticket' %(self.author)
728	return True
729	else:
730	return False
731
732
733	# Is the updater in the CC?
734	try:
735	cc_list = tkt['cc'].split(',')
736	for cc in cc_list:
737	if self.email_addr.lower() in cc.strip():
738
739	if self.DEBUG:
740	print 'ALLOW, %s is in the CC' %(self.email_addr)
741
742	return True
743
744	except KeyError:
745	return False
746
747	def check_permission(self, tkt, action):
748	"""
749	check if the reporter has the right permission for the action:
750	- TICKET_CREATE
751	- TICKET_MODIFY
752
753	There are three models:
754	- None : no checking at all
755	- trac : check the permission via trac permission model
756	- email2trac: ....
757	"""
758	if self.VERBOSE:
759	print "VB: check_permission"
760
761	if self.TICKET_PERMISSION_SYSTEM in ['trac']:
762
763	perm = PermissionSystem(self.env)
764	if perm.check_permission(action, self.author):
765	return True
766	else:
767	return False
768
769	elif self.TICKET_PERMISSION_SYSTEM in ['update_restricted_to_participants']:
770	if action in ['TICKET_MODIFY']:
771	return (self.check_permission_participants(tkt))
772	else:
773	return True
774
775	# Default is to allow everybody ticket updates and ticket creation
776	else:
777	return True
778
779
780	def update_ticket_fields(self, ticket, user_dict, use_default=None):
781	"""
782	This will update the ticket fields. It will check if the
783	given fields are known and if the right values are specified
784	It will only update the ticket field value:
785	- If the field is known
786	- If the value supplied is valid for the ticket field.
787	If not then there are two options:
788	1) Skip the value (use_default=None)
789	2) Set default value for field (use_default=1)
790	"""
791	if self.VERBOSE:
792	print "VB: update_ticket_fields"
793
794	# Build a system dictionary from the ticket fields
795	# with field as index and option as value
796	#
797	sys_dict = dict()
798	for field in ticket.fields:
799	try:
800	sys_dict[field['name']] = field['options']
801
802	except KeyError:
803	sys_dict[field['name']] = None
804	pass
805
806	## Check user supplied fields an compare them with the
807	# system one's
808	#
809	for field,value in user_dict.items():
810	if self.DEBUG >= 10:
811	s = 'TD: user_field\t %s = %s' %(field,value)
812	self.print_unicode(s)
813
814	## To prevent mail loop
815	#
816	if field == 'cc':
817
818	cc_list = user_dict['cc'].split(',')
819
820	if self.trac_smtp_from in cc_list:
821	if self.DEBUG > 10:
822	print 'TD: MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from)
823	cc_list.remove(self.trac_smtp_from)
824
825	value = ','.join(cc_list)
826
827
828	if sys_dict.has_key(field):
829
830	# Check if value is an allowed system option, if TypeError then
831	# every value is allowed
832	#
833	try:
834	if value in sys_dict[field]:
835	ticket[field] = value
836	else:
837	# Must we set a default if value is not allowed
838	#
839	if use_default:
840	value = self.get_config('ticket', 'default_%s' %(field) )
841
842	except TypeError:
843	pass
844
845	## Only set if we have a value
846	#
847	if value:
848	ticket[field] = value
849
850	if self.DEBUG >= 10:
851	s = 'ticket_field\t %s = %s' %(field, ticket[field])
852	self.print_unicode(s)
853
854	def ticket_update(self, m, id, spam):
855	"""
856	If the current email is a reply to an existing ticket, this function
857	will append the contents of this email to that ticket, instead of
858	creating a new one.
859	"""
860	if self.VERBOSE:
861	print "VB: ticket_update: %s" %id
862
863	# Must we update ticket fields
864	#
865	update_fields = dict()
866	try:
867	id, keywords = string.split(id, '?')
868
869	update_fields = self.str_to_dict(keywords)
870
871	# Strip '#'
872	#
873	self.id = int(id[1:])
874
875	except ValueError:
876
877	# Strip '#'
878	#
879	self.id = int(id[1:])
880
881	if self.VERBOSE:
882	print "VB: ticket_update: %s" %id
883
884
885	# When is the change committed
886	#
887	if self.VERSION < 0.11:
888	when = int(time.time())
889	else:
890	utc = UTC()
891	when = datetime.now(utc)
892
893	try:
894	tkt = Ticket(self.env, self.id, self.db)
895
896	except util.TracError, detail:
897
898	# Not a valid ticket
899
900	self.id = None
901	return False
902
903	# Check the permission of the reporter
904	#
905	if self.TICKET_PERMISSION_SYSTEM:
906	if not self.check_permission(tkt, 'TICKET_MODIFY'):
907	print 'Reporter: %s has no permission to modify tickets' %self.author
908	return False
909
910	# How many changes has this ticket
911	cnum = len(tkt.get_changelog())
912
913
914	# reopen the ticket if it is was closed
915	# We must use the ticket workflow framework
916	#
917	if tkt['status'] in ['closed'] and self.EMAIL_TRIGGERS_WORKFLOW:
918
919	#print controller.actions['reopen']
920	#
921	# As reference
922	# req = Mock(href=Href('/'), abs_href=Href('http://www.example.com/'), authname='anonymous', perm=MockPerm(), args={})
923	#
924	#a = controller.render_ticket_action_control(req, tkt, 'reopen')
925	#print 'controller : ', a
926	#
927	#b = controller.get_all_status()
928	#print 'get all status: ', b
929	#
930	#b = controller.get_ticket_changes(req, tkt, 'reopen')
931	#print 'get_ticket_changes :', b
932
933	if self.WORKFLOW and (self.VERSION >= 0.11 ) :
934	from trac.ticket.default_workflow import ConfigurableTicketWorkflow
935	from trac.test import Mock, MockPerm
936
937	req = Mock(authname='anonymous', perm=MockPerm(), args={})
938
939	controller = ConfigurableTicketWorkflow(self.env)
940	fields = controller.get_ticket_changes(req, tkt, self.WORKFLOW)
941
942	if self.DEBUG:
943	print 'TD: Workflow ticket update fields: ', fields
944
945	for key in fields.keys():
946	tkt[key] = fields[key]
947
948	else:
949	tkt['status'] = 'reopened'
950	tkt['resolution'] = ''
951
952	# Must we update some ticket fields properties via subjectline
953	#
954	if update_fields:
955	self.update_ticket_fields(tkt, update_fields)
956
957	message_parts = self.get_message_parts(m)
958	message_parts = self.unique_attachment_names(message_parts)
959
960	# Must we update some ticket fields properties via body_text
961	#
962	if self.properties:
963	self.update_ticket_fields(tkt, self.properties)
964
965	if self.EMAIL_HEADER:
966	message_parts.insert(0, self.email_header_txt(m))
967
968	body_text = self.body_text(message_parts)
969
970	error_with_attachments = self.attach_attachments(message_parts)
971
972	if body_text.strip() or update_fields or self.properties:
973	if self.DRY_RUN:
974	print 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) ', self.author, cnum
975	else:
976	if error_with_attachments:
977	body_text = '%s\\%s' %(error_with_attachments, body_text)
978
979	tkt.save_changes(self.author, body_text, when, None, str(cnum))
980
981
982	if not spam:
983	self.notify(tkt, False, when)
984
985	return True
986
987	def set_ticket_fields(self, ticket):
988	"""
989	set the ticket fields to value specified
990	- /etc/email2trac.conf with <prefix>_<field>
991	- trac default values, trac.ini
992	"""
993	user_dict = dict()
994
995	for field in ticket.fields:
996
997	name = field['name']
998
999	## default trac value
1000	#
1001	if not field.get('custom'):
1002	value = self.get_config('ticket', 'default_%s' %(name) )
1003	else:
1004	## Else we get the default value for reporter
1005	#
1006	value = field.get('value')
1007	options = field.get('options')
1008
1009	if value and options and (value not in options):
1010	value = options[int(value)]
1011
1012	if self.DEBUG > 10:
1013	s = 'TD: trac.ini name %s = %s' %(name, value)
1014	self.print_unicode(s)
1015
1016	## email2trac.conf settings
1017	#
1018	prefix = self.parameters['ticket_prefix']
1019	try:
1020	value = self.parameters['%s_%s' %(prefix, name)]
1021	if self.DEBUG > 10:
1022	s = 'TD: email2trac.conf %s = %s ' %(name, value)
1023	self.print_unicode(s)
1024
1025	except KeyError, detail:
1026	pass
1027
1028	if self.DEBUG:
1029	s = 'TD: user_dict[%s] = %s' %(name, value)
1030	self.print_unicode(s)
1031
1032	if value:
1033	user_dict[name] = value
1034
1035	self.update_ticket_fields(ticket, user_dict, use_default=1)
1036
1037	if 'status' not in user_dict.keys():
1038	ticket['status'] = 'new'
1039
1040
1041	def ticket_update_by_subject(self, subject):
1042	"""
1043	This list of Re: prefixes is probably incomplete. Taken from
1044	wikipedia. Here is how the subject is matched
1045	- Re: <subject>
1046	- Re: (<Mail list label>:)+ <subject>
1047
1048	So we must have the last column
1049	"""
1050	if self.VERBOSE:
1051	print "VB: ticket_update_by_subject()"
1052
1053	matched_id = None
1054	if self.TICKET_UPDATE and self.TICKET_UPDATE_BY_SUBJECT:
1055
1056	SUBJECT_RE = re.compile(r'^(RE\|AW\|VS\|SV):(.:)\s(.)', re.IGNORECASE)
1057	result = SUBJECT_RE.search(subject)
1058
1059	if result:
1060	# This is a reply
1061	orig_subject = result.group(3)
1062
1063	if self.DEBUG:
1064	print 'TD: subject search string: %s' %(orig_subject)
1065
1066	cursor = self.db.cursor()
1067	summaries = [orig_subject, '%%: %s' % orig_subject]
1068
1069	##
1070	# Convert days to seconds
1071	lookback = int(time.mktime(time.gmtime())) - \
1072	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK * 24 * 3600
1073
1074
1075	for summary in summaries:
1076	if self.DEBUG:
1077	print 'TD: Looking for summary matching: "%s"' % summary
1078	sql = """SELECT id FROM ticket
1079	WHERE changetime >= %s AND summary LIKE %s
1080	ORDER BY changetime DESC"""
1081	cursor.execute(sql, [lookback, summary.strip()])
1082
1083	for row in cursor:
1084	(matched_id,) = row
1085	if self.DEBUG:
1086	print 'TD: Found matching ticket id: %d' % matched_id
1087	break
1088
1089	if matched_id:
1090	matched_id = '#%d' % matched_id
1091	return matched_id
1092
1093	return matched_id
1094
1095
1096	def new_ticket(self, msg, subject, spam, set_fields = None):
1097	"""
1098	Create a new ticket
1099	"""
1100	if self.VERBOSE:
1101	print "VB: function new_ticket()"
1102
1103	tkt = Ticket(self.env)
1104
1105	self.set_reply_fields(tkt, msg)
1106
1107	self.set_ticket_fields(tkt)
1108
1109	# Check the permission of the reporter
1110	#
1111	if self.TICKET_PERMISSION_SYSTEM:
1112	if not self.check_permission(tkt, 'TICKET_CREATE'):
1113	print 'Reporter: %s has no permission to create tickets' %self.author
1114	return False
1115
1116	# Old style setting for component, will be removed
1117	#
1118	if spam:
1119	tkt['component'] = 'Spam'
1120
1121	elif self.parameters.has_key('component'):
1122	tkt['component'] = self.parameters['component']
1123
1124	if not msg['Subject']:
1125	tkt['summary'] = u'(No subject)'
1126	else:
1127	tkt['summary'] = subject
1128
1129
1130	if set_fields:
1131	rest, keywords = string.split(set_fields, '?')
1132
1133	if keywords:
1134	update_fields = self.str_to_dict(keywords)
1135	self.update_ticket_fields(tkt, update_fields)
1136
1137	# produce e-mail like header
1138	#
1139	head = ''
1140	if self.EMAIL_HEADER > 0:
1141	head = self.email_header_txt(msg)
1142
1143	message_parts = self.get_message_parts(msg)
1144
1145	# Must we update some ticket fields properties via body_text
1146	#
1147	if self.properties:
1148	self.update_ticket_fields(tkt, self.properties)
1149
1150	if self.DEBUG:
1151	print 'TD: self.get_message_parts ',
1152	print message_parts
1153
1154	message_parts = self.unique_attachment_names(message_parts)
1155	if self.DEBUG:
1156	print 'TD: self.unique_attachment_names',
1157	print message_parts
1158
1159	if self.EMAIL_HEADER > 0:
1160	message_parts.insert(0, self.email_header_txt(msg))
1161
1162	body_text = self.body_text(message_parts)
1163
1164	tkt['description'] = body_text
1165
1166	#when = int(time.time())
1167	#
1168	utc = UTC()
1169	when = datetime.now(utc)
1170
1171	if not self.DRY_RUN:
1172	self.id = tkt.insert()
1173
1174	changed = False
1175	comment = ''
1176
1177	# some routines in trac are dependend on ticket id
1178	# like alternate notify template
1179	#
1180	if self.notify_template:
1181	tkt['id'] = self.id
1182	changed = True
1183
1184	## Rewrite the description if we have mailto enabled
1185	#
1186	if self.MAILTO:
1187	changed = True
1188	comment = u'\nadded mailto line\n'
1189	mailto = self.html_mailto_link( m['Subject'])
1190
1191	tkt['description'] = u'%s\r\n%s%s\r\n' \
1192	%(head, mailto, body_text)
1193
1194	## Save the attachments to the ticket
1195	#
1196	error_with_attachments = self.attach_attachments(message_parts)
1197
1198	if error_with_attachments:
1199	changed = True
1200	comment = '%s\n%s\n' %(comment, error_with_attachments)
1201
1202	if changed:
1203	if self.DRY_RUN:
1204	print 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
1205	else:
1206	tkt.save_changes(tkt['reporter'], comment)
1207	#print tkt.get_changelog(self.db, when)
1208
1209	if not spam:
1210	self.notify(tkt, True)
1211
1212
1213	def attach_attachments(self, message_parts, update=False):
1214	'''
1215	save any attachments as files in the ticket's directory
1216	'''
1217	if self.VERBOSE:
1218	print "VB: attach_attachments()"
1219
1220	if self.DRY_RUN:
1221	print "DRY_RUN: no attachments attached to tickets"
1222	return ''
1223
1224	count = 0
1225
1226	# Get Maxium attachment size
1227	#
1228	max_size = int(self.get_config('attachment', 'max_size'))
1229	status = None
1230
1231	for item in message_parts:
1232	# Skip body parts
1233	if not isinstance(item, tuple):
1234	continue
1235
1236	(original, filename, part) = item
1237	#
1238	# We have to determine the size so we use this temporary solution. we must escape it
1239	# else we get UnicodeErrors.
1240	#
1241	path, fd = util.create_unique_file(os.path.join(self.TMPDIR, util.text.unicode_quote(filename)))
1242	text = part.get_payload(decode=1)
1243	if not text:
1244	text = '(None)'
1245	fd.write(text)
1246	fd.close()
1247
1248	# get the file_size
1249	#
1250	stats = os.lstat(path)
1251	file_size = stats[stat.ST_SIZE]
1252
1253	# Check if the attachment size is allowed
1254	#
1255	if (max_size != -1) and (file_size > max_size):
1256	status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1257	%(status, original, file_size, max_size)
1258
1259	os.unlink(path)
1260	continue
1261	else:
1262	count = count + 1
1263
1264	# Insert the attachment
1265	#
1266	fd = open(path, 'rb')
1267	if self.system == 'discussion':
1268	att = attachment.Attachment(self.env, 'discussion', 'topic/%s'
1269	% (self.id,))
1270	else:
1271	att = attachment.Attachment(self.env, 'ticket', self.id)
1272
1273	# This will break the ticket_update system, the body_text is vaporized
1274	# ;-(
1275	#
1276	if not update:
1277	att.author = self.author
1278	att.description = self.email_to_unicode('Added by email2trac')
1279
1280	try:
1281	att.insert(filename, fd, file_size)
1282	except OSError, detail:
1283	status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
1284
1285	# Remove the created temporary filename
1286	#
1287	fd.close()
1288	os.unlink(path)
1289
1290	## return error
1291	#
1292	return status
1293
1294	########## Fullblog functions #################################################
1295
1296	def blog(self, id):
1297	"""
1298	The blog create/update function
1299	"""
1300	# import the modules
1301	#
1302	from tracfullblog.core import FullBlogCore
1303	from tracfullblog.model import BlogPost, BlogComment
1304	from trac.test import Mock, MockPerm
1305
1306	# instantiate blog core
1307	blog = FullBlogCore(self.env)
1308	req = Mock(authname='anonymous', perm=MockPerm(), args={})
1309
1310	if id:
1311
1312	# update blog
1313	#
1314	comment = BlogComment(self.env, id)
1315	comment.author = self.author
1316
1317	message_parts = self.get_message_parts(m)
1318	comment.comment = self.body_text(message_parts)
1319
1320	blog.create_comment(req, comment)
1321
1322	else:
1323	# create blog
1324	#
1325	import time
1326	post = BlogPost(self.env, 'blog_'+time.strftime("%Y%m%d%H%M%S", time.gmtime()))
1327
1328	#post = BlogPost(self.env, blog._get_default_postname(self.env))
1329
1330	post.author = self.author
1331	post.title = self.email_to_unicode(m['Subject'])
1332
1333	message_parts = self.get_message_parts(m)
1334	post.body = self.body_text(message_parts)
1335
1336	blog.create_post(req, post, self.author, u'Created by email2trac', False)
1337
1338
1339	########## Discussion functions ##############################################
1340
1341	def discussion_topic(self, content, subject):
1342
1343	# Import modules.
1344	from tracdiscussion.api import DiscussionApi
1345	from trac.util.datefmt import to_timestamp, utc
1346
1347	if self.DEBUG:
1348	print 'TD: Creating a new topic in forum:', self.id
1349
1350	# Get dissussion API component.
1351	api = self.env[DiscussionApi]
1352	context = self._create_context(content, subject)
1353
1354	# Get forum for new topic.
1355	forum = api.get_forum(context, self.id)
1356
1357	if not forum and self.DEBUG:
1358	print 'ERROR: Replied forum doesn\'t exist'
1359
1360	# Prepare topic.
1361	topic = {'forum' : forum['id'],
1362	'subject' : context.subject,
1363	'time': to_timestamp(datetime.now(utc)),
1364	'author' : self.author,
1365	'subscribers' : [self.email_addr],
1366	'body' : self.body_text(context.content_parts)}
1367
1368	# Add topic to DB and commit it.
1369	self._add_topic(api, context, topic)
1370	self.db.commit()
1371
1372	def discussion_topic_reply(self, content, subject):
1373
1374	# Import modules.
1375	from tracdiscussion.api import DiscussionApi
1376	from trac.util.datefmt import to_timestamp, utc
1377
1378	if self.DEBUG:
1379	print 'TD: Replying to discussion topic', self.id
1380
1381	# Get dissussion API component.
1382	api = self.env[DiscussionApi]
1383	context = self._create_context(content, subject)
1384
1385	# Get replied topic.
1386	topic = api.get_topic(context, self.id)
1387
1388	if not topic and self.DEBUG:
1389	print 'ERROR: Replied topic doesn\'t exist'
1390
1391	# Prepare message.
1392	message = {'forum' : topic['forum'],
1393	'topic' : topic['id'],
1394	'replyto' : -1,
1395	'time' : to_timestamp(datetime.now(utc)),
1396	'author' : self.author,
1397	'body' : self.body_text(context.content_parts)}
1398
1399	# Add message to DB and commit it.
1400	self._add_message(api, context, message)
1401	self.db.commit()
1402
1403	def discussion_message_reply(self, content, subject):
1404
1405	# Import modules.
1406	from tracdiscussion.api import DiscussionApi
1407	from trac.util.datefmt import to_timestamp, utc
1408
1409	if self.DEBUG:
1410	print 'TD: Replying to discussion message', self.id
1411
1412	# Get dissussion API component.
1413	api = self.env[DiscussionApi]
1414	context = self._create_context(content, subject)
1415
1416	# Get replied message.
1417	message = api.get_message(context, self.id)
1418
1419	if not message and self.DEBUG:
1420	print 'ERROR: Replied message doesn\'t exist'
1421
1422	# Prepare message.
1423	message = {'forum' : message['forum'],
1424	'topic' : message['topic'],
1425	'replyto' : message['id'],
1426	'time' : to_timestamp(datetime.now(utc)),
1427	'author' : self.author,
1428	'body' : self.body_text(context.content_parts)}
1429
1430	# Add message to DB and commit it.
1431	self._add_message(api, context, message)
1432	self.db.commit()
1433
1434	def _create_context(self, content, subject):
1435
1436	# Import modules.
1437	from trac.mimeview import Context
1438	from trac.web.api import Request
1439	from trac.perm import PermissionCache
1440
1441	# TODO: Read server base URL from config.
1442	# Create request object to mockup context creation.
1443	#
1444	environ = {'SERVER_PORT' : 80,
1445	'SERVER_NAME' : 'test',
1446	'REQUEST_METHOD' : 'POST',
1447	'wsgi.url_scheme' : 'http',
1448	'wsgi.input' : sys.stdin}
1449	chrome = {'links': {},
1450	'scripts': [],
1451	'ctxtnav': [],
1452	'warnings': [],
1453	'notices': []}
1454
1455	if self.env.base_url_for_redirect:
1456	environ['trac.base_url'] = self.env.base_url
1457
1458	req = Request(environ, None)
1459	req.chrome = chrome
1460	req.tz = 'missing'
1461	req.authname = self.author
1462	req.perm = PermissionCache(self.env, self.author)
1463
1464	# Create and return context.
1465	context = Context.from_request(req)
1466	context.realm = 'discussion-email2trac'
1467	context.cursor = self.db.cursor()
1468	context.content = content
1469	context.subject = subject
1470
1471	# Read content parts from content.
1472	context.content_parts = self.get_message_parts(content)
1473	context.content_parts = self.unique_attachment_names(
1474	context.content_parts)
1475
1476	return context
1477
1478	def _add_topic(self, api, context, topic):
1479	context.req.perm.assert_permission('DISCUSSION_APPEND')
1480
1481	# Filter topic.
1482	for discussion_filter in api.discussion_filters:
1483	accept, topic_or_error = discussion_filter.filter_topic(
1484	context, topic)
1485	if accept:
1486	topic = topic_or_error
1487	else:
1488	raise TracError(topic_or_error)
1489
1490	# Add a new topic.
1491	api.add_topic(context, topic)
1492
1493	# Get inserted topic with new ID.
1494	topic = api.get_topic_by_time(context, topic['time'])
1495
1496	# Attach attachments.
1497	self.id = topic['id']
1498	self.attach_attachments(context.content_parts, True)
1499
1500	# Notify change listeners.
1501	for listener in api.topic_change_listeners:
1502	listener.topic_created(context, topic)
1503
1504	def _add_message(self, api, context, message):
1505	context.req.perm.assert_permission('DISCUSSION_APPEND')
1506
1507	# Filter message.
1508	for discussion_filter in api.discussion_filters:
1509	accept, message_or_error = discussion_filter.filter_message(
1510	context, message)
1511	if accept:
1512	message = message_or_error
1513	else:
1514	raise TracError(message_or_error)
1515
1516	# Add message.
1517	api.add_message(context, message)
1518
1519	# Get inserted message with new ID.
1520	message = api.get_message_by_time(context, message['time'])
1521
1522	# Attach attachments.
1523	self.id = message['topic']
1524	self.attach_attachments(context.content_parts, True)
1525
1526	# Notify change listeners.
1527	for listener in api.message_change_listeners:
1528	listener.message_created(context, message)
1529
1530	########## MAIN function ######################################################
1531
1532	def parse(self, fp):
1533	"""
1534	"""
1535	if self.VERBOSE:
1536	print "VB: main function parse()"
1537	global m
1538
1539	m = email.message_from_file(fp)
1540
1541	if not m:
1542	if self.DEBUG:
1543	print "TD: This is not a valid email message format"
1544	return
1545
1546	# Work around lack of header folding in Python; see http://bugs.python.org/issue4696
1547	try:
1548	m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
1549	except AttributeError, detail:
1550	pass
1551
1552	if self.DEBUG > 1: # save the entire e-mail message text
1553	self.save_email_for_debug(m, True)
1554
1555	self.db = self.env.get_db_cnx()
1556	self.get_sender_info(m)
1557
1558	if not self.email_header_acl('white_list', self.email_addr, True):
1559	if self.DEBUG > 1 :
1560	print 'Message rejected : %s not in white list' %(self.email_addr)
1561	return False
1562
1563	if self.email_header_acl('black_list', self.email_addr, False):
1564	if self.DEBUG > 1 :
1565	print 'Message rejected : %s in black list' %(self.email_addr)
1566	return False
1567
1568	if not self.email_header_acl('recipient_list', self.to_email_addr, True):
1569	if self.DEBUG > 1 :
1570	print 'Message rejected : %s not in recipient list' %(self.to_email_addr)
1571	return False
1572
1573	# If drop the message
1574	#
1575	if self.spam(m) == 'drop':
1576	return False
1577
1578	elif self.spam(m) == 'spam':
1579	spam_msg = True
1580	else:
1581	spam_msg = False
1582
1583	if not m['Subject']:
1584	subject = 'No Subject'
1585	else:
1586	subject = self.email_to_unicode(m['Subject'])
1587
1588	if self.DEBUG:
1589	print "TD:", subject
1590
1591	#
1592	# [hic] #1529: Re: LRZ
1593	# [hic] #1529?owner=bas,priority=medium: Re: LRZ
1594	#
1595	ticket_regex = r'''
1596	(?P<new_fields>[#][?].*)
1597	\|(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?.*)?:)
1598	'''
1599	# Check if FullBlogPlugin is installed
1600	#
1601	blog_enabled = None
1602	blog_regex = ''
1603	if self.get_config('components', 'tracfullblog.*') in ['enabled']:
1604	blog_enabled = True
1605	blog_regex = '''\|(?P<blog>blog:(?P<blog_id>\w*))'''
1606
1607
1608	# Check if DiscussionPlugin is installed
1609	#
1610	discussion_enabled = None
1611	discussion_regex = ''
1612	if self.get_config('components', 'tracdiscussion.api.*') in ['enabled']:
1613	discussion_enabled = True
1614	discussion_regex = r'''
1615	\|(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
1616	\|(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
1617	\|(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
1618	'''
1619
1620
1621	regex_str = ticket_regex + blog_regex + discussion_regex
1622	SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
1623
1624	# Find out if this is a ticket, a blog or a discussion
1625	#
1626	result = SYSTEM_RE.search(subject)
1627
1628	if result:
1629	# update ticket + fields
1630	#
1631	if result.group('reply') and self.TICKET_UPDATE:
1632	self.system = 'ticket'
1633
1634	# Skip the last ':' character
1635	#
1636	if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
1637	self.new_ticket(m, subject, spam_msg)
1638
1639	# New ticket + fields
1640	#
1641	elif result.group('new_fields'):
1642	self.system = 'ticket'
1643	self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
1644
1645	if blog_enabled:
1646	if result.group('blog'):
1647	self.system = 'blog'
1648	self.blog(result.group('blog_id'))
1649
1650	if discussion_enabled:
1651	# New topic.
1652	#
1653	if result.group('forum'):
1654	self.system = 'discussion'
1655	self.id = int(result.group('forum_id'))
1656	self.discussion_topic(m, subject[result.end('forum'):])
1657
1658	# Reply to topic.
1659	#
1660	elif result.group('topic'):
1661	self.system = 'discussion'
1662	self.id = int(result.group('topic_id'))
1663	self.discussion_topic_reply(m, subject[result.end('topic'):])
1664
1665	# Reply to topic message.
1666	#
1667	elif result.group('message'):
1668	self.system = 'discussion'
1669	self.id = int(result.group('message_id'))
1670	self.discussion_message_reply(m, subject[result.end('message'):])
1671
1672	else:
1673	self.system = 'ticket'
1674	result = self.ticket_update_by_subject(subject)
1675	if result:
1676	if not self.ticket_update(m, result, spam_msg):
1677	self.new_ticket(m, subject, spam_msg)
1678	else:
1679	# No update by subject, so just create a new ticket
1680	self.new_ticket(m, subject, spam_msg)
1681
1682
1683	########## BODY TEXT functions ###########################################################
1684
1685	def strip_signature(self, text):
1686	"""
1687	Strip signature from message, inspired by Mailman software
1688	"""
1689	body = []
1690	for line in text.splitlines():
1691	if line == '-- ':
1692	break
1693	body.append(line)
1694
1695	return ('\n'.join(body))
1696
1697	def reflow(self, text, delsp = 0):
1698	"""
1699	Reflow the message based on the format="flowed" specification (RFC 3676)
1700	"""
1701	flowedlines = []
1702	quotelevel = 0
1703	prevflowed = 0
1704
1705	for line in text.splitlines():
1706	from re import match
1707
1708	# Figure out the quote level and the content of the current line
1709	m = match('(>)( ?)(.)', line)
1710	linequotelevel = len(m.group(1))
1711	line = m.group(3)
1712
1713	# Determine whether this line is flowed
1714	if line and line != '-- ' and line[-1] == ' ':
1715	flowed = 1
1716	else:
1717	flowed = 0
1718
1719	if flowed and delsp and line and line[-1] == ' ':
1720	line = line[:-1]
1721
1722	# If the previous line is flowed, append this line to it
1723	if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1724	flowedlines[-1] += line
1725	# Otherwise, start a new line
1726	else:
1727	flowedlines.append('>' * linequotelevel + line)
1728
1729	prevflowed = flowed
1730
1731
1732	return '\n'.join(flowedlines)
1733
1734	def strip_quotes(self, text):
1735	"""
1736	Strip quotes from message by Nicolas Mendoza
1737	"""
1738	body = []
1739	for line in text.splitlines():
1740	if line.startswith(self.EMAIL_QUOTE):
1741	continue
1742	body.append(line)
1743
1744	return ('\n'.join(body))
1745
1746	def inline_properties(self, text):
1747	"""
1748	Parse text if we use inline keywords to set ticket fields
1749	"""
1750	if self.DEBUG:
1751	print 'TD: inline_properties function'
1752
1753	properties = dict()
1754	body = list()
1755
1756	INLINE_EXP = re.compile('\s[@]\s([a-zA-Z]+)\s:(.)$')
1757
1758	for line in text.splitlines():
1759	match = INLINE_EXP.match(line)
1760	if match:
1761	keyword, value = match.groups()
1762	self.properties[keyword] = value.strip()
1763	if self.DEBUG:
1764	print "TD: inline properties: %s : %s" %(keyword,value)
1765	else:
1766	body.append(line)
1767
1768	return '\n'.join(body)
1769
1770
1771	def wrap_text(self, text, replace_whitespace = False):
1772	"""
1773	Will break a lines longer then given length into several small
1774	lines of size given length
1775	"""
1776	import textwrap
1777
1778	LINESEPARATOR = '\n'
1779	reformat = ''
1780
1781	for s in text.split(LINESEPARATOR):
1782	tmp = textwrap.fill(s,self.USE_TEXTWRAP)
1783	if tmp:
1784	reformat = '%s\n%s' %(reformat,tmp)
1785	else:
1786	reformat = '%s\n' %reformat
1787
1788	return reformat
1789
1790	# Python2.4 and higher
1791	#
1792	#return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
1793	#
1794
1795	########## EMAIL attachements functions ###########################################################
1796
1797	def inline_part(self, part):
1798	"""
1799	"""
1800	if self.VERBOSE:
1801	print "VB: inline_part()"
1802
1803	return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
1804
1805	def get_message_parts(self, msg):
1806	"""
1807	parses the email message and returns a list of body parts and attachments
1808	body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
1809	"""
1810	if self.VERBOSE:
1811	print "VB: get_message_parts()"
1812
1813	message_parts = list()
1814
1815	ALTERNATIVE_MULTIPART = False
1816
1817	for part in msg.walk():
1818	if self.DEBUG:
1819	print 'TD: Message part: Main-Type: %s' % part.get_content_maintype()
1820	print 'TD: Message part: Content-Type: %s' % part.get_content_type()
1821
1822	## Check content type
1823	#
1824	if part.get_content_type() in self.STRIP_CONTENT_TYPES:
1825
1826	if self.DEBUG:
1827	print "TD: A %s attachment named '%s' was skipped" %(part.get_content_type(), part.get_filename())
1828
1829	continue
1830
1831	## Catch some mulitpart execptions
1832	#
1833	if part.get_content_type() == 'multipart/alternative':
1834	ALTERNATIVE_MULTIPART = True
1835	continue
1836
1837	## Skip multipart containers
1838	#
1839	if part.get_content_maintype() == 'multipart':
1840	if self.DEBUG:
1841	print "TD: Skipping multipart container"
1842	continue
1843
1844	## Check if this is an inline part. It's inline if there is co Cont-Disp header, or if there is one and it says "inline"
1845	#
1846	inline = self.inline_part(part)
1847
1848	## Drop HTML message
1849	#
1850	if ALTERNATIVE_MULTIPART and self.DROP_ALTERNATIVE_HTML_VERSION:
1851	if part.get_content_type() == 'text/html':
1852	if self.DEBUG:
1853	print "TD: Skipping alternative HTML message"
1854
1855	ALTERNATIVE_MULTIPART = False
1856	continue
1857
1858	## Inline text parts are where the body is
1859	#
1860	if part.get_content_type() == 'text/plain' and inline:
1861	if self.DEBUG:
1862	print 'TD: Inline body part'
1863
1864	# Try to decode, if fails then do not decode
1865	#
1866	body_text = part.get_payload(decode=1)
1867	if not body_text:
1868	body_text = part.get_payload(decode=0)
1869
1870	format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
1871	delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
1872
1873	if self.REFLOW and not self.VERBATIM_FORMAT and format == 'flowed':
1874	body_text = self.reflow(body_text, delsp == 'yes')
1875
1876	if self.STRIP_SIGNATURE:
1877	body_text = self.strip_signature(body_text)
1878
1879	if self.STRIP_QUOTES:
1880	body_text = self.strip_quotes(body_text)
1881
1882	if self.INLINE_PROPERTIES:
1883	body_text = self.inline_properties(body_text)
1884
1885	if self.USE_TEXTWRAP:
1886	body_text = self.wrap_text(body_text)
1887
1888	## Get contents charset (iso-8859-15 if not defined in mail headers)
1889	#
1890	charset = part.get_content_charset()
1891	if not charset:
1892	charset = 'iso-8859-15'
1893
1894	try:
1895	ubody_text = unicode(body_text, charset)
1896
1897	except UnicodeError, detail:
1898	ubody_text = unicode(body_text, 'iso-8859-15')
1899
1900	except LookupError, detail:
1901	ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
1902
1903	if self.VERBATIM_FORMAT:
1904	message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
1905	else:
1906	message_parts.append('%s' %ubody_text)
1907	else:
1908	if self.DEBUG:
1909	s = 'TD: Filename: %s' % part.get_filename()
1910	self.print_unicode(s)
1911
1912	##
1913	# First try to use email header function to convert filename.
1914	# If this fails the use the plan filename
1915	try:
1916	filename = self.email_to_unicode(part.get_filename())
1917	except UnicodeEncodeError, detail:
1918	filename = part.get_filename()
1919
1920	message_parts.append((filename, part))
1921
1922	return message_parts
1923
1924	def unique_attachment_names(self, message_parts):
1925	"""
1926	"""
1927	renamed_parts = []
1928	attachment_names = set()
1929
1930	for item in message_parts:
1931
1932	## If not an attachment, leave it alone
1933	#
1934	if not isinstance(item, tuple):
1935	renamed_parts.append(item)
1936	continue
1937
1938	(filename, part) = item
1939
1940	## If no filename, use a default one
1941	#
1942	if not filename:
1943	filename = 'untitled-part'
1944
1945	# Guess the extension from the content type, use non strict mode
1946	# some additional non-standard but commonly used MIME types
1947	# are also recognized
1948	#
1949	ext = mimetypes.guess_extension(part.get_content_type(), False)
1950	if not ext:
1951	ext = '.bin'
1952
1953	filename = '%s%s' % (filename, ext)
1954
1955	## Discard relative paths for windows/unix in attachment names
1956	#
1957	#filename = filename.replace('\\', '/').replace(':', '/')
1958	filename = filename.replace('\\', '_')
1959	filename = filename.replace('/', '_')
1960
1961	#
1962	# We try to normalize the filename to utf-8 NFC if we can.
1963	# Files uploaded from OS X might be in NFD.
1964	# Check python version and then try it
1965	#
1966	#if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
1967	# try:
1968	# filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
1969	# except TypeError:
1970	# pass
1971
1972	# Make the filename unique for this ticket
1973	num = 0
1974	unique_filename = filename
1975	dummy_filename, ext = os.path.splitext(filename)
1976
1977	while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
1978	num += 1
1979	unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
1980
1981	if self.DEBUG:
1982	s = 'TD: Attachment with filename %s will be saved as %s' % (filename, unique_filename)
1983	self.print_unicode(s)
1984
1985	attachment_names.add(unique_filename)
1986
1987	renamed_parts.append((filename, unique_filename, part))
1988
1989	return renamed_parts
1990
1991
1992	def attachment_exists(self, filename):
1993
1994	if self.DEBUG:
1995	s = 'TD: attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
1996	self.print_unicode(s)
1997
1998	# We have no valid ticket id
1999	#
2000	if not self.id:
2001	return False
2002
2003	try:
2004	if self.system == 'discussion':
2005	att = attachment.Attachment(self.env, 'discussion', 'ticket/%s'
2006	% (self.id,), filename)
2007	else:
2008	att = attachment.Attachment(self.env, 'ticket', self.id,
2009	filename)
2010	return True
2011	except attachment.ResourceNotFound:
2012	return False
2013
2014	########## TRAC Ticket Text ###########################################################
2015
2016	def body_text(self, message_parts):
2017	body_text = []
2018
2019	for part in message_parts:
2020	# Plain text part, append it
2021	if not isinstance(part, tuple):
2022	body_text.extend(part.strip().splitlines())
2023	body_text.append("")
2024	continue
2025
2026	(original, filename, part) = part
2027	inline = self.inline_part(part)
2028
2029	if part.get_content_maintype() == 'image' and inline:
2030	if self.system != 'discussion':
2031	body_text.append('[[Image(%s)]]' % filename)
2032	body_text.append("")
2033	else:
2034	if self.system != 'discussion':
2035	body_text.append('[attachment:"%s"]' % filename)
2036	body_text.append("")
2037
2038	body_text = '\r\n'.join(body_text)
2039	return body_text
2040
2041	def html_mailto_link(self, subject):
2042	"""
2043	This function returns a HTML mailto tag with the ticket id and author email address
2044	"""
2045	if not self.author:
2046	author = self.email_addr
2047	else:
2048	author = self.author
2049
2050	# use urllib to escape the chars
2051	#
2052	s = 'mailto:%s?Subject=%s&Cc=%s' %(
2053	urllib.quote(self.email_addr),
2054	urllib.quote('Re: #%s: %s' %(self.id, subject)),
2055	urllib.quote(self.MAILTO_CC)
2056	)
2057
2058	s = '\r\n{{{\r\n#!html\r\n<a\r\n href="%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
2059	return s
2060
2061	########## TRAC notify section ###########################################################
2062
2063	def notify(self, tkt, new=True, modtime=0):
2064	"""
2065	A wrapper for the TRAC notify function. So we can use templates
2066	"""
2067	if self.VERBOSE:
2068	print "VB: notify()"
2069
2070	if self.DRY_RUN:
2071	print 'DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter']
2072	return
2073	try:
2074
2075	#from trac.ticket.web_ui import TicketModule
2076	#from trac.ticket.notification import TicketNotificationSystem
2077	#ticket_sys = TicketNotificationSystem(self.env)
2078	#a = TicketModule(self.env)
2079	#print a.__dict__
2080	#tn_sys = TicketNotificationSystem(self.env)
2081	#print tn_sys
2082	#print tn_sys.__dict__
2083	#sys.exit(0)
2084
2085	# create false {abs_}href properties, to trick Notify()
2086	#
2087	if not (self.VERSION in [0.11, 0.12]):
2088	self.env.abs_href = Href(self.get_config('project', 'url'))
2089	self.env.href = Href(self.get_config('project', 'url'))
2090
2091
2092	tn = TicketNotifyEmail(self.env)
2093
2094	if self.notify_template:
2095
2096	if self.VERSION >= 0.11:
2097
2098	from trac.web.chrome import Chrome
2099
2100	if self.notify_template_update and not new:
2101	tn.template_name = self.notify_template_update
2102	else:
2103	tn.template_name = self.notify_template
2104
2105	tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
2106
2107	else:
2108
2109	tn.template_name = self.notify_template;
2110
2111	tn.notify(tkt, new, modtime)
2112
2113	except Exception, e:
2114	print 'TD: Failure sending notification on creation of ticket #%s: %s' %(self.id, e)
2115
2116
2117
2118	########## Parse Config File ###########################################################
2119
2120	def ReadConfig(file, name):
2121	"""
2122	Parse the config file
2123	"""
2124	if not os.path.isfile(file):
2125	print 'File %s does not exist' %file
2126	sys.exit(1)
2127
2128	config = trac_config.Configuration(file)
2129
2130	# Use given project name else use defaults
2131	#
2132	if name:
2133	sections = config.sections()
2134	if not name in sections:
2135	print "Not a valid project name: %s" %name
2136	print "Valid names: %s" %sections
2137	sys.exit(1)
2138
2139	project = dict()
2140	for option, value in config.options(name):
2141	project[option] = value
2142
2143	else:
2144	# use some trac internals to get the defaults
2145	#
2146	project = config.parser.defaults()
2147
2148	return project
2149
2150
2151	if __name__ == '__main__':
2152	# Default config file
2153	#
2154	configfile = '@email2trac_conf@'
2155	project = ''
2156	component = ''
2157	ticket_prefix = 'default'
2158	dry_run = None
2159	verbose = None
2160
2161	ENABLE_SYSLOG = 0
2162
2163	SHORT_OPT = 'chf:np:t:v'
2164	LONG_OPT = ['component=', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=', 'verbose']
2165
2166	try:
2167	opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
2168	except getopt.error,detail:
2169	print __doc__
2170	print detail
2171	sys.exit(1)
2172
2173	project_name = None
2174	for opt,value in opts:
2175	if opt in [ '-h', '--help']:
2176	print __doc__
2177	sys.exit(0)
2178	elif opt in ['-c', '--component']:
2179	component = value
2180	elif opt in ['-f', '--file']:
2181	configfile = value
2182	elif opt in ['-n', '--dry-run']:
2183	dry_run = True
2184	elif opt in ['-p', '--project']:
2185	project_name = value
2186	elif opt in ['-t', '--ticket_prefix']:
2187	ticket_prefix = value
2188	elif opt in ['-v', '--verbose']:
2189	verbose = True
2190
2191	settings = ReadConfig(configfile, project_name)
2192	if not settings.has_key('project'):
2193	print __doc__
2194	print 'No Trac project is defined in the email2trac config file.'
2195	sys.exit(1)
2196
2197	if component:
2198	settings['component'] = component
2199
2200	# The default prefix for ticket values in email2trac.conf
2201	#
2202	settings['ticket_prefix'] = ticket_prefix
2203	settings['dry_run'] = dry_run
2204	settings['verbose'] = verbose
2205
2206
2207	# Determine major trac version used to be in email2trac.conf
2208	# Quick hack for 0.12
2209	#
2210	version = '0.%s' %(trac_version.split('.')[1])
2211	if version.startswith('0.12'):
2212	version = '0.12'
2213
2214	if verbose:
2215	print "Found trac version: %s" %(version)
2216
2217	#debug HvB
2218	#print settings
2219
2220	try:
2221	if version == '0.10':
2222	from trac import attachment
2223	from trac.env import Environment
2224	from trac.ticket import Ticket
2225	from trac.web.href import Href
2226	from trac import util
2227	#
2228	# return util.text.to_unicode(str)
2229	#
2230	# see http://projects.edgewall.com/trac/changeset/2799
2231	from trac.ticket.notification import TicketNotifyEmail
2232	from trac import config as trac_config
2233	from trac.core import TracError
2234
2235	elif version == '0.11':
2236	from trac import attachment
2237	from trac.env import Environment
2238	from trac.ticket import Ticket
2239	from trac.web.href import Href
2240	from trac import config as trac_config
2241	from trac import util
2242	from trac.core import TracError
2243	from trac.perm import PermissionSystem
2244
2245	#
2246	# return util.text.to_unicode(str)
2247	#
2248	# see http://projects.edgewall.com/trac/changeset/2799
2249	from trac.ticket.notification import TicketNotifyEmail
2250
2251	elif version == '0.12':
2252	from trac import attachment
2253	from trac.env import Environment
2254	from trac.ticket import Ticket
2255	from trac.web.href import Href
2256	from trac import config as trac_config
2257	from trac import util
2258	from trac.core import TracError
2259	from trac.perm import PermissionSystem
2260
2261	#
2262	# return util.text.to_unicode(str)
2263	#
2264	# see http://projects.edgewall.com/trac/changeset/2799
2265	from trac.ticket.notification import TicketNotifyEmail
2266
2267
2268	else:
2269	print 'TRAC version %s is not supported' %version
2270	sys.exit(1)
2271
2272	if settings.has_key('enable_syslog'):
2273	if SYSLOG_AVAILABLE:
2274	ENABLE_SYSLOG = float(settings['enable_syslog'])
2275
2276
2277	# Must be set before environment is created
2278	#
2279	if settings.has_key('python_egg_cache'):
2280	python_egg_cache = str(settings['python_egg_cache'])
2281	os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
2282
2283
2284	if int(settings['debug']) > 0:
2285	print 'Loading environment', settings['project']
2286
2287	env = Environment(settings['project'], create=0)
2288
2289	tktparser = TicketEmailParser(env, settings, float(version))
2290	tktparser.parse(sys.stdin)
2291
2292	# Catch all errors ans log to SYSLOG if we have enabled this
2293	# else stdout
2294	#
2295	except Exception, error:
2296	if ENABLE_SYSLOG:
2297	syslog.openlog('email2trac', syslog.LOG_NOWAIT)
2298
2299	etype, evalue, etb = sys.exc_info()
2300	for e in traceback.format_exception(etype, evalue, etb):
2301	syslog.syslog(e)
2302
2303	syslog.closelog()
2304	else:
2305	traceback.print_exc()
2306
2307	if m:
2308	tktparser.save_email_for_debug(m, True)
2309
2310
2311	sys.exit(1)
2312	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format