Context Navigation

source: trunk/email2trac.py.in @ 408

Last change on this file since 408 was 408, checked in by bas, 14 years ago
port self.DEBUG output to the new logging module
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 60.9 KB

Line
1	#!@PYTHON@
2	# Copyright (C) 2002
3	#
4	# This file is part of the email2trac utils
5	#
6	# This program is free software; you can redistribute it and/or modify it
7	# under the terms of the GNU General Public License as published by the
8	# Free Software Foundation; either version 2, or (at your option) any
9	# later version.
10	#
11	# This program is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with this program; if not, write to the Free Software
18	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
19	#
20	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
21	#
22	"""
23	email2trac.py -- Email tickets to Trac.
24
25	A simple MTA filter to create Trac tickets from inbound emails.
26
27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28	Copyright 2005, Edgewall Software
29
30	Authors:
31	Bas van der Vlies <basv@sara.nl>
32	Walter de Jong <walter@sara.nl>
33
34	The scripts reads emails from stdin and inserts directly into a Trac database.
35
36	How to use
37	----------
38	* See https://subtrac.sara.nl/oss/email2trac/
39
40	* Create an config file:
41	[DEFAULT] # REQUIRED
42	project : /data/trac/test # REQUIRED
43	debug : 1 # OPTIONAL, if set print some DEBUG info
44
45	[jouvin] # OPTIONAL project declaration, if set both fields necessary
46	project : /data/trac/jouvin # use -p\|--project jouvin.
47
48	* default config file is : /etc/email2trac.conf
49
50	* Commandline opions:
51	-h,--help
52	-f,--file <configuration file>
53	-n,--dry-run
54	-p, --project <project name>
55	-t, --ticket_prefix <name>
56
57	SVN Info:
58	$Id: email2trac.py.in 408 2010-07-19 15:23:08Z bas $
59	"""
60	import os
61	import sys
62	import string
63	import getopt
64	import stat
65	import time
66	import email
67	import email.Iterators
68	import email.Header
69	import re
70	import urllib
71	import unicodedata
72	from stat import *
73	import mimetypes
74	import traceback
75	import logging
76	import logging.handlers
77	import UserDict
78
79
80	from trac import __version__ as trac_version
81
82	# Will fail where unavailable, e.g. Windows
83	#
84	try:
85	import syslog
86	SYSLOG_AVAILABLE = True
87	except ImportError:
88	SYSLOG_AVAILABLE = False
89
90	from datetime import tzinfo, timedelta, datetime
91	from trac import config as trac_config
92
93	# Some global variables
94	#
95	trac_default_version = '0.11'
96	m = None
97
98	# A UTC class needed for trac version 0.11, added by
99	# tbaschak at ktc dot mb dot ca
100	#
101	class UTC(tzinfo):
102	"""UTC"""
103	ZERO = timedelta(0)
104	HOUR = timedelta(hours=1)
105
106	def utcoffset(self, dt):
107	return self.ZERO
108
109	def tzname(self, dt):
110	return "UTC"
111
112	def dst(self, dt):
113	return self.ZERO
114
115	class SaraDict(UserDict.UserDict):
116	def __init__(self, dictin = None):
117	UserDict.UserDict.__init__(self)
118	self.name = None
119
120	if dictin:
121	if dictin.has_key('name'):
122	self.name = dictin['name']
123	del dictin['name']
124	self.data = dictin
125
126	def get_value(self, key):
127	if self.has_key(key):
128	return self[key]
129	else:
130	return None
131
132	def __repr__(self):
133	return repr(self.data)
134
135	def __str__(self):
136	return str(self.data)
137
138	def __getattr__(self, name):
139	"""
140	override the class attribute get method. Return the value
141	from the Userdict
142	"""
143	if self.data.has_key(name):
144	return self.data[name]
145	else:
146	return None
147
148	def __setattr__(self, name, value):
149	"""
150	override the class attribute set method only when the UserDict
151	has set its class attribute
152	"""
153	if self.__dict__.has_key('data'):
154	self.data[name] = value
155	else:
156	self.__dict__[name] = value
157
158	def __iter__(self):
159	return iter(self.data.keys())
160
161	class TicketEmailParser(object):
162	env = None
163	comment = '> '
164
165	def __init__(self, env, parameters, version):
166	self.env = env
167
168	# Database connection
169	#
170	self.db = None
171
172	# Save parameters
173	#
174	self.parameters = parameters
175
176	# Some useful mail constants
177	#
178	self.email_name = None
179	self.email_addr = None
180	self.email_from = None
181	self.author = None
182	self.id = None
183
184	self.STRIP_CONTENT_TYPES = list()
185
186	self.VERSION = version
187
188	self.get_config = self.env.config.get
189
190	self.setup_log()
191	self.setup_parameters()
192
193
194	def setup_log(self):
195	"""
196	Setup loging
197
198	Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
199	would be interpreted by the ConfigParser itself.
200
201	"""
202	if not self.parameters.log_type:
203	self.parameters.log_type = 'syslog'
204
205	if self.parameters.log_type == 'file':
206	if not os.path.isabs(parameters.log_file):
207	self.parameters.log_file = os.path.join(self.env.path, 'log')
208	self.log_handler = logging.FileHandler()
209
210	elif self.parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
211	# Requires win32 extensions
212	self.log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
213
214	elif self.parameters.log_type in ('syslog', 'unix'):
215	self.log_handler = logging.handlers.SysLogHandler('/dev/log')
216
217	elif self.parameters.log_type in ('stderr'):
218	self.log_handler = logging.StreamHandler(sys.stderr)
219
220	else:
221	self.log_handler = logging.handlers.BufferingHandler(0)
222
223
224	if self.parameters.log_format:
225	self.parameters.log_format = self.parameters.log_format.replace('$(', '%(')
226	else:
227	self.parameters.log_format = 'Email2trac: %(message)s'
228
229	self.logger = logging.getLogger('email2trac')
230
231	## Setup debug level
232	#
233	if not self.parameters.log_level:
234	self.parameters.log_level = 'CRITICAL'
235	else:
236	self.parameters.log_level = self.parameters.log_level.upper()
237
238	if self.parameters.log_level in ['DEBUG', 'ALL'] and self.parameters.debug > 0:
239	self.logger.setLevel(logging.DEBUG)
240
241	elif self.parameters.log_level in ['INFO'] or self.parameters.verbose:
242	self.logger.setLevel(logging.INFO)
243
244	elif self.parameters.log_level in ['WARNING']:
245	self.logger.setLevel(logging.WARNING)
246
247	elif self.parameters.log_level in ['ERROR']:
248	self.logger.setLevel(logging.ERROR)
249
250	elif self.parameters.log_level in ['CRITICAL']:
251	self.logger.setLevel(logging.CRITICAL)
252
253	else:
254	self.logger.setLevel(logging.ERROR)
255
256	self.log_formatter = logging.Formatter(self.parameters.log_format)
257	self.log_handler.setFormatter(self.log_formatter)
258
259	self.logger.addHandler(self.log_handler)
260
261
262	def setup_parameters(self):
263	if self.parameters.has_key('umask'):
264	os.umask(int(self.parameters['umask'], 8))
265
266	if self.parameters.debug:
267	self.parameters.debug = int(self.parameters.debug)
268	else:
269	self.parameters.debug = 0
270
271	if self.parameters.has_key('mailto_link'):
272	self.MAILTO = int(self.parameters['mailto_link'])
273	if self.parameters.has_key('mailto_cc'):
274	self.MAILTO_CC = self.parameters['mailto_cc']
275	else:
276	self.MAILTO_CC = ''
277	else:
278	self.MAILTO = 0
279
280	if self.parameters.has_key('spam_level'):
281	self.SPAM_LEVEL = int(self.parameters['spam_level'])
282	else:
283	self.SPAM_LEVEL = 0
284
285	if self.parameters.has_key('spam_header'):
286	self.SPAM_HEADER = self.parameters['spam_header']
287	else:
288	self.SPAM_HEADER = 'X-Spam-Score'
289
290	if self.parameters.has_key('email_quote'):
291	self.EMAIL_QUOTE = str(self.parameters['email_quote'])
292	else:
293	self.EMAIL_QUOTE = '> '
294
295	if self.parameters.has_key('email_header'):
296	self.EMAIL_HEADER = int(self.parameters['email_header'])
297	else:
298	self.EMAIL_HEADER = 0
299
300	if self.parameters.has_key('alternate_notify_template'):
301	self.notify_template = str(self.parameters['alternate_notify_template'])
302	else:
303	self.notify_template = None
304
305	if self.parameters.has_key('alternate_notify_template_update'):
306	self.notify_template_update = str(self.parameters['alternate_notify_template_update'])
307	else:
308	self.notify_template_update = None
309
310	if self.parameters.has_key('reply_all'):
311	self.REPLY_ALL = int(self.parameters['reply_all'])
312	else:
313	self.REPLY_ALL = 0
314
315	if self.parameters.has_key('ticket_permission_system'):
316	self.TICKET_PERMISSION_SYSTEM = str(self.parameters['ticket_permission_system'])
317	else:
318	self.TICKET_PERMISSION_SYSTEM = None
319
320	if self.parameters.has_key('ticket_update'):
321	self.TICKET_UPDATE = int(self.parameters['ticket_update'])
322	else:
323	self.TICKET_UPDATE = 0
324
325	if self.parameters.has_key('ticket_update_by_subject'):
326	self.TICKET_UPDATE_BY_SUBJECT = int(self.parameters['ticket_update_by_subject'])
327	else:
328	self.TICKET_UPDATE_BY_SUBJECT = 0
329
330	if self.parameters.has_key('ticket_update_by_subject_lookback'):
331	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK = int(self.parameters['ticket_update_by_subject_lookback'])
332	else:
333	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK = 30
334
335	if self.parameters.has_key('drop_spam'):
336	self.DROP_SPAM = int(self.parameters['drop_spam'])
337	else:
338	self.DROP_SPAM = 0
339
340	if self.parameters.has_key('verbatim_format'):
341	self.VERBATIM_FORMAT = int(self.parameters['verbatim_format'])
342	else:
343	self.VERBATIM_FORMAT = 1
344
345	if self.parameters.has_key('reflow'):
346	self.REFLOW = int(self.parameters['reflow'])
347	else:
348	self.REFLOW = 1
349
350	if self.parameters.has_key('drop_alternative_html_version'):
351	self.DROP_ALTERNATIVE_HTML_VERSION = int(self.parameters['drop_alternative_html_version'])
352	else:
353	self.DROP_ALTERNATIVE_HTML_VERSION = 0
354
355	if self.parameters.has_key('strip_signature'):
356	self.STRIP_SIGNATURE = int(self.parameters['strip_signature'])
357	else:
358	self.STRIP_SIGNATURE = 0
359
360	if self.parameters.has_key('strip_quotes'):
361	self.STRIP_QUOTES = int(self.parameters['strip_quotes'])
362	else:
363	self.STRIP_QUOTES = 0
364
365	self.properties = dict()
366	if self.parameters.has_key('inline_properties'):
367	self.INLINE_PROPERTIES = int(self.parameters['inline_properties'])
368	else:
369	self.INLINE_PROPERTIES = 0
370
371	if self.parameters.has_key('use_textwrap'):
372	self.USE_TEXTWRAP = int(self.parameters['use_textwrap'])
373	else:
374	self.USE_TEXTWRAP = 0
375
376	if self.parameters.has_key('binhex'):
377	self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
378
379	if self.parameters.has_key('applesingle'):
380	self.STRIP_CONTENT_TYPES.append('application/applefile')
381
382	if self.parameters.has_key('appledouble'):
383	self.STRIP_CONTENT_TYPES.append('application/applefile')
384
385	if self.parameters.has_key('strip_content_types'):
386	items = self.parameters['strip_content_types'].split(',')
387	for item in items:
388	self.STRIP_CONTENT_TYPES.append(item.strip())
389
390	self.WORKFLOW = None
391	if self.parameters.has_key('workflow'):
392	self.WORKFLOW = self.parameters['workflow']
393
394	# Use OS independend functions
395	#
396	self.TMPDIR = os.path.normcase('/tmp')
397	if self.parameters.has_key('tmpdir'):
398	self.TMPDIR = os.path.normcase(str(self.parameters['tmpdir']))
399
400	if self.parameters.has_key('ignore_trac_user_settings'):
401	self.IGNORE_TRAC_USER_SETTINGS = int(self.parameters['ignore_trac_user_settings'])
402	else:
403	self.IGNORE_TRAC_USER_SETTINGS = 0
404
405	if self.parameters.has_key('email_triggers_workflow'):
406	self.EMAIL_TRIGGERS_WORKFLOW = int(self.parameters['email_triggers_workflow'])
407	else:
408	self.EMAIL_TRIGGERS_WORKFLOW = 1
409
410	if self.parameters.has_key('subject_field_separator'):
411	self.SUBJECT_FIELD_SEPARATOR = self.parameters['subject_field_separator'].strip()
412	else:
413	self.SUBJECT_FIELD_SEPARATOR = '&'
414
415	self.trac_smtp_from = self.get_config('notification', 'smtp_from')
416
417	self.system = None
418
419	########## Email Header Functions ###########################################################
420
421	def spam(self, message):
422	"""
423	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
424	# Note if Spam_level then '*' are included
425	"""
426	spam = False
427	if message.has_key(self.SPAM_HEADER):
428	spam_l = string.split(message[self.SPAM_HEADER])
429
430	try:
431	number = spam_l[0].count('*')
432	except IndexError, detail:
433	number = 0
434
435	if number >= self.SPAM_LEVEL:
436	spam = True
437
438	# treat virus mails as spam
439	#
440	elif message.has_key('X-Virus-found'):
441	spam = True
442
443	# How to handle SPAM messages
444	#
445	if self.DROP_SPAM and spam:
446	if self.parameters.debug > 2 :
447	set.logger.debug('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' % self.SPAM_LEVEL)
448
449	return 'drop'
450
451	elif spam:
452
453	return 'Spam'
454
455	else:
456
457	return False
458
459	def email_header_acl(self, keyword, header_field, default):
460	"""
461	This function wil check if the email address is allowed or denied
462	to send mail to the ticket list
463	"""
464	try:
465	mail_addresses = self.parameters[keyword]
466
467	# Check if we have an empty string
468	#
469	if not mail_addresses:
470	return default
471
472	except KeyError, detail:
473	if self.parameters.debug > 2 :
474	self.logger.debug('%s not defined, all messages are allowed.' %(keyword))
475
476	return default
477
478	mail_addresses = string.split(mail_addresses, ',')
479
480	for entry in mail_addresses:
481	entry = entry.strip()
482	TO_RE = re.compile(entry, re.VERBOSE\|re.IGNORECASE)
483	result = TO_RE.search(header_field)
484	if result:
485	return True
486
487	return False
488
489	def email_header_txt(self, m):
490	"""
491	Display To and CC addresses in description field
492	"""
493	s = ''
494
495	if m['To'] and len(m['To']) > 0:
496	s = "'''To:''' %s\r\n" %(m['To'])
497	if m['Cc'] and len(m['Cc']) > 0:
498	s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
499
500	return self.email_to_unicode(s)
501
502
503	def get_sender_info(self, message):
504	"""
505	Get the default author name and email address from the message
506	"""
507
508	self.email_to = self.email_to_unicode(message['to'])
509	self.to_name, self.to_email_addr = email.Utils.parseaddr (self.email_to)
510
511	self.email_from = self.email_to_unicode(message['from'])
512	self.email_name, self.email_addr = email.Utils.parseaddr(self.email_from)
513
514	## Trac can not handle author's name that contains spaces
515	# and forbid the ticket email address as author field
516
517	if self.email_addr == self.trac_smtp_from:
518	if self.email_name:
519	self.author = self.email_name
520	else:
521	self.author = "email2trac"
522	else:
523	self.author = self.email_addr
524
525	if self.IGNORE_TRAC_USER_SETTINGS:
526	return
527
528	# Is this a registered user, use email address as search key:
529	# result:
530	# u : login name
531	# n : Name that the user has set in the settings tab
532	# e : email address that the user has set in the settings tab
533	#
534	users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
535	if e and (e.lower() == self.email_addr.lower()) ]
536
537	if len(users) == 1:
538	self.email_from = users[0][0]
539	self.author = users[0][0]
540
541	def set_reply_fields(self, ticket, message):
542	"""
543	Set all the right fields for a new ticket
544	"""
545	self.logger.info('function set_reply_fields')
546
547	## Only use name or email adress
548	#ticket['reporter'] = self.email_from
549	ticket['reporter'] = self.author
550
551
552	# Put all CC-addresses in ticket CC field
553	#
554	if self.REPLY_ALL:
555
556	email_cc = ''
557
558	cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
559
560	if not cc_addrs:
561	return
562
563	## Build a list of forbidden CC addresses
564	#
565	#to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
566	#to_list = list()
567	#for n,e in to_addrs:
568	# to_list.append(e)
569
570	# Always Remove reporter email address from cc-list
571	#
572	try:
573	cc_addrs.remove((self.author, self.email_addr))
574	except ValueError, detail:
575	pass
576
577	for name,addr in cc_addrs:
578
579	## Prevent mail loop
580	#
581	#if addr in to_list:
582
583	if addr == self.trac_smtp_from:
584	if self.parameters.debug:
585	self.logger.debug("Skipping %s mail address for CC-field" %(addr))
586	continue
587
588	if email_cc:
589	email_cc = '%s, %s' %(email_cc, addr)
590	else:
591	email_cc = addr
592
593	if email_cc:
594	if self.parameters.debug:
595	self.logger.debug('set_reply_fields: %s' %email_cc)
596
597	ticket['cc'] = self.email_to_unicode(email_cc)
598
599
600	########## DEBUG functions ###########################################################
601
602	def debug_body(self, message_body, tempfile=False):
603	if tempfile:
604	import tempfile
605	body_file = tempfile.mktemp('.email2trac')
606	else:
607	body_file = os.path.join(self.TMPDIR, 'body.txt')
608
609	if self.parameters.dry_run:
610	print 'DRY-RUN: not saving body to %s' %(body_file)
611	return
612
613	print 'TD: writing body to %s' %(body_file)
614	fx = open(body_file, 'wb')
615	if not message_body:
616	message_body = '(None)'
617
618	message_body = message_body.encode('utf-8')
619	#message_body = unicode(message_body, 'iso-8859-15')
620
621	fx.write(message_body)
622	fx.close()
623	try:
624	os.chmod(body_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
625	except OSError:
626	pass
627
628	def debug_attachments(self, message_parts):
629	"""
630	"""
631	self.logger.info('function debug_attachments')
632
633	n = 0
634	for item in message_parts:
635	# Skip inline text parts
636	if not isinstance(item, tuple):
637	continue
638
639	(original, filename, part) = item
640
641	n = n + 1
642	print 'TD: part%d: Content-Type: %s' % (n, part.get_content_type())
643
644	s = 'TD: part%d: filename: %s' %(n, filename)
645	self.print_unicode(s)
646
647	## Forbidden chars
648	#
649	filename = filename.replace('\\', '_')
650	filename = filename.replace('/', '_')
651
652
653	part_file = os.path.join(self.TMPDIR, filename)
654	s = 'TD: writing part%d (%s)' % (n,part_file)
655	self.print_unicode(s)
656
657	if self.parameters.dry_run:
658	print 'DRY_RUN: NOT saving attachments'
659	continue
660
661	part_file = util.text.unicode_quote(part_file)
662
663	fx = open(part_file, 'wb')
664	text = part.get_payload(decode=1)
665
666	if not text:
667	text = '(None)'
668
669	fx.write(text)
670	fx.close()
671
672	try:
673	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
674	except OSError:
675	pass
676
677	def save_email_for_debug(self, message, tempfile=False):
678
679	if tempfile:
680	import tempfile
681	msg_file = tempfile.mktemp('.email2trac')
682	else:
683	#msg_file = '/var/tmp/msg.txt'
684	msg_file = os.path.join(self.TMPDIR, 'msg.txt')
685
686	if self.parameters.dry_run:
687	print 'DRY_RUN: NOT saving email message to %s' %(msg_file)
688	else:
689	print 'TD: saving email to %s' %(msg_file)
690
691	fx = open(msg_file, 'wb')
692	fx.write('%s' % message)
693	fx.close()
694
695	try:
696	os.chmod(msg_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
697	except OSError:
698	pass
699
700	message_parts = self.get_message_parts(message)
701	message_parts = self.unique_attachment_names(message_parts)
702	body_text = self.body_text(message_parts)
703	self.debug_body(body_text, True)
704	self.debug_attachments(message_parts)
705
706	########## Conversion functions ###########################################################
707
708	def email_to_unicode(self, message_str):
709	"""
710	Email has 7 bit ASCII code, convert it to unicode with the charset
711	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
712	understands it.
713	"""
714	self.logger.info("function email_to_unicode")
715
716	results = email.Header.decode_header(message_str)
717
718	s = None
719	for text,format in results:
720	if format:
721	try:
722	temp = unicode(text, format)
723	except UnicodeError, detail:
724	# This always works
725	#
726	temp = unicode(text, 'iso-8859-15')
727	except LookupError, detail:
728	#text = 'ERROR: Could not find charset: %s, please install' %format
729	#temp = unicode(text, 'iso-8859-15')
730	temp = message_str
731
732	else:
733	temp = string.strip(text)
734	temp = unicode(text, 'iso-8859-15')
735
736	if s:
737	s = '%s %s' %(s, temp)
738	else:
739	s = '%s' %temp
740
741	#s = s.encode('utf-8')
742	return s
743
744	def str_to_dict(self, s):
745	"""
746	Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
747	"""
748	self.logger.info("function str_to_dict")
749
750	fields = string.split(s, self.SUBJECT_FIELD_SEPARATOR)
751
752	result = dict()
753	for field in fields:
754	try:
755	index, value = string.split(field, '=')
756
757	# We can not change the description of a ticket via the subject
758	# line. The description is the body of the email
759	#
760	if index.lower() in ['description']:
761	continue
762
763	if value:
764	result[index.lower()] = value
765
766	except ValueError:
767	pass
768	return result
769
770	def print_unicode(self,s):
771	"""
772	This function prints unicode strings uif possible else it will quote it
773	"""
774	try:
775	self.logger.debug(s)
776	except UnicodeEncodeError, detail:
777	self.logger.debug(util.text.unicode_quote(s))
778
779	########## TRAC ticket functions ###########################################################
780
781	def check_permission_participants(self, tkt):
782	"""
783	Check if the mailer is allowed to update the ticket
784	"""
785	self.logger.info('function check_permission_participants')
786
787	if tkt['reporter'].lower() in [self.author, self.email_addr]:
788	if self.parameters.debug:
789	self.logger.debug('ALLOW, %s is the ticket reporter' %(self.email_addr))
790
791	return True
792
793	perm = PermissionSystem(self.env)
794	if perm.check_permission('TICKET_MODIFY', self.author):
795	if self.parameters.debug:
796	self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
797
798	return True
799
800	else:
801	return False
802
803
804	# Is the updater in the CC?
805	try:
806	cc_list = tkt['cc'].split(',')
807	for cc in cc_list:
808	if self.email_addr.lower() in cc.strip():
809
810	if self.parameters.debug:
811	self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
812
813	return True
814
815	except KeyError:
816	return False
817
818	def check_permission(self, tkt, action):
819	"""
820	check if the reporter has the right permission for the action:
821	- TICKET_CREATE
822	- TICKET_MODIFY
823
824	There are three models:
825	- None : no checking at all
826	- trac : check the permission via trac permission model
827	- email2trac: ....
828	"""
829	self.logger.info("function check_permission")
830
831	if self.TICKET_PERMISSION_SYSTEM in ['trac']:
832
833	perm = PermissionSystem(self.env)
834	if perm.check_permission(action, self.author):
835	return True
836	else:
837	return False
838
839	elif self.TICKET_PERMISSION_SYSTEM in ['update_restricted_to_participants']:
840	if action in ['TICKET_MODIFY']:
841	return (self.check_permission_participants(tkt))
842	else:
843	return True
844
845	# Default is to allow everybody ticket updates and ticket creation
846	else:
847	return True
848
849
850	def update_ticket_fields(self, ticket, user_dict, use_default=None):
851	"""
852	This will update the ticket fields. It will check if the
853	given fields are known and if the right values are specified
854	It will only update the ticket field value:
855	- If the field is known
856	- If the value supplied is valid for the ticket field.
857	If not then there are two options:
858	1) Skip the value (use_default=None)
859	2) Set default value for field (use_default=1)
860	"""
861	self.logger.info("function update_ticket_fields")
862
863	# Build a system dictionary from the ticket fields
864	# with field as index and option as value
865	#
866	sys_dict = dict()
867	for field in ticket.fields:
868	try:
869	sys_dict[field['name']] = field['options']
870
871	except KeyError:
872	sys_dict[field['name']] = None
873	pass
874
875	## Check user supplied fields an compare them with the
876	# system one's
877	#
878	for field,value in user_dict.items():
879	if self.parameters.debug:
880	s = 'TD: user_field\t %s = %s' %(field,value)
881	self.print_unicode(s)
882
883	## To prevent mail loop
884	#
885	if field == 'cc':
886
887	cc_list = user_dict['cc'].split(',')
888
889	if self.trac_smtp_from in cc_list:
890	if self.parameters.debug:
891	self.logger.debug('TD: MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
892
893	cc_list.remove(self.trac_smtp_from)
894
895	value = ','.join(cc_list)
896
897
898	if sys_dict.has_key(field):
899
900	# Check if value is an allowed system option, if TypeError then
901	# every value is allowed
902	#
903	try:
904	if value in sys_dict[field]:
905	ticket[field] = value
906	else:
907	# Must we set a default if value is not allowed
908	#
909	if use_default:
910	value = self.get_config('ticket', 'default_%s' %(field) )
911
912	except TypeError:
913	pass
914
915	## Only set if we have a value
916	#
917	if value:
918	ticket[field] = value
919
920	if self.parameters.debug:
921	s = 'ticket_field\t %s = %s' %(field, ticket[field])
922	self.print_unicode(s)
923
924	def ticket_update(self, m, id, spam):
925	"""
926	If the current email is a reply to an existing ticket, this function
927	will append the contents of this email to that ticket, instead of
928	creating a new one.
929	"""
930	self.logger.info("function ticket_update")
931
932	# Must we update ticket fields
933	#
934	update_fields = dict()
935	try:
936	id, keywords = string.split(id, '?')
937
938	update_fields = self.str_to_dict(keywords)
939
940	# Strip '#'
941	#
942	self.id = int(id[1:])
943
944	except ValueError:
945
946	# Strip '#'
947	#
948	self.id = int(id[1:])
949
950	self.logger.info("function ticket_update id %s" %id)
951
952	# When is the change committed
953	#
954	if self.VERSION < 0.11:
955	when = int(time.time())
956	else:
957	utc = UTC()
958	when = datetime.now(utc)
959
960	try:
961	tkt = Ticket(self.env, self.id, self.db)
962
963	except util.TracError, detail:
964
965	# Not a valid ticket
966
967	self.id = None
968	return False
969
970	# Check the permission of the reporter
971	#
972	if self.TICKET_PERMISSION_SYSTEM:
973	if not self.check_permission(tkt, 'TICKET_MODIFY'):
974	print 'Reporter: %s has no permission to modify tickets' %self.author
975	return False
976
977	# How many changes has this ticket
978	cnum = len(tkt.get_changelog())
979
980
981	# reopen the ticket if it is was closed
982	# We must use the ticket workflow framework
983	#
984	if tkt['status'] in ['closed'] and self.EMAIL_TRIGGERS_WORKFLOW:
985
986	#print controller.actions['reopen']
987	#
988	# As reference
989	# req = Mock(href=Href('/'), abs_href=Href('http://www.example.com/'), authname='anonymous', perm=MockPerm(), args={})
990	#
991	#a = controller.render_ticket_action_control(req, tkt, 'reopen')
992	#print 'controller : ', a
993	#
994	#b = controller.get_all_status()
995	#print 'get all status: ', b
996	#
997	#b = controller.get_ticket_changes(req, tkt, 'reopen')
998	#print 'get_ticket_changes :', b
999
1000	if self.WORKFLOW and (self.VERSION >= 0.11 ) :
1001	from trac.ticket.default_workflow import ConfigurableTicketWorkflow
1002	from trac.test import Mock, MockPerm
1003
1004	req = Mock(authname='anonymous', perm=MockPerm(), args={})
1005
1006	controller = ConfigurableTicketWorkflow(self.env)
1007	fields = controller.get_ticket_changes(req, tkt, self.WORKFLOW)
1008
1009	if self.parameters.debug:
1010	self.logger.debug('Workflow ticket update fields: ')
1011
1012	for key in fields.keys():
1013	if self.parameters.debug:
1014	self.logger.debug('\t %s : %s' %(key, fields[key]))
1015
1016	tkt[key] = fields[key]
1017
1018	else:
1019	tkt['status'] = 'reopened'
1020	tkt['resolution'] = ''
1021
1022	# Must we update some ticket fields properties via subjectline
1023	#
1024	if update_fields:
1025	self.update_ticket_fields(tkt, update_fields)
1026
1027	message_parts = self.get_message_parts(m)
1028	message_parts = self.unique_attachment_names(message_parts)
1029
1030	# Must we update some ticket fields properties via body_text
1031	#
1032	if self.properties:
1033	self.update_ticket_fields(tkt, self.properties)
1034
1035	if self.EMAIL_HEADER:
1036	message_parts.insert(0, self.email_header_txt(m))
1037
1038	body_text = self.body_text(message_parts)
1039
1040	error_with_attachments = self.attach_attachments(message_parts)
1041
1042	if body_text.strip() or update_fields or self.properties:
1043	if self.parameters.dry_run:
1044	print 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) ', self.author, cnum
1045	else:
1046	if error_with_attachments:
1047	body_text = '%s\\%s' %(error_with_attachments, body_text)
1048
1049	tkt.save_changes(self.author, body_text, when, None, str(cnum))
1050
1051
1052	if not spam:
1053	self.notify(tkt, False, when)
1054
1055	return True
1056
1057	def set_ticket_fields(self, ticket):
1058	"""
1059	set the ticket fields to value specified
1060	- /etc/email2trac.conf with <prefix>_<field>
1061	- trac default values, trac.ini
1062	"""
1063	user_dict = dict()
1064
1065	for field in ticket.fields:
1066
1067	name = field['name']
1068
1069	## default trac value
1070	#
1071	if not field.get('custom'):
1072	value = self.get_config('ticket', 'default_%s' %(name) )
1073	else:
1074	## Else we get the default value for reporter
1075	#
1076	value = field.get('value')
1077	options = field.get('options')
1078
1079	if value and options and (value not in options):
1080	value = options[int(value)]
1081
1082	if self.parameters.debug:
1083	s = 'TD: trac.ini name %s = %s' %(name, value)
1084	self.print_unicode(s)
1085
1086	## email2trac.conf settings
1087	#
1088	prefix = self.parameters['ticket_prefix']
1089	try:
1090	value = self.parameters['%s_%s' %(prefix, name)]
1091	if self.parameters.debug > 10:
1092	s = 'TD: email2trac.conf %s = %s ' %(name, value)
1093	self.print_unicode(s)
1094
1095	except KeyError, detail:
1096	pass
1097
1098	if self.parameters.debug:
1099	s = 'TD: user_dict[%s] = %s' %(name, value)
1100	self.print_unicode(s)
1101
1102	if value:
1103	user_dict[name] = value
1104
1105	self.update_ticket_fields(ticket, user_dict, use_default=1)
1106
1107	if 'status' not in user_dict.keys():
1108	ticket['status'] = 'new'
1109
1110
1111	def ticket_update_by_subject(self, subject):
1112	"""
1113	This list of Re: prefixes is probably incomplete. Taken from
1114	wikipedia. Here is how the subject is matched
1115	- Re: <subject>
1116	- Re: (<Mail list label>:)+ <subject>
1117
1118	So we must have the last column
1119	"""
1120	self.logger.info('function ticket_update_by_subject')
1121
1122	matched_id = None
1123	if self.TICKET_UPDATE and self.TICKET_UPDATE_BY_SUBJECT:
1124
1125	SUBJECT_RE = re.compile(r'^(RE\|AW\|VS\|SV):(.:)\s(.)', re.IGNORECASE)
1126	result = SUBJECT_RE.search(subject)
1127
1128	if result:
1129	# This is a reply
1130	orig_subject = result.group(3)
1131
1132	if self.parameters.debug:
1133	self.logger.debug('subject search string: %s' %(orig_subject))
1134
1135	cursor = self.db.cursor()
1136	summaries = [orig_subject, '%%: %s' % orig_subject]
1137
1138	##
1139	# Convert days to seconds
1140	lookback = int(time.mktime(time.gmtime())) - \
1141	self.TICKET_UPDATE_BY_SUBJECT_LOOKBACK * 24 * 3600
1142
1143
1144	for summary in summaries:
1145	if self.parameters.debug:
1146	self.logger.debug('Looking for summary matching: "%s"' % summary)
1147
1148	sql = """SELECT id FROM ticket
1149	WHERE changetime >= %s AND summary LIKE %s
1150	ORDER BY changetime DESC"""
1151	cursor.execute(sql, [lookback, summary.strip()])
1152
1153	for row in cursor:
1154	(matched_id,) = row
1155
1156	if self.parameters.debug:
1157	self.logger.debug('Found matching ticket id: %d' % matched_id)
1158
1159	break
1160
1161	if matched_id:
1162	matched_id = '#%d' % matched_id
1163	return matched_id
1164
1165	return matched_id
1166
1167
1168	def new_ticket(self, msg, subject, spam, set_fields = None):
1169	"""
1170	Create a new ticket
1171	"""
1172	self.logger.info('function new_ticket')
1173
1174	tkt = Ticket(self.env)
1175
1176	self.set_reply_fields(tkt, msg)
1177
1178	self.set_ticket_fields(tkt)
1179
1180	# Check the permission of the reporter
1181	#
1182	if self.TICKET_PERMISSION_SYSTEM:
1183	if not self.check_permission(tkt, 'TICKET_CREATE'):
1184	self.logger.error('Reporter: %s has no permission to create tickets' %self.author)
1185	return False
1186
1187	# Old style setting for component, will be removed
1188	#
1189	if spam:
1190	tkt['component'] = 'Spam'
1191
1192	elif self.parameters.has_key('component'):
1193	tkt['component'] = self.parameters['component']
1194
1195	if not msg['Subject']:
1196	tkt['summary'] = u'(No subject)'
1197	else:
1198	tkt['summary'] = subject
1199
1200
1201	if set_fields:
1202	rest, keywords = string.split(set_fields, '?')
1203
1204	if keywords:
1205	update_fields = self.str_to_dict(keywords)
1206	self.update_ticket_fields(tkt, update_fields)
1207
1208	# produce e-mail like header
1209	#
1210	head = ''
1211	if self.EMAIL_HEADER > 0:
1212	head = self.email_header_txt(msg)
1213
1214	message_parts = self.get_message_parts(msg)
1215
1216	# Must we update some ticket fields properties via body_text
1217	#
1218	if self.properties:
1219	self.update_ticket_fields(tkt, self.properties)
1220
1221	message_parts = self.unique_attachment_names(message_parts)
1222
1223	if self.EMAIL_HEADER > 0:
1224	message_parts.insert(0, self.email_header_txt(msg))
1225
1226	body_text = self.body_text(message_parts)
1227
1228	tkt['description'] = body_text
1229
1230	#when = int(time.time())
1231	#
1232	utc = UTC()
1233	when = datetime.now(utc)
1234
1235	if self.parameters.dry_run:
1236	print 'DRY_RUN: tkt.insert()'
1237	else:
1238	self.id = tkt.insert()
1239
1240	changed = False
1241	comment = ''
1242
1243	# some routines in trac are dependend on ticket id
1244	# like alternate notify template
1245	#
1246	if self.notify_template:
1247	tkt['id'] = self.id
1248	changed = True
1249
1250	## Rewrite the description if we have mailto enabled
1251	#
1252	if self.MAILTO:
1253	changed = True
1254	comment = u'\nadded mailto line\n'
1255	mailto = self.html_mailto_link( m['Subject'])
1256
1257	tkt['description'] = u'%s\r\n%s%s\r\n' \
1258	%(head, mailto, body_text)
1259
1260	## Save the attachments to the ticket
1261	#
1262	error_with_attachments = self.attach_attachments(message_parts)
1263
1264	if error_with_attachments:
1265	changed = True
1266	comment = '%s\n%s\n' %(comment, error_with_attachments)
1267
1268	if changed:
1269	if self.parameters.dry_run:
1270	print 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
1271	else:
1272	tkt.save_changes(tkt['reporter'], comment)
1273	#print tkt.get_changelog(self.db, when)
1274
1275	if not spam:
1276	self.notify(tkt, True)
1277
1278
1279	def attach_attachments(self, message_parts, update=False):
1280	'''
1281	save any attachments as files in the ticket's directory
1282	'''
1283	self.logger.info('function attach_attachments()')
1284
1285	if self.parameters.dry_run:
1286	print "DRY_RUN: no attachments attached to tickets"
1287	return ''
1288
1289	count = 0
1290
1291	# Get Maxium attachment size
1292	#
1293	max_size = int(self.get_config('attachment', 'max_size'))
1294	status = None
1295
1296	for item in message_parts:
1297	# Skip body parts
1298	if not isinstance(item, tuple):
1299	continue
1300
1301	(original, filename, part) = item
1302	#
1303	# We have to determine the size so we use this temporary solution. we must escape it
1304	# else we get UnicodeErrors.
1305	#
1306	path, fd = util.create_unique_file(os.path.join(self.TMPDIR, util.text.unicode_quote(filename)))
1307	text = part.get_payload(decode=1)
1308	if not text:
1309	text = '(None)'
1310	fd.write(text)
1311	fd.close()
1312
1313	# get the file_size
1314	#
1315	stats = os.lstat(path)
1316	file_size = stats[stat.ST_SIZE]
1317
1318	# Check if the attachment size is allowed
1319	#
1320	if (max_size != -1) and (file_size > max_size):
1321	status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1322	%(status, original, file_size, max_size)
1323
1324	os.unlink(path)
1325	continue
1326	else:
1327	count = count + 1
1328
1329	# Insert the attachment
1330	#
1331	fd = open(path, 'rb')
1332	if self.system == 'discussion':
1333	att = attachment.Attachment(self.env, 'discussion', 'topic/%s'
1334	% (self.id,))
1335	else:
1336	att = attachment.Attachment(self.env, 'ticket', self.id)
1337
1338	# This will break the ticket_update system, the body_text is vaporized
1339	# ;-(
1340	#
1341	if not update:
1342	att.author = self.author
1343	att.description = self.email_to_unicode('Added by email2trac')
1344
1345	try:
1346	att.insert(filename, fd, file_size)
1347	except OSError, detail:
1348	status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
1349
1350	# Remove the created temporary filename
1351	#
1352	fd.close()
1353	os.unlink(path)
1354
1355	## return error
1356	#
1357	return status
1358
1359	########## Fullblog functions #################################################
1360
1361	def blog(self, id):
1362	"""
1363	The blog create/update function
1364	"""
1365	# import the modules
1366	#
1367	from tracfullblog.core import FullBlogCore
1368	from tracfullblog.model import BlogPost, BlogComment
1369	from trac.test import Mock, MockPerm
1370
1371	# instantiate blog core
1372	blog = FullBlogCore(self.env)
1373	req = Mock(authname='anonymous', perm=MockPerm(), args={})
1374
1375	if id:
1376
1377	# update blog
1378	#
1379	comment = BlogComment(self.env, id)
1380	comment.author = self.author
1381
1382	message_parts = self.get_message_parts(m)
1383	comment.comment = self.body_text(message_parts)
1384
1385	blog.create_comment(req, comment)
1386
1387	else:
1388	# create blog
1389	#
1390	import time
1391	post = BlogPost(self.env, 'blog_'+time.strftime("%Y%m%d%H%M%S", time.gmtime()))
1392
1393	#post = BlogPost(self.env, blog._get_default_postname(self.env))
1394
1395	post.author = self.author
1396	post.title = self.email_to_unicode(m['Subject'])
1397
1398	message_parts = self.get_message_parts(m)
1399	post.body = self.body_text(message_parts)
1400
1401	blog.create_post(req, post, self.author, u'Created by email2trac', False)
1402
1403
1404	########## Discussion functions ##############################################
1405
1406	def discussion_topic(self, content, subject):
1407
1408	# Import modules.
1409	from tracdiscussion.api import DiscussionApi
1410	from trac.util.datefmt import to_timestamp, utc
1411
1412	if self.parameters.debug:
1413	self.logger.debug('Creating a new topic in forum:', self.id)
1414
1415	# Get dissussion API component.
1416	api = self.env[DiscussionApi]
1417	context = self._create_context(content, subject)
1418
1419	# Get forum for new topic.
1420	forum = api.get_forum(context, self.id)
1421
1422	if not forum and self.parameters.debug:
1423	self.logger.debug("ERROR: Replied forum doesn't exist")
1424
1425	# Prepare topic.
1426	topic = {'forum' : forum['id'],
1427	'subject' : context.subject,
1428	'time': to_timestamp(datetime.now(utc)),
1429	'author' : self.author,
1430	'subscribers' : [self.email_addr],
1431	'body' : self.body_text(context.content_parts)}
1432
1433	# Add topic to DB and commit it.
1434	self._add_topic(api, context, topic)
1435	self.db.commit()
1436
1437	def discussion_topic_reply(self, content, subject):
1438
1439	# Import modules.
1440	from tracdiscussion.api import DiscussionApi
1441	from trac.util.datefmt import to_timestamp, utc
1442
1443	if self.parameters.debug:
1444	self.logger.debug('Replying to discussion topic', self.id)
1445
1446	# Get dissussion API component.
1447	api = self.env[DiscussionApi]
1448	context = self._create_context(content, subject)
1449
1450	# Get replied topic.
1451	topic = api.get_topic(context, self.id)
1452
1453	if not topic and self.parameters.debug:
1454	self.logger.debug("ERROR: Replied topic doesn't exist")
1455
1456	# Prepare message.
1457	message = {'forum' : topic['forum'],
1458	'topic' : topic['id'],
1459	'replyto' : -1,
1460	'time' : to_timestamp(datetime.now(utc)),
1461	'author' : self.author,
1462	'body' : self.body_text(context.content_parts)}
1463
1464	# Add message to DB and commit it.
1465	self._add_message(api, context, message)
1466	self.db.commit()
1467
1468	def discussion_message_reply(self, content, subject):
1469
1470	# Import modules.
1471	from tracdiscussion.api import DiscussionApi
1472	from trac.util.datefmt import to_timestamp, utc
1473
1474	if self.parameters.debug:
1475	self.loggger.debug('TD: Replying to discussion message', self.id)
1476
1477	# Get dissussion API component.
1478	api = self.env[DiscussionApi]
1479	context = self._create_context(content, subject)
1480
1481	# Get replied message.
1482	message = api.get_message(context, self.id)
1483
1484	if not message and self.parameters.debug:
1485	self.logger.debug("ERROR: Replied message doesn't exist")
1486
1487	# Prepare message.
1488	message = {'forum' : message['forum'],
1489	'topic' : message['topic'],
1490	'replyto' : message['id'],
1491	'time' : to_timestamp(datetime.now(utc)),
1492	'author' : self.author,
1493	'body' : self.body_text(context.content_parts)}
1494
1495	# Add message to DB and commit it.
1496	self._add_message(api, context, message)
1497	self.db.commit()
1498
1499	def _create_context(self, content, subject):
1500
1501	# Import modules.
1502	from trac.mimeview import Context
1503	from trac.web.api import Request
1504	from trac.perm import PermissionCache
1505
1506	# TODO: Read server base URL from config.
1507	# Create request object to mockup context creation.
1508	#
1509	environ = {'SERVER_PORT' : 80,
1510	'SERVER_NAME' : 'test',
1511	'REQUEST_METHOD' : 'POST',
1512	'wsgi.url_scheme' : 'http',
1513	'wsgi.input' : sys.stdin}
1514	chrome = {'links': {},
1515	'scripts': [],
1516	'ctxtnav': [],
1517	'warnings': [],
1518	'notices': []}
1519
1520	if self.env.base_url_for_redirect:
1521	environ['trac.base_url'] = self.env.base_url
1522
1523	req = Request(environ, None)
1524	req.chrome = chrome
1525	req.tz = 'missing'
1526	req.authname = self.author
1527	req.perm = PermissionCache(self.env, self.author)
1528
1529	# Create and return context.
1530	context = Context.from_request(req)
1531	context.realm = 'discussion-email2trac'
1532	context.cursor = self.db.cursor()
1533	context.content = content
1534	context.subject = subject
1535
1536	# Read content parts from content.
1537	context.content_parts = self.get_message_parts(content)
1538	context.content_parts = self.unique_attachment_names(
1539	context.content_parts)
1540
1541	return context
1542
1543	def _add_topic(self, api, context, topic):
1544	context.req.perm.assert_permission('DISCUSSION_APPEND')
1545
1546	# Filter topic.
1547	for discussion_filter in api.discussion_filters:
1548	accept, topic_or_error = discussion_filter.filter_topic(
1549	context, topic)
1550	if accept:
1551	topic = topic_or_error
1552	else:
1553	raise TracError(topic_or_error)
1554
1555	# Add a new topic.
1556	api.add_topic(context, topic)
1557
1558	# Get inserted topic with new ID.
1559	topic = api.get_topic_by_time(context, topic['time'])
1560
1561	# Attach attachments.
1562	self.id = topic['id']
1563	self.attach_attachments(context.content_parts, True)
1564
1565	# Notify change listeners.
1566	for listener in api.topic_change_listeners:
1567	listener.topic_created(context, topic)
1568
1569	def _add_message(self, api, context, message):
1570	context.req.perm.assert_permission('DISCUSSION_APPEND')
1571
1572	# Filter message.
1573	for discussion_filter in api.discussion_filters:
1574	accept, message_or_error = discussion_filter.filter_message(
1575	context, message)
1576	if accept:
1577	message = message_or_error
1578	else:
1579	raise TracError(message_or_error)
1580
1581	# Add message.
1582	api.add_message(context, message)
1583
1584	# Get inserted message with new ID.
1585	message = api.get_message_by_time(context, message['time'])
1586
1587	# Attach attachments.
1588	self.id = message['topic']
1589	self.attach_attachments(context.content_parts, True)
1590
1591	# Notify change listeners.
1592	for listener in api.message_change_listeners:
1593	listener.message_created(context, message)
1594
1595	########## MAIN function ######################################################
1596
1597	def parse(self, fp):
1598	"""
1599	"""
1600	self.logger.info('Main function parse')
1601	global m
1602
1603	m = email.message_from_file(fp)
1604
1605	if not m:
1606	if self.parameters.debug:
1607	self.logger.debug('This is not a valid email message format')
1608
1609	return
1610
1611	# Work around lack of header folding in Python; see http://bugs.python.org/issue4696
1612	try:
1613	m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
1614	except AttributeError, detail:
1615	pass
1616
1617	if self.parameters.debug: # save the entire e-mail message text
1618	self.save_email_for_debug(m, True)
1619
1620	self.db = self.env.get_db_cnx()
1621	self.get_sender_info(m)
1622
1623	if not self.email_header_acl('white_list', self.email_addr, True):
1624	if self.parameters.debug:
1625	self.logger.debug('Message rejected : %s not in white list' %(self.email_addr))
1626
1627	return False
1628
1629	if self.email_header_acl('black_list', self.email_addr, False):
1630	if self.parameters.debug:
1631	self.logger.debug('Message rejected : %s in black list' %(self.email_addr))
1632
1633	return False
1634
1635	if not self.email_header_acl('recipient_list', self.to_email_addr, True):
1636	if self.parameters.debug:
1637	self.logger.debug('Message rejected : %s not in recipient list' %(self.to_email_addr))
1638
1639	return False
1640
1641	# If drop the message
1642	#
1643	if self.spam(m) == 'drop':
1644	return False
1645
1646	elif self.spam(m) == 'spam':
1647	spam_msg = True
1648	else:
1649	spam_msg = False
1650
1651	if not m['Subject']:
1652	subject = 'No Subject'
1653	else:
1654	subject = self.email_to_unicode(m['Subject'])
1655
1656	if self.parameters.debug:
1657	self.logger.debug('subject: %s' %subject)
1658
1659	#
1660	# [hic] #1529: Re: LRZ
1661	# [hic] #1529?owner=bas,priority=medium: Re: LRZ
1662	#
1663	ticket_regex = r'''
1664	(?P<new_fields>[#][?].*)
1665	\|(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?.*)?:)
1666	'''
1667	# Check if FullBlogPlugin is installed
1668	#
1669	blog_enabled = None
1670	blog_regex = ''
1671	if self.get_config('components', 'tracfullblog.*') in ['enabled']:
1672	blog_enabled = True
1673	blog_regex = '''\|(?P<blog>blog:(?P<blog_id>\w*))'''
1674
1675
1676	# Check if DiscussionPlugin is installed
1677	#
1678	discussion_enabled = None
1679	discussion_regex = ''
1680	if self.get_config('components', 'tracdiscussion.api.*') in ['enabled']:
1681	discussion_enabled = True
1682	discussion_regex = r'''
1683	\|(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
1684	\|(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
1685	\|(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
1686	'''
1687
1688
1689	regex_str = ticket_regex + blog_regex + discussion_regex
1690	SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
1691
1692	# Find out if this is a ticket, a blog or a discussion
1693	#
1694	result = SYSTEM_RE.search(subject)
1695
1696	if result:
1697	# update ticket + fields
1698	#
1699	if result.group('reply') and self.TICKET_UPDATE:
1700	self.system = 'ticket'
1701
1702	# Skip the last ':' character
1703	#
1704	if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
1705	self.new_ticket(m, subject, spam_msg)
1706
1707	# New ticket + fields
1708	#
1709	elif result.group('new_fields'):
1710	self.system = 'ticket'
1711	self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
1712
1713	if blog_enabled:
1714	if result.group('blog'):
1715	self.system = 'blog'
1716	self.blog(result.group('blog_id'))
1717
1718	if discussion_enabled:
1719	# New topic.
1720	#
1721	if result.group('forum'):
1722	self.system = 'discussion'
1723	self.id = int(result.group('forum_id'))
1724	self.discussion_topic(m, subject[result.end('forum'):])
1725
1726	# Reply to topic.
1727	#
1728	elif result.group('topic'):
1729	self.system = 'discussion'
1730	self.id = int(result.group('topic_id'))
1731	self.discussion_topic_reply(m, subject[result.end('topic'):])
1732
1733	# Reply to topic message.
1734	#
1735	elif result.group('message'):
1736	self.system = 'discussion'
1737	self.id = int(result.group('message_id'))
1738	self.discussion_message_reply(m, subject[result.end('message'):])
1739
1740	else:
1741	self.system = 'ticket'
1742	result = self.ticket_update_by_subject(subject)
1743	if result:
1744	if not self.ticket_update(m, result, spam_msg):
1745	self.new_ticket(m, subject, spam_msg)
1746	else:
1747	# No update by subject, so just create a new ticket
1748	self.new_ticket(m, subject, spam_msg)
1749
1750
1751	########## BODY TEXT functions ###########################################################
1752
1753	def strip_signature(self, text):
1754	"""
1755	Strip signature from message, inspired by Mailman software
1756	"""
1757	body = []
1758	for line in text.splitlines():
1759	if line == '-- ':
1760	break
1761	body.append(line)
1762
1763	return ('\n'.join(body))
1764
1765	def reflow(self, text, delsp = 0):
1766	"""
1767	Reflow the message based on the format="flowed" specification (RFC 3676)
1768	"""
1769	flowedlines = []
1770	quotelevel = 0
1771	prevflowed = 0
1772
1773	for line in text.splitlines():
1774	from re import match
1775
1776	# Figure out the quote level and the content of the current line
1777	m = match('(>)( ?)(.)', line)
1778	linequotelevel = len(m.group(1))
1779	line = m.group(3)
1780
1781	# Determine whether this line is flowed
1782	if line and line != '-- ' and line[-1] == ' ':
1783	flowed = 1
1784	else:
1785	flowed = 0
1786
1787	if flowed and delsp and line and line[-1] == ' ':
1788	line = line[:-1]
1789
1790	# If the previous line is flowed, append this line to it
1791	if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1792	flowedlines[-1] += line
1793	# Otherwise, start a new line
1794	else:
1795	flowedlines.append('>' * linequotelevel + line)
1796
1797	prevflowed = flowed
1798
1799
1800	return '\n'.join(flowedlines)
1801
1802	def strip_quotes(self, text):
1803	"""
1804	Strip quotes from message by Nicolas Mendoza
1805	"""
1806	body = []
1807	for line in text.splitlines():
1808	if line.startswith(self.EMAIL_QUOTE):
1809	continue
1810	body.append(line)
1811
1812	return ('\n'.join(body))
1813
1814	def inline_properties(self, text):
1815	"""
1816	Parse text if we use inline keywords to set ticket fields
1817	"""
1818	if self.parameters.debug:
1819	self.logger.info('function inline_properties')
1820
1821	properties = dict()
1822	body = list()
1823
1824	INLINE_EXP = re.compile('\s[@]\s([a-zA-Z]+)\s:(.)$')
1825
1826	for line in text.splitlines():
1827	match = INLINE_EXP.match(line)
1828	if match:
1829	keyword, value = match.groups()
1830	self.properties[keyword] = value.strip()
1831
1832	if self.parameters.debug:
1833	self.logger.debug('inline properties: %s : %s' %(keyword,value))
1834	else:
1835	body.append(line)
1836
1837	return '\n'.join(body)
1838
1839
1840	def wrap_text(self, text, replace_whitespace = False):
1841	"""
1842	Will break a lines longer then given length into several small
1843	lines of size given length
1844	"""
1845	import textwrap
1846
1847	LINESEPARATOR = '\n'
1848	reformat = ''
1849
1850	for s in text.split(LINESEPARATOR):
1851	tmp = textwrap.fill(s,self.USE_TEXTWRAP)
1852	if tmp:
1853	reformat = '%s\n%s' %(reformat,tmp)
1854	else:
1855	reformat = '%s\n' %reformat
1856
1857	return reformat
1858
1859	# Python2.4 and higher
1860	#
1861	#return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
1862	#
1863
1864	########## EMAIL attachements functions ###########################################################
1865
1866	def inline_part(self, part):
1867	"""
1868	"""
1869	self.logger.info('function inline_part()')
1870
1871	return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
1872
1873	def get_message_parts(self, msg):
1874	"""
1875	parses the email message and returns a list of body parts and attachments
1876	body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
1877	"""
1878	self.logger.info('function get_message_parts()')
1879
1880	message_parts = list()
1881
1882	ALTERNATIVE_MULTIPART = False
1883
1884	for part in msg.walk():
1885	if self.parameters.debug:
1886	self.logger.debug('Message part: Main-Type: %s' % part.get_content_maintype())
1887	self.logger.debug('Message part: Content-Type: %s' % part.get_content_type())
1888
1889	## Check content type
1890	#
1891	if part.get_content_type() in self.STRIP_CONTENT_TYPES:
1892
1893	if self.parameters.debug:
1894	self.logger.debug("A %s attachment named '%s' was skipped" %(part.get_content_type(), part.get_filename()))
1895
1896	continue
1897
1898	## Catch some mulitpart execptions
1899	#
1900	if part.get_content_type() == 'multipart/alternative':
1901	ALTERNATIVE_MULTIPART = True
1902	continue
1903
1904	## Skip multipart containers
1905	#
1906	if part.get_content_maintype() == 'multipart':
1907	if self.parameters.debug:
1908	self.logger.debug("Skipping multipart container")
1909	continue
1910
1911	## Check if this is an inline part. It's inline if there is co Cont-Disp header, or if there is one and it says "inline"
1912	#
1913	inline = self.inline_part(part)
1914
1915	## Drop HTML message
1916	#
1917	if ALTERNATIVE_MULTIPART and self.DROP_ALTERNATIVE_HTML_VERSION:
1918	if part.get_content_type() == 'text/html':
1919	if self.parameters.debug:
1920	self.logger.debug('Skipping alternative HTML message')
1921
1922	ALTERNATIVE_MULTIPART = False
1923	continue
1924
1925	## Inline text parts are where the body is
1926	#
1927	if part.get_content_type() == 'text/plain' and inline:
1928	if self.parameters.debug:
1929	self.logger.debug(' Inline body part')
1930
1931	# Try to decode, if fails then do not decode
1932	#
1933	body_text = part.get_payload(decode=1)
1934	if not body_text:
1935	body_text = part.get_payload(decode=0)
1936
1937	format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
1938	delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
1939
1940	if self.REFLOW and not self.VERBATIM_FORMAT and format == 'flowed':
1941	body_text = self.reflow(body_text, delsp == 'yes')
1942
1943	if self.STRIP_SIGNATURE:
1944	body_text = self.strip_signature(body_text)
1945
1946	if self.STRIP_QUOTES:
1947	body_text = self.strip_quotes(body_text)
1948
1949	if self.INLINE_PROPERTIES:
1950	body_text = self.inline_properties(body_text)
1951
1952	if self.USE_TEXTWRAP:
1953	body_text = self.wrap_text(body_text)
1954
1955	## Get contents charset (iso-8859-15 if not defined in mail headers)
1956	#
1957	charset = part.get_content_charset()
1958	if not charset:
1959	charset = 'iso-8859-15'
1960
1961	try:
1962	ubody_text = unicode(body_text, charset)
1963
1964	except UnicodeError, detail:
1965	ubody_text = unicode(body_text, 'iso-8859-15')
1966
1967	except LookupError, detail:
1968	ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
1969
1970	if self.VERBATIM_FORMAT:
1971	message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
1972	else:
1973	message_parts.append('%s' %ubody_text)
1974	else:
1975	if self.parameters.debug:
1976	s = 'TD: Filename: %s' % part.get_filename()
1977	self.print_unicode(s)
1978
1979	##
1980	# First try to use email header function to convert filename.
1981	# If this fails the use the plan filename
1982	try:
1983	filename = self.email_to_unicode(part.get_filename())
1984	except UnicodeEncodeError, detail:
1985	filename = part.get_filename()
1986
1987	message_parts.append((filename, part))
1988
1989	return message_parts
1990
1991	def unique_attachment_names(self, message_parts):
1992	"""
1993	"""
1994	renamed_parts = []
1995	attachment_names = set()
1996
1997	for item in message_parts:
1998
1999	## If not an attachment, leave it alone
2000	#
2001	if not isinstance(item, tuple):
2002	renamed_parts.append(item)
2003	continue
2004
2005	(filename, part) = item
2006
2007	## If no filename, use a default one
2008	#
2009	if not filename:
2010	filename = 'untitled-part'
2011
2012	# Guess the extension from the content type, use non strict mode
2013	# some additional non-standard but commonly used MIME types
2014	# are also recognized
2015	#
2016	ext = mimetypes.guess_extension(part.get_content_type(), False)
2017	if not ext:
2018	ext = '.bin'
2019
2020	filename = '%s%s' % (filename, ext)
2021
2022	## Discard relative paths for windows/unix in attachment names
2023	#
2024	#filename = filename.replace('\\', '/').replace(':', '/')
2025	filename = filename.replace('\\', '_')
2026	filename = filename.replace('/', '_')
2027
2028	#
2029	# We try to normalize the filename to utf-8 NFC if we can.
2030	# Files uploaded from OS X might be in NFD.
2031	# Check python version and then try it
2032	#
2033	#if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
2034	# try:
2035	# filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
2036	# except TypeError:
2037	# pass
2038
2039	# Make the filename unique for this ticket
2040	num = 0
2041	unique_filename = filename
2042	dummy_filename, ext = os.path.splitext(filename)
2043
2044	while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
2045	num += 1
2046	unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
2047
2048	if self.parameters.debug:
2049	s = 'TD: Attachment with filename %s will be saved as %s' % (filename, unique_filename)
2050	self.print_unicode(s)
2051
2052	attachment_names.add(unique_filename)
2053
2054	renamed_parts.append((filename, unique_filename, part))
2055
2056	return renamed_parts
2057
2058
2059	def attachment_exists(self, filename):
2060
2061	if self.parameters.debug:
2062	s = 'TD: attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
2063	self.print_unicode(s)
2064
2065	# We have no valid ticket id
2066	#
2067	if not self.id:
2068	return False
2069
2070	try:
2071	if self.system == 'discussion':
2072	att = attachment.Attachment(self.env, 'discussion', 'ticket/%s'
2073	% (self.id,), filename)
2074	else:
2075	att = attachment.Attachment(self.env, 'ticket', self.id,
2076	filename)
2077	return True
2078	except attachment.ResourceNotFound:
2079	return False
2080
2081	########## TRAC Ticket Text ###########################################################
2082
2083	def body_text(self, message_parts):
2084	body_text = []
2085
2086	for part in message_parts:
2087	# Plain text part, append it
2088	if not isinstance(part, tuple):
2089	body_text.extend(part.strip().splitlines())
2090	body_text.append("")
2091	continue
2092
2093	(original, filename, part) = part
2094	inline = self.inline_part(part)
2095
2096	if part.get_content_maintype() == 'image' and inline:
2097	if self.system != 'discussion':
2098	body_text.append('[[Image(%s)]]' % filename)
2099	body_text.append("")
2100	else:
2101	if self.system != 'discussion':
2102	body_text.append('[attachment:"%s"]' % filename)
2103	body_text.append("")
2104
2105	body_text = '\r\n'.join(body_text)
2106	return body_text
2107
2108	def html_mailto_link(self, subject):
2109	"""
2110	This function returns a HTML mailto tag with the ticket id and author email address
2111	"""
2112	if not self.author:
2113	author = self.email_addr
2114	else:
2115	author = self.author
2116
2117	# use urllib to escape the chars
2118	#
2119	s = 'mailto:%s?Subject=%s&Cc=%s' %(
2120	urllib.quote(self.email_addr),
2121	urllib.quote('Re: #%s: %s' %(self.id, subject)),
2122	urllib.quote(self.MAILTO_CC)
2123	)
2124
2125	s = '\r\n{{{\r\n#!html\r\n<a\r\n href="%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
2126	return s
2127
2128	########## TRAC notify section ###########################################################
2129
2130	def notify(self, tkt, new=True, modtime=0):
2131	"""
2132	A wrapper for the TRAC notify function. So we can use templates
2133	"""
2134	self.logger.info('function notify()')
2135
2136	if self.parameters.dry_run:
2137	print 'DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter']
2138	return
2139	try:
2140
2141	#from trac.ticket.web_ui import TicketModule
2142	#from trac.ticket.notification import TicketNotificationSystem
2143	#ticket_sys = TicketNotificationSystem(self.env)
2144	#a = TicketModule(self.env)
2145	#print a.__dict__
2146	#tn_sys = TicketNotificationSystem(self.env)
2147	#print tn_sys
2148	#print tn_sys.__dict__
2149	#sys.exit(0)
2150
2151	# create false {abs_}href properties, to trick Notify()
2152	#
2153	if not (self.VERSION in [0.11, 0.12]):
2154	self.env.abs_href = Href(self.get_config('project', 'url'))
2155	self.env.href = Href(self.get_config('project', 'url'))
2156
2157
2158	tn = TicketNotifyEmail(self.env)
2159
2160	if self.notify_template:
2161
2162	if self.VERSION >= 0.11:
2163
2164	from trac.web.chrome import Chrome
2165
2166	if self.notify_template_update and not new:
2167	tn.template_name = self.notify_template_update
2168	else:
2169	tn.template_name = self.notify_template
2170
2171	tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
2172
2173	else:
2174
2175	tn.template_name = self.notify_template;
2176
2177	tn.notify(tkt, new, modtime)
2178
2179	except Exception, e:
2180	print 'TD: Failure sending notification on creation of ticket #%s: %s' %(self.id, e)
2181
2182
2183
2184	########## Parse Config File ###########################################################
2185
2186	def ReadConfig(file, name):
2187	"""
2188	Parse the config file
2189	"""
2190	if not os.path.isfile(file):
2191	print 'File %s does not exist' %file
2192	sys.exit(1)
2193
2194	config = trac_config.Configuration(file)
2195
2196	# Use given project name else use defaults
2197	#
2198	if name:
2199	sections = config.sections()
2200	if not name in sections:
2201	print "Not a valid project name: %s" %name
2202	print "Valid names: %s" %sections
2203	sys.exit(1)
2204
2205	project = SaraDict()
2206	for option, value in config.options(name):
2207	project[option] = value
2208
2209	else:
2210	# use some trac internals to get the defaults
2211	#
2212	tmp = config.parser.defaults()
2213	project = SaraDict()
2214
2215	for option,value in tmp.items():
2216	project[option] = value
2217
2218	return project
2219
2220
2221	if __name__ == '__main__':
2222	# Default config file
2223	#
2224	configfile = '@email2trac_conf@'
2225	project = ''
2226	component = ''
2227	ticket_prefix = 'default'
2228	dry_run = None
2229	verbose = None
2230
2231	ENABLE_SYSLOG = 0
2232
2233	SHORT_OPT = 'chf:np:t:v'
2234	LONG_OPT = ['component=', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=', 'verbose']
2235
2236	try:
2237	opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
2238	except getopt.error,detail:
2239	print __doc__
2240	print detail
2241	sys.exit(1)
2242
2243	project_name = None
2244	for opt,value in opts:
2245	if opt in [ '-h', '--help']:
2246	print __doc__
2247	sys.exit(0)
2248	elif opt in ['-c', '--component']:
2249	component = value
2250	elif opt in ['-f', '--file']:
2251	configfile = value
2252	elif opt in ['-n', '--dry-run']:
2253	dry_run = True
2254	elif opt in ['-p', '--project']:
2255	project_name = value
2256	elif opt in ['-t', '--ticket_prefix']:
2257	ticket_prefix = value
2258	elif opt in ['-v', '--verbose']:
2259	verbose = True
2260
2261	settings = ReadConfig(configfile, project_name)
2262	if not settings.has_key('project'):
2263	print __doc__
2264	print 'No Trac project is defined in the email2trac config file.'
2265	sys.exit(1)
2266
2267	if component:
2268	settings['component'] = component
2269
2270	# The default prefix for ticket values in email2trac.conf
2271	#
2272	settings['ticket_prefix'] = ticket_prefix
2273	settings['dry_run'] = dry_run
2274	settings['verbose'] = verbose
2275
2276
2277	# Determine major trac version used to be in email2trac.conf
2278	# Quick hack for 0.12
2279	#
2280	version = '0.%s' %(trac_version.split('.')[1])
2281	if version.startswith('0.12'):
2282	version = '0.12'
2283
2284	if verbose:
2285	print "Found trac version: %s" %(version)
2286
2287	#debug HvB
2288	#print settings
2289
2290	try:
2291	if version == '0.10':
2292	from trac import attachment
2293	from trac.env import Environment
2294	from trac.ticket import Ticket
2295	from trac.web.href import Href
2296	from trac import util
2297	#
2298	# return util.text.to_unicode(str)
2299	#
2300	# see http://projects.edgewall.com/trac/changeset/2799
2301	from trac.ticket.notification import TicketNotifyEmail
2302	from trac import config as trac_config
2303	from trac.core import TracError
2304
2305	elif version == '0.11':
2306	from trac import attachment
2307	from trac.env import Environment
2308	from trac.ticket import Ticket
2309	from trac.web.href import Href
2310	from trac import config as trac_config
2311	from trac import util
2312	from trac.core import TracError
2313	from trac.perm import PermissionSystem
2314
2315	#
2316	# return util.text.to_unicode(str)
2317	#
2318	# see http://projects.edgewall.com/trac/changeset/2799
2319	from trac.ticket.notification import TicketNotifyEmail
2320
2321	elif version == '0.12':
2322	from trac import attachment
2323	from trac.env import Environment
2324	from trac.ticket import Ticket
2325	from trac.web.href import Href
2326	from trac import config as trac_config
2327	from trac import util
2328	from trac.core import TracError
2329	from trac.perm import PermissionSystem
2330
2331	#
2332	# return util.text.to_unicode(str)
2333	#
2334	# see http://projects.edgewall.com/trac/changeset/2799
2335	from trac.ticket.notification import TicketNotifyEmail
2336
2337
2338	else:
2339	print 'TRAC version %s is not supported' %version
2340	sys.exit(1)
2341
2342	if settings.has_key('enable_syslog'):
2343	if SYSLOG_AVAILABLE:
2344	ENABLE_SYSLOG = float(settings['enable_syslog'])
2345
2346
2347	# Must be set before environment is created
2348	#
2349	if settings.has_key('python_egg_cache'):
2350	python_egg_cache = str(settings['python_egg_cache'])
2351	os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
2352
2353
2354	if int(settings['debug']) > 0:
2355	print 'Loading environment', settings['project']
2356
2357	env = Environment(settings['project'], create=0)
2358
2359	tktparser = TicketEmailParser(env, settings, float(version))
2360	tktparser.parse(sys.stdin)
2361
2362	# Catch all errors ans log to SYSLOG if we have enabled this
2363	# else stdout
2364	#
2365	except Exception, error:
2366	if ENABLE_SYSLOG:
2367	syslog.openlog('email2trac', syslog.LOG_NOWAIT)
2368
2369	etype, evalue, etb = sys.exc_info()
2370	for e in traceback.format_exception(etype, evalue, etb):
2371	syslog.syslog(e)
2372
2373	syslog.closelog()
2374	else:
2375	traceback.print_exc()
2376
2377	if m:
2378	tktparser.save_email_for_debug(m, True)
2379
2380
2381	sys.exit(1)
2382	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format