Context Navigation

source: trunk/email2trac.py.in @ 458

Last change on this file since 458 was 456, checked in by bas, 14 years ago
Mail was lost when ticket_update was disabled and ticket reply was detected
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 57.5 KB

Line
1	#!@PYTHON@
2	# Copyright (C) 2002
3	#
4	# This file is part of the email2trac utils
5	#
6	# This program is free software; you can redistribute it and/or modify it
7	# under the terms of the GNU General Public License as published by the
8	# Free Software Foundation; either version 2, or (at your option) any
9	# later version.
10	#
11	# This program is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with this program; if not, write to the Free Software
18	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
19	#
20	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
21	#
22	"""
23	email2trac.py -- Email tickets to Trac.
24
25	A simple MTA filter to create Trac tickets from inbound emails.
26
27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28	Copyright 2005, Edgewall Software
29
30	Authors:
31	Bas van der Vlies <basv@sara.nl>
32	Walter de Jong <walter@sara.nl>
33
34	The scripts reads emails from stdin and inserts directly into a Trac database.
35
36	How to use
37	----------
38	* See https://subtrac.sara.nl/oss/email2trac/
39
40	* Create an config file:
41	[DEFAULT] # REQUIRED
42	project : /data/trac/test # REQUIRED
43	debug : 1 # OPTIONAL, if set print some DEBUG info
44
45	[jouvin] # OPTIONAL project declaration, if set both fields necessary
46	project : /data/trac/jouvin # use -p\|--project jouvin.
47
48	* default config file is : /etc/email2trac.conf
49
50	* Commandline opions:
51	-h,--help
52	-d, --debug
53	-f,--file <configuration file>
54	-n,--dry-run
55	-p, --project <project name>
56	-t, --ticket_prefix <name>
57
58	SVN Info:
59	$Id: email2trac.py.in 456 2010-07-23 12:40:36Z bas $
60	"""
61	import os
62	import sys
63	import string
64	import getopt
65	import time
66	import email
67	import email.Iterators
68	import email.Header
69	import re
70	import urllib
71	import unicodedata
72	import mimetypes
73	import traceback
74	import logging
75	import logging.handlers
76	import UserDict
77
78	from datetime import tzinfo, timedelta, datetime
79	from stat import *
80
81
82	from trac import __version__ as trac_version
83	from trac import config as trac_config
84
85	# Some global variables
86	#
87	trac_default_version = '0.11'
88	m = None
89
90	class SaraDict(UserDict.UserDict):
91	def __init__(self, dictin = None):
92	UserDict.UserDict.__init__(self)
93	self.name = None
94
95	if dictin:
96	if dictin.has_key('name'):
97	self.name = dictin['name']
98	del dictin['name']
99	self.data = dictin
100
101	def get_value(self, name):
102	if self.has_key(name):
103	return self[name]
104	else:
105	return None
106
107	def __repr__(self):
108	return repr(self.data)
109
110	def __str__(self):
111	return str(self.data)
112
113	def __getattr__(self, name):
114	"""
115	override the class attribute get method. Return the value
116	from the dictionary
117	"""
118	if self.data.has_key(name):
119	return self.data[name]
120	else:
121	return None
122
123	def __setattr__(self, name, value):
124	"""
125	override the class attribute set method only when the UserDict
126	has set its class attribute
127	"""
128	if self.__dict__.has_key('data'):
129	self.data[name] = value
130	else:
131	self.__dict__[name] = value
132
133	def __iter__(self):
134	return iter(self.data.keys())
135
136	class TicketEmailParser(object):
137	env = None
138	comment = '> '
139
140	def __init__(self, env, parameters, logger, version):
141	self.env = env
142
143	# Database connection
144	#
145	self.db = None
146
147	# Save parameters
148	#
149	self.parameters = parameters
150	self.logger = logger
151
152	# Some useful mail constants
153	#
154	self.email_name = None
155	self.email_addr = None
156	self.email_from = None
157	self.author = None
158	self.id = None
159
160	self.STRIP_CONTENT_TYPES = list()
161
162	## fields properties via body_text
163	#
164	self.properties = dict()
165
166	self.VERSION = version
167
168	self.get_config = self.env.config.get
169
170	## init function ##
171	#
172	self.setup_parameters()
173
174	def setup_parameters(self):
175	if self.parameters.umask:
176	os.umask(self.parameters.umask)
177
178	if not self.parameters.spam_level:
179	self.parameters.spam_level = 0
180
181	if not self.parameters.spam_header:
182	self.parameters.spam_header = 'X-Spam-Score'
183
184	if not self.parameters.email_quote:
185	self.parameters.email_quote = '> '
186
187	if not self.parameters.ticket_update_by_subject_lookback:
188	self.parameters.ticket_update_by_subject_lookback = 30
189
190	if self.parameters.verbatim_format == None:
191	self.parameters.verbatim_format = 1
192
193	if self.parameters.reflow == None:
194	self.parameters.reflow = 1
195
196	if self.parameters.binhex:
197	self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
198
199	if self.parameters.applesingle:
200	self.STRIP_CONTENT_TYPES.append('application/applefile')
201
202	if self.parameters.appledouble:
203	self.STRIP_CONTENT_TYPES.append('application/applefile')
204
205	if self.parameters.strip_content_types:
206	items = self.parameters.strip_content_types.split(',')
207	for item in items:
208	self.STRIP_CONTENT_TYPES.append(item.strip())
209
210	if self.parameters.tmpdir:
211	self.parameters.tmpdir = os.path.normcase(str(self.parameters['tmpdir']))
212	else:
213	self.parameters.tmpdir = os.path.normcase('/tmp')
214
215	if self.parameters.email_triggers_workflow == None:
216	self.parameters.email_triggers_workflow = 1
217
218	if not self.parameters.subject_field_separator:
219	self.parameters.subject_field_separator = '&'
220	else:
221	self.parameters.subject_field_separator = self.parameters.subject_field_separator.strip()
222
223	self.trac_smtp_from = self.get_config('notification', 'smtp_from')
224
225	self.system = None
226
227	########## Email Header Functions ###########################################################
228
229	def spam(self, message):
230	"""
231	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
232	# Note if Spam_level then '*' are included
233	"""
234	spam = False
235	if message.has_key(self.parameters.spam_header):
236	spam_l = string.split(message[self.parameters.spam_header])
237
238	try:
239	number = spam_l[0].count('*')
240	except IndexError, detail:
241	number = 0
242
243	if number >= self.parameters.spam_level:
244	spam = True
245
246	# treat virus mails as spam
247	#
248	elif message.has_key('X-Virus-found'):
249	spam = True
250
251	# How to handle SPAM messages
252	#
253	if self.parameters.drop_spam and spam:
254
255	self.logger.info('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' %self.parameters.spam_level)
256	return 'drop'
257
258	elif spam:
259
260	return 'Spam'
261	else:
262
263	return False
264
265	def email_header_acl(self, keyword, header_field, default):
266	"""
267	This function wil check if the email address is allowed or denied
268	to send mail to the ticket list
269	"""
270	self.logger.debug('function email_header_acl: %s' %keyword)
271
272	try:
273	mail_addresses = self.parameters[keyword]
274
275	# Check if we have an empty string
276	#
277	if not mail_addresses:
278	return default
279
280	except KeyError, detail:
281	self.logger.debug('%s not defined, all messages are allowed.' %(keyword))
282
283	return default
284
285	mail_addresses = string.split(mail_addresses, ',')
286
287	for entry in mail_addresses:
288	entry = entry.strip()
289	TO_RE = re.compile(entry, re.VERBOSE\|re.IGNORECASE)
290	result = TO_RE.search(header_field)
291	if result:
292	return True
293
294	return False
295
296	def email_header_txt(self, m):
297	"""
298	Display To and CC addresses in description field
299	"""
300	s = ''
301
302	if m['To'] and len(m['To']) > 0:
303	s = "'''To:''' %s\r\n" %(m['To'])
304	if m['Cc'] and len(m['Cc']) > 0:
305	s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
306
307	return self.email_to_unicode(s)
308
309
310	def get_sender_info(self, message):
311	"""
312	Get the default author name and email address from the message
313	"""
314
315	self.email_to = self.email_to_unicode(message['to'])
316	self.to_name, self.to_email_addr = email.Utils.parseaddr (self.email_to)
317
318	self.email_from = self.email_to_unicode(message['from'])
319	self.email_name, self.email_addr = email.Utils.parseaddr(self.email_from)
320
321	## Trac can not handle author's name that contains spaces
322	# and forbid the ticket email address as author field
323
324	if self.email_addr == self.trac_smtp_from:
325	if self.email_name:
326	self.author = self.email_name
327	else:
328	self.author = "email2trac"
329	else:
330	self.author = self.email_addr
331
332	if self.parameters.ignore_trac_user_settings:
333	return
334
335	# Is this a registered user, use email address as search key:
336	# result:
337	# u : login name
338	# n : Name that the user has set in the settings tab
339	# e : email address that the user has set in the settings tab
340	#
341	users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
342	if e and (e.lower() == self.email_addr.lower()) ]
343
344	if len(users) == 1:
345	self.email_from = users[0][0]
346	self.author = users[0][0]
347
348	def set_reply_fields(self, ticket, message):
349	"""
350	Set all the right fields for a new ticket
351	"""
352	self.logger.debug('function set_reply_fields')
353
354	## Only use name or email adress
355	#ticket['reporter'] = self.email_from
356	ticket['reporter'] = self.author
357
358
359	# Put all CC-addresses in ticket CC field
360	#
361	if self.parameters.reply_all:
362
363	email_cc = ''
364
365	cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
366
367	if not cc_addrs:
368	return
369
370	## Build a list of forbidden CC addresses
371	#
372	#to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
373	#to_list = list()
374	#for n,e in to_addrs:
375	# to_list.append(e)
376
377	# Always Remove reporter email address from cc-list
378	#
379	try:
380	cc_addrs.remove((self.author, self.email_addr))
381	except ValueError, detail:
382	pass
383
384	for name,addr in cc_addrs:
385
386	## Prevent mail loop
387	#
388	#if addr in to_list:
389
390	if addr == self.trac_smtp_from:
391	self.logger.debug("Skipping %s mail address for CC-field" %(addr))
392	continue
393
394	if email_cc:
395	email_cc = '%s, %s' %(email_cc, addr)
396	else:
397	email_cc = addr
398
399	if email_cc:
400	self.logger.debug('set_reply_fields: %s' %email_cc)
401
402	ticket['cc'] = self.email_to_unicode(email_cc)
403
404
405	########## DEBUG functions ###########################################################
406
407	def debug_body(self, message_body, tempfile=False):
408	if tempfile:
409	import tempfile
410	body_file = tempfile.mktemp('.email2trac')
411	else:
412	body_file = os.path.join(self.parameters.tmpdir, 'body.txt')
413
414	if self.parameters.dry_run:
415	print 'DRY-RUN: not saving body to %s' %(body_file)
416	return
417
418	print 'writing body to %s' %(body_file)
419	fx = open(body_file, 'wb')
420	if not message_body:
421	message_body = '(None)'
422
423	message_body = message_body.encode('utf-8')
424	#message_body = unicode(message_body, 'iso-8859-15')
425
426	fx.write(message_body)
427	fx.close()
428	try:
429	os.chmod(body_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
430	except OSError:
431	pass
432
433	def debug_attachments(self, message_parts):
434	"""
435	"""
436	self.logger.debug('function debug_attachments')
437
438	n = 0
439	for item in message_parts:
440	# Skip inline text parts
441	if not isinstance(item, tuple):
442	continue
443
444	(original, filename, part) = item
445
446	n = n + 1
447	print 'part%d: Content-Type: %s' % (n, part.get_content_type())
448
449	s = 'part%d: filename: %s' %(n, filename)
450	self.print_unicode(s)
451
452	## Forbidden chars
453	#
454	filename = filename.replace('\\', '_')
455	filename = filename.replace('/', '_')
456
457
458	part_file = os.path.join(self.parameters.tmpdir, filename)
459	s = 'writing part%d (%s)' % (n,part_file)
460	self.print_unicode(s)
461
462	if self.parameters.dry_run:
463	print 'DRY_RUN: NOT saving attachments'
464	continue
465
466	part_file = util.text.unicode_quote(part_file)
467
468	fx = open(part_file, 'wb')
469	text = part.get_payload(decode=1)
470
471	if not text:
472	text = '(None)'
473
474	fx.write(text)
475	fx.close()
476
477	try:
478	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
479	except OSError:
480	pass
481
482	def save_email_for_debug(self, message, tempfile=False):
483
484	if tempfile:
485	import tempfile
486	msg_file = tempfile.mktemp('.email2trac')
487	else:
488	#msg_file = '/var/tmp/msg.txt'
489	msg_file = os.path.join(self.parameters.tmpdir, 'msg.txt')
490
491	if self.parameters.dry_run:
492	print 'DRY_RUN: NOT saving email message to %s' %(msg_file)
493	else:
494	print 'saving email to %s' %(msg_file)
495
496	fx = open(msg_file, 'wb')
497	fx.write('%s' % message)
498	fx.close()
499
500	try:
501	os.chmod(msg_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
502	except OSError:
503	pass
504
505	message_parts = self.get_message_parts(message)
506	message_parts = self.unique_attachment_names(message_parts)
507	body_text = self.body_text(message_parts)
508	self.debug_body(body_text, True)
509	self.debug_attachments(message_parts)
510
511	########## Conversion functions ###########################################################
512
513	def email_to_unicode(self, message_str):
514	"""
515	Email has 7 bit ASCII code, convert it to unicode with the charset
516	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
517	understands it.
518	"""
519	self.logger.debug("function email_to_unicode")
520
521	results = email.Header.decode_header(message_str)
522
523	s = None
524	for text,format in results:
525	if format:
526	try:
527	temp = unicode(text, format)
528	except UnicodeError, detail:
529	# This always works
530	#
531	temp = unicode(text, 'iso-8859-15')
532	except LookupError, detail:
533	#text = 'ERROR: Could not find charset: %s, please install' %format
534	#temp = unicode(text, 'iso-8859-15')
535	temp = message_str
536
537	else:
538	temp = string.strip(text)
539	temp = unicode(text, 'iso-8859-15')
540
541	if s:
542	s = '%s %s' %(s, temp)
543	else:
544	s = '%s' %temp
545
546	#s = s.encode('utf-8')
547	return s
548
549	def str_to_dict(self, s):
550	"""
551	Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
552	"""
553	self.logger.debug("function str_to_dict")
554
555	fields = string.split(s, self.parameters.subject_field_separator)
556
557	result = dict()
558	for field in fields:
559	try:
560	index, value = string.split(field, '=')
561
562	# We can not change the description of a ticket via the subject
563	# line. The description is the body of the email
564	#
565	if index.lower() in ['description']:
566	continue
567
568	if value:
569	result[index.lower()] = value
570
571	except ValueError:
572	pass
573	return result
574
575	def print_unicode(self,s):
576	"""
577	This function prints unicode strings uif possible else it will quote it
578	"""
579	try:
580	self.logger.debug(s)
581	except UnicodeEncodeError, detail:
582	self.logger.debug(util.text.unicode_quote(s))
583
584	########## TRAC ticket functions ###########################################################
585
586	def check_permission_participants(self, tkt):
587	"""
588	Check if the mailer is allowed to update the ticket
589	"""
590	self.logger.debug('function check_permission_participants')
591
592	if tkt['reporter'].lower() in [self.author, self.email_addr]:
593	self.logger.debug('ALLOW, %s is the ticket reporter' %(self.email_addr))
594
595	return True
596
597	perm = PermissionSystem(self.env)
598	if perm.check_permission('TICKET_MODIFY', self.author):
599	self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
600
601	return True
602
603	else:
604	return False
605
606
607	# Is the updater in the CC?
608	try:
609	cc_list = tkt['cc'].split(',')
610	for cc in cc_list:
611	if self.email_addr.lower() in cc.strip():
612	self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
613
614	return True
615
616	except KeyError:
617	return False
618
619	def check_permission(self, tkt, action):
620	"""
621	check if the reporter has the right permission for the action:
622	- TICKET_CREATE
623	- TICKET_MODIFY
624
625	There are three models:
626	- None : no checking at all
627	- trac : check the permission via trac permission model
628	- email2trac: ....
629	"""
630	self.logger.debug("function check_permission")
631
632	if self.parameters.ticket_permission_system in ['trac']:
633
634	perm = PermissionSystem(self.env)
635	if perm.check_permission(action, self.author):
636	return True
637	else:
638	return False
639
640	elif self.parameters.ticket_permission_system in ['update_restricted_to_participants']:
641	if action in ['TICKET_MODIFY']:
642	return (self.check_permission_participants(tkt))
643	else:
644	return True
645
646	# Default is to allow everybody ticket updates and ticket creation
647	else:
648	return True
649
650
651	def update_ticket_fields(self, ticket, user_dict, use_default=None):
652	"""
653	This will update the ticket fields. It will check if the
654	given fields are known and if the right values are specified
655	It will only update the ticket field value:
656	- If the field is known
657	- If the value supplied is valid for the ticket field.
658	If not then there are two options:
659	1) Skip the value (use_default=None)
660	2) Set default value for field (use_default=1)
661	"""
662	self.logger.debug("function update_ticket_fields")
663
664	# Build a system dictionary from the ticket fields
665	# with field as index and option as value
666	#
667	sys_dict = dict()
668	for field in ticket.fields:
669	try:
670	sys_dict[field['name']] = field['options']
671
672	except KeyError:
673	sys_dict[field['name']] = None
674	pass
675
676	## Check user supplied fields an compare them with the
677	# system one's
678	#
679	for field,value in user_dict.items():
680	if self.parameters.debug:
681	s = 'user_field\t %s = %s' %(field,value)
682	self.print_unicode(s)
683
684	## To prevent mail loop
685	#
686	if field == 'cc':
687
688	cc_list = user_dict['cc'].split(',')
689
690	if self.trac_smtp_from in cc_list:
691	self.logger.debug('MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
692
693	cc_list.remove(self.trac_smtp_from)
694
695	value = ','.join(cc_list)
696
697
698	if sys_dict.has_key(field):
699
700	# Check if value is an allowed system option, if TypeError then
701	# every value is allowed
702	#
703	try:
704	if value in sys_dict[field]:
705	ticket[field] = value
706	else:
707	# Must we set a default if value is not allowed
708	#
709	if use_default:
710	value = self.get_config('ticket', 'default_%s' %(field) )
711
712	except TypeError:
713	pass
714
715	## Only set if we have a value
716	#
717	if value:
718	ticket[field] = value
719
720	if self.parameters.debug:
721	s = 'ticket_field\t %s = %s' %(field, ticket[field])
722	self.print_unicode(s)
723
724	def ticket_update(self, m, id, spam):
725	"""
726	If the current email is a reply to an existing ticket, this function
727	will append the contents of this email to that ticket, instead of
728	creating a new one.
729	"""
730	self.logger.debug("function ticket_update")
731
732	if not self.parameters.ticket_update:
733	self.logger.debug("ticket_update disabled")
734	return False
735
736	# Must we update ticket fields
737	#
738	update_fields = dict()
739	try:
740	id, keywords = string.split(id, '?')
741
742	update_fields = self.str_to_dict(keywords)
743
744	# Strip '#'
745	#
746	self.id = int(id[1:])
747
748	except ValueError:
749
750	# Strip '#'
751	#
752	self.id = int(id[1:])
753
754	self.logger.debug("ticket_update id %s" %id)
755
756	# When is the change committed
757	#
758	if self.VERSION < 0.11:
759	when = int(time.time())
760	else:
761	when = datetime.now(util.datefmt.utc)
762
763	try:
764	tkt = Ticket(self.env, self.id, self.db)
765
766	except util.TracError, detail:
767
768	# Not a valid ticket
769
770	self.id = None
771	return False
772
773	# Check the permission of the reporter
774	#
775	if self.parameters.ticket_permission_system:
776	if not self.check_permission(tkt, 'TICKET_MODIFY'):
777	self.logger.info('Reporter: %s has no permission to modify tickets' %self.author)
778	return False
779
780	# How many changes has this ticket
781	cnum = len(tkt.get_changelog())
782
783
784	# reopen the ticket if it is was closed
785	# We must use the ticket workflow framework
786	#
787	if tkt['status'] in ['closed'] and self.parameters.email_triggers_workflow:
788
789	#print controller.actions['reopen']
790	#
791	# As reference
792	# req = Mock(href=Href('/'), abs_href=Href('http://www.example.com/'), authname='anonymous', perm=MockPerm(), args={})
793	#
794	#a = controller.render_ticket_action_control(req, tkt, 'reopen')
795	#print 'controller : ', a
796	#
797	#b = controller.get_all_status()
798	#print 'get all status: ', b
799	#
800	#b = controller.get_ticket_changes(req, tkt, 'reopen')
801	#print 'get_ticket_changes :', b
802
803	if self.parameters.workflow and (self.VERSION >= 0.11 ) :
804	from trac.ticket.default_workflow import ConfigurableTicketWorkflow
805	from trac.test import Mock, MockPerm
806
807	req = Mock(authname='anonymous', perm=MockPerm(), args={})
808
809	controller = ConfigurableTicketWorkflow(self.env)
810	fields = controller.get_ticket_changes(req, tkt, self.parameters.workflow)
811
812	self.logger.debug('Workflow ticket update fields: ')
813
814	for key in fields.keys():
815	self.logger.debug('\t %s : %s' %(key, fields[key]))
816	tkt[key] = fields[key]
817
818	else:
819	tkt['status'] = 'reopened'
820	tkt['resolution'] = ''
821
822	# Must we update some ticket fields properties via subjectline
823	#
824	if update_fields:
825	self.update_ticket_fields(tkt, update_fields)
826
827	message_parts = self.get_message_parts(m)
828	message_parts = self.unique_attachment_names(message_parts)
829
830	# Must we update some ticket fields properties via body_text
831	#
832	if self.properties:
833	self.update_ticket_fields(tkt, self.properties)
834
835	if self.parameters.email_header:
836	message_parts.insert(0, self.email_header_txt(m))
837
838	body_text = self.body_text(message_parts)
839
840	error_with_attachments = self.attach_attachments(message_parts)
841
842	if body_text.strip() or update_fields or self.properties:
843	if self.parameters.dry_run:
844	print 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) ', self.author, cnum
845	else:
846	if error_with_attachments:
847	body_text = '%s\\%s' %(error_with_attachments, body_text)
848	self.logger.debug('tkt.save_changes(%s, %d)' %(self.author, cnum))
849	tkt.save_changes(self.author, body_text, when, None, str(cnum))
850
851
852	if not spam:
853	self.notify(tkt, False, when)
854
855	return True
856
857	def set_ticket_fields(self, ticket):
858	"""
859	set the ticket fields to value specified
860	- /etc/email2trac.conf with <prefix>_<field>
861	- trac default values, trac.ini
862	"""
863	self.logger.debug('function set_ticket_fields')
864
865	user_dict = dict()
866
867	for field in ticket.fields:
868
869	name = field['name']
870
871	## default trac value
872	#
873	if not field.get('custom'):
874	value = self.get_config('ticket', 'default_%s' %(name) )
875	else:
876	## Else we get the default value for reporter
877	#
878	value = field.get('value')
879	options = field.get('options')
880
881	if value and options and (value not in options):
882	value = options[int(value)]
883
884	if self.parameters.debug:
885	s = 'trac[%s] = %s' %(name, value)
886	self.print_unicode(s)
887
888	## email2trac.conf settings
889	#
890	prefix = self.parameters['ticket_prefix']
891	try:
892	value = self.parameters['%s_%s' %(prefix, name)]
893	if self.parameters.debug > 10:
894	s = 'email2trac[%s] = %s ' %(name, value)
895	self.print_unicode(s)
896
897	except KeyError, detail:
898	pass
899
900	if self.parameters.debug:
901	s = 'used %s = %s' %(name, value)
902	self.print_unicode(s)
903
904	if value:
905	user_dict[name] = value
906
907	self.update_ticket_fields(ticket, user_dict, use_default=1)
908
909	if 'status' not in user_dict.keys():
910	ticket['status'] = 'new'
911
912
913	def ticket_update_by_subject(self, subject):
914	"""
915	This list of Re: prefixes is probably incomplete. Taken from
916	wikipedia. Here is how the subject is matched
917	- Re: <subject>
918	- Re: (<Mail list label>:)+ <subject>
919
920	So we must have the last column
921	"""
922	self.logger.debug('function ticket_update_by_subject')
923
924	matched_id = None
925	if self.parameters.ticket_update and self.parameters.ticket_update_by_subject:
926
927	SUBJECT_RE = re.compile(r'^(RE\|AW\|VS\|SV):(.:)\s(.)', re.IGNORECASE)
928	result = SUBJECT_RE.search(subject)
929
930	if result:
931	# This is a reply
932	orig_subject = result.group(3)
933
934	self.logger.debug('subject search string: %s' %(orig_subject))
935
936	cursor = self.db.cursor()
937	summaries = [orig_subject, '%%: %s' % orig_subject]
938
939	##
940	# Convert days to seconds
941	lookback = int(time.mktime(time.gmtime())) - \
942	self.parameters.ticket_update_by_subject_lookback * 24 * 3600
943
944
945	for summary in summaries:
946	self.logger.debug('Looking for summary matching: "%s"' % summary)
947
948	sql = """SELECT id FROM ticket
949	WHERE changetime >= %s AND summary LIKE %s
950	ORDER BY changetime DESC"""
951	cursor.execute(sql, [lookback, summary.strip()])
952
953	for row in cursor:
954	(matched_id,) = row
955
956	self.logger.debug('Found matching ticket id: %d' % matched_id)
957
958	break
959
960	if matched_id:
961	matched_id = '#%d' % matched_id
962	return matched_id
963
964	return matched_id
965
966
967	def new_ticket(self, msg, subject, spam, set_fields = None):
968	"""
969	Create a new ticket
970	"""
971	self.logger.debug('function new_ticket')
972
973	tkt = Ticket(self.env)
974
975	self.set_reply_fields(tkt, msg)
976
977	self.set_ticket_fields(tkt)
978
979	# Check the permission of the reporter
980	#
981	if self.parameters.ticket_permission_system:
982	if not self.check_permission(tkt, 'TICKET_CREATE'):
983	self.logger.info('Reporter: %s has no permission to create tickets' %self.author)
984	return False
985
986	# Old style setting for component, will be removed
987	#
988	if spam:
989	tkt['component'] = 'Spam'
990
991	elif self.parameters.has_key('component'):
992	tkt['component'] = self.parameters['component']
993
994	if not msg['Subject']:
995	tkt['summary'] = u'(No subject)'
996	else:
997	tkt['summary'] = subject
998
999
1000	if set_fields:
1001	rest, keywords = string.split(set_fields, '?')
1002
1003	if keywords:
1004	update_fields = self.str_to_dict(keywords)
1005	self.update_ticket_fields(tkt, update_fields)
1006
1007
1008	message_parts = self.get_message_parts(msg)
1009
1010	# Must we update some ticket fields properties via body_text
1011	#
1012	if self.properties:
1013	self.update_ticket_fields(tkt, self.properties)
1014
1015	message_parts = self.unique_attachment_names(message_parts)
1016
1017	# produce e-mail like header
1018	#
1019	head = ''
1020	if self.parameters.email_header:
1021	head = self.email_header_txt(msg)
1022	message_parts.insert(0, head)
1023
1024	body_text = self.body_text(message_parts)
1025
1026	tkt['description'] = body_text
1027
1028	# When is the change committed
1029	#
1030	if self.VERSION < 0.11:
1031	when = int(time.time())
1032	else:
1033	when = datetime.now(util.datefmt.utc)
1034
1035	if self.parameters.dry_run:
1036	print 'DRY_RUN: tkt.insert()'
1037	else:
1038	self.id = tkt.insert()
1039
1040	changed = False
1041	comment = ''
1042
1043	# some routines in trac are dependend on ticket id
1044	# like alternate notify template
1045	#
1046	if self.parameters.alternate_notify_template:
1047	tkt['id'] = self.id
1048	changed = True
1049
1050	## Rewrite the description if we have mailto enabled
1051	#
1052	if self.parameters.mailto_link:
1053	changed = True
1054	comment = u'\nadded mailto line\n'
1055	mailto = self.html_mailto_link( m['Subject'])
1056
1057	tkt['description'] = u'%s\r\n%s%s\r\n' \
1058	%(head, mailto, body_text)
1059
1060	## Save the attachments to the ticket
1061	#
1062	error_with_attachments = self.attach_attachments(message_parts)
1063
1064	if error_with_attachments:
1065	changed = True
1066	comment = '%s\n%s\n' %(comment, error_with_attachments)
1067
1068	if changed:
1069	if self.parameters.dry_run:
1070	print 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
1071	else:
1072	tkt.save_changes(tkt['reporter'], comment)
1073	#print tkt.get_changelog(self.db, when)
1074
1075	if not spam:
1076	self.notify(tkt, True)
1077
1078
1079	def attach_attachments(self, message_parts, update=False):
1080	'''
1081	save any attachments as files in the ticket's directory
1082	'''
1083	self.logger.debug('function attach_attachments()')
1084
1085	if self.parameters.dry_run:
1086	print "DRY_RUN: no attachments attached to tickets"
1087	return ''
1088
1089	count = 0
1090
1091	# Get Maxium attachment size
1092	#
1093	max_size = int(self.get_config('attachment', 'max_size'))
1094	status = None
1095
1096	for item in message_parts:
1097	# Skip body parts
1098	if not isinstance(item, tuple):
1099	continue
1100
1101	(original, filename, part) = item
1102	#
1103	# We have to determine the size so we use this temporary solution. we must escape it
1104	# else we get UnicodeErrors.
1105	#
1106	path, fd = util.create_unique_file(os.path.join(self.parameters.tmpdir, util.text.unicode_quote(filename)))
1107	text = part.get_payload(decode=1)
1108	if not text:
1109	text = '(None)'
1110	fd.write(text)
1111	fd.close()
1112
1113	# get the file_size
1114	#
1115	stats = os.lstat(path)
1116	file_size = stats[ST_SIZE]
1117
1118	# Check if the attachment size is allowed
1119	#
1120	if (max_size != -1) and (file_size > max_size):
1121	status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1122	%(status, original, file_size, max_size)
1123
1124	os.unlink(path)
1125	continue
1126	else:
1127	count = count + 1
1128
1129	# Insert the attachment
1130	#
1131	fd = open(path, 'rb')
1132	if self.system == 'discussion':
1133	att = attachment.Attachment(self.env, 'discussion', 'topic/%s'
1134	% (self.id,))
1135	else:
1136	self.logger.debug('Attach %s to ticket %d' %(util.text.unicode_quote(filename), self.id))
1137	att = attachment.Attachment(self.env, 'ticket', self.id)
1138
1139	# This will break the ticket_update system, the body_text is vaporized
1140	# ;-(
1141	#
1142	if not update:
1143	att.author = self.author
1144	att.description = self.email_to_unicode('Added by email2trac')
1145
1146	try:
1147	self.logger.debug('Insert atachment')
1148	att.insert(filename, fd, file_size)
1149	except OSError, detail:
1150	self.logger.info('%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail))
1151	status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
1152
1153	# Remove the created temporary filename
1154	#
1155	fd.close()
1156	os.unlink(path)
1157
1158	## return error
1159	#
1160	return status
1161
1162	########## Fullblog functions #################################################
1163
1164	def blog(self, id):
1165	"""
1166	The blog create/update function
1167	"""
1168	# import the modules
1169	#
1170	from tracfullblog.core import FullBlogCore
1171	from tracfullblog.model import BlogPost, BlogComment
1172	from trac.test import Mock, MockPerm
1173
1174	# instantiate blog core
1175	blog = FullBlogCore(self.env)
1176	req = Mock(authname='anonymous', perm=MockPerm(), args={})
1177
1178	if id:
1179
1180	# update blog
1181	#
1182	comment = BlogComment(self.env, id)
1183	comment.author = self.author
1184
1185	message_parts = self.get_message_parts(m)
1186	comment.comment = self.body_text(message_parts)
1187
1188	blog.create_comment(req, comment)
1189
1190	else:
1191	# create blog
1192	#
1193	import time
1194	post = BlogPost(self.env, 'blog_'+time.strftime("%Y%m%d%H%M%S", time.gmtime()))
1195
1196	#post = BlogPost(self.env, blog._get_default_postname(self.env))
1197
1198	post.author = self.author
1199	post.title = self.email_to_unicode(m['Subject'])
1200
1201	message_parts = self.get_message_parts(m)
1202	post.body = self.body_text(message_parts)
1203
1204	blog.create_post(req, post, self.author, u'Created by email2trac', False)
1205
1206
1207	########## Discussion functions ##############################################
1208
1209	def discussion_topic(self, content, subject):
1210
1211	# Import modules.
1212	from tracdiscussion.api import DiscussionApi
1213	from trac.util.datefmt import to_timestamp, utc
1214
1215	self.logger.debug('Creating a new topic in forum:', self.id)
1216
1217	# Get dissussion API component.
1218	api = self.env[DiscussionApi]
1219	context = self._create_context(content, subject)
1220
1221	# Get forum for new topic.
1222	forum = api.get_forum(context, self.id)
1223
1224	if not forum:
1225	self.logger.error("ERROR: Replied forum doesn't exist")
1226
1227	# Prepare topic.
1228	topic = {'forum' : forum['id'],
1229	'subject' : context.subject,
1230	'time': to_timestamp(datetime.now(utc)),
1231	'author' : self.author,
1232	'subscribers' : [self.email_addr],
1233	'body' : self.body_text(context.content_parts)}
1234
1235	# Add topic to DB and commit it.
1236	self._add_topic(api, context, topic)
1237	self.db.commit()
1238
1239	def discussion_topic_reply(self, content, subject):
1240
1241	# Import modules.
1242	from tracdiscussion.api import DiscussionApi
1243	from trac.util.datefmt import to_timestamp, utc
1244
1245	self.logger.debug('Replying to discussion topic', self.id)
1246
1247	# Get dissussion API component.
1248	api = self.env[DiscussionApi]
1249	context = self._create_context(content, subject)
1250
1251	# Get replied topic.
1252	topic = api.get_topic(context, self.id)
1253
1254	if not topic:
1255	self.logger.error("ERROR: Replied topic doesn't exist")
1256
1257	# Prepare message.
1258	message = {'forum' : topic['forum'],
1259	'topic' : topic['id'],
1260	'replyto' : -1,
1261	'time' : to_timestamp(datetime.now(utc)),
1262	'author' : self.author,
1263	'body' : self.body_text(context.content_parts)}
1264
1265	# Add message to DB and commit it.
1266	self._add_message(api, context, message)
1267	self.db.commit()
1268
1269	def discussion_message_reply(self, content, subject):
1270
1271	# Import modules.
1272	from tracdiscussion.api import DiscussionApi
1273	from trac.util.datefmt import to_timestamp, utc
1274
1275	self.loggger.debug('Replying to discussion message', self.id)
1276
1277	# Get dissussion API component.
1278	api = self.env[DiscussionApi]
1279	context = self._create_context(content, subject)
1280
1281	# Get replied message.
1282	message = api.get_message(context, self.id)
1283
1284	if not message:
1285	self.logger.error("ERROR: Replied message doesn't exist")
1286
1287	# Prepare message.
1288	message = {'forum' : message['forum'],
1289	'topic' : message['topic'],
1290	'replyto' : message['id'],
1291	'time' : to_timestamp(datetime.now(utc)),
1292	'author' : self.author,
1293	'body' : self.body_text(context.content_parts)}
1294
1295	# Add message to DB and commit it.
1296	self._add_message(api, context, message)
1297	self.db.commit()
1298
1299	def _create_context(self, content, subject):
1300
1301	# Import modules.
1302	from trac.mimeview import Context
1303	from trac.web.api import Request
1304	from trac.perm import PermissionCache
1305
1306	# TODO: Read server base URL from config.
1307	# Create request object to mockup context creation.
1308	#
1309	environ = {'SERVER_PORT' : 80,
1310	'SERVER_NAME' : 'test',
1311	'REQUEST_METHOD' : 'POST',
1312	'wsgi.url_scheme' : 'http',
1313	'wsgi.input' : sys.stdin}
1314	chrome = {'links': {},
1315	'scripts': [],
1316	'ctxtnav': [],
1317	'warnings': [],
1318	'notices': []}
1319
1320	if self.env.base_url_for_redirect:
1321	environ['trac.base_url'] = self.env.base_url
1322
1323	req = Request(environ, None)
1324	req.chrome = chrome
1325	req.tz = 'missing'
1326	req.authname = self.author
1327	req.perm = PermissionCache(self.env, self.author)
1328
1329	# Create and return context.
1330	context = Context.from_request(req)
1331	context.realm = 'discussion-email2trac'
1332	context.cursor = self.db.cursor()
1333	context.content = content
1334	context.subject = subject
1335
1336	# Read content parts from content.
1337	context.content_parts = self.get_message_parts(content)
1338	context.content_parts = self.unique_attachment_names(
1339	context.content_parts)
1340
1341	return context
1342
1343	def _add_topic(self, api, context, topic):
1344	context.req.perm.assert_permission('DISCUSSION_APPEND')
1345
1346	# Filter topic.
1347	for discussion_filter in api.discussion_filters:
1348	accept, topic_or_error = discussion_filter.filter_topic(
1349	context, topic)
1350	if accept:
1351	topic = topic_or_error
1352	else:
1353	raise TracError(topic_or_error)
1354
1355	# Add a new topic.
1356	api.add_topic(context, topic)
1357
1358	# Get inserted topic with new ID.
1359	topic = api.get_topic_by_time(context, topic['time'])
1360
1361	# Attach attachments.
1362	self.id = topic['id']
1363	self.attach_attachments(context.content_parts, True)
1364
1365	# Notify change listeners.
1366	for listener in api.topic_change_listeners:
1367	listener.topic_created(context, topic)
1368
1369	def _add_message(self, api, context, message):
1370	context.req.perm.assert_permission('DISCUSSION_APPEND')
1371
1372	# Filter message.
1373	for discussion_filter in api.discussion_filters:
1374	accept, message_or_error = discussion_filter.filter_message(
1375	context, message)
1376	if accept:
1377	message = message_or_error
1378	else:
1379	raise TracError(message_or_error)
1380
1381	# Add message.
1382	api.add_message(context, message)
1383
1384	# Get inserted message with new ID.
1385	message = api.get_message_by_time(context, message['time'])
1386
1387	# Attach attachments.
1388	self.id = message['topic']
1389	self.attach_attachments(context.content_parts, True)
1390
1391	# Notify change listeners.
1392	for listener in api.message_change_listeners:
1393	listener.message_created(context, message)
1394
1395	########## MAIN function ######################################################
1396
1397	def parse(self, fp):
1398	"""
1399	"""
1400	self.logger.debug('Main function parse')
1401	global m
1402
1403	m = email.message_from_file(fp)
1404
1405	if not m:
1406	self.logger.debug('This is not a valid email message format')
1407	return
1408
1409	# Work around lack of header folding in Python; see http://bugs.python.org/issue4696
1410	try:
1411	m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
1412	except AttributeError, detail:
1413	pass
1414
1415	if self.parameters.debug: # save the entire e-mail message text
1416	self.save_email_for_debug(m, True)
1417
1418	self.db = self.env.get_db_cnx()
1419	self.get_sender_info(m)
1420
1421	if not self.email_header_acl('white_list', self.email_addr, True):
1422	self.logger.info('Message rejected : %s not in white list' %(self.email_addr))
1423	return False
1424
1425	if self.email_header_acl('black_list', self.email_addr, False):
1426	self.logger.info('Message rejected : %s in black list' %(self.email_addr))
1427	return False
1428
1429	if not self.email_header_acl('recipient_list', self.to_email_addr, True):
1430	self.logger.info('Message rejected : %s not in recipient list' %(self.to_email_addr))
1431	return False
1432
1433	# If spam drop the message
1434	#
1435	if self.spam(m) == 'drop':
1436	return False
1437
1438	elif self.spam(m) == 'spam':
1439	spam_msg = True
1440	else:
1441	spam_msg = False
1442
1443	if not m['Subject']:
1444	subject = 'No Subject'
1445	else:
1446	subject = self.email_to_unicode(m['Subject'])
1447
1448	self.logger.debug('subject: %s' %subject)
1449
1450	#
1451	# [hic] #1529: Re: LRZ
1452	# [hic] #1529?owner=bas,priority=medium: Re: LRZ
1453	#
1454	ticket_regex = r'''
1455	(?P<new_fields>[#][?].*)
1456	\|(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?.*)?:)
1457	'''
1458	# Check if FullBlogPlugin is installed
1459	#
1460	blog_enabled = None
1461	blog_regex = ''
1462	if self.get_config('components', 'tracfullblog.*') in ['enabled']:
1463	blog_enabled = True
1464	blog_regex = '''\|(?P<blog>blog:(?P<blog_id>\w*))'''
1465
1466
1467	# Check if DiscussionPlugin is installed
1468	#
1469	discussion_enabled = None
1470	discussion_regex = ''
1471	if self.get_config('components', 'tracdiscussion.api.*') in ['enabled']:
1472	discussion_enabled = True
1473	discussion_regex = r'''
1474	\|(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
1475	\|(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
1476	\|(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
1477	'''
1478
1479
1480	regex_str = ticket_regex + blog_regex + discussion_regex
1481	SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
1482
1483	# Find out if this is a ticket, a blog or a discussion
1484	#
1485	result = SYSTEM_RE.search(subject)
1486
1487	if result:
1488	# update ticket + fields
1489	#
1490	#if result.group('reply') and self.parameters.ticket_update:
1491	if result.group('reply'):
1492	self.system = 'ticket'
1493
1494	# Skip the last ':' character
1495	#
1496	if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
1497	self.new_ticket(m, subject, spam_msg)
1498
1499	# New ticket + fields
1500	#
1501	elif result.group('new_fields'):
1502	self.system = 'ticket'
1503	self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
1504
1505	if blog_enabled:
1506	if result.group('blog'):
1507	self.system = 'blog'
1508	self.blog(result.group('blog_id'))
1509
1510	if discussion_enabled:
1511	# New topic.
1512	#
1513	if result.group('forum'):
1514	self.system = 'discussion'
1515	self.id = int(result.group('forum_id'))
1516	self.discussion_topic(m, subject[result.end('forum'):])
1517
1518	# Reply to topic.
1519	#
1520	elif result.group('topic'):
1521	self.system = 'discussion'
1522	self.id = int(result.group('topic_id'))
1523	self.discussion_topic_reply(m, subject[result.end('topic'):])
1524
1525	# Reply to topic message.
1526	#
1527	elif result.group('message'):
1528	self.system = 'discussion'
1529	self.id = int(result.group('message_id'))
1530	self.discussion_message_reply(m, subject[result.end('message'):])
1531
1532	else:
1533	self.system = 'ticket'
1534	result = self.ticket_update_by_subject(subject)
1535	if result:
1536	if not self.ticket_update(m, result, spam_msg):
1537	self.new_ticket(m, subject, spam_msg)
1538	else:
1539	# No update by subject, so just create a new ticket
1540	self.new_ticket(m, subject, spam_msg)
1541
1542
1543	########## BODY TEXT functions ###########################################################
1544
1545	def strip_signature(self, text):
1546	"""
1547	Strip signature from message, inspired by Mailman software
1548	"""
1549	body = []
1550	for line in text.splitlines():
1551	if line == '-- ':
1552	break
1553	body.append(line)
1554
1555	return ('\n'.join(body))
1556
1557	def reflow(self, text, delsp = 0):
1558	"""
1559	Reflow the message based on the format="flowed" specification (RFC 3676)
1560	"""
1561	flowedlines = []
1562	quotelevel = 0
1563	prevflowed = 0
1564
1565	for line in text.splitlines():
1566	from re import match
1567
1568	# Figure out the quote level and the content of the current line
1569	m = match('(>)( ?)(.)', line)
1570	linequotelevel = len(m.group(1))
1571	line = m.group(3)
1572
1573	# Determine whether this line is flowed
1574	if line and line != '-- ' and line[-1] == ' ':
1575	flowed = 1
1576	else:
1577	flowed = 0
1578
1579	if flowed and delsp and line and line[-1] == ' ':
1580	line = line[:-1]
1581
1582	# If the previous line is flowed, append this line to it
1583	if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1584	flowedlines[-1] += line
1585	# Otherwise, start a new line
1586	else:
1587	flowedlines.append('>' * linequotelevel + line)
1588
1589	prevflowed = flowed
1590
1591
1592	return '\n'.join(flowedlines)
1593
1594	def strip_quotes(self, text):
1595	"""
1596	Strip quotes from message by Nicolas Mendoza
1597	"""
1598	body = []
1599	for line in text.splitlines():
1600	if line.startswith(self.parameters.email_quote):
1601	continue
1602	body.append(line)
1603
1604	return ('\n'.join(body))
1605
1606	def inline_properties(self, text):
1607	"""
1608	Parse text if we use inline keywords to set ticket fields
1609	"""
1610	self.logger.debug('function inline_properties')
1611
1612	properties = dict()
1613	body = list()
1614
1615	INLINE_EXP = re.compile('\s[@]\s([a-zA-Z]+)\s:(.)$')
1616
1617	for line in text.splitlines():
1618	match = INLINE_EXP.match(line)
1619	if match:
1620	keyword, value = match.groups()
1621	self.properties[keyword] = value.strip()
1622
1623	self.logger.debug('inline properties: %s : %s' %(keyword,value))
1624
1625	else:
1626	body.append(line)
1627
1628	return '\n'.join(body)
1629
1630
1631	def wrap_text(self, text, replace_whitespace = False):
1632	"""
1633	Will break a lines longer then given length into several small
1634	lines of size given length
1635	"""
1636	import textwrap
1637
1638	LINESEPARATOR = '\n'
1639	reformat = ''
1640
1641	for s in text.split(LINESEPARATOR):
1642	tmp = textwrap.fill(s, self.parameters.use_textwrap)
1643	if tmp:
1644	reformat = '%s\n%s' %(reformat,tmp)
1645	else:
1646	reformat = '%s\n' %reformat
1647
1648	return reformat
1649
1650	# Python2.4 and higher
1651	#
1652	#return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
1653	#
1654
1655	########## EMAIL attachements functions ###########################################################
1656
1657	def inline_part(self, part):
1658	"""
1659	"""
1660	self.logger.debug('function inline_part()')
1661
1662	return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
1663
1664	def get_message_parts(self, msg):
1665	"""
1666	parses the email message and returns a list of body parts and attachments
1667	body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
1668	"""
1669	self.logger.debug('function get_message_parts()')
1670
1671	message_parts = list()
1672
1673	ALTERNATIVE_MULTIPART = False
1674
1675	for part in msg.walk():
1676	self.logger.debug('Message part: Main-Type: %s' % part.get_content_maintype())
1677	self.logger.debug('Message part: Content-Type: %s' % part.get_content_type())
1678
1679	## Check content type
1680	#
1681	if part.get_content_type() in self.STRIP_CONTENT_TYPES:
1682	self.logger.debug("A %s attachment named '%s' was skipped" %(part.get_content_type(), part.get_filename()))
1683	continue
1684
1685	## Catch some mulitpart execptions
1686	#
1687	if part.get_content_type() == 'multipart/alternative':
1688	ALTERNATIVE_MULTIPART = True
1689	continue
1690
1691	## Skip multipart containers
1692	#
1693	if part.get_content_maintype() == 'multipart':
1694	self.logger.debug("Skipping multipart container")
1695
1696	continue
1697
1698	## Check if this is an inline part. It's inline if there is co Cont-Disp header, or if there is one and it says "inline"
1699	#
1700	inline = self.inline_part(part)
1701
1702	## Drop HTML message
1703	#
1704	if ALTERNATIVE_MULTIPART and self.parameters.drop_alternative_html_version:
1705	if part.get_content_type() == 'text/html':
1706	self.logger.debug('Skipping alternative HTML message')
1707	ALTERNATIVE_MULTIPART = False
1708	continue
1709
1710	## Inline text parts are where the body is
1711	#
1712	if part.get_content_type() == 'text/plain' and inline:
1713	self.logger.debug(' Inline body part')
1714
1715	# Try to decode, if fails then do not decode
1716	#
1717	body_text = part.get_payload(decode=1)
1718	if not body_text:
1719	body_text = part.get_payload(decode=0)
1720
1721	format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
1722	delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
1723
1724	if self.parameters.reflow and not self.parameters.verbatim_format and format == 'flowed':
1725	body_text = self.reflow(body_text, delsp == 'yes')
1726
1727	if self.parameters.strip_signature:
1728	body_text = self.strip_signature(body_text)
1729
1730	if self.parameters.strip_quotes:
1731	body_text = self.strip_quotes(body_text)
1732
1733	if self.parameters.inline_properties:
1734	body_text = self.inline_properties(body_text)
1735
1736	if self.parameters.use_textwrap:
1737	body_text = self.wrap_text(body_text)
1738
1739	## Get contents charset (iso-8859-15 if not defined in mail headers)
1740	#
1741	charset = part.get_content_charset()
1742	if not charset:
1743	charset = 'iso-8859-15'
1744
1745	try:
1746	ubody_text = unicode(body_text, charset)
1747
1748	except UnicodeError, detail:
1749	ubody_text = unicode(body_text, 'iso-8859-15')
1750
1751	except LookupError, detail:
1752	ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
1753
1754	if self.parameters.verbatim_format:
1755	message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
1756	else:
1757	message_parts.append('%s' %ubody_text)
1758	else:
1759	if self.parameters.debug:
1760	s = ' Filename: %s' % part.get_filename()
1761	self.print_unicode(s)
1762
1763	##
1764	# First try to use email header function to convert filename.
1765	# If this fails the use the plan filename
1766	try:
1767	filename = self.email_to_unicode(part.get_filename())
1768	except UnicodeEncodeError, detail:
1769	filename = part.get_filename()
1770
1771	message_parts.append((filename, part))
1772
1773	return message_parts
1774
1775	def unique_attachment_names(self, message_parts):
1776	"""
1777	"""
1778	renamed_parts = []
1779	attachment_names = set()
1780
1781	for item in message_parts:
1782
1783	## If not an attachment, leave it alone
1784	#
1785	if not isinstance(item, tuple):
1786	renamed_parts.append(item)
1787	continue
1788
1789	(filename, part) = item
1790
1791	## If no filename, use a default one
1792	#
1793	if not filename:
1794	filename = 'untitled-part'
1795
1796	# Guess the extension from the content type, use non strict mode
1797	# some additional non-standard but commonly used MIME types
1798	# are also recognized
1799	#
1800	ext = mimetypes.guess_extension(part.get_content_type(), False)
1801	if not ext:
1802	ext = '.bin'
1803
1804	filename = '%s%s' % (filename, ext)
1805
1806	## Discard relative paths for windows/unix in attachment names
1807	#
1808	#filename = filename.replace('\\', '/').replace(':', '/')
1809	filename = filename.replace('\\', '_')
1810	filename = filename.replace('/', '_')
1811
1812	#
1813	# We try to normalize the filename to utf-8 NFC if we can.
1814	# Files uploaded from OS X might be in NFD.
1815	# Check python version and then try it
1816	#
1817	#if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
1818	# try:
1819	# filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
1820	# except TypeError:
1821	# pass
1822
1823	# Make the filename unique for this ticket
1824	num = 0
1825	unique_filename = filename
1826	dummy_filename, ext = os.path.splitext(filename)
1827
1828	while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
1829	num += 1
1830	unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
1831
1832	if self.parameters.debug:
1833	s = 'Attachment with filename %s will be saved as %s' % (filename, unique_filename)
1834	self.print_unicode(s)
1835
1836	attachment_names.add(unique_filename)
1837
1838	renamed_parts.append((filename, unique_filename, part))
1839
1840	return renamed_parts
1841
1842
1843	def attachment_exists(self, filename):
1844
1845	if self.parameters.debug:
1846	s = 'attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
1847	self.print_unicode(s)
1848
1849	# We have no valid ticket id
1850	#
1851	if not self.id:
1852	return False
1853
1854	try:
1855	if self.system == 'discussion':
1856	att = attachment.Attachment(self.env, 'discussion', 'ticket/%s'
1857	% (self.id,), filename)
1858	else:
1859	att = attachment.Attachment(self.env, 'ticket', self.id,
1860	filename)
1861	return True
1862	except attachment.ResourceNotFound:
1863	return False
1864
1865	########## TRAC Ticket Text ###########################################################
1866
1867	def body_text(self, message_parts):
1868	body_text = []
1869
1870	for part in message_parts:
1871	# Plain text part, append it
1872	if not isinstance(part, tuple):
1873	body_text.extend(part.strip().splitlines())
1874	body_text.append("")
1875	continue
1876
1877	(original, filename, part) = part
1878	inline = self.inline_part(part)
1879
1880	if part.get_content_maintype() == 'image' and inline:
1881	if self.system != 'discussion':
1882	body_text.append('[[Image(%s)]]' % filename)
1883	body_text.append("")
1884	else:
1885	if self.system != 'discussion':
1886	body_text.append('[attachment:"%s"]' % filename)
1887	body_text.append("")
1888
1889	body_text = '\r\n'.join(body_text)
1890	return body_text
1891
1892	def html_mailto_link(self, subject):
1893	"""
1894	This function returns a HTML mailto tag with the ticket id and author email address
1895	"""
1896	if not self.author:
1897	author = self.email_addr
1898	else:
1899	author = self.author
1900
1901	if not self.parameters.mailto_cc:
1902	self.parameters.mailto_cc = ''
1903
1904	# use urllib to escape the chars
1905	#
1906	s = 'mailto:%s?Subject=%s&Cc=%s' %(
1907	urllib.quote(self.email_addr),
1908	urllib.quote('Re: #%s: %s' %(self.id, subject)),
1909	urllib.quote(self.parameters.mailto_cc)
1910	)
1911
1912	s = '\r\n{{{\r\n#!html\r\n<a\r\n href="%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
1913	return s
1914
1915	########## TRAC notify section ###########################################################
1916
1917	def notify(self, tkt, new=True, modtime=0):
1918	"""
1919	A wrapper for the TRAC notify function. So we can use templates
1920	"""
1921	self.logger.debug('function notify()')
1922
1923	if self.parameters.dry_run:
1924	print 'DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter']
1925	return
1926	try:
1927
1928	#from trac.ticket.web_ui import TicketModule
1929	#from trac.ticket.notification import TicketNotificationSystem
1930	#ticket_sys = TicketNotificationSystem(self.env)
1931	#a = TicketModule(self.env)
1932	#print a.__dict__
1933	#tn_sys = TicketNotificationSystem(self.env)
1934	#print tn_sys
1935	#print tn_sys.__dict__
1936	#sys.exit(0)
1937
1938	# create false {abs_}href properties, to trick Notify()
1939	#
1940	if not (self.VERSION in [0.11, 0.12]):
1941	self.env.abs_href = Href(self.get_config('project', 'url'))
1942	self.env.href = Href(self.get_config('project', 'url'))
1943
1944
1945	tn = TicketNotifyEmail(self.env)
1946
1947	if self.parameters.alternate_notify_template:
1948
1949	if self.VERSION >= 0.11:
1950
1951	from trac.web.chrome import Chrome
1952
1953	if self.parameters.alternate_notify_template_update and not new:
1954	tn.template_name = self.parameters.alternate_notify_template_update
1955	else:
1956	tn.template_name = self.parameters.alternate_notify_template
1957
1958	tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
1959
1960	else:
1961
1962	tn.template_name = self.parameters.alternate_notify_template
1963
1964	tn.notify(tkt, new, modtime)
1965
1966	except Exception, e:
1967	self.logger.error('Failure sending notification on creation of ticket #%s: %s' %(self.id, e))
1968
1969
1970
1971	########## Parse Config File ###########################################################
1972
1973	def ReadConfig(file, name):
1974	"""
1975	Parse the config file
1976	"""
1977	if not os.path.isfile(file):
1978	print 'File %s does not exist' %file
1979	sys.exit(1)
1980
1981	config = trac_config.Configuration(file)
1982
1983	# Use given project name else use defaults
1984	#
1985	if name:
1986	sections = config.sections()
1987	if not name in sections:
1988	print "Not a valid project name: %s" %name
1989	print "Valid names: %s" %sections
1990	sys.exit(1)
1991
1992	project = SaraDict()
1993	for option, value in config.options(name):
1994	try:
1995	project[option] = int(value)
1996	except ValueError:
1997	project[option] = value
1998
1999	else:
2000	# use some trac internals to get the defaults
2001	#
2002	tmp = config.parser.defaults()
2003	project = SaraDict()
2004
2005	for option, value in tmp.items():
2006	try:
2007	project[option] = int(value)
2008	except ValueError:
2009	project[option] = value
2010
2011	return project
2012
2013	########## Setup Logging ###############################################################
2014
2015	def setup_log(parameters, project_name, interactive=None):
2016	"""
2017	Setup loging
2018
2019	Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
2020	would be interpreted by the ConfigParser itself.
2021	"""
2022	logger = logging.getLogger('email2trac %s' %project_name)
2023
2024	if interactive:
2025	parameters.log_type = 'stderr'
2026
2027	if not parameters.log_type:
2028	parameters.log_type = 'syslog'
2029
2030	if parameters.log_type == 'file':
2031
2032	if not parameters.log_file:
2033	parameters.log_file = 'email2trac.log'
2034
2035	if not os.path.isabs(parameters.log_file):
2036	import tempfile
2037	parameters.log_file = os.path.join(tempfile.gettempdir(), parameters.log_file)
2038
2039	log_handler = logging.FileHandler(parameters.log_file)
2040
2041	elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
2042	# Requires win32 extensions
2043	log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
2044
2045	elif parameters.log_type in ('syslog', 'unix'):
2046	log_handler = logging.handlers.SysLogHandler('/dev/log')
2047
2048	elif parameters.log_type in ('stderr'):
2049	log_handler = logging.StreamHandler(sys.stderr)
2050
2051	else:
2052	log_handler = logging.handlers.BufferingHandler(0)
2053
2054	if parameters.log_format:
2055	parameters.log_format = parameters.log_format.replace('$(', '%(')
2056	else:
2057	parameters.log_format = '%(name)s: %(message)s'
2058
2059	log_formatter = logging.Formatter(parameters.log_format)
2060	log_handler.setFormatter(log_formatter)
2061	logger.addHandler(log_handler)
2062
2063	if (parameters.log_level in ['DEBUG', 'ALL']) or (parameters.debug > 0):
2064	logger.setLevel(logging.DEBUG)
2065
2066	elif parameters.log_level in ['INFO'] or parameters.verbose:
2067	logger.setLevel(logging.INFO)
2068
2069	elif parameters.log_level in ['WARNING']:
2070	logger.setLevel(logging.WARNING)
2071
2072	elif parameters.log_level in ['ERROR']:
2073	logger.setLevel(logging.ERROR)
2074
2075	elif parameters.log_level in ['CRITICAL']:
2076	logger.setLevel(logging.CRITICAL)
2077
2078	else:
2079	logger.setLevel(logging.INFO)
2080
2081	return logger
2082
2083
2084	if __name__ == '__main__':
2085	# Default config file
2086	#
2087	configfile = '@email2trac_conf@'
2088	project = ''
2089	component = ''
2090	ticket_prefix = 'default'
2091	dry_run = None
2092	verbose = None
2093	debug_interactive = None
2094
2095	SHORT_OPT = 'cdhf:np:t:v'
2096	LONG_OPT = ['component=', 'debug', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=', 'verbose']
2097
2098	try:
2099	opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
2100	except getopt.error,detail:
2101	print __doc__
2102	print detail
2103	sys.exit(1)
2104
2105	project_name = None
2106	for opt,value in opts:
2107	if opt in [ '-h', '--help']:
2108	print __doc__
2109	sys.exit(0)
2110	elif opt in ['-c', '--component']:
2111	component = value
2112	elif opt in ['-d', '--debug']:
2113	debug_interactive = 1
2114	elif opt in ['-f', '--file']:
2115	configfile = value
2116	elif opt in ['-n', '--dry-run']:
2117	dry_run = True
2118	elif opt in ['-p', '--project']:
2119	project_name = value
2120	elif opt in ['-t', '--ticket_prefix']:
2121	ticket_prefix = value
2122	elif opt in ['-v', '--verbose']:
2123	verbose = True
2124
2125	settings = ReadConfig(configfile, project_name)
2126
2127	# The default prefix for ticket values in email2trac.conf
2128	#
2129	settings.ticket_prefix = ticket_prefix
2130	settings.dry_run = dry_run
2131	settings.verbose = verbose
2132
2133	if not settings.debug and debug_interactive:
2134	settings.debug = debug_interactive
2135
2136	if not settings.project:
2137	print __doc__
2138	print 'No Trac project is defined in the email2trac config file.'
2139	sys.exit(1)
2140
2141	logger = setup_log(settings, os.path.basename(settings.project), debug_interactive)
2142
2143	if component:
2144	settings['component'] = component
2145
2146	# Determine major trac version used to be in email2trac.conf
2147	# Quick hack for 0.12
2148	#
2149	version = '0.%s' %(trac_version.split('.')[1])
2150	if version.startswith('0.12'):
2151	version = '0.12'
2152
2153	logger.debug("Found trac version: %s" %(version))
2154
2155	try:
2156	if version == '0.10':
2157	from trac import attachment
2158	from trac.env import Environment
2159	from trac.ticket import Ticket
2160	from trac.web.href import Href
2161	from trac import util
2162	#
2163	# return util.text.to_unicode(str)
2164	#
2165	# see http://projects.edgewall.com/trac/changeset/2799
2166	from trac.ticket.notification import TicketNotifyEmail
2167	from trac import config as trac_config
2168	from trac.core import TracError
2169
2170	elif version == '0.11':
2171	from trac import attachment
2172	from trac.env import Environment
2173	from trac.ticket import Ticket
2174	from trac.web.href import Href
2175	from trac import config as trac_config
2176	from trac import util
2177	from trac.core import TracError
2178	from trac.perm import PermissionSystem
2179
2180	#
2181	# return util.text.to_unicode(str)
2182	#
2183	# see http://projects.edgewall.com/trac/changeset/2799
2184	from trac.ticket.notification import TicketNotifyEmail
2185
2186	elif version == '0.12':
2187	from trac import attachment
2188	from trac.env import Environment
2189	from trac.ticket import Ticket
2190	from trac.web.href import Href
2191	from trac import config as trac_config
2192	from trac import util
2193	from trac.core import TracError
2194	from trac.perm import PermissionSystem
2195
2196	#
2197	# return util.text.to_unicode(str)
2198	#
2199	# see http://projects.edgewall.com/trac/changeset/2799
2200	from trac.ticket.notification import TicketNotifyEmail
2201
2202
2203	else:
2204	logger.error('TRAC version %s is not supported' %version)
2205	sys.exit(1)
2206
2207	# Must be set before environment is created
2208	#
2209	if settings.has_key('python_egg_cache'):
2210	python_egg_cache = str(settings['python_egg_cache'])
2211	os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
2212
2213	if settings.debug > 0:
2214	logger.debug('Loading environment %s', settings.project)
2215
2216	env = Environment(settings['project'], create=0)
2217
2218	tktparser = TicketEmailParser(env, settings, logger, float(version))
2219	tktparser.parse(sys.stdin)
2220
2221	# Catch all errors ans log to SYSLOG if we have enabled this
2222	# else stdout
2223	#
2224	except Exception, error:
2225
2226	etype, evalue, etb = sys.exc_info()
2227	for e in traceback.format_exception(etype, evalue, etb):
2228	logger.critical(e)
2229
2230	if m:
2231	tktparser.save_email_for_debug(m, True)
2232
2233	sys.exit(1)
2234	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format