Context Navigation

source: trunk/email2trac.py.in @ 486

Last change on this file since 486 was 486, checked in by bas, 14 years ago
Fixed the calculation of number of ticket changes, closes #223
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 59.8 KB

Line
1	#!@PYTHON@
2	# Copyright (C) 2002
3	#
4	# This file is part of the email2trac utils
5	#
6	# This program is free software; you can redistribute it and/or modify it
7	# under the terms of the GNU General Public License as published by the
8	# Free Software Foundation; either version 2, or (at your option) any
9	# later version.
10	#
11	# This program is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with this program; if not, write to the Free Software
18	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
19	#
20	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
21	#
22	"""
23	email2trac.py -- Email tickets to Trac.
24
25	A simple MTA filter to create Trac tickets from inbound emails.
26
27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28	Copyright 2005, Edgewall Software
29
30	Authors:
31	Bas van der Vlies <basv@sara.nl>
32	Walter de Jong <walter@sara.nl>
33
34	The scripts reads emails from stdin and inserts directly into a Trac database.
35
36	How to use
37	----------
38	* See https://subtrac.sara.nl/oss/email2trac/
39
40	* Create an config file:
41	[DEFAULT] # REQUIRED
42	project : /data/trac/test # REQUIRED
43	debug : 1 # OPTIONAL, if set print some DEBUG info
44
45	[jouvin] # OPTIONAL project declaration, if set both fields necessary
46	project : /data/trac/jouvin # use -p\|--project jouvin.
47
48	* default config file is : /etc/email2trac.conf
49
50	* Commandline opions:
51	-h,--help
52	-d, --debug
53	-f,--file <configuration file>
54	-n,--dry-run
55	-p, --project <project name>
56	-t, --ticket_prefix <name>
57
58	SVN Info:
59	$Id: email2trac.py.in 486 2010-10-11 09:07:31Z bas $
60	"""
61	import os
62	import sys
63	import string
64	import getopt
65	import time
66	import email
67	import email.Iterators
68	import email.Header
69	import re
70	import urllib
71	import unicodedata
72	import mimetypes
73	import traceback
74	import logging
75	import logging.handlers
76	import UserDict
77
78	from datetime import tzinfo, timedelta, datetime
79	from stat import *
80
81
82	from trac import __version__ as trac_version
83	from trac import config as trac_config
84
85	# Some global variables
86	#
87	m = None
88
89	class SaraDict(UserDict.UserDict):
90	def __init__(self, dictin = None):
91	UserDict.UserDict.__init__(self)
92	self.name = None
93
94	if dictin:
95	if dictin.has_key('name'):
96	self.name = dictin['name']
97	del dictin['name']
98	self.data = dictin
99
100	def get_value(self, name):
101	if self.has_key(name):
102	return self[name]
103	else:
104	return None
105
106	def __repr__(self):
107	return repr(self.data)
108
109	def __str__(self):
110	return str(self.data)
111
112	def __getattr__(self, name):
113	"""
114	override the class attribute get method. Return the value
115	from the dictionary
116	"""
117	if self.data.has_key(name):
118	return self.data[name]
119	else:
120	return None
121
122	def __setattr__(self, name, value):
123	"""
124	override the class attribute set method only when the UserDict
125	has set its class attribute
126	"""
127	if self.__dict__.has_key('data'):
128	self.data[name] = value
129	else:
130	self.__dict__[name] = value
131
132	def __iter__(self):
133	return iter(self.data.keys())
134
135	class TicketEmailParser(object):
136	env = None
137	comment = '> '
138
139	def __init__(self, env, parameters, logger, version):
140	self.env = env
141
142	# Database connection
143	#
144	self.db = None
145
146	# Save parameters
147	#
148	self.parameters = parameters
149	self.logger = logger
150
151	# Some useful mail constants
152	#
153	self.email_name = None
154	self.email_addr = None
155	self.email_from = None
156	self.author = None
157	self.id = None
158
159	self.STRIP_CONTENT_TYPES = list()
160
161	## fields properties via body_text
162	#
163	self.properties = dict()
164
165	self.VERSION = version
166
167	self.get_config = self.env.config.get
168
169	## init function ##
170	#
171	self.setup_parameters()
172
173	def setup_parameters(self):
174	if self.parameters.umask:
175	os.umask(self.parameters.umask)
176
177	if not self.parameters.spam_level:
178	self.parameters.spam_level = 0
179
180	if not self.parameters.spam_header:
181	self.parameters.spam_header = 'X-Spam-Score'
182
183	if not self.parameters.email_quote:
184	self.parameters.email_quote = '> '
185
186	if not self.parameters.ticket_update_by_subject_lookback:
187	self.parameters.ticket_update_by_subject_lookback = 30
188
189	if self.parameters.verbatim_format == None:
190	self.parameters.verbatim_format = 1
191
192	if self.parameters.reflow == None:
193	self.parameters.reflow = 1
194
195	if self.parameters.binhex:
196	self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
197
198	if self.parameters.applesingle:
199	self.STRIP_CONTENT_TYPES.append('application/applefile')
200
201	if self.parameters.appledouble:
202	self.STRIP_CONTENT_TYPES.append('application/applefile')
203
204	if self.parameters.strip_content_types:
205	items = self.parameters.strip_content_types.split(',')
206	for item in items:
207	self.STRIP_CONTENT_TYPES.append(item.strip())
208
209	if self.parameters.tmpdir:
210	self.parameters.tmpdir = os.path.normcase(str(self.parameters['tmpdir']))
211	else:
212	self.parameters.tmpdir = os.path.normcase('/tmp')
213
214	if self.parameters.email_triggers_workflow == None:
215	self.parameters.email_triggers_workflow = 1
216
217	if not self.parameters.subject_field_separator:
218	self.parameters.subject_field_separator = '&'
219	else:
220	self.parameters.subject_field_separator = self.parameters.subject_field_separator.strip()
221
222	self.trac_smtp_from = self.get_config('notification', 'smtp_from')
223	self.smtp_default_domain = self.get_config('notification', 'smtp_default_domain')
224
225
226	self.system = None
227
228	########## Email Header Functions ###########################################################
229
230	def spam(self, message):
231	"""
232	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
233	# Note if Spam_level then '*' are included
234	"""
235	spam = False
236	if message.has_key(self.parameters.spam_header):
237	spam_l = string.split(message[self.parameters.spam_header])
238
239	try:
240	number = spam_l[0].count('*')
241	except IndexError, detail:
242	number = 0
243
244	if number >= self.parameters.spam_level:
245	spam = True
246
247	# treat virus mails as spam
248	#
249	elif message.has_key('X-Virus-found'):
250	spam = True
251
252	# How to handle SPAM messages
253	#
254	if self.parameters.drop_spam and spam:
255
256	self.logger.info('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' %self.parameters.spam_level)
257	return 'drop'
258
259	elif spam:
260
261	return 'Spam'
262	else:
263
264	return False
265
266	def email_header_acl(self, keyword, header_field, default):
267	"""
268	This function wil check if the email address is allowed or denied
269	to send mail to the ticket list
270	"""
271	self.logger.debug('function email_header_acl: %s' %keyword)
272
273	try:
274	mail_addresses = self.parameters[keyword]
275
276	# Check if we have an empty string
277	#
278	if not mail_addresses:
279	return default
280
281	except KeyError, detail:
282	self.logger.debug('%s not defined, all messages are allowed.' %(keyword))
283
284	return default
285
286	mail_addresses = string.split(mail_addresses, ',')
287
288	for entry in mail_addresses:
289	entry = entry.strip()
290	TO_RE = re.compile(entry, re.VERBOSE\|re.IGNORECASE)
291	result = TO_RE.search(header_field)
292	if result:
293	return True
294
295	return False
296
297	def email_header_txt(self, m):
298	"""
299	Display To and CC addresses in description field
300	"""
301	s = ''
302
303	if m['To'] and len(m['To']) > 0:
304	s = "'''To:''' %s\r\n" %(m['To'])
305	if m['Cc'] and len(m['Cc']) > 0:
306	s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
307
308	return self.email_to_unicode(s)
309
310
311	def get_sender_info(self, message):
312	"""
313	Get the default author name and email address from the message
314	"""
315
316	self.email_to = self.email_to_unicode(message['to'])
317	self.to_name, self.to_email_addr = email.Utils.parseaddr (self.email_to)
318
319	self.email_from = self.email_to_unicode(message['from'])
320	self.email_name, self.email_addr = email.Utils.parseaddr(self.email_from)
321
322	## Trac can not handle author's name that contains spaces
323	#
324	if self.email_addr == self.trac_smtp_from:
325	if self.email_name:
326	self.author = self.email_name
327	else:
328	self.author = "email2trac"
329	else:
330	self.author = self.email_addr
331
332	if self.parameters.ignore_trac_user_settings:
333	return
334
335	# Is this a registered user, use email address as search key:
336	# result:
337	# u : login name
338	# n : Name that the user has set in the settings tab
339	# e : email address that the user has set in the settings tab
340	#
341	users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
342	if (
343	(e and (e.lower() == self.email_addr.lower())) or
344	(u + '@' + self.smtp_default_domain.lower() == self.email_addr.lower())
345	)
346	]
347
348	if len(users) >= 1:
349	self.email_from = users[0][0]
350	self.author = users[0][0]
351
352	def set_reply_fields(self, ticket, message):
353	"""
354	Set all the right fields for a new ticket
355	"""
356	self.logger.debug('function set_reply_fields')
357
358	## Only use name or email adress
359	#ticket['reporter'] = self.email_from
360	ticket['reporter'] = self.author
361
362
363	# Put all CC-addresses in ticket CC field
364	#
365	if self.parameters.reply_all:
366
367	email_cc = ''
368
369	cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
370
371	if not cc_addrs:
372	return
373
374	## Build a list of forbidden CC addresses
375	#
376	#to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
377	#to_list = list()
378	#for n,e in to_addrs:
379	# to_list.append(e)
380
381	# Always Remove reporter email address from cc-list
382	#
383	try:
384	cc_addrs.remove((self.author, self.email_addr))
385	except ValueError, detail:
386	pass
387
388	for name,addr in cc_addrs:
389
390	## Prevent mail loop
391	#
392	#if addr in to_list:
393
394	if addr == self.trac_smtp_from:
395	self.logger.debug("Skipping %s mail address for CC-field" %(addr))
396	continue
397
398	if email_cc:
399	email_cc = '%s, %s' %(email_cc, addr)
400	else:
401	email_cc = addr
402
403	if email_cc:
404	self.logger.debug('set_reply_fields: %s' %email_cc)
405
406	ticket['cc'] = self.email_to_unicode(email_cc)
407
408
409	########## DEBUG functions ###########################################################
410
411	def debug_body(self, message_body, tempfile=False):
412	if tempfile:
413	import tempfile
414	body_file = tempfile.mktemp('.email2trac')
415	else:
416	body_file = os.path.join(self.parameters.tmpdir, 'body.txt')
417
418	if self.parameters.dry_run:
419	print 'DRY-RUN: not saving body to %s' %(body_file)
420	return
421
422	print 'writing body to %s' %(body_file)
423	fx = open(body_file, 'wb')
424	if not message_body:
425	message_body = '(None)'
426
427	message_body = message_body.encode('utf-8')
428	#message_body = unicode(message_body, 'iso-8859-15')
429
430	fx.write(message_body)
431	fx.close()
432	try:
433	os.chmod(body_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
434	except OSError:
435	pass
436
437	def debug_attachments(self, message_parts):
438	"""
439	"""
440	self.logger.debug('function debug_attachments')
441
442	n = 0
443	for item in message_parts:
444	# Skip inline text parts
445	if not isinstance(item, tuple):
446	continue
447
448	(original, filename, part) = item
449
450	n = n + 1
451	print 'part%d: Content-Type: %s' % (n, part.get_content_type())
452
453	s = 'part%d: filename: %s' %(n, filename)
454	self.print_unicode(s)
455
456	## Forbidden chars
457	#
458	filename = filename.replace('\\', '_')
459	filename = filename.replace('/', '_')
460
461
462	part_file = os.path.join(self.parameters.tmpdir, filename)
463	s = 'writing part%d (%s)' % (n,part_file)
464	self.print_unicode(s)
465
466	if self.parameters.dry_run:
467	print 'DRY_RUN: NOT saving attachments'
468	continue
469
470	part_file = util.text.unicode_quote(part_file)
471
472	fx = open(part_file, 'wb')
473	text = part.get_payload(decode=1)
474
475	if not text:
476	text = '(None)'
477
478	fx.write(text)
479	fx.close()
480
481	try:
482	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
483	except OSError:
484	pass
485
486	def save_email_for_debug(self, message, create_tempfile=False):
487
488	if create_tempfile:
489	import tempfile
490	msg_file = tempfile.mktemp('.email2trac')
491	else:
492	#msg_file = '/var/tmp/msg.txt'
493	msg_file = os.path.join(self.parameters.tmpdir, 'msg.txt')
494
495	if self.parameters.dry_run:
496	print 'DRY_RUN: NOT saving email message to %s' %(msg_file)
497	else:
498	print 'saving email to %s' %(msg_file)
499
500	fx = open(msg_file, 'wb')
501	fx.write('%s' % message)
502	fx.close()
503
504	try:
505	os.chmod(msg_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
506	except OSError:
507	pass
508
509	message_parts = self.get_message_parts(message)
510	message_parts = self.unique_attachment_names(message_parts)
511	body_text = self.body_text(message_parts)
512	self.debug_body(body_text, True)
513	self.debug_attachments(message_parts)
514
515	########## Conversion functions ###########################################################
516
517	def email_to_unicode(self, message_str):
518	"""
519	Email has 7 bit ASCII code, convert it to unicode with the charset
520	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
521	understands it.
522	"""
523	self.logger.debug("function email_to_unicode")
524
525	results = email.Header.decode_header(message_str)
526
527	s = None
528	for text,format in results:
529	if format:
530	try:
531	temp = unicode(text, format)
532	except UnicodeError, detail:
533	# This always works
534	#
535	temp = unicode(text, 'iso-8859-15')
536	except LookupError, detail:
537	#text = 'ERROR: Could not find charset: %s, please install' %format
538	#temp = unicode(text, 'iso-8859-15')
539	temp = message_str
540
541	else:
542	temp = string.strip(text)
543	temp = unicode(text, 'iso-8859-15')
544
545	if s:
546	s = '%s %s' %(s, temp)
547	else:
548	s = '%s' %temp
549
550	#s = s.encode('utf-8')
551	return s
552
553	def str_to_dict(self, s):
554	"""
555	Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
556	"""
557	self.logger.debug("function str_to_dict")
558
559	fields = string.split(s, self.parameters.subject_field_separator)
560
561	result = dict()
562	for field in fields:
563	try:
564	index, value = string.split(field, '=')
565
566	# We can not change the description of a ticket via the subject
567	# line. The description is the body of the email
568	#
569	if index.lower() in ['description']:
570	continue
571
572	if value:
573	result[index.lower()] = value
574
575	except ValueError:
576	pass
577	return result
578
579	def print_unicode(self,s):
580	"""
581	This function prints unicode strings uif possible else it will quote it
582	"""
583	try:
584	self.logger.debug(s)
585	except UnicodeEncodeError, detail:
586	self.logger.debug(util.text.unicode_quote(s))
587
588	########## TRAC ticket functions ###########################################################
589
590	def check_permission_participants(self, tkt, action):
591	"""
592	Check if the mailer is allowed to update the ticket
593	"""
594	self.logger.debug('function check_permission_participants')
595
596	if tkt['reporter'].lower() in [self.author, self.email_addr]:
597	self.logger.debug('ALLOW, %s is the ticket reporter' %(self.email_addr))
598
599	return True
600
601	perm = PermissionSystem(self.env)
602	if perm.check_permission(action, self.author):
603	self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
604
605	return True
606
607	# Is the updater in the CC?
608	try:
609	cc_list = tkt['cc'].split(',')
610	for cc in cc_list:
611	if self.email_addr.lower() in cc.strip():
612	self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
613
614	return True
615
616	except KeyError:
617	pass
618
619	return False
620
621	def check_permission(self, tkt, action):
622	"""
623	check if the reporter has the right permission for the action:
624	- TICKET_CREATE
625	- TICKET_MODIFY
626	- TICKET_APPEND
627	- TICKET_CHGPROP
628
629	There are three models:
630	- None : no checking at all
631	- trac : check the permission via trac permission model
632	- email2trac: ....
633	"""
634	self.logger.debug("function check_permission")
635
636	if self.parameters.ticket_permission_system in ['trac']:
637
638	perm = PermissionSystem(self.env)
639	if perm.check_permission(action, self.author):
640	return True
641	else:
642	return False
643
644	elif self.parameters.ticket_permission_system in ['update_restricted_to_participants']:
645	return (self.check_permission_participants(tkt, action))
646
647	## Default is to allow everybody ticket updates and ticket creation
648	#
649	else:
650	return True
651
652
653	def update_ticket_fields(self, ticket, user_dict, new=None):
654	"""
655	This will update the ticket fields. It will check if the
656	given fields are known and if the right values are specified
657	It will only update the ticket field value:
658	- If the field is known
659	- If the value supplied is valid for the ticket field.
660	If not then there are two options:
661	1) Skip the value (new_ticket=None)
662	2) Set default value for field (new_ticket=1)
663	"""
664	self.logger.debug("function update_ticket_fields")
665
666	## Check only permission model on ticket updates
667	#
668	if not new:
669	if self.parameters.ticket_permission_system:
670	if not self.check_permission(ticket, 'TICKET_CHGPROP'):
671	self.logger.info('Reporter: %s has no permission to change ticket properties' %self.author)
672	return False
673
674	## Build a system dictionary from the ticket fields
675	# with field as index and option as value
676	#
677	sys_dict = dict()
678	for field in ticket.fields:
679	try:
680	sys_dict[field['name']] = field['options']
681
682	except KeyError:
683	sys_dict[field['name']] = None
684	pass
685
686	## Check user supplied fields an compare them with the
687	# system one's
688	#
689	for field,value in user_dict.items():
690	if self.parameters.debug:
691	s = 'user_field\t %s = %s' %(field,value)
692	self.print_unicode(s)
693
694	## To prevent mail loop
695	#
696	if field == 'cc':
697
698	cc_list = user_dict['cc'].split(',')
699
700	if self.trac_smtp_from in cc_list:
701	self.logger.debug('MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
702
703	cc_list.remove(self.trac_smtp_from)
704
705	value = ','.join(cc_list)
706
707
708	if sys_dict.has_key(field):
709
710	## Check if value is an allowed system option, if TypeError then
711	# every value is allowed
712	#
713	try:
714	if value in sys_dict[field]:
715	ticket[field] = value
716	else:
717	## Must we set a default if value is not allowed
718	#
719	if new:
720	value = self.get_config('ticket', 'default_%s' %(field) )
721
722	except TypeError:
723	pass
724
725	## Only set if we have a value
726	#
727	if value:
728	ticket[field] = value
729
730	if self.parameters.debug:
731	s = 'ticket_field\t %s = %s' %(field, ticket[field])
732	self.print_unicode(s)
733
734	def ticket_update(self, m, id, spam):
735	"""
736	If the current email is a reply to an existing ticket, this function
737	will append the contents of this email to that ticket, instead of
738	creating a new one.
739	"""
740	self.logger.debug("function ticket_update")
741
742	if not self.parameters.ticket_update:
743	self.logger.debug("ticket_update disabled")
744	return False
745
746	## Must we update ticket fields
747	#
748	update_fields = dict()
749	try:
750	id, keywords = string.split(id, '?')
751
752	update_fields = self.str_to_dict(keywords)
753
754	## Strip '#'
755	#
756	self.id = int(id[1:])
757
758	except ValueError:
759
760	## Strip '#'
761	#
762	self.id = int(id[1:])
763
764	self.logger.debug("ticket_update id %s" %id)
765
766	## When is the change committed
767	#
768	if self.VERSION < 0.11:
769	when = int(time.time())
770	else:
771	when = datetime.now(util.datefmt.utc)
772
773	try:
774	tkt = Ticket(self.env, self.id, self.db)
775
776	except util.TracError, detail:
777
778	## Not a valid ticket
779	#
780	self.id = None
781	return False
782
783	## Check the permission of the reporter
784	#
785	if self.parameters.ticket_permission_system:
786	if not self.check_permission(tkt, 'TICKET_APPEND'):
787	self.logger.info('Reporter: %s has no permission to add comments or attachments to tickets' %self.author)
788	return False
789
790	## How many changes has this ticket
791	#
792	# cnum = len(tkt.get_changelog())
793	grouped = TicketModule(self.env).grouped_changelog_entries(tkt, self.db)
794	cnum = sum(1 for e in grouped) + 1
795
796
797	## reopen the ticket if it is was closed
798	# We must use the ticket workflow framework
799	#
800	if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
801
802	self.logger.debug('Workflow ticket update fields: ')
803
804	from trac.ticket.default_workflow import ConfigurableTicketWorkflow
805	from trac.test import Mock, MockPerm
806
807	req = Mock(authname=self.author, perm=MockPerm(), args={})
808	try:
809	workflow = self.parameters['workflow_%s' %tkt['status']]
810	except KeyError:
811	## fallback for compability (Will be deprecated)
812	#
813	if tkt['status'] in ['closed']:
814	workflow = self.parameters.workflow
815	else:
816	workflow = None
817
818	controller = ConfigurableTicketWorkflow(self.env)
819	#print controller.actions
820	#print controller.actions.keys()
821	#print controller.get_ticket_actions(req, tkt)
822	#print controller.actions[workflow]
823	#print controller.actions[workflow]['permissions'] is a list
824
825	if workflow:
826
827	if self.parameters.ticket_permission_system:
828
829	if self.check_permission(tkt, controller.actions[workflow]['permissions'][0]):
830	fields = controller.get_ticket_changes(req, tkt, workflow)
831	else:
832	fields = dict()
833	self.logger.info('Reporter: %s has no permission to trigger workflow' %self.author)
834
835	else:
836	fields = controller.get_ticket_changes(req, tkt, workflow)
837
838	for key in fields.keys():
839	self.logger.debug('\t %s : %s' %(key, fields[key]))
840	tkt[key] = fields[key]
841
842	## Old pre 0.11 situation
843	#
844	elif self.parameters.email_triggers_workflow:
845
846	self.logger.debug('email triggers workflow pre trac 0.11')
847
848	if tkt['status'] in ['closed']:
849	tkt['status'] = 'reopened'
850	tkt['resolution'] = ''
851
852	else:
853	self.logger.debug('email triggers workflow disabled')
854
855	## Must we update some ticket fields properties via subjectline
856	#
857	if update_fields:
858	self.update_ticket_fields(tkt, update_fields)
859
860	message_parts = self.get_message_parts(m)
861	message_parts = self.unique_attachment_names(message_parts)
862
863	## Must we update some ticket fields properties via body_text
864	#
865	if self.properties:
866	self.update_ticket_fields(tkt, self.properties)
867
868	if self.parameters.email_header:
869	message_parts.insert(0, self.email_header_txt(m))
870
871	body_text = self.body_text(message_parts)
872
873	error_with_attachments = self.attach_attachments(message_parts)
874
875	if body_text.strip() or update_fields or self.properties:
876	if self.parameters.dry_run:
877	print 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) ', self.author, cnum
878	else:
879	if error_with_attachments:
880	body_text = '%s\\%s' %(error_with_attachments, body_text)
881	self.logger.debug('tkt.save_changes(%s, %d)' %(self.author, cnum))
882	tkt.save_changes(self.author, body_text, when, None, str(cnum))
883
884
885	if not spam:
886	self.notify(tkt, False, when)
887
888	return True
889
890	def set_ticket_fields(self, ticket):
891	"""
892	set the ticket fields to value specified
893	- /etc/email2trac.conf with <prefix>_<field>
894	- trac default values, trac.ini
895	"""
896	self.logger.debug('function set_ticket_fields')
897
898	user_dict = dict()
899
900	for field in ticket.fields:
901
902	name = field['name']
903
904	## default trac value
905	#
906	if not field.get('custom'):
907	value = self.get_config('ticket', 'default_%s' %(name) )
908	if (name in ['resolution']) and (value in ['fixed']):
909	value = None
910	else:
911	## Else we get the default value for reporter
912	#
913	value = field.get('value')
914	options = field.get('options')
915
916	if value and options and (value not in options):
917	value = options[int(value)]
918
919	if self.parameters.debug:
920	s = 'trac[%s] = %s' %(name, value)
921	self.print_unicode(s)
922
923	## email2trac.conf settings
924	#
925	prefix = self.parameters.ticket_prefix
926	try:
927	value = self.parameters['%s_%s' %(prefix, name)]
928	if self.parameters.debug:
929	s = 'email2trac[%s] = %s ' %(name, value)
930	self.print_unicode(s)
931
932	except KeyError, detail:
933	pass
934
935	if value:
936	user_dict[name] = value
937	if self.parameters.debug:
938	s = 'used %s = %s' %(name, value)
939	self.print_unicode(s)
940
941	self.update_ticket_fields(ticket, user_dict, new=1)
942
943	if 'status' not in user_dict.keys():
944	ticket['status'] = 'new'
945
946
947	def ticket_update_by_subject(self, subject):
948	"""
949	This list of Re: prefixes is probably incomplete. Taken from
950	wikipedia. Here is how the subject is matched
951	- Re: <subject>
952	- Re: (<Mail list label>:)+ <subject>
953
954	So we must have the last column
955	"""
956	self.logger.debug('function ticket_update_by_subject')
957
958	matched_id = None
959	if self.parameters.ticket_update and self.parameters.ticket_update_by_subject:
960
961	SUBJECT_RE = re.compile(r'^(RE\|AW\|VS\|SV):(.:)\s(.)', re.IGNORECASE)
962	result = SUBJECT_RE.search(subject)
963
964	if result:
965	## This is a reply
966	#
967	orig_subject = result.group(3)
968
969	self.logger.debug('subject search string: %s' %(orig_subject))
970
971	cursor = self.db.cursor()
972	summaries = [orig_subject, '%%: %s' % orig_subject]
973
974	## Convert days to seconds
975	#
976	lookback = int(time.mktime(time.gmtime())) - \
977	self.parameters.ticket_update_by_subject_lookback * 24 * 3600
978
979
980	for summary in summaries:
981	self.logger.debug('Looking for summary matching: "%s"' % summary)
982
983	sql = """SELECT id FROM ticket
984	WHERE changetime >= %s AND summary LIKE %s
985	ORDER BY changetime DESC"""
986	cursor.execute(sql, [lookback, summary.strip()])
987
988	for row in cursor:
989	(matched_id,) = row
990
991	self.logger.debug('Found matching ticket id: %d' % matched_id)
992
993	break
994
995	if matched_id:
996	matched_id = '#%d' % matched_id
997	return matched_id
998
999	return matched_id
1000
1001
1002	def new_ticket(self, msg, subject, spam, set_fields = None):
1003	"""
1004	Create a new ticket
1005	"""
1006	self.logger.debug('function new_ticket')
1007
1008	tkt = Ticket(self.env)
1009
1010	self.set_reply_fields(tkt, msg)
1011
1012	self.set_ticket_fields(tkt)
1013
1014	## Check the permission of the reporter
1015	#
1016	if self.parameters.ticket_permission_system:
1017	if not self.check_permission(tkt, 'TICKET_CREATE'):
1018	self.logger.info('Reporter: %s has no permission to create tickets' %self.author)
1019	return False
1020
1021	## Old style setting for component, will be removed
1022	#
1023	if spam:
1024	tkt['component'] = 'Spam'
1025
1026	elif self.parameters.has_key('component'):
1027	tkt['component'] = self.parameters['component']
1028
1029	if not msg['Subject']:
1030	tkt['summary'] = u'(No subject)'
1031	else:
1032	tkt['summary'] = subject
1033
1034
1035	if set_fields:
1036	rest, keywords = string.split(set_fields, '?')
1037
1038	if keywords:
1039	update_fields = self.str_to_dict(keywords)
1040	self.update_ticket_fields(tkt, update_fields)
1041
1042
1043	message_parts = self.get_message_parts(msg)
1044
1045	## Must we update some ticket fields properties via body_text
1046	#
1047	if self.properties:
1048	self.update_ticket_fields(tkt, self.properties)
1049
1050	message_parts = self.unique_attachment_names(message_parts)
1051
1052	## produce e-mail like header
1053	#
1054	head = ''
1055	if self.parameters.email_header:
1056	head = self.email_header_txt(msg)
1057	message_parts.insert(0, head)
1058
1059	body_text = self.body_text(message_parts)
1060
1061	tkt['description'] = body_text
1062
1063	## When is the change committed
1064	#
1065	if self.VERSION < 0.11:
1066	when = int(time.time())
1067	else:
1068	when = datetime.now(util.datefmt.utc)
1069
1070	if self.parameters.dry_run:
1071	print 'DRY_RUN: tkt.insert()'
1072	else:
1073	self.id = tkt.insert()
1074
1075	changed = False
1076	comment = ''
1077
1078	## some routines in trac are dependend on ticket id
1079	# like alternate notify template
1080	#
1081	if self.parameters.alternate_notify_template:
1082	tkt['id'] = self.id
1083	changed = True
1084
1085	## Rewrite the description if we have mailto enabled
1086	#
1087	if self.parameters.mailto_link:
1088	changed = True
1089	comment = u'\nadded mailto line\n'
1090	mailto = self.html_mailto_link( m['Subject'])
1091
1092	tkt['description'] = u'%s\r\n%s%s\r\n' \
1093	%(head, mailto, body_text)
1094
1095	## Save the attachments to the ticket
1096	#
1097	error_with_attachments = self.attach_attachments(message_parts)
1098
1099	if error_with_attachments:
1100	changed = True
1101	comment = '%s\n%s\n' %(comment, error_with_attachments)
1102
1103	if changed:
1104	if self.parameters.dry_run:
1105	print 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
1106	else:
1107	tkt.save_changes(tkt['reporter'], comment)
1108	#print tkt.get_changelog(self.db, when)
1109
1110	if not spam:
1111	self.notify(tkt, True)
1112
1113
1114	def attach_attachments(self, message_parts, update=False):
1115	'''
1116	save any attachments as files in the ticket's directory
1117	'''
1118	self.logger.debug('function attach_attachments()')
1119
1120	if self.parameters.dry_run:
1121	print "DRY_RUN: no attachments attached to tickets"
1122	return ''
1123
1124	count = 0
1125
1126	## Get Maxium attachment size
1127	#
1128	max_size = int(self.get_config('attachment', 'max_size'))
1129	status = None
1130
1131	for item in message_parts:
1132	## Skip body parts
1133	#
1134	if not isinstance(item, tuple):
1135	continue
1136
1137	(original, filename, part) = item
1138
1139	## We have to determine the size so we use this temporary solution. we must escape it
1140	# else we get UnicodeErrors.
1141	#
1142	path, fd = util.create_unique_file(os.path.join(self.parameters.tmpdir, util.text.unicode_quote(filename)))
1143	text = part.get_payload(decode=1)
1144	if not text:
1145	text = '(None)'
1146	fd.write(text)
1147	fd.close()
1148
1149	## get the file_size
1150	#
1151	stats = os.lstat(path)
1152	file_size = stats[ST_SIZE]
1153
1154	## Check if the attachment size is allowed
1155	#
1156	if (max_size != -1) and (file_size > max_size):
1157	status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1158	%(status, original, file_size, max_size)
1159
1160	os.unlink(path)
1161	continue
1162	else:
1163	count = count + 1
1164
1165	## Insert the attachment
1166	#
1167	fd = open(path, 'rb')
1168	if self.system == 'discussion':
1169	att = attachment.Attachment(self.env, 'discussion', 'topic/%s'
1170	% (self.id,))
1171	else:
1172	self.logger.debug('Attach %s to ticket %d' %(util.text.unicode_quote(filename), self.id))
1173	att = attachment.Attachment(self.env, 'ticket', self.id)
1174
1175	## This will break the ticket_update system, the body_text is vaporized
1176	# ;-(
1177	#
1178	if not update:
1179	att.author = self.author
1180	att.description = self.email_to_unicode('Added by email2trac')
1181
1182	try:
1183	self.logger.debug('Insert atachment')
1184	att.insert(filename, fd, file_size)
1185	except OSError, detail:
1186	self.logger.info('%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail))
1187	status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
1188
1189	## Remove the created temporary filename
1190	#
1191	fd.close()
1192	os.unlink(path)
1193
1194	## return error
1195	#
1196	return status
1197
1198	########## Fullblog functions #################################################
1199
1200	def blog(self, id):
1201	"""
1202	The blog create/update function
1203	"""
1204	## import the modules
1205	#
1206	from tracfullblog.core import FullBlogCore
1207	from tracfullblog.model import BlogPost, BlogComment
1208	from trac.test import Mock, MockPerm
1209
1210	## instantiate blog core
1211	#
1212	blog = FullBlogCore(self.env)
1213	req = Mock(authname='anonymous', perm=MockPerm(), args={})
1214
1215	if id:
1216
1217	## update blog
1218	#
1219	comment = BlogComment(self.env, id)
1220	comment.author = self.author
1221
1222	message_parts = self.get_message_parts(m)
1223	comment.comment = self.body_text(message_parts)
1224
1225	blog.create_comment(req, comment)
1226
1227	else:
1228	## create blog
1229	#
1230	import time
1231	post = BlogPost(self.env, 'blog_'+time.strftime("%Y%m%d%H%M%S", time.gmtime()))
1232
1233	#post = BlogPost(self.env, blog._get_default_postname(self.env))
1234
1235	post.author = self.author
1236	post.title = self.email_to_unicode(m['Subject'])
1237
1238	message_parts = self.get_message_parts(m)
1239	post.body = self.body_text(message_parts)
1240
1241	blog.create_post(req, post, self.author, u'Created by email2trac', False)
1242
1243
1244	########## Discussion functions ##############################################
1245
1246	def discussion_topic(self, content, subject):
1247
1248	## Import modules.
1249	#
1250	from tracdiscussion.api import DiscussionApi
1251	from trac.util.datefmt import to_timestamp, utc
1252
1253	self.logger.debug('Creating a new topic in forum:', self.id)
1254
1255	## Get dissussion API component.
1256	#
1257	api = self.env[DiscussionApi]
1258	context = self._create_context(content, subject)
1259
1260	## Get forum for new topic.
1261	#
1262	forum = api.get_forum(context, self.id)
1263
1264	if not forum:
1265	self.logger.error("ERROR: Replied forum doesn't exist")
1266
1267	## Prepare topic.
1268	#
1269	topic = {'forum' : forum['id'],
1270	'subject' : context.subject,
1271	'time': to_timestamp(datetime.now(utc)),
1272	'author' : self.author,
1273	'subscribers' : [self.email_addr],
1274	'body' : self.body_text(context.content_parts)}
1275
1276	## Add topic to DB and commit it.
1277	#
1278	self._add_topic(api, context, topic)
1279	self.db.commit()
1280
1281	def discussion_topic_reply(self, content, subject):
1282
1283	## Import modules.
1284	#
1285	from tracdiscussion.api import DiscussionApi
1286	from trac.util.datefmt import to_timestamp, utc
1287
1288	self.logger.debug('Replying to discussion topic', self.id)
1289
1290	## Get dissussion API component.
1291	#
1292	api = self.env[DiscussionApi]
1293	context = self._create_context(content, subject)
1294
1295	## Get replied topic.
1296	#
1297	topic = api.get_topic(context, self.id)
1298
1299	if not topic:
1300	self.logger.error("ERROR: Replied topic doesn't exist")
1301
1302	## Prepare message.
1303	#
1304	message = {'forum' : topic['forum'],
1305	'topic' : topic['id'],
1306	'replyto' : -1,
1307	'time' : to_timestamp(datetime.now(utc)),
1308	'author' : self.author,
1309	'body' : self.body_text(context.content_parts)}
1310
1311	## Add message to DB and commit it.
1312	#
1313	self._add_message(api, context, message)
1314	self.db.commit()
1315
1316	def discussion_message_reply(self, content, subject):
1317
1318	## Import modules.
1319	#
1320	from tracdiscussion.api import DiscussionApi
1321	from trac.util.datefmt import to_timestamp, utc
1322
1323	self.loggger.debug('Replying to discussion message', self.id)
1324
1325	## Get dissussion API component.
1326	#
1327	api = self.env[DiscussionApi]
1328	context = self._create_context(content, subject)
1329
1330	## Get replied message.
1331	#
1332	message = api.get_message(context, self.id)
1333
1334	if not message:
1335	self.logger.error("ERROR: Replied message doesn't exist")
1336
1337	## Prepare message.
1338	#
1339	message = {'forum' : message['forum'],
1340	'topic' : message['topic'],
1341	'replyto' : message['id'],
1342	'time' : to_timestamp(datetime.now(utc)),
1343	'author' : self.author,
1344	'body' : self.body_text(context.content_parts)}
1345
1346	## Add message to DB and commit it.
1347	#
1348	self._add_message(api, context, message)
1349	self.db.commit()
1350
1351	def _create_context(self, content, subject):
1352
1353	## Import modules.
1354	#
1355	from trac.mimeview import Context
1356	from trac.web.api import Request
1357	from trac.perm import PermissionCache
1358
1359	## TODO: Read server base URL from config.
1360	# Create request object to mockup context creation.
1361	#
1362	environ = {'SERVER_PORT' : 80,
1363	'SERVER_NAME' : 'test',
1364	'REQUEST_METHOD' : 'POST',
1365	'wsgi.url_scheme' : 'http',
1366	'wsgi.input' : sys.stdin}
1367	chrome = {'links': {},
1368	'scripts': [],
1369	'ctxtnav': [],
1370	'warnings': [],
1371	'notices': []}
1372
1373	if self.env.base_url_for_redirect:
1374	environ['trac.base_url'] = self.env.base_url
1375
1376	req = Request(environ, None)
1377	req.chrome = chrome
1378	req.tz = 'missing'
1379	req.authname = self.author
1380	req.perm = PermissionCache(self.env, self.author)
1381
1382	## Create and return context.
1383	#
1384	context = Context.from_request(req)
1385	context.realm = 'discussion-email2trac'
1386	context.cursor = self.db.cursor()
1387	context.content = content
1388	context.subject = subject
1389
1390	## Read content parts from content.
1391	#
1392	context.content_parts = self.get_message_parts(content)
1393	context.content_parts = self.unique_attachment_names(
1394	context.content_parts)
1395
1396	return context
1397
1398	def _add_topic(self, api, context, topic):
1399	context.req.perm.assert_permission('DISCUSSION_APPEND')
1400
1401	## Filter topic.
1402	#
1403	for discussion_filter in api.discussion_filters:
1404	accept, topic_or_error = discussion_filter.filter_topic(
1405	context, topic)
1406	if accept:
1407	topic = topic_or_error
1408	else:
1409	raise TracError(topic_or_error)
1410
1411	## Add a new topic.
1412	#
1413	api.add_topic(context, topic)
1414
1415	## Get inserted topic with new ID.
1416	#
1417	topic = api.get_topic_by_time(context, topic['time'])
1418
1419	## Attach attachments.
1420	#
1421	self.id = topic['id']
1422	self.attach_attachments(context.content_parts, True)
1423
1424	## Notify change listeners.
1425	#
1426	for listener in api.topic_change_listeners:
1427	listener.topic_created(context, topic)
1428
1429	def _add_message(self, api, context, message):
1430	context.req.perm.assert_permission('DISCUSSION_APPEND')
1431
1432	## Filter message.
1433	#
1434	for discussion_filter in api.discussion_filters:
1435	accept, message_or_error = discussion_filter.filter_message(
1436	context, message)
1437	if accept:
1438	message = message_or_error
1439	else:
1440	raise TracError(message_or_error)
1441
1442	## Add message.
1443	#
1444	api.add_message(context, message)
1445
1446	## Get inserted message with new ID.
1447	#
1448	message = api.get_message_by_time(context, message['time'])
1449
1450	## Attach attachments.
1451	#
1452	self.id = message['topic']
1453	self.attach_attachments(context.content_parts, True)
1454
1455	## Notify change listeners.
1456	#
1457	for listener in api.message_change_listeners:
1458	listener.message_created(context, message)
1459
1460	########## MAIN function ######################################################
1461
1462	def parse(self, fp):
1463	"""
1464	"""
1465	self.logger.debug('Main function parse')
1466	global m
1467
1468	m = email.message_from_file(fp)
1469
1470	if not m:
1471	self.logger.debug('This is not a valid email message format')
1472	return
1473
1474	## Work around lack of header folding in Python; see http://bugs.python.org/issue4696
1475	#
1476	try:
1477	m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
1478	except AttributeError, detail:
1479	pass
1480
1481	if self.parameters.debug: # save the entire e-mail message text
1482	self.save_email_for_debug(m, True)
1483
1484	self.db = self.env.get_db_cnx()
1485	self.get_sender_info(m)
1486
1487	if not self.email_header_acl('white_list', self.email_addr, True):
1488	self.logger.info('Message rejected : %s not in white list' %(self.email_addr))
1489	return False
1490
1491	if self.email_header_acl('black_list', self.email_addr, False):
1492	self.logger.info('Message rejected : %s in black list' %(self.email_addr))
1493	return False
1494
1495	if not self.email_header_acl('recipient_list', self.to_email_addr, True):
1496	self.logger.info('Message rejected : %s not in recipient list' %(self.to_email_addr))
1497	return False
1498
1499	## If spam drop the message
1500	#
1501	if self.spam(m) == 'drop':
1502	return False
1503
1504	elif self.spam(m) == 'spam':
1505	spam_msg = True
1506	else:
1507	spam_msg = False
1508
1509	if not m['Subject']:
1510	subject = 'No Subject'
1511	else:
1512	subject = self.email_to_unicode(m['Subject'])
1513
1514	self.logger.debug('subject: %s' %subject)
1515
1516	## [hic] #1529: Re: LRZ
1517	# [hic] #1529?owner=bas,priority=medium: Re: LRZ
1518	#
1519	ticket_regex = r'''
1520	(?P<new_fields>[#][?].*)
1521	\|(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?.*)?:)
1522	'''
1523	## Check if FullBlogPlugin is installed
1524	#
1525	blog_enabled = None
1526	blog_regex = ''
1527	if self.get_config('components', 'tracfullblog.*') in ['enabled']:
1528	blog_enabled = True
1529	blog_regex = '''\|(?P<blog>blog:(?P<blog_id>\w*))'''
1530
1531
1532	## Check if DiscussionPlugin is installed
1533	#
1534	discussion_enabled = None
1535	discussion_regex = ''
1536	if self.get_config('components', 'tracdiscussion.api.*') in ['enabled']:
1537	discussion_enabled = True
1538	discussion_regex = r'''
1539	\|(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
1540	\|(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
1541	\|(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
1542	'''
1543
1544
1545	regex_str = ticket_regex + blog_regex + discussion_regex
1546	SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
1547
1548	## Find out if this is a ticket, a blog or a discussion
1549	#
1550	result = SYSTEM_RE.search(subject)
1551
1552	if result:
1553	## update ticket + fields
1554	#
1555	if result.group('reply'):
1556	self.system = 'ticket'
1557
1558	## Skip the last ':' character
1559	#
1560	if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
1561	self.new_ticket(m, subject, spam_msg)
1562
1563	## New ticket + fields
1564	#
1565	elif result.group('new_fields'):
1566	self.system = 'ticket'
1567	self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
1568
1569	if blog_enabled:
1570	if result.group('blog'):
1571	self.system = 'blog'
1572	self.blog(result.group('blog_id'))
1573
1574	if discussion_enabled:
1575	## New topic.
1576	#
1577	if result.group('forum'):
1578	self.system = 'discussion'
1579	self.id = int(result.group('forum_id'))
1580	self.discussion_topic(m, subject[result.end('forum'):])
1581
1582	## Reply to topic.
1583	#
1584	elif result.group('topic'):
1585	self.system = 'discussion'
1586	self.id = int(result.group('topic_id'))
1587	self.discussion_topic_reply(m, subject[result.end('topic'):])
1588
1589	## Reply to topic message.
1590	#
1591	elif result.group('message'):
1592	self.system = 'discussion'
1593	self.id = int(result.group('message_id'))
1594	self.discussion_message_reply(m, subject[result.end('message'):])
1595
1596	else:
1597	self.system = 'ticket'
1598	result = self.ticket_update_by_subject(subject)
1599	if result:
1600	if not self.ticket_update(m, result, spam_msg):
1601	self.new_ticket(m, subject, spam_msg)
1602	else:
1603	## No update by subject, so just create a new ticket
1604	#
1605	self.new_ticket(m, subject, spam_msg)
1606
1607
1608	########## BODY TEXT functions ###########################################################
1609
1610	def strip_signature(self, text):
1611	"""
1612	Strip signature from message, inspired by Mailman software
1613	"""
1614	self.logger.debug('function strip_signature')
1615
1616	body = []
1617	for line in text.splitlines():
1618	if line == '-- ':
1619	break
1620	body.append(line)
1621
1622	return ('\n'.join(body))
1623
1624	def reflow(self, text, delsp = 0):
1625	"""
1626	Reflow the message based on the format="flowed" specification (RFC 3676)
1627	"""
1628	flowedlines = []
1629	quotelevel = 0
1630	prevflowed = 0
1631
1632	for line in text.splitlines():
1633	from re import match
1634
1635	## Figure out the quote level and the content of the current line
1636	#
1637	m = match('(>)( ?)(.)', line)
1638	linequotelevel = len(m.group(1))
1639	line = m.group(3)
1640
1641	## Determine whether this line is flowed
1642	#
1643	if line and line != '-- ' and line[-1] == ' ':
1644	flowed = 1
1645	else:
1646	flowed = 0
1647
1648	if flowed and delsp and line and line[-1] == ' ':
1649	line = line[:-1]
1650
1651	## If the previous line is flowed, append this line to it
1652	#
1653	if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1654	flowedlines[-1] += line
1655
1656	## Otherwise, start a new line
1657	#
1658	else:
1659	flowedlines.append('>' * linequotelevel + line)
1660
1661	prevflowed = flowed
1662
1663
1664	return '\n'.join(flowedlines)
1665
1666	def strip_quotes(self, text):
1667	"""
1668	Strip quotes from message by Nicolas Mendoza
1669	"""
1670	self.logger.debug('function strip_quotes')
1671
1672	body = []
1673	for line in text.splitlines():
1674	try:
1675
1676	if line.startswith(self.parameters.email_quote):
1677	continue
1678
1679	except UnicodeDecodeError:
1680
1681	tmp_line = self.email_to_unicode(line)
1682	if tmp_line.startswith(self.parameters.email_quote):
1683	continue
1684
1685	body.append(line)
1686
1687	return ('\n'.join(body))
1688
1689	def inline_properties(self, text):
1690	"""
1691	Parse text if we use inline keywords to set ticket fields
1692	"""
1693	self.logger.debug('function inline_properties')
1694
1695	properties = dict()
1696	body = list()
1697
1698	INLINE_EXP = re.compile('\s[@]\s([a-zA-Z]+)\s:(.)$')
1699
1700	for line in text.splitlines():
1701	match = INLINE_EXP.match(line)
1702	if match:
1703	keyword, value = match.groups()
1704	self.properties[keyword] = value.strip()
1705
1706	self.logger.debug('inline properties: %s : %s' %(keyword,value))
1707
1708	else:
1709	body.append(line)
1710
1711	return '\n'.join(body)
1712
1713
1714	def wrap_text(self, text, replace_whitespace = False):
1715	"""
1716	Will break a lines longer then given length into several small
1717	lines of size given length
1718	"""
1719	import textwrap
1720
1721	LINESEPARATOR = '\n'
1722	reformat = ''
1723
1724	for s in text.split(LINESEPARATOR):
1725	tmp = textwrap.fill(s, self.parameters.use_textwrap)
1726	if tmp:
1727	reformat = '%s\n%s' %(reformat,tmp)
1728	else:
1729	reformat = '%s\n' %reformat
1730
1731	return reformat
1732
1733	# Python2.4 and higher
1734	#
1735	#return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
1736	#
1737
1738	########## EMAIL attachements functions ###########################################################
1739
1740	def inline_part(self, part):
1741	"""
1742	"""
1743	self.logger.debug('function inline_part()')
1744
1745	return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
1746
1747	def get_message_parts(self, msg):
1748	"""
1749	parses the email message and returns a list of body parts and attachments
1750	body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
1751	"""
1752	self.logger.debug('function get_message_parts()')
1753
1754	message_parts = list()
1755
1756	ALTERNATIVE_MULTIPART = False
1757
1758	for part in msg.walk():
1759	self.logger.debug('Message part: Main-Type: %s' % part.get_content_maintype())
1760	self.logger.debug('Message part: Content-Type: %s' % part.get_content_type())
1761
1762	## Check content type
1763	#
1764	if part.get_content_type() in self.STRIP_CONTENT_TYPES:
1765	self.logger.debug("A %s attachment named '%s' was skipped" %(part.get_content_type(), part.get_filename()))
1766	continue
1767
1768	## Catch some mulitpart execptions
1769	#
1770	if part.get_content_type() == 'multipart/alternative':
1771	ALTERNATIVE_MULTIPART = True
1772	continue
1773
1774	## Skip multipart containers
1775	#
1776	if part.get_content_maintype() == 'multipart':
1777	self.logger.debug("Skipping multipart container")
1778
1779	continue
1780
1781	## Check if this is an inline part. It's inline if there is co Cont-Disp header, or if there is one and it says "inline"
1782	#
1783	inline = self.inline_part(part)
1784
1785	## Drop HTML message
1786	#
1787	if ALTERNATIVE_MULTIPART and self.parameters.drop_alternative_html_version:
1788	if part.get_content_type() == 'text/html':
1789	self.logger.debug('Skipping alternative HTML message')
1790	ALTERNATIVE_MULTIPART = False
1791	continue
1792
1793	## Inline text parts are where the body is
1794	#
1795	if part.get_content_type() == 'text/plain' and inline:
1796	self.logger.debug(' Inline body part')
1797
1798	## Try to decode, if fails then do not decode
1799	#
1800	body_text = part.get_payload(decode=1)
1801	if not body_text:
1802	body_text = part.get_payload(decode=0)
1803
1804	format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
1805	delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
1806
1807	if self.parameters.reflow and not self.parameters.verbatim_format and format == 'flowed':
1808	body_text = self.reflow(body_text, delsp == 'yes')
1809
1810	if self.parameters.strip_signature:
1811	body_text = self.strip_signature(body_text)
1812
1813	if self.parameters.strip_quotes:
1814	body_text = self.strip_quotes(body_text)
1815
1816	if self.parameters.inline_properties:
1817	body_text = self.inline_properties(body_text)
1818
1819	if self.parameters.use_textwrap:
1820	body_text = self.wrap_text(body_text)
1821
1822	## Get contents charset (iso-8859-15 if not defined in mail headers)
1823	#
1824	charset = part.get_content_charset()
1825	if not charset:
1826	charset = 'iso-8859-15'
1827
1828	try:
1829	ubody_text = unicode(body_text, charset)
1830
1831	except UnicodeError, detail:
1832	ubody_text = unicode(body_text, 'iso-8859-15')
1833
1834	except LookupError, detail:
1835	ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
1836
1837	if self.parameters.verbatim_format:
1838	message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
1839	else:
1840	message_parts.append('%s' %ubody_text)
1841	else:
1842	if self.parameters.debug:
1843	s = ' Filename: %s' % part.get_filename()
1844	self.print_unicode(s)
1845
1846	## First try to use email header function to convert filename.
1847	# If this fails the use the plan filename
1848	#
1849	try:
1850	filename = self.email_to_unicode(part.get_filename())
1851	except UnicodeEncodeError, detail:
1852	filename = part.get_filename()
1853
1854	message_parts.append((filename, part))
1855
1856	return message_parts
1857
1858	def unique_attachment_names(self, message_parts):
1859	"""
1860	"""
1861	renamed_parts = []
1862	attachment_names = set()
1863
1864	for item in message_parts:
1865
1866	## If not an attachment, leave it alone
1867	#
1868	if not isinstance(item, tuple):
1869	renamed_parts.append(item)
1870	continue
1871
1872	(filename, part) = item
1873
1874	## If no filename, use a default one
1875	#
1876	if not filename:
1877	filename = 'untitled-part'
1878
1879	## Guess the extension from the content type, use non strict mode
1880	# some additional non-standard but commonly used MIME types
1881	# are also recognized
1882	#
1883	ext = mimetypes.guess_extension(part.get_content_type(), False)
1884	if not ext:
1885	ext = '.bin'
1886
1887	filename = '%s%s' % (filename, ext)
1888
1889	## Discard relative paths for windows/unix in attachment names
1890	#
1891	#filename = filename.replace('\\', '/').replace(':', '/')
1892	filename = filename.replace('\\', '_')
1893	filename = filename.replace('/', '_')
1894
1895	## remove linefeed char
1896	#
1897	for forbidden_char in ['\r', '\n']:
1898	filename = filename.replace(forbidden_char,'')
1899
1900	## We try to normalize the filename to utf-8 NFC if we can.
1901	# Files uploaded from OS X might be in NFD.
1902	# Check python version and then try it
1903	#
1904	#if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
1905	# try:
1906	# filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
1907	# except TypeError:
1908	# pass
1909
1910	## Make the filename unique for this ticket
1911	#
1912	num = 0
1913	unique_filename = filename
1914	dummy_filename, ext = os.path.splitext(filename)
1915
1916	while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
1917	num += 1
1918	unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
1919
1920	if self.parameters.debug:
1921	s = 'Attachment with filename %s will be saved as %s' % (filename, unique_filename)
1922	self.print_unicode(s)
1923
1924	attachment_names.add(unique_filename)
1925
1926	renamed_parts.append((filename, unique_filename, part))
1927
1928	return renamed_parts
1929
1930
1931	def attachment_exists(self, filename):
1932
1933	if self.parameters.debug:
1934	s = 'attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
1935	self.print_unicode(s)
1936
1937	## We have no valid ticket id
1938	#
1939	if not self.id:
1940	return False
1941
1942	try:
1943	if self.system == 'discussion':
1944	att = attachment.Attachment(self.env, 'discussion', 'ticket/%s'
1945	% (self.id,), filename)
1946	else:
1947	att = attachment.Attachment(self.env, 'ticket', self.id,
1948	filename)
1949	return True
1950	except attachment.ResourceNotFound:
1951	return False
1952
1953	########## TRAC Ticket Text ###########################################################
1954
1955	def body_text(self, message_parts):
1956	body_text = []
1957
1958	for part in message_parts:
1959
1960	## Plain text part, append it
1961	#
1962	if not isinstance(part, tuple):
1963	body_text.extend(part.strip().splitlines())
1964	body_text.append("")
1965	continue
1966
1967	(original, filename, part) = part
1968	inline = self.inline_part(part)
1969
1970	if part.get_content_maintype() == 'image' and inline:
1971	if self.system != 'discussion':
1972	body_text.append('[[Image(%s)]]' % filename)
1973	body_text.append("")
1974	else:
1975	if self.system != 'discussion':
1976	body_text.append('[attachment:"%s"]' % filename)
1977	body_text.append("")
1978
1979	body_text = '\r\n'.join(body_text)
1980	return body_text
1981
1982	def html_mailto_link(self, subject):
1983	"""
1984	This function returns a HTML mailto tag with the ticket id and author email address
1985	"""
1986	if not self.author:
1987	author = self.email_addr
1988	else:
1989	author = self.author
1990
1991	if not self.parameters.mailto_cc:
1992	self.parameters.mailto_cc = ''
1993
1994	## use urllib to escape the chars
1995	#
1996	s = 'mailto:%s?Subject=%s&Cc=%s' %(
1997	urllib.quote(self.email_addr),
1998	urllib.quote('Re: #%s: %s' %(self.id, subject)),
1999	urllib.quote(self.parameters.mailto_cc)
2000	)
2001
2002	s = '\r\n{{{\r\n#!html\r\n<a\r\n href="%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
2003	return s
2004
2005	########## TRAC notify section ###########################################################
2006
2007	def notify(self, tkt, new=True, modtime=0):
2008	"""
2009	A wrapper for the TRAC notify function. So we can use templates
2010	"""
2011	self.logger.debug('function notify()')
2012
2013	if self.parameters.dry_run:
2014	print 'DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter']
2015	return
2016	try:
2017
2018	#from trac.ticket.web_ui import TicketModule
2019	#from trac.ticket.notification import TicketNotificationSystem
2020	#ticket_sys = TicketNotificationSystem(self.env)
2021	#a = TicketModule(self.env)
2022	#print a.__dict__
2023	#tn_sys = TicketNotificationSystem(self.env)
2024	#print tn_sys
2025	#print tn_sys.__dict__
2026	#sys.exit(0)
2027
2028	## create false {abs_}href properties, to trick Notify()
2029	#
2030	if not (self.VERSION in [0.11, 0.12]):
2031	self.env.abs_href = Href(self.get_config('project', 'url'))
2032	self.env.href = Href(self.get_config('project', 'url'))
2033
2034
2035	tn = TicketNotifyEmail(self.env)
2036
2037	if self.parameters.alternate_notify_template:
2038
2039	if self.VERSION >= 0.11:
2040
2041	from trac.web.chrome import Chrome
2042
2043	if self.parameters.alternate_notify_template_update and not new:
2044	tn.template_name = self.parameters.alternate_notify_template_update
2045	else:
2046	tn.template_name = self.parameters.alternate_notify_template
2047
2048	tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
2049
2050	else:
2051
2052	tn.template_name = self.parameters.alternate_notify_template
2053
2054	tn.notify(tkt, new, modtime)
2055
2056	except Exception, e:
2057	self.logger.error('Failure sending notification on creation of ticket #%s: %s' %(self.id, e))
2058
2059
2060
2061	########## Parse Config File ###########################################################
2062
2063	def ReadConfig(file, name):
2064	"""
2065	Parse the config file
2066	"""
2067	if not os.path.isfile(file):
2068	print 'File %s does not exist' %file
2069	sys.exit(1)
2070
2071	config = trac_config.Configuration(file)
2072
2073	parentdir = config.get('DEFAULT', 'parentdir')
2074	sections = config.sections()
2075
2076	## use some trac internals to get the defaults
2077	#
2078	tmp = config.parser.defaults()
2079	project = SaraDict()
2080
2081	for option, value in tmp.items():
2082	try:
2083	project[option] = int(value)
2084	except ValueError:
2085	project[option] = value
2086
2087	if name:
2088	if name in sections:
2089	project = SaraDict()
2090	for option, value in config.options(name):
2091	try:
2092	project[option] = int(value)
2093	except ValueError:
2094	project[option] = value
2095
2096	elif not parentdir:
2097	print "Not a valid project name: %s, valid names are: %s" %(name, sections)
2098	print "or set parentdir in the [DEFAULT] section"
2099	sys.exit(1)
2100
2101	## If parentdir the set project dir to parentdir + name
2102	#
2103	if not project.has_key('project'):
2104	if not parentdir:
2105	print "You must set project or parentdir in your configuration file"
2106	sys.exit(1)
2107	elif not name:
2108	print "You must set project in your configuration file"
2109	else:
2110	project['project'] = os.path.join(parentdir, name)
2111
2112	return project
2113
2114	########## Setup Logging ###############################################################
2115
2116	def setup_log(parameters, project_name, interactive=None):
2117	"""
2118	Setup loging
2119
2120	Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
2121	would be interpreted by the ConfigParser itself.
2122	"""
2123	logger = logging.getLogger('email2trac %s' %project_name)
2124
2125	if interactive:
2126	parameters.log_type = 'stderr'
2127
2128	if not parameters.log_type:
2129	parameters.log_type = 'syslog'
2130
2131	if parameters.log_type == 'file':
2132
2133	if not parameters.log_file:
2134	parameters.log_file = 'email2trac.log'
2135
2136	if not os.path.isabs(parameters.log_file):
2137	import tempfile
2138	parameters.log_file = os.path.join(tempfile.gettempdir(), parameters.log_file)
2139
2140	log_handler = logging.FileHandler(parameters.log_file)
2141
2142	elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
2143	## Requires win32 extensions
2144	#
2145	log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
2146
2147	elif parameters.log_type in ('syslog', 'unix'):
2148	log_handler = logging.handlers.SysLogHandler('/dev/log')
2149
2150	elif parameters.log_type in ('stderr'):
2151	log_handler = logging.StreamHandler(sys.stderr)
2152
2153	else:
2154	log_handler = logging.handlers.BufferingHandler(0)
2155
2156	if parameters.log_format:
2157	parameters.log_format = parameters.log_format.replace('$(', '%(')
2158	else:
2159	parameters.log_format = '%(name)s: %(message)s'
2160
2161	log_formatter = logging.Formatter(parameters.log_format)
2162	log_handler.setFormatter(log_formatter)
2163	logger.addHandler(log_handler)
2164
2165	if (parameters.log_level in ['DEBUG', 'ALL']) or (parameters.debug > 0):
2166	logger.setLevel(logging.DEBUG)
2167
2168	elif parameters.log_level in ['INFO'] or parameters.verbose:
2169	logger.setLevel(logging.INFO)
2170
2171	elif parameters.log_level in ['WARNING']:
2172	logger.setLevel(logging.WARNING)
2173
2174	elif parameters.log_level in ['ERROR']:
2175	logger.setLevel(logging.ERROR)
2176
2177	elif parameters.log_level in ['CRITICAL']:
2178	logger.setLevel(logging.CRITICAL)
2179
2180	else:
2181	logger.setLevel(logging.INFO)
2182
2183	return logger
2184
2185
2186	if __name__ == '__main__':
2187	## Default config file
2188	#
2189	configfile = '@email2trac_conf@'
2190	project = ''
2191	component = ''
2192	ticket_prefix = 'default'
2193	dry_run = None
2194	verbose = None
2195	debug_interactive = None
2196
2197	SHORT_OPT = 'cdhf:np:t:v'
2198	LONG_OPT = ['component=', 'debug', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=', 'verbose']
2199
2200	try:
2201	opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
2202	except getopt.error,detail:
2203	print __doc__
2204	print detail
2205	sys.exit(1)
2206
2207	project_name = None
2208	for opt,value in opts:
2209	if opt in [ '-h', '--help']:
2210	print __doc__
2211	sys.exit(0)
2212	elif opt in ['-c', '--component']:
2213	component = value
2214	elif opt in ['-d', '--debug']:
2215	debug_interactive = 1
2216	elif opt in ['-f', '--file']:
2217	configfile = value
2218	elif opt in ['-n', '--dry-run']:
2219	dry_run = True
2220	elif opt in ['-p', '--project']:
2221	project_name = value
2222	elif opt in ['-t', '--ticket_prefix']:
2223	ticket_prefix = value
2224	elif opt in ['-v', '--verbose']:
2225	verbose = True
2226
2227	settings = ReadConfig(configfile, project_name)
2228
2229	## The default prefix for ticket values in email2trac.conf
2230	#
2231	settings.ticket_prefix = ticket_prefix
2232	settings.dry_run = dry_run
2233	settings.verbose = verbose
2234
2235	if not settings.debug and debug_interactive:
2236	settings.debug = debug_interactive
2237
2238	if not settings.project:
2239	print __doc__
2240	print 'No Trac project is defined in the email2trac config file.'
2241	sys.exit(1)
2242
2243	logger = setup_log(settings, os.path.basename(settings.project), debug_interactive)
2244
2245	if component:
2246	settings['component'] = component
2247
2248	## Determine major trac version used to be in email2trac.conf
2249	# Quick hack for 0.12
2250	#
2251	version = '0.%s' %(trac_version.split('.')[1])
2252	if version.startswith('0.12'):
2253	version = '0.12'
2254
2255	logger.debug("Found trac version: %s" %(version))
2256
2257	try:
2258	if version == '0.10':
2259	from trac import attachment
2260	from trac.env import Environment
2261	from trac.ticket import Ticket
2262	from trac.web.href import Href
2263	from trac import util
2264	from trac.ticket.web_ui import TicketModule
2265
2266	#
2267	# return util.text.to_unicode(str)
2268	#
2269	# see http://projects.edgewall.com/trac/changeset/2799
2270	from trac.ticket.notification import TicketNotifyEmail
2271	from trac import config as trac_config
2272	from trac.core import TracError
2273
2274	elif version == '0.11':
2275	from trac import attachment
2276	from trac.env import Environment
2277	from trac.ticket import Ticket
2278	from trac.web.href import Href
2279	from trac import config as trac_config
2280	from trac import util
2281	from trac.core import TracError
2282	from trac.perm import PermissionSystem
2283	from trac.ticket.web_ui import TicketModule
2284
2285	#
2286	# return util.text.to_unicode(str)
2287	#
2288	# see http://projects.edgewall.com/trac/changeset/2799
2289	from trac.ticket.notification import TicketNotifyEmail
2290
2291	elif version == '0.12':
2292	from trac import attachment
2293	from trac.env import Environment
2294	from trac.ticket import Ticket
2295	from trac.web.href import Href
2296	from trac import config as trac_config
2297	from trac import util
2298	from trac.core import TracError
2299	from trac.perm import PermissionSystem
2300	from trac.ticket.web_ui import TicketModule
2301
2302	#
2303	# return util.text.to_unicode(str)
2304	#
2305	# see http://projects.edgewall.com/trac/changeset/2799
2306	from trac.ticket.notification import TicketNotifyEmail
2307
2308
2309	else:
2310	logger.error('TRAC version %s is not supported' %version)
2311	sys.exit(1)
2312
2313	## Must be set before environment is created
2314	#
2315	if settings.has_key('python_egg_cache'):
2316	python_egg_cache = str(settings['python_egg_cache'])
2317	os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
2318
2319	if settings.debug > 0:
2320	logger.debug('Loading environment %s', settings.project)
2321
2322	try:
2323	env = Environment(settings['project'], create=0)
2324	except IOError, detail:
2325	print "Trac project does not exists: %s" %(settings['project'])
2326	sys.exit(1)
2327
2328	tktparser = TicketEmailParser(env, settings, logger, float(version))
2329	tktparser.parse(sys.stdin)
2330
2331	## Catch all errors and use the logging module
2332	#
2333	except Exception, error:
2334
2335	etype, evalue, etb = sys.exc_info()
2336	for e in traceback.format_exception(etype, evalue, etb):
2337	logger.critical(e)
2338
2339	if m:
2340	tktparser.save_email_for_debug(m, True)
2341
2342	sys.exit(1)
2343	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format