Context Navigation

source: trunk/email2trac.py.in @ 646

Last change on this file since 646 was 646, checked in by bas, 10 years ago
Fixid agilo ticket model import for old and new versions, closes #330
Property svn:executable set to ``* Property svn:keywords set to `Id`
File size: 93.3 KB

Line
1	#!@PYTHON@
2	#
3	# This file is part of the email2trac utils
4	#
5	# Copyright 2002 SURFsara
6	#
7	# Licensed under the Apache License, Version 2.0 (the "License");
8	# you may not use this file except in compliance with the License.
9	# You may obtain a copy of the License at
10	#
11	# http://www.apache.org/licenses/LICENSE-2.0
12	#
13	# Unless required by applicable law or agreed to in writing, software
14	# distributed under the License is distributed on an "AS IS" BASIS,
15	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	# See the License for the specific language governing permissions and
17	# limitations under the License.
18	#
19	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
20	#
21	"""
22	email2trac.py -- Email -> TRAC tickets
23
24	A MTA filter to create Trac tickets from inbound emails.
25
26	first proof of concept from:
27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28	Copyright 2005, Edgewall Software
29
30	Authors:
31	Bas van der Vlies <bas.vandervlies@surfsara.nl>
32	Walter de Jong <walter.dejong@surfsara.nl>
33
34	How to use
35	----------
36	* See https://oss.trac.surfsara.nl/email2trac/
37
38	* Commandline opions:
39	-h,--help
40	-A, --agilo
41	-B, --bloodhound_product
42	-d, --debug
43	-E, --virtualenv <path>
44	-f,--file <configuration file>
45	-n,--dry-run
46	-p, --project <project name>
47	-t, --ticket_prefix <name>
48	-v, --verbose
49
50	SVN Info:
51	$Id: email2trac.py.in 646 2014-01-09 15:01:15Z bas $
52	"""
53	import os
54	import sys
55	import string
56	import getopt
57	import time
58	import email
59	import email.Iterators
60	import email.Header
61	import re
62	import urllib
63	import unicodedata
64	import mimetypes
65	import traceback
66	import logging
67	import logging.handlers
68	import UserDict
69	import tempfile
70
71	from datetime import timedelta, datetime
72	from stat import *
73
74	## Some global variables
75	#
76	m = None
77
78	class SaraDict(UserDict.UserDict):
79	def __init__(self, dictin = None):
80	UserDict.UserDict.__init__(self)
81	self.name = None
82
83	if dictin:
84	if dictin.has_key('name'):
85	self.name = dictin['name']
86	del dictin['name']
87	self.data = dictin
88
89	def get_value(self, name):
90	if self.has_key(name):
91	return self[name]
92	else:
93	return None
94
95	def __repr__(self):
96	return repr(self.data)
97
98	def __str__(self):
99	return str(self.data)
100
101	def __getattr__(self, name):
102	"""
103	override the class attribute get method. Return the value
104	from the dictionary
105	"""
106	if self.data.has_key(name):
107	return self.data[name]
108	else:
109	return None
110
111	def __setattr__(self, name, value):
112	"""
113	override the class attribute set method only when the UserDict
114	has set its class attribute
115	"""
116	if self.__dict__.has_key('data'):
117	self.data[name] = value
118	else:
119	self.__dict__[name] = value
120
121	def __iter__(self):
122	return iter(self.data.keys())
123
124
125
126	class TicketEmailParser(object):
127	env = None
128	comment = '> '
129
130	def __init__(self, env, parameters, logger, version):
131	self.env = env
132
133	# Database connection
134	#
135	self.db = None
136
137	# Save parameters
138	#
139	self.parameters = parameters
140	self.logger = logger
141
142	# Some useful mail constants
143	#
144	self.email_name = None
145	self.email_addr = None
146	self.email_from = None
147	self.author = None
148	self.id = None
149
150	## Will be set to True if the user has ben registered
151	#
152	self.allow_registered_user = False
153
154	self.STRIP_CONTENT_TYPES = list()
155
156	## fields properties via body_text
157	#
158	self.properties = dict()
159
160	self.VERSION = version
161
162	self.get_config = self.env.config.get
163
164	## init function ##
165	#
166	self.setup_parameters()
167
168	def setup_parameters(self):
169	if self.parameters.umask:
170	os.umask(self.parameters.umask)
171
172	if not self.parameters.spam_level:
173	self.parameters.spam_level = 0
174
175	if not self.parameters.spam_header:
176	self.parameters.spam_header = 'X-Spam-Score'
177
178	if not self.parameters.email_quote:
179	self.parameters.email_quote = '^> .*'
180	self.parameters.skip_line_regex = '^> .*'
181
182	if not self.parameters.ticket_update_by_subject_lookback:
183	self.parameters.ticket_update_by_subject_lookback = 30
184
185	if self.parameters.verbatim_format == None:
186	self.parameters.verbatim_format = 1
187
188	if self.parameters.reflow == None:
189	self.parameters.reflow = 1
190
191	if self.parameters.binhex:
192	self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
193
194	if self.parameters.applesingle:
195	self.STRIP_CONTENT_TYPES.append('application/applefile')
196
197	if self.parameters.appledouble:
198	self.STRIP_CONTENT_TYPES.append('application/applefile')
199
200	if self.parameters.strip_content_types:
201	items = self.parameters.strip_content_types.split(',')
202	for item in items:
203	self.STRIP_CONTENT_TYPES.append(item.strip())
204
205	if self.parameters.tmpdir:
206	self.parameters.tmpdir = os.path.normcase(str(self.parameters['tmpdir']))
207	else:
208	self.parameters.tmpdir = os.path.normcase('/tmp')
209
210	if self.parameters.email_triggers_workflow == None:
211	self.parameters.email_triggers_workflow = 1
212
213	if not self.parameters.subject_field_separator:
214	self.parameters.subject_field_separator = '&'
215	else:
216	self.parameters.subject_field_separator = self.parameters.subject_field_separator.strip()
217
218	if not self.parameters.strip_signature_regex:
219	self.parameters.strip_signature_regex = '^-- $'
220	self.parameters.cut_line_regex = '^-- $'
221
222	self.trac_smtp_from = self.get_config('notification', 'smtp_from')
223	self.smtp_default_domain = self.get_config('notification', 'smtp_default_domain')
224	self.smtp_replyto = self.get_config('notification', 'smtp_replyto')
225	self.trac_smtp_always_cc = self.get_config('notification', 'smtp_always_cc')
226	self.trac_smtp_always_bcc = self.get_config('notification', 'smtp_always_bcc')
227
228	self.system = None
229
230	########## Email Header Functions ###########################################################
231
232	def spam(self, message):
233	"""
234	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
235	# Note if Spam_level then '*' are included
236	"""
237	self.logger.debug('function spam')
238
239	spam = False
240	if message.has_key(self.parameters.spam_header):
241	spam_l = string.split(message[self.parameters.spam_header])
242	#self.logger.info('Spam header: %s' %(message[self.parameters.spam_header]))
243
244	try:
245	number = spam_l[0].count('*')
246	except IndexError, detail:
247	number = 0
248
249	if number >= self.parameters.spam_level:
250	spam = True
251
252	## treat virus mails as spam
253	#
254	elif message.has_key('X-Virus-found'):
255	spam = True
256
257	## Astaro firewall spam handling
258	#
259	elif message.get('X-Spam-Flag') == "YES" and message.get('X-Spam-Result') == "Spam":
260	spam = True
261
262	## How to handle SPAM messages
263	#
264	if self.parameters.drop_spam and spam:
265
266	self.logger.info('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' %self.parameters.spam_level)
267	return 'drop'
268
269	elif spam:
270
271	return 'Spam'
272	else:
273
274	return False
275
276	def email_header_acl(self, keyword, message_field, default):
277	"""
278	This function wil check if the email address is allowed or denied
279	to send mail to the ticket list
280	"""
281	self.logger.debug('function email_header_acl: %s' %keyword)
282
283	try:
284	mail_addresses = self.parameters[keyword]
285
286	# Check if we have an empty string
287	#
288	if not mail_addresses:
289	return default
290
291	except KeyError, detail:
292	self.logger.debug('\t %s not defined, all messages are allowed.' %(keyword))
293
294	return default
295
296	mail_addresses = string.split(mail_addresses, ',')
297	message_addresses = string.split(message_field, ',')
298
299	for entry in mail_addresses:
300	entry = entry.strip()
301	TO_RE = re.compile(entry, re.VERBOSE\|re.IGNORECASE)
302
303	for addr in message_addresses:
304	addr = addr.strip()
305
306	if self.parameters.compare_function_list in [ 'matches', 'match']:
307	s = '\t%s matches %s' %(addr, entry)
308	result = TO_RE.match(addr)
309	else:
310	s = '\t%s contains %s' %(addr, entry)
311	result = TO_RE.search(addr)
312
313	if result:
314	self.logger.debug(s)
315	return True
316
317	return False
318
319	def email_header_txt(self, m):
320	"""
321	Display To and CC addresses in description field
322	"""
323	s = ''
324
325	if m['To'] and len(m['To']) > 0:
326	s = "'''To:''' %s\r\n" %(m['To'])
327	if m['Cc'] and len(m['Cc']) > 0:
328	s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
329
330	return self.email_to_unicode(s)
331
332
333	def get_sender_info(self, message):
334	"""
335	Get the default author name and email address from the message
336	"""
337	self.logger.debug('function get_sender_info')
338
339	to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
340	self.email_to_addrs = list()
341
342	for n,e in to_addrs:
343	self.email_to_addrs.append(e)
344
345	self.email_to_addrs = ','.join(self.email_to_addrs)
346
347
348	self.email_name, self.email_addr = email.Utils.parseaddr(message['from'])
349
350	## decode email name can contain charset
351	#
352	self.email_name = self.email_to_unicode(self.email_name)
353
354	dstr = '\t email name: %s, email address: %s' %(self.email_name, self.email_addr)
355	self.logger.debug(dstr)
356
357	## Trac can not handle author's name that contains spaces
358	#
359	if self.email_addr.lower() == self.trac_smtp_from.lower():
360	if self.email_name:
361	self.author = self.email_name
362	else:
363	self.author = "email2trac"
364	else:
365	self.author = self.email_addr
366
367	if self.parameters.ignore_trac_user_settings:
368	return
369
370	# Is this a registered user, use email address as search key:
371	# result:
372	# u : login name
373	# n : Name that the user has set in the settings tab
374	# e : email address that the user has set in the settings tab
375	#
376	users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
377	if (
378	(e and (e.lower() == self.email_addr.lower())) or
379	(u and (u.lower() + '@' + self.smtp_default_domain.lower() == self.email_addr.lower()))
380	)
381	]
382
383	if len(users) >= 1:
384	self.email_from = users[0][2]
385	self.author = users[0][0]
386
387	self.logger.debug('\tget_sender_info: found registered user: %s' %(self.author) )
388
389	if self.parameters.white_list_registered_users:
390	self.logger.debug('\tget_sender_info: added registered user to white list')
391	self.allow_registered_user = True
392
393
394	def set_cc_fields(self, ticket, message, update=False):
395	"""
396	Set all the right fields for a new ticket
397	"""
398	self.logger.debug('function set_cc_fields')
399
400
401	# Put all CC-addresses in ticket CC field
402	#
403	if self.parameters.reply_all:
404
405	ticket_cc = ''
406
407	msg_cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
408
409	if not msg_cc_addrs:
410	return
411
412	if update:
413	self.logger.debug("\tupdate ticket cc-field")
414	ticket_cc = ticket['cc']
415	ticket_cc_list = ticket_cc.split(',')
416
417	for name,addr in msg_cc_addrs:
418
419	## Prevent mail loop
420	#
421	if addr == self.trac_smtp_from:
422	self.logger.debug("\tSkipping %s email address for CC-field, same as smtp_from address in trac.ini " %(addr))
423	continue
424
425	## Alwyas remove reporter email address
426	#
427	elif addr == self.email_addr:
428	self.logger.debug("\tSkipping reporter email address for CC-field")
429	continue
430
431	## Always remove the always_cc address
432	#
433	elif addr == self.trac_smtp_always_cc:
434	self.logger.debug("\tSkipping smtp_always_cc email address for CC-field")
435	continue
436
437	## Always remove the always_bcc address
438	#
439	elif addr == self.trac_smtp_always_bcc:
440	self.logger.debug("\tSkipping smtp_always_bcc email address for CC-field")
441	continue
442
443	## Skip also addr in cc_black_list (email2trac.conf)
444	#
445	if self.email_header_acl('cc_black_list', addr, False):
446	self.logger.debug("\tSkipping address for CC-field due to cc_black_list")
447	continue
448
449	else:
450	## On update, prevent adding duplicates
451	#
452	if update:
453	if addr in ticket_cc_list:
454	continue
455
456	if ticket_cc:
457	ticket_cc = '%s, %s' %(ticket_cc, addr)
458	else:
459	ticket_cc = addr
460
461	if ticket_cc:
462	self.logger.debug('\tCC fields: %s' %ticket_cc)
463	ticket['cc'] = self.email_to_unicode(ticket_cc)
464
465
466	def acl_list_from_file(self, f, keyword):
467	"""
468	Read the email address from a file
469	"""
470	self.logger.debug('function acl_list_from_file %s : %s' %(f, keyword))
471
472	if not os.path.isfile(f):
473	self.logger.error('%s_file: %s does not exists' %(keyword, f) )
474	else:
475	## read whole file and replace '\n' with ''
476	#
477	addr_l = open(f, 'r').readlines()
478	s = ','.join(addr_l).replace('\n','')
479
480	try:
481	self.parameters[keyword] = "%s,%s" %(self.parameters[keyword], s)
482	except KeyError, detail:
483	self.parameters[keyword] = s
484
485	########## DEBUG functions ###########################################################
486
487	def debug_body(self, message_body, temp_file):
488	"""
489	"""
490	self.logger.debug('function debug_body:')
491
492	body_file = "%s.body" %(temp_file)
493	fx = open(body_file, 'wb')
494
495	if self.parameters.dry_run:
496	self.logger.info('DRY-RUN: not saving body to %s' %(body_file))
497	return
498
499	self.logger.debug('writing body to %s' %(body_file))
500	if not message_body:
501	message_body = '(None)'
502
503	message_body = message_body.encode('utf-8')
504
505	fx.write(message_body)
506	fx.close()
507	try:
508	os.chmod(body_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
509	except OSError:
510	pass
511
512	def debug_attachments(self, message_parts, temp_file):
513	"""
514	"""
515	self.logger.debug('function debug_attachments')
516
517	n = 0
518	for item in message_parts:
519
520	## Skip inline text parts
521	#
522	if not isinstance(item, tuple):
523	continue
524
525	(original, filename, part) = item
526	self.logger.debug('\t part%d: Content-Type: %s' % (n, part.get_content_type()) )
527
528	dummy_filename, ext = os.path.splitext(filename)
529
530	n = n + 1
531	part_name = 'part%d%s' %(n, ext)
532	part_file = "%s.%s" %(temp_file, part_name)
533
534	s = 'writing %s: filename: %s' %(part_file, filename)
535	self.print_unicode(s)
536
537	## Forbidden chars, just write part files instead of names
538	#
539	#filename = filename.replace('\\', '_')
540	#filename = filename.replace('/', '_')
541	#filename = filename + '.att_email2trac'
542	# part_file = os.path.join(self.parameters.tmpdir, filename)
543	#part_file = util.text.unicode_quote(part_file)
544	#self.print_unicode(part_file)
545
546	if self.parameters.dry_run:
547	self.logger.info('DRY_RUN: NOT saving attachments')
548	continue
549
550	fx = open(part_file, 'wb')
551	text = part.get_payload(decode=1)
552
553	if not text:
554	text = '(None)'
555
556	fx.write(text)
557	fx.close()
558
559	try:
560	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
561	except OSError:
562	pass
563
564	def save_email_for_debug(self, message, project_name, create_tempfile=False):
565
566	if self.parameters.dry_run:
567	self.logger.debug('DRY_RUN: NOT saving email message')
568	return
569
570	if create_tempfile:
571	(fd, tmp_file) = tempfile.mkstemp('.%s.email2trac' %project_name)
572	fx = os.fdopen(fd, 'wb')
573	else:
574	tmp_file = os.path.join(self.parameters.tmpdir, 'msg.%s.email2trac' %project_name)
575	fx = open(tmp_file, 'wb')
576
577
578	self.logger.debug('saving email to %s' %(tmp_file))
579	fx.write('%s' % message)
580	fx.close()
581
582	try:
583	os.chmod(tmp_file, S_IRWXU\|S_IRWXG\|S_IRWXO)
584	except OSError:
585	pass
586
587	message_parts = self.get_message_parts(message)
588	message_parts = self.unique_attachment_names(message_parts)
589	body_text = self.get_body_text(message_parts)
590	self.debug_body(body_text, tmp_file)
591	self.debug_attachments(message_parts, tmp_file)
592
593	########## Conversion functions ###########################################################
594
595	def email_to_unicode(self, message_str):
596	"""
597	Email has 7 bit ASCII code, convert it to unicode with the charset
598	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
599	understands it.
600	"""
601	self.logger.debug("function email_to_unicode")
602
603	results = email.Header.decode_header(message_str)
604
605	s = None
606	for text,format in results:
607	if format:
608	try:
609	temp = unicode(text, format)
610	except UnicodeError, detail:
611	# This always works
612	#
613	msg = 'ERROR: Could not find charset: %s, please install' %format
614	self.logger.error(msg)
615	temp = unicode(text, 'iso-8859-15')
616	except LookupError, detail:
617	msg = 'ERROR: Could not find charset: %s, please install' %format
618	self.logger.error(msg)
619	#temp = unicode(text, 'iso-8859-15')
620	temp = message_str
621
622	else:
623	temp = string.strip(text)
624	temp = unicode(text, 'iso-8859-15')
625
626	if s:
627	s = '%s %s' %(s, temp)
628	else:
629	s = '%s' %temp
630
631	return s
632
633	def str_to_dict(self, s):
634	"""
635	Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
636	"""
637	self.logger.debug("function str_to_dict")
638
639	fields = string.split(s, self.parameters.subject_field_separator)
640
641	result = dict()
642	for field in fields:
643	try:
644	index, value = string.split(field, '=')
645
646	# We can not change the description of a ticket via the subject
647	# line. The description is the body of the email
648	#
649	if index.lower() in ['description']:
650	continue
651
652	if value:
653	result[index.lower()] = value
654
655	except ValueError:
656	pass
657	return result
658
659	def print_unicode(self,s):
660	"""
661	This function prints unicode strings if possible else it will quote it
662	"""
663	try:
664	self.logger.debug(s)
665	except UnicodeEncodeError, detail:
666	self.logger.debug(util.text.unicode_quote(s))
667
668
669	def html_2_txt(self, data):
670	"""
671	Various routines to convert html syntax to valid trac wiki syntax
672	"""
673	self.logger.debug('function html_2_txt')
674
675	## This routine make an safe html that can be include
676	# in trac, but no further text processing can be done
677	#
678	# try:
679	# from lxml.html.clean import Cleaner
680	# tags_rm = list()
681	# tags_rm.append('body')
682	#
683	# cleaner = Cleaner(remove_tags=tags_rm )
684	# parsed_data = cleaner.clean_html(data)
685	# parsed_data = '\n{{{\n#!html\n' + parsed_data + '\n}}}\n'
686	#
687	# return parsed_data
688	#
689	# except ImportError::
690	# pass
691
692	parsed_data = None
693	if self.parameters.html2text_cmd:
694	(fd, tmp_file) = tempfile.mkstemp('email2trac.html')
695	f = os.fdopen(fd, 'w')
696
697	cmd = '%s %s' %(self.parameters.html2text_cmd, tmp_file)
698	self.logger.debug('\t html2text conversion %s'%(cmd))
699
700	if self.parameters.dry_run:
701	self.logger.info('DRY_RUN: html2text conversion command: %s\n' %(cmd))
702
703	else:
704	f.write(data)
705	f.close()
706
707	lines = os.popen(cmd).readlines()
708	parsed_data = ''.join(lines)
709
710	os.unlink(tmp_file)
711
712	else:
713	self.logger.debug('\t No html2text conversion tool specified in email2trac.conf')
714
715	return parsed_data
716
717	def check_filename_length(self, name):
718	"""
719	To bypass a bug in Trac
720	check if the filename lenght is not larger then OS limit.
721	yes : return truncated filename
722	no : return unmodified filename
723	"""
724	self.logger.debug('function check_filename_length: ')
725
726	if not name:
727	return 'None'
728
729	dummy_filename, ext = os.path.splitext(name)
730
731	## Trac uses this format
732	#
733	try:
734	quote_format = util.text.unicode_quote(dummy_filename)
735
736	except UnicodeDecodeError, detail:
737	## last resort convert to unicode
738	#
739	dummy_filename = util.text.to_unicode(dummy_filename)
740	quote_format = util.text.unicode_quote(dummy_filename)
741
742	## Determine max filename lenght
743	#
744	try:
745	filemax_length = os.pathconf('/', 'PC_NAME_MAX')
746	except AttributeError, detail:
747	filemax_lenght = 240
748
749	if len(quote_format) <= filemax_length:
750	return name
751
752	else:
753	## Truncate file to filemax_length and reserve room for extension
754	# We must break on a boundry
755	#
756	length = filemax_length - 6
757
758	for i in range(0,10):
759
760	truncated = quote_format[ : (length - i)]
761
762	try:
763	unqoute_truncated = util.text.unicode_unquote(truncated)
764	unqoute_truncated = unqoute_truncated + ext
765
766	self.print_unicode('\t ' + unqoute_truncated)
767
768	break
769
770	except UnicodeDecodeError, detail:
771	continue
772
773
774	return unqoute_truncated
775
776	########## TRAC ticket functions ###########################################################
777
778	def mail_workflow(self, tkt):
779	"""
780	"""
781	self.logger.debug('function mail_workflow: ')
782
783	req = Mock(authname=self.author, perm=MockPerm(), args={})
784	ticket_system = TicketSystem(self.env)
785
786	try:
787	workflow = self.parameters['workflow_%s' %tkt['status'].lower()]
788
789	except KeyError:
790	## fallback for compability (Will be deprecated)
791	# workflow can be none.
792	#
793	workflow = None
794	if tkt['status'] in ['closed']:
795	workflow = self.parameters.workflow
796
797	if workflow:
798
799	## process all workflow implementations
800	#
801	tkt_module = TicketModule(self.env)
802	field_changes, problems = tkt_module.get_ticket_changes(req, tkt, workflow)
803
804	for field in field_changes.keys():
805
806	## We have already processed these fields
807	#
808	if not field in ['summary', 'description']:
809	s = 'workflow : %s, field %s : %s, by %s' \
810	%(workflow, field, field_changes[field]['new'],field_changes[field]['by'] )
811	self.logger.debug(s)
812
813	tkt[field] = field_changes[field]['new']
814
815	return True
816
817	else:
818	return False
819
820	def check_permission_participants(self, tkt, action):
821	"""
822	Check if the mailer is allowed to update the ticket
823	"""
824	self.logger.debug('function check_permission_participants %s')
825
826	if tkt['reporter'].lower() in [self.author.lower(), self.email_addr.lower()]:
827	self.logger.debug('ALLOW, %s is the ticket reporter' %(self.email_addr))
828
829	return True
830
831	perm = PermissionSystem(self.env)
832	if perm.check_permission(action, self.author):
833	self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
834
835	return True
836
837	# Is the updater in the CC?
838	try:
839	cc_list = tkt['cc'].split(',')
840	for cc in cc_list:
841	if self.email_addr.lower() in cc.lower.strip():
842	self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
843
844	return True
845
846	except KeyError:
847	pass
848
849	return False
850
851	def check_permission(self, tkt, action):
852	"""
853	check if the reporter has the right permission for the action:
854	- TICKET_CREATE
855	- TICKET_MODIFY
856	- TICKET_APPEND
857	- TICKET_CHGPROP
858
859	There are three models:
860	- None : no checking at all
861	- trac : check the permission via trac permission model
862	- email2trac: ....
863	"""
864	self.logger.debug("function check_permission: %s" %(action))
865
866	if self.parameters.ticket_permission_system in ['trac']:
867
868	perm = PermissionCache(self.env, self.author)
869	if perm.has_permission(action):
870	return True
871	else:
872	return False
873
874	elif self.parameters.ticket_permission_system in ['update_restricted_to_participants']:
875	return (self.check_permission_participants(tkt, action))
876
877	## Default is to allow everybody ticket updates and ticket creation
878	#
879	else:
880	return True
881
882
883	def update_ticket_fields(self, ticket, user_dict, new=None):
884	"""
885	This will update the ticket fields. It will check if the
886	given fields are known and if the right values are specified
887	It will only update the ticket field value:
888	- If the field is known
889	- If the value supplied is valid for the ticket field.
890	If not then there are two options:
891	1) Skip the value (new=None)
892	2) Set default value for field (new=1)
893	"""
894	self.logger.debug("function update_ticket_fields")
895
896	if self.parameters.bh_product:
897	if 'product' in user_dict:
898	if user_dict['product'] != ticket.env.product.prefix:
899	self.logging.warning("bloodhound products cannot be changed "
900	"- ignoring")
901	user_dict.pop('product')
902
903	## Check only permission model on ticket updates
904	#
905	if not new:
906	if self.parameters.ticket_permission_system:
907	if not self.check_permission(ticket, 'TICKET_CHGPROP'):
908	self.logger.info('Reporter: %s has no permission to change ticket properties' %self.author)
909	return False
910
911	## Build a system dictionary from the ticket fields
912	# with field as index and option as value
913	#
914	sys_dict = dict()
915
916	for field in ticket.fields:
917
918	try:
919	sys_dict[field['name']] = field['options']
920
921	except KeyError:
922	sys_dict[field['name']] = None
923	pass
924
925	## Check user supplied fields an compare them with the
926	# system one's
927	#
928	for field,value in user_dict.items():
929
930	s = 'user_field\t %s = %s' %(field,value)
931	self.print_unicode(s)
932
933	if not field in sys_dict.keys():
934	self.logger.debug('%s is not a valid field for tickets' %(field))
935	continue
936
937	## To prevent mail loop
938	#
939	if field == 'cc':
940
941	cc_list = user_dict['cc'].split(',')
942
943	if self.trac_smtp_from in cc_list:
944	self.logger.debug('MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
945
946	cc_list.remove(self.trac_smtp_from)
947
948	value = ','.join(cc_list)
949
950
951	## Check if every value is allowed for this field
952	#
953	if sys_dict[field]:
954
955	if value in sys_dict[field]:
956	ticket[field] = value
957	else:
958	## Must we set a default if value is not allowed
959	#
960	if new:
961	value = self.get_config('ticket', 'default_%s' %(field) )
962
963	else:
964
965	ticket[field] = value
966
967	s = 'ticket_field\t %s = %s' %(field, ticket[field])
968	self.print_unicode(s)
969
970	def ticket_update(self, m, id, spam):
971	"""
972	If the current email is a reply to an existing ticket, this function
973	will append the contents of this email to that ticket, instead of
974	creating a new one.
975	"""
976	self.logger.debug("function ticket_update")
977
978	if not self.parameters.ticket_update:
979	self.logger.debug("ticket_update disabled")
980	return False
981
982	## Must we update ticket fields
983	#
984	update_fields = dict()
985	try:
986	id, keywords = string.split(id, '?')
987
988	update_fields = self.str_to_dict(keywords)
989
990	## Strip '#'
991	#
992	self.id = int(id[1:])
993
994	except ValueError:
995
996	## Strip '#'
997	#
998	self.id = int(id[1:])
999
1000	self.logger.debug("\tticket id: %s" %id)
1001
1002	## When is the change committed
1003	#
1004	if self.VERSION < 0.11:
1005	when = int(time.time())
1006	else:
1007	when = datetime.now(util.datefmt.utc)
1008
1009	try:
1010	tkt = Ticket(self.env, self.id, self.db)
1011
1012	except util.TracError, detail:
1013
1014	## Not a valid ticket
1015	#
1016	self.logger.info("\tCreating a new ticket, ticket id: %s does not exists" %id)
1017	self.id = None
1018	return False
1019
1020	## Check the permission of the reporter
1021	#
1022	if self.parameters.ticket_permission_system:
1023	if not self.check_permission(tkt, 'TICKET_APPEND'):
1024	self.logger.info('Reporter: %s has no permission to add comments or attachments to tickets' %self.author)
1025	return False
1026
1027	## How many changes has this ticket
1028	#
1029	grouped = TicketModule(self.env).grouped_changelog_entries(tkt, self.db)
1030	cnum = sum(1 for e in grouped) + 1
1031
1032
1033	## reopen the ticket if it is was closed
1034	# We must use the ticket workflow framework
1035	#
1036	if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
1037
1038	if not self.mail_workflow(tkt):
1039
1040	if tkt['status'] in ['closed']:
1041	tkt['status'] = 'reopened'
1042	tkt['resolution'] = ''
1043
1044	## Old pre 0.11 situation
1045	#
1046	elif self.parameters.email_triggers_workflow:
1047
1048	self.logger.debug('\temail triggers workflow pre trac 0.11')
1049
1050	if tkt['status'] in ['closed']:
1051	tkt['status'] = 'reopened'
1052	tkt['resolution'] = ''
1053
1054	else:
1055	self.logger.debug('\temail triggers workflow disabled')
1056
1057	## Must we update some ticket fields properties via subject line
1058	#
1059	if update_fields:
1060	self.update_ticket_fields(tkt, update_fields)
1061
1062
1063	message_parts = self.get_message_parts(m)
1064	message_parts = self.unique_attachment_names(message_parts)
1065
1066	## Must we update some ticket fields properties via inline comments
1067	# in body_text
1068	#
1069	if self.properties:
1070	self.update_ticket_fields(tkt, self.properties)
1071
1072	## Must we update the CC ticket field
1073	#
1074	self.set_cc_fields(tkt, m, update=True)
1075
1076	if self.parameters.email_header:
1077	message_parts.insert(0, self.email_header_txt(m))
1078
1079	body_text = self.get_body_text(message_parts)
1080
1081	error_with_attachments = self.attach_attachments(message_parts)
1082
1083	if body_text.strip() or update_fields or self.properties:
1084
1085	if self.parameters.dry_run:
1086	s = 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) %s %s' %(self.author, cnum)
1087	self.logger.info(s)
1088
1089	else:
1090	if error_with_attachments:
1091	body_text = '%s\\%s' %(error_with_attachments, body_text)
1092
1093	self.logger.debug('\ttkt.save_changes(%s, %d)' %(self.author, cnum))
1094	tkt.save_changes(self.author, body_text, when, None, str(cnum))
1095
1096	if not spam:
1097	self.notify(tkt, False, when)
1098
1099	return True
1100
1101	def set_ticket_fields(self, ticket):
1102	"""
1103	set the ticket fields to value specified
1104	- /etc/email2trac.conf with <prefix>_<field>
1105	- trac default values, trac.ini
1106	"""
1107	self.logger.debug('function set_ticket_fields')
1108
1109	user_dict = dict()
1110
1111	for field in ticket.fields:
1112
1113	name = field['name']
1114
1115	## default trac value
1116	#
1117	if not field.get('custom'):
1118	value = self.get_config('ticket', 'default_%s' %(name) )
1119
1120	## skip this field can only be set by email2trac.conf
1121	#
1122	if name in ['resolution']:
1123	value = None
1124
1125	else:
1126	## Else get the default value for reporter
1127	#
1128	value = field.get('value')
1129	options = field.get('options')
1130
1131	if value and options and (value not in options):
1132	value = options[int(value)]
1133
1134	s = 'trac[%s] = %s' %(name, value)
1135	self.print_unicode(s)
1136
1137	## email2trac.conf settings
1138	#
1139	prefix = self.parameters.ticket_prefix
1140	try:
1141	value = self.parameters['%s_%s' %(prefix, name)]
1142
1143	s = 'email2trac[%s] = %s ' %(name, value)
1144	self.print_unicode(s)
1145
1146	except KeyError, detail:
1147	pass
1148
1149	if value:
1150	user_dict[name] = value
1151
1152	s = 'used %s = %s' %(name, value)
1153	self.print_unicode(s)
1154
1155	self.update_ticket_fields(ticket, user_dict, new=1)
1156
1157	if 'status' not in user_dict.keys():
1158	ticket['status'] = 'new'
1159
1160
1161	def ticket_update_by_subject(self, subject):
1162	"""
1163	This list of Re: prefixes is probably incomplete. Taken from
1164	wikipedia. Here is how the subject is matched
1165	- Re: <subject>
1166	- Re: (<Mail list label>:)+ <subject>
1167
1168	So we must have the last column
1169	"""
1170	self.logger.debug('function ticket_update_by_subject')
1171
1172	found_id = None
1173	if self.parameters.ticket_update and self.parameters.ticket_update_by_subject:
1174
1175	SUBJECT_RE = re.compile(r'^(?:(?:RE\|AW\|VS\|SV\|FW\|FWD):\s)+(.)', re.IGNORECASE)
1176	result = SUBJECT_RE.search(subject)
1177
1178	if result:
1179	## This is a reply
1180	#
1181	orig_subject = result.group(1)
1182
1183	self.logger.debug('subject search string: %s' %(orig_subject))
1184
1185	cursor = self.db.cursor()
1186	summaries = [orig_subject, '%%: %s' % orig_subject]
1187
1188	## Time resolution is in micoseconds
1189	#
1190	search_date = datetime.now(util.datefmt.utc) - timedelta(days=self.parameters.ticket_update_by_subject_lookback)
1191
1192	if self.VERSION < 0.12:
1193	lookback = util.datefmt.to_timestamp(search_date)
1194	else:
1195	lookback = util.datefmt.to_utimestamp(search_date)
1196
1197	for summary in summaries:
1198	self.logger.debug('Looking for summary matching: "%s"' % summary)
1199
1200	sql = """SELECT id, reporter FROM ticket
1201	WHERE changetime >= %s AND summary LIKE %s
1202	ORDER BY changetime DESC"""
1203
1204	cursor.execute(sql, [lookback, summary.strip()])
1205
1206	for row in cursor:
1207
1208	(matched_id, sql_reporter) = row
1209
1210	## Save first entry.
1211	#
1212	if not found_id:
1213	found_id = matched_id
1214
1215	## If subject and reporter are the same. The we certainly have found the right ticket
1216	#
1217	if sql_reporter == self.author:
1218	self.logger.debug('Found matching reporter: %s with ticket id: %d' %(sql_reporter, matched_id))
1219	found_id = matched_id
1220	break
1221
1222	if found_id:
1223	self.logger.debug('Found matching ticket id: %d' % found_id)
1224	found_id = '#%d' % found_id
1225	return (found_id, orig_subject)
1226
1227	return (found_id, subject)
1228
1229
1230	def new_ticket(self, msg, subject, spam, set_fields = None):
1231	"""
1232	Create a new ticket
1233	"""
1234	self.logger.debug('function new_ticket')
1235
1236	tkt = Ticket(self.env)
1237
1238	## self.author can be email address of an username
1239	#
1240	tkt['reporter'] = self.author
1241
1242	self.set_cc_fields(tkt, msg)
1243
1244	self.set_ticket_fields(tkt)
1245
1246	## Check the permission of the reporter
1247	#
1248	if self.parameters.ticket_permission_system:
1249	if not self.check_permission(tkt, 'TICKET_CREATE'):
1250	self.logger.info('Reporter: %s has no permission to create tickets' %self.author)
1251	return False
1252
1253	## Old style setting for component, will be removed
1254	#
1255	if spam:
1256	tkt['component'] = 'Spam'
1257
1258	elif self.parameters.has_key('component'):
1259	tkt['component'] = self.parameters['component']
1260
1261	if not msg['Subject']:
1262	tkt['summary'] = u'(No subject)'
1263	else:
1264	tkt['summary'] = subject
1265
1266
1267	if set_fields:
1268	rest, keywords = string.split(set_fields, '?')
1269
1270	if keywords:
1271	update_fields = self.str_to_dict(keywords)
1272	self.update_ticket_fields(tkt, update_fields)
1273
1274
1275	message_parts = self.get_message_parts(msg, True)
1276
1277	## Must we update some ticket fields properties via body_text
1278	#
1279	if self.properties:
1280	self.update_ticket_fields(tkt, self.properties)
1281
1282	message_parts = self.unique_attachment_names(message_parts)
1283
1284	## produce e-mail like header
1285	#
1286	head = ''
1287	if self.parameters.email_header:
1288	head = self.email_header_txt(msg)
1289	message_parts.insert(0, head)
1290
1291	body_text = self.get_body_text(message_parts)
1292
1293	tkt['description'] = body_text
1294
1295	## When is the change committed
1296	#
1297	if self.VERSION < 0.11:
1298	when = int(time.time())
1299	else:
1300	when = datetime.now(util.datefmt.utc)
1301
1302	if self.parameters.dry_run:
1303	self.logger.info('DRY_RUN: tkt.insert()')
1304	else:
1305	self.id = tkt.insert()
1306
1307	changed = False
1308	comment = ''
1309
1310	## some routines in trac are dependend on ticket id
1311	# like alternate notify template
1312	#
1313	if self.parameters.alternate_notify_template:
1314	tkt['id'] = self.id
1315	changed = True
1316
1317	## Rewrite the description if we have mailto enabled
1318	#
1319	if self.parameters.mailto_link:
1320	changed = True
1321	comment = u'\nadded mailto line\n'
1322
1323	#mailto = self.html_mailto_link( m['Subject'])
1324	mailto = self.html_mailto_link(subject)
1325
1326	tkt['description'] = u'%s\r\n%s%s\r\n' \
1327	%(head, mailto, body_text)
1328
1329	## Save the attachments to the ticket
1330	#
1331	error_with_attachments = self.attach_attachments(message_parts)
1332
1333	if error_with_attachments:
1334	changed = True
1335	comment = '%s\n%s\n' %(comment, error_with_attachments)
1336
1337	if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
1338	if self.mail_workflow(tkt):
1339	changed = True
1340
1341	if changed:
1342	if self.parameters.dry_run:
1343	s = 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
1344	self.logger.info(s)
1345
1346	else:
1347	tkt.save_changes(tkt['reporter'], comment)
1348
1349	if not spam:
1350	self.notify(tkt, True)
1351
1352
1353	def attach_attachments(self, message_parts, update=False):
1354	'''
1355	save any attachments as files in the ticket's directory
1356	'''
1357	self.logger.debug('function attach_attachments()')
1358
1359	if self.parameters.dry_run:
1360	self.logger.debug("DRY_RUN: no attachments attached to tickets")
1361	return ''
1362
1363	count = 0
1364
1365	## Get Maxium attachment size
1366	#
1367	max_size = int(self.get_config('attachment', 'max_size'))
1368	status = None
1369
1370	for item in message_parts:
1371	## Skip body parts
1372	#
1373	if not isinstance(item, tuple):
1374	continue
1375
1376	(original, filename, part) = item
1377
1378	## We have to determine the size so we use this temporary solution.
1379	#
1380	path, fd = util.create_unique_file(os.path.join(self.parameters.tmpdir, 'email2trac_tmp.att'))
1381	text = part.get_payload(decode=1)
1382	if not text:
1383	text = '(None)'
1384	fd.write(text)
1385	fd.close()
1386
1387	## get the file_size
1388	#
1389	stats = os.lstat(path)
1390	file_size = stats[ST_SIZE]
1391
1392	## Check if the attachment size is allowed
1393	#
1394	if (max_size != -1) and (file_size > max_size):
1395	status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1396	%(status, original, file_size, max_size)
1397
1398	os.unlink(path)
1399	continue
1400	else:
1401	count = count + 1
1402
1403	## Insert the attachment
1404	#
1405	fd = open(path, 'rb')
1406	if self.system == 'discussion':
1407	att = attachment.Attachment(self.env, 'discussion', 'topic/%s' % (self.id,))
1408	elif self.system == 'blog':
1409	att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,))
1410	else:
1411	s = 'Attach %s to ticket %d' %(filename, self.id)
1412	self.print_unicode(s)
1413	att = attachment.Attachment(self.env, 'ticket', self.id)
1414
1415	## This will break the ticket_update system, the body_text is vaporized
1416	# ;-(
1417	#
1418	if not update:
1419	att.author = self.author
1420	att.description = self.email_to_unicode('Added by email2trac')
1421
1422	try:
1423
1424	self.logger.debug('Insert atachment')
1425	att.insert(filename, fd, file_size)
1426
1427	except OSError, detail:
1428
1429	self.logger.info('%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail))
1430	status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
1431
1432	## Remove the created temporary filename
1433	#
1434	fd.close()
1435	os.unlink(path)
1436
1437	## return error
1438	#
1439	return status
1440
1441	########## Fullblog functions #################################################
1442
1443	def blog(self, msg, subject, id, params):
1444	"""
1445	The blog create/update function
1446	"""
1447	## import the modules
1448	#
1449	from tracfullblog.core import FullBlogCore
1450	from tracfullblog.model import BlogPost, BlogComment
1451
1452	## instantiate blog core
1453	#
1454	blog = FullBlogCore(self.env)
1455	req = Mock(authname='anonymous', perm=MockPerm(), args={})
1456
1457	## parameters from the subject
1458	#
1459	params = self.str_to_dict((params or '').lstrip('?'))
1460
1461	## preferably get the time from the email date header, or else
1462	# use current date and time
1463	date = email.Utils.parsedate_tz(msg.get('date'))
1464	if date:
1465	dt = util.datefmt.to_datetime(email.Utils.mktime_tz(date), util.datefmt.utc)
1466	else:
1467	self.logger.warn("No valid date header found")
1468	dt = util.datefmt.to_datetime(None, util.datefmt.utc)
1469
1470	## blog entry affected
1471	#
1472	self.id = id or util.datefmt.format_datetime(dt, "%Y%m%d%H%M%S", util.datefmt.utc)
1473
1474	## check wether a blog post exists
1475	#
1476	post = BlogPost(self.env, self.id)
1477	force_update = self.properties.get('update', params.get('update'))
1478
1479	## message parts
1480	#
1481	message_parts = self.get_message_parts(msg)
1482	message_parts = self.unique_attachment_names(message_parts)
1483
1484	if post.get_versions() and not force_update:
1485
1486	## add comment to blog entry
1487	#
1488	comment = BlogComment(self.env, self.id)
1489	comment.author = self.properties.get('author', params.get('author', self.author))
1490	comment.comment = self.get_body_text(message_parts)
1491	comment.time = dt
1492
1493	if self.parameters.dry_run:
1494	self.logger.info('DRY-RUN: not adding comment for blog entry "%s"' % id)
1495	return
1496	warnings = blog.create_comment(req, comment)
1497
1498	else:
1499	## create or update blog entry
1500	#
1501	post.author = self.properties.get('author', params.get('author', self.author))
1502	post.categories = self.properties.get('categories', params.get('categories', ''))
1503	post.title = subject.strip()
1504	post.publish_time = dt
1505	post.body = self.get_body_text(message_parts)
1506
1507	if self.parameters.dry_run:
1508	self.logger.info('DRY-RUN: not creating blog entry "%s"' % post.title)
1509	return
1510	warnings = blog.create_post(req, post, self.author, u'Created by email2trac', False)
1511
1512	## check for problems
1513	#
1514	if warnings:
1515	raise TracError(', '.join('blog:%s:%s' % (w[0], w[1]) for w in warnings))
1516
1517	## all seems well, attach attachments
1518	#
1519	self.attach_attachments(message_parts)
1520
1521
1522	########## Discussion functions ##############################################
1523
1524	def discussion_topic(self, content, subject):
1525
1526	## Import modules.
1527	#
1528	from tracdiscussion.api import DiscussionApi
1529	from trac.util.datefmt import to_timestamp, utc
1530
1531	self.logger.debug('Creating a new topic in forum:', self.id)
1532
1533	## Get dissussion API component.
1534	#
1535	api = self.env[DiscussionApi]
1536	context = self._create_context(content, subject)
1537
1538	## Get forum for new topic.
1539	#
1540	forum = api.get_forum(context, self.id)
1541
1542	if not forum:
1543	self.logger.error("ERROR: Replied forum doesn't exist")
1544
1545	## Prepare topic.
1546	#
1547	topic = {'forum' : forum['id'],
1548	'subject' : context.subject,
1549	'time': to_timestamp(datetime.now(utc)),
1550	'author' : self.author,
1551	'subscribers' : [self.email_addr],
1552	'body' : self.get_body_text(context.content_parts)}
1553
1554	## Add topic to DB and commit it.
1555	#
1556	self._add_topic(api, context, topic)
1557	self.db.commit()
1558
1559	def discussion_topic_reply(self, content, subject):
1560
1561	## Import modules.
1562	#
1563	from tracdiscussion.api import DiscussionApi
1564	from trac.util.datefmt import to_timestamp, utc
1565
1566	self.logger.debug('Replying to discussion topic', self.id)
1567
1568	## Get dissussion API component.
1569	#
1570	api = self.env[DiscussionApi]
1571	context = self._create_context(content, subject)
1572
1573	## Get replied topic.
1574	#
1575	topic = api.get_topic(context, self.id)
1576
1577	if not topic:
1578	self.logger.error("ERROR: Replied topic doesn't exist")
1579
1580	## Prepare message.
1581	#
1582	message = {'forum' : topic['forum'],
1583	'topic' : topic['id'],
1584	'replyto' : -1,
1585	'time' : to_timestamp(datetime.now(utc)),
1586	'author' : self.author,
1587	'body' : self.get_body_text(context.content_parts)}
1588
1589	## Add message to DB and commit it.
1590	#
1591	self._add_message(api, context, message)
1592	self.db.commit()
1593
1594	def discussion_message_reply(self, content, subject):
1595
1596	## Import modules.
1597	#
1598	from tracdiscussion.api import DiscussionApi
1599	from trac.util.datefmt import to_timestamp, utc
1600
1601	self.logger.debug('Replying to discussion message', self.id)
1602
1603	## Get dissussion API component.
1604	#
1605	api = self.env[DiscussionApi]
1606	context = self._create_context(content, subject)
1607
1608	## Get replied message.
1609	#
1610	message = api.get_message(context, self.id)
1611
1612	if not message:
1613	self.logger.error("ERROR: Replied message doesn't exist")
1614
1615	## Prepare message.
1616	#
1617	message = {'forum' : message['forum'],
1618	'topic' : message['topic'],
1619	'replyto' : message['id'],
1620	'time' : to_timestamp(datetime.now(utc)),
1621	'author' : self.author,
1622	'body' : self.get_body_text(context.content_parts)}
1623
1624	## Add message to DB and commit it.
1625	#
1626	self._add_message(api, context, message)
1627	self.db.commit()
1628
1629	def _create_context(self, content, subject):
1630
1631	## Import modules.
1632	#
1633	from trac.mimeview import Context
1634	from trac.web.api import Request
1635	from trac.perm import PermissionCache
1636
1637	## TODO: Read server base URL from config.
1638	# Create request object to mockup context creation.
1639	#
1640	environ = {'SERVER_PORT' : 80,
1641	'SERVER_NAME' : 'test',
1642	'REQUEST_METHOD' : 'POST',
1643	'wsgi.url_scheme' : 'http',
1644	'wsgi.input' : sys.stdin}
1645	chrome = {'links': {},
1646	'scripts': [],
1647	'ctxtnav': [],
1648	'warnings': [],
1649	'notices': []}
1650
1651	if self.env.base_url_for_redirect:
1652	environ['trac.base_url'] = self.env.base_url
1653
1654	req = Request(environ, None)
1655	req.chrome = chrome
1656	req.tz = 'missing'
1657	req.authname = self.author
1658	req.perm = PermissionCache(self.env, self.author)
1659	req.locale = None
1660
1661	## Create and return context.
1662	#
1663	context = Context.from_request(req)
1664	context.realm = 'discussion-email2trac'
1665	context.cursor = self.db.cursor()
1666	context.content = content
1667	context.subject = subject
1668
1669	## Read content parts from content.
1670	#
1671	context.content_parts = self.get_message_parts(content)
1672	context.content_parts = self.unique_attachment_names(
1673	context.content_parts)
1674
1675	return context
1676
1677	def _add_topic(self, api, context, topic):
1678	context.req.perm.assert_permission('DISCUSSION_APPEND')
1679
1680	## Filter topic.
1681	#
1682	for discussion_filter in api.discussion_filters:
1683	accept, topic_or_error = discussion_filter.filter_topic(
1684	context, topic)
1685	if accept:
1686	topic = topic_or_error
1687	else:
1688	raise TracError(topic_or_error)
1689
1690	## Add a new topic.
1691	#
1692	api.add_topic(context, topic)
1693
1694	## Get inserted topic with new ID.
1695	#
1696	topic = api.get_topic_by_time(context, topic['time'])
1697
1698	## Attach attachments.
1699	#
1700	self.id = topic['id']
1701	self.attach_attachments(context.content_parts, True)
1702
1703	## Notify change listeners.
1704	#
1705	for listener in api.topic_change_listeners:
1706	listener.topic_created(context, topic)
1707
1708	def _add_message(self, api, context, message):
1709	context.req.perm.assert_permission('DISCUSSION_APPEND')
1710
1711	## Filter message.
1712	#
1713	for discussion_filter in api.discussion_filters:
1714	accept, message_or_error = discussion_filter.filter_message(
1715	context, message)
1716	if accept:
1717	message = message_or_error
1718	else:
1719	raise TracError(message_or_error)
1720
1721	## Add message.
1722	#
1723	api.add_message(context, message)
1724
1725	## Get inserted message with new ID.
1726	#
1727	message = api.get_message_by_time(context, message['time'])
1728
1729	## Attach attachments.
1730	#
1731
1732	self.attach_attachments(context.content_parts, True)
1733
1734	## Notify change listeners.
1735	#
1736	for listener in api.message_change_listeners:
1737	listener.message_created(context, message)
1738
1739	########## MAIN function ######################################################
1740
1741	def parse(self, fp):
1742	"""
1743	"""
1744	self.logger.debug('Main function parse')
1745	global m
1746
1747	m = email.message_from_file(fp)
1748
1749	if not m:
1750	self.logger.debug('This is not a valid email message format')
1751	return
1752
1753	## Work around lack of header folding in Python; see http://bugs.python.org/issue4696
1754	#
1755	try:
1756	m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
1757	except AttributeError, detail:
1758	pass
1759
1760	if self.parameters.debug: # save the entire e-mail message text
1761	self.save_email_for_debug(m, self.parameters.project_name, True)
1762
1763	if self.VERSION <= 0.11:
1764	self.db = self.env.get_db_cnx()
1765	else:
1766	self.db = self.env.get_read_db()
1767
1768	self.get_sender_info(m)
1769
1770
1771	if self.parameters.white_list_file:
1772	self.acl_list_from_file(self.parameters.white_list_file, 'white_list')
1773
1774	if not ( self.email_header_acl('white_list', self.email_addr, True) or self.allow_registered_user ) :
1775
1776	self.logger.info('Message rejected : %s not in white list' %(self.email_addr))
1777	return False
1778
1779	if self.email_header_acl('black_list', self.email_addr, False):
1780	self.logger.info('Message rejected : %s in black list' %(self.email_addr))
1781	return False
1782
1783	if not self.email_header_acl('recipient_list', self.email_to_addrs, True):
1784	self.logger.info('Message rejected : %s not in recipient list' %(self.email_to_addrs))
1785	return False
1786
1787	## If spam drop the message
1788	#
1789	if self.spam(m) == 'drop':
1790	return False
1791
1792	elif self.spam(m) == 'spam':
1793	spam_msg = True
1794	else:
1795	spam_msg = False
1796
1797	if not m['Subject']:
1798	subject = 'No Subject'
1799	else:
1800	subject = self.email_to_unicode(m['Subject'])
1801
1802	## Bug in python logging module <2.6
1803	#
1804	self.logger.info('subject: %s' %repr(subject))
1805
1806	## First try the new method and then fall back to old method
1807	#
1808	if not self.parse_delivered_to_field(m, subject, spam_msg):
1809	self.parse_subject_field(m, subject, spam_msg)
1810
1811	def parse_delivered_to_field(self, m, subject, spam_msg):
1812	"""
1813	"""
1814	self.logger.debug('function parse_delivered_to_field')
1815
1816	## Ticket id is in Delivered-To Field. Trac notify must send this
1817	# eg: is+390@surfsara.nl
1818
1819	## Only run if this parameter is se
1820	#
1821	if not self.parameters.recipient_delimiter:
1822	return False
1823
1824	try:
1825
1826	#print self.parameters.project_name
1827	self.logger.debug('Delivered To: %s' %m['Delivered-To'])
1828
1829	id = m['Delivered-To']
1830	id = id.split(self.parameters.recipient_delimiter)[1]
1831	id = id.split('@')[0]
1832
1833	self.logger.debug('\t Found ticket id: %s' %id)
1834
1835	if not self.ticket_update(m, id, spam_msg):
1836	return False
1837
1838	except KeyError, detail:
1839	pass
1840	except IndexError, detail:
1841	pass
1842
1843	return False
1844
1845	def parse_subject_field(self, m, subject, spam_msg):
1846	"""
1847	"""
1848	self.logger.debug('function parse_subject_header')
1849
1850	## [hic] #1529: Re: LRZ
1851	# [hic] #1529?owner=bas,priority=medium: Re: LRZ
1852	#
1853	ticket_regex = r'''
1854	(?P<new_fields>[#][?].*)
1855	\|(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?.*)?:)
1856	'''
1857
1858	## Check if FullBlogPlugin is installed
1859	#
1860	blog_enabled = None
1861	blog_regex = ''
1862	if self.get_config('components', 'tracfullblog.*') in ['enabled']:
1863	self.logger.debug('Trac BLOG support enabled')
1864	blog_enabled = True
1865	blog_regex = '''\|(?P<blog>blog(?P<blog_params>[?][^:])?:(?P<blog_id>\S))'''
1866
1867
1868	## Check if DiscussionPlugin is installed
1869	#
1870	discussion_enabled = None
1871	discussion_regex = ''
1872	if self.get_config('components', 'tracdiscussion.api.discussionapi') in ['enabled']:
1873	self.logger.debug('Trac Discussion support enabled')
1874	discussion_enabled = True
1875	discussion_regex = r'''
1876	\|(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
1877	\|(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
1878	\|(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
1879	'''
1880
1881
1882	regex_str = ticket_regex + blog_regex + discussion_regex
1883	SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
1884
1885	## Find out if this is a ticket, a blog or a discussion
1886	#
1887	result = SYSTEM_RE.search(subject)
1888
1889	if result:
1890	## update ticket + fields
1891	#
1892	if result.group('reply'):
1893	self.system = 'ticket'
1894
1895	## Skip the last ':' character
1896	#
1897	if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
1898	self.new_ticket(m, subject, spam_msg)
1899
1900	## New ticket + fields
1901	#
1902	elif result.group('new_fields'):
1903	self.system = 'ticket'
1904	self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
1905
1906	if blog_enabled:
1907	if result.group('blog'):
1908	self.system = 'blog'
1909	self.blog(m, subject[result.end('blog'):], result.group('blog_id'), result.group('blog_params'))
1910
1911	if discussion_enabled:
1912	## New topic.
1913	#
1914	if result.group('forum'):
1915	self.system = 'discussion'
1916	self.id = int(result.group('forum_id'))
1917	self.discussion_topic(m, subject[result.end('forum'):])
1918
1919	## Reply to topic.
1920	#
1921	elif result.group('topic'):
1922	self.system = 'discussion'
1923	self.id = int(result.group('topic_id'))
1924	self.discussion_topic_reply(m, subject[result.end('topic'):])
1925
1926	## Reply to topic message.
1927	#
1928	elif result.group('message'):
1929	self.system = 'discussion'
1930	self.id = int(result.group('message_id'))
1931	self.discussion_message_reply(m, subject[result.end('message'):])
1932
1933	else:
1934
1935	self.system = 'ticket'
1936	(matched_id, subject) = self.ticket_update_by_subject(subject)
1937
1938	if matched_id:
1939
1940	if not self.ticket_update(m, matched_id, spam_msg):
1941	self.new_ticket(m, subject, spam_msg)
1942
1943	else:
1944	## No update by subject, so just create a new ticket
1945	#
1946	self.new_ticket(m, subject, spam_msg)
1947
1948
1949	########## BODY TEXT functions ###########################################################
1950
1951	def strip_signature(self, text):
1952	"""
1953	Strip signature from message, inspired by Mailman software
1954	"""
1955	self.logger.debug('function strip_signature: %s' %self.parameters.strip_signature_regex)
1956
1957	body = []
1958
1959	STRIP_RE = re.compile( self.parameters.strip_signature_regex )
1960	for line in text.splitlines():
1961
1962	match = STRIP_RE.match(line)
1963	if match:
1964	self.logger.debug('\t"%s " matched, skiping rest of message' %line)
1965	break
1966
1967	body.append(line)
1968
1969	return ('\n'.join(body))
1970
1971	def reflow(self, text, delsp = 0):
1972	"""
1973	Reflow the message based on the format="flowed" specification (RFC 3676)
1974	"""
1975	flowedlines = []
1976	quotelevel = 0
1977	prevflowed = 0
1978
1979	for line in text.splitlines():
1980	from re import match
1981
1982	## Figure out the quote level and the content of the current line
1983	#
1984	m = match('(>)( ?)(.)', line)
1985	linequotelevel = len(m.group(1))
1986	line = m.group(3)
1987
1988	## Determine whether this line is flowed
1989	#
1990	if line and line != '-- ' and line[-1] == ' ':
1991	flowed = 1
1992	else:
1993	flowed = 0
1994
1995	if flowed and delsp and line and line[-1] == ' ':
1996	line = line[:-1]
1997
1998	## If the previous line is flowed, append this line to it
1999	#
2000	if prevflowed and line != '-- ' and linequotelevel == quotelevel:
2001	flowedlines[-1] += line
2002
2003	## Otherwise, start a new line
2004	#
2005	else:
2006	flowedlines.append('>' * linequotelevel + line)
2007
2008	prevflowed = flowed
2009
2010
2011	return '\n'.join(flowedlines)
2012
2013	def strip_quotes(self, text):
2014	"""
2015	Strip quotes from message by Nicolas Mendoza
2016	"""
2017	self.logger.debug('function strip_quotes: %s' %self.parameters.email_quote)
2018
2019	body = []
2020
2021	STRIP_RE = re.compile( self.parameters.email_quote )
2022
2023	for line in text.splitlines():
2024
2025	try:
2026
2027	match = STRIP_RE.match(line)
2028	if match:
2029	self.logger.debug('\t"%s " matched, skipping rest of message' %line)
2030	continue
2031
2032	except UnicodeDecodeError:
2033
2034	tmp_line = self.email_to_unicode(line)
2035
2036	match = STRIP_RE.match(tmp_line)
2037	if match:
2038	self.logger.debug('\t"%s " matched, skipping rest of message' %line)
2039	continue
2040
2041	body.append(line)
2042
2043	return ('\n'.join(body))
2044
2045	def inline_properties(self, text):
2046	"""
2047	Parse text if we use inline keywords to set ticket fields
2048	"""
2049	self.logger.debug('function inline_properties')
2050
2051	properties = dict()
2052	body = list()
2053
2054	INLINE_EXP = re.compile('\s[@]\s(\w+)\s:(.)$')
2055
2056	for line in text.splitlines():
2057	match = INLINE_EXP.match(line)
2058	if match:
2059	keyword, value = match.groups()
2060
2061	if self.parameters.inline_properties_first_wins:
2062	if keyword in self.properties.keys():
2063	continue
2064
2065	self.properties[keyword] = value.strip()
2066	self.logger.debug('\tinline properties: %s : %s' %(keyword,value))
2067
2068	else:
2069	body.append(line)
2070
2071	return '\n'.join(body)
2072
2073
2074	def wrap_text(self, text, replace_whitespace = False):
2075	"""
2076	Will break a lines longer then given length into several small
2077	lines of size given length
2078	"""
2079	import textwrap
2080
2081	LINESEPARATOR = '\n'
2082	reformat = ''
2083
2084	for s in text.split(LINESEPARATOR):
2085	tmp = textwrap.fill(s, self.parameters.use_textwrap)
2086	if tmp:
2087	reformat = '%s\n%s' %(reformat,tmp)
2088	else:
2089	reformat = '%s\n' %reformat
2090
2091	return reformat
2092
2093	# Python2.4 and higher
2094	#
2095	#return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
2096	#
2097
2098	########## EMAIL attachements functions ###########################################################
2099
2100	def inline_part(self, part):
2101	"""
2102	"""
2103	self.logger.debug('function inline_part()')
2104
2105	return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
2106
2107	def get_message_parts(self, msg, new_email=False):
2108	"""
2109	parses the email message and returns a list of body parts and attachments
2110	body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
2111	"""
2112	self.logger.debug('function get_message_parts()')
2113
2114	message_parts = list()
2115
2116	ALTERNATIVE_MULTIPART = False
2117
2118	for part in msg.walk():
2119	content_maintype = part.get_content_maintype()
2120	content_type = part.get_content_type()
2121
2122	self.logger.debug('\t Message part: Main-Type: %s' % content_maintype)
2123	self.logger.debug('\t Message part: Content-Type: %s' % content_type)
2124
2125	## Check content type
2126	#
2127	if content_type in self.STRIP_CONTENT_TYPES:
2128	self.logger.debug("\t A %s attachment named '%s' was skipped" %(content_type, part.get_filename()))
2129	continue
2130
2131	## Catch some mulitpart execptions
2132	#
2133	if content_type == 'multipart/alternative':
2134	ALTERNATIVE_MULTIPART = True
2135	continue
2136
2137	## Skip multipart containers
2138	#
2139	if content_maintype == 'multipart':
2140	self.logger.debug("\t Skipping multipart container")
2141	continue
2142
2143	## Check if this is an inline part. It's inline if there is co Cont-Disp header,
2144	# or if there is one and it says "inline"
2145	#
2146	inline = self.inline_part(part)
2147
2148	## Drop HTML message
2149	#
2150	if ALTERNATIVE_MULTIPART and self.parameters.drop_alternative_html_version:
2151
2152	if content_type == 'text/html':
2153	self.logger.debug('\t Skipping alternative HTML message')
2154	ALTERNATIVE_MULTIPART = False
2155	continue
2156
2157
2158	#if self.VERSION < 1.0:
2159	# filename = part.get_filename()
2160
2161	## convert 7 bit filename to 8 bit unicode
2162	#
2163	raw_filename = part.get_filename()
2164	filename = self.email_to_unicode(raw_filename);
2165
2166	s = '\t unicode filename: %s' %(filename)
2167	self.print_unicode(s)
2168	self.logger.debug('\t raw filename: %s' %repr(raw_filename))
2169
2170	if self.VERSION < 1.0:
2171	filename = self.check_filename_length(filename)
2172
2173	## Save all non plain text message as attachment
2174	#
2175	if not content_type in ['text/plain']:
2176
2177	message_parts.append( (filename, part) )
2178
2179	## We only convert html messages
2180	#
2181	if not content_type == 'text/html':
2182	self.logger.debug('\t Appending %s (%s)' %(repr(filename), content_type))
2183	continue
2184
2185
2186	## We have an text or html message
2187	#
2188	if not inline:
2189	self.logger.debug('\t Appending %s (%s), not an inline messsage part' %(repr(filename), content_type))
2190	message_parts.append( (filename, part) )
2191	continue
2192
2193	## Try to decode message part. We have a html or plain text messafe
2194	#
2195	body_text = part.get_payload(decode=1)
2196	if not body_text:
2197	body_text = part.get_payload(decode=0)
2198
2199	## Try to convert html message
2200	#
2201	if content_type == 'text/html':
2202
2203	body_text = self.html_2_txt(body_text)
2204	if not body_text:
2205	continue
2206
2207	format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
2208	delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
2209
2210	if self.parameters.reflow and not self.parameters.verbatim_format and format == 'flowed':
2211	body_text = self.reflow(body_text, delsp == 'yes')
2212
2213	if new_email and self.parameters.only_strip_on_update:
2214	self.logger.debug('Skip signature/quote stripping for new messages')
2215	else:
2216	if self.parameters.strip_signature:
2217	body_text = self.strip_signature(body_text)
2218
2219	if self.parameters.strip_quotes:
2220	body_text = self.strip_quotes(body_text)
2221
2222	if self.parameters.inline_properties:
2223	body_text = self.inline_properties(body_text)
2224
2225	if self.parameters.use_textwrap:
2226	body_text = self.wrap_text(body_text)
2227
2228	## Get contents charset (iso-8859-15 if not defined in mail headers)
2229	#
2230	charset = part.get_content_charset()
2231	if not charset:
2232	charset = 'iso-8859-15'
2233
2234	try:
2235	ubody_text = unicode(body_text, charset)
2236
2237	except UnicodeError, detail:
2238	ubody_text = unicode(body_text, 'iso-8859-15')
2239
2240	except LookupError, detail:
2241	ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
2242
2243	if self.parameters.verbatim_format:
2244	message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
2245	else:
2246	message_parts.append('%s' %ubody_text)
2247
2248	return message_parts
2249
2250	def unique_attachment_names(self, message_parts):
2251	"""
2252	Make sure we have unique names attachments:
2253	- check if it contains illegal characters
2254	- Rename "None" filenames to "untitled-part"
2255	"""
2256	self.logger.debug('function unique_attachment_names()')
2257	renamed_parts = []
2258	attachment_names = set()
2259
2260	for item in message_parts:
2261
2262	## If not an attachment, leave it alone
2263	#
2264	if not isinstance(item, tuple):
2265	renamed_parts.append(item)
2266	continue
2267
2268	(filename, part) = item
2269
2270	## If filename = None, use a default one
2271	#
2272	if filename in [ 'None']:
2273	filename = 'untitled-part'
2274	self.logger.info('\t Rename filename "None" to: %s' %filename)
2275
2276	## Guess the extension from the content type, use non strict mode
2277	# some additional non-standard but commonly used MIME types
2278	# are also recognized
2279	#
2280	ext = mimetypes.guess_extension(part.get_content_type(), False)
2281	if not ext:
2282	ext = '.bin'
2283
2284	filename = '%s%s' % (filename, ext)
2285
2286	## Discard relative paths for windows/unix in attachment names
2287	#
2288	filename = filename.replace('\\', '_')
2289	filename = filename.replace('/', '_')
2290
2291	## remove linefeed char
2292	#
2293	for forbidden_char in ['\r', '\n']:
2294	filename = filename.replace(forbidden_char,'')
2295
2296	## We try to normalize the filename to utf-8 NFC if we can.
2297	# Files uploaded from OS X might be in NFD.
2298	# Check python version and then try it
2299	#
2300	#if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
2301	# try:
2302	# filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
2303	# except TypeError:
2304	# pass
2305
2306	## Make the filename unique for this ticket
2307	#
2308	num = 0
2309	unique_filename = filename
2310	dummy_filename, ext = os.path.splitext(filename)
2311
2312	while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
2313	num += 1
2314	unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
2315
2316	s = '\t Attachment with filename %s will be saved as %s' % (filename, unique_filename)
2317	self.print_unicode(s)
2318
2319	attachment_names.add(unique_filename)
2320
2321	renamed_parts.append((filename, unique_filename, part))
2322
2323	return renamed_parts
2324
2325
2326	def attachment_exists(self, filename):
2327
2328	self.logger.debug("function attachment_exists")
2329
2330	s = '\t check if attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
2331	self.print_unicode(s)
2332
2333	## Do we have a valid ticket id
2334	#
2335	if not self.id:
2336	return False
2337
2338	try:
2339	if self.system == 'discussion':
2340
2341	att = attachment.Attachment(self.env, 'discussion', 'ticket/%s' % (self.id,), filename)
2342
2343	elif self.system == 'blog':
2344
2345	att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,), filename)
2346
2347	else:
2348
2349	att = attachment.Attachment(self.env, 'ticket', self.id, filename)
2350
2351	return True
2352
2353	except attachment.ResourceNotFound:
2354
2355	return False
2356
2357	########## TRAC Ticket Text ###########################################################
2358
2359	def get_body_text(self, message_parts):
2360	"""
2361	"""
2362	self.logger.debug('function get_body_text()')
2363
2364	body_text = []
2365
2366	for part in message_parts:
2367
2368	## Plain text part, append it
2369	#
2370	if not isinstance(part, tuple):
2371	body_text.extend(part.strip().splitlines())
2372	body_text.append("")
2373	continue
2374
2375	(original, filename, part) = part
2376	inline = self.inline_part(part)
2377
2378	## Skip generation of attachment link if html is converted to text
2379	#
2380	if part.get_content_type() == 'text/html' and self.parameters.html2text_cmd and inline:
2381	s = 'Skipping attachment link for html part: %s' %(filename)
2382	self.print_unicode(s)
2383	continue
2384
2385	if part.get_content_maintype() == 'image' and inline:
2386
2387	if self.system != 'discussion':
2388	s = 'wiki image link for: %s' %(filename)
2389	self.print_unicode(s)
2390	body_text.append('[[Image(%s)]]' % filename)
2391
2392	body_text.append("")
2393
2394	else:
2395
2396	if self.system != 'discussion':
2397
2398	s = 'wiki attachment link for: %s' %(filename)
2399	self.print_unicode(s)
2400	body_text.append('[attachment:"%s"]' % filename)
2401
2402	body_text.append("")
2403
2404	## Convert list body_texts to string
2405	#
2406	body_text = '\r\n'.join(body_text)
2407	return body_text
2408
2409	def html_mailto_link(self, subject):
2410	"""
2411	This function returns a HTML mailto tag with the ticket id and author email address
2412	"""
2413	self.logger.debug("function html_mailto_link")
2414
2415	if not self.author:
2416	author = self.email_addr
2417	else:
2418	author = self.author
2419
2420	if not self.parameters.mailto_cc:
2421	self.parameters.mailto_cc = ''
2422
2423	## Bug in urllib.quote function
2424	#
2425	if isinstance(subject, unicode):
2426	subject = subject.encode('utf-8')
2427
2428	## use urllib to escape the chars
2429	#
2430	s = '%s?Subject=%s&cc=%s' %(
2431	urllib.quote(self.email_addr),
2432	urllib.quote('Re: #%s: %s' %(self.id, subject)),
2433	urllib.quote(self.parameters.mailto_cc)
2434	)
2435
2436	if self.VERSION in [ 0.10 ]:
2437	s = '\r\n{{{\r\n#!html\r\n<a\r\n href="mailto:%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
2438	else:
2439	s = '[mailto:"%s" Reply to: %s]' %(s, author)
2440
2441	self.logger.debug("\tmailto link %s" %s)
2442	return s
2443
2444	########## TRAC notify section ###########################################################
2445
2446	def notify(self, tkt, new=True, modtime=0):
2447	"""
2448	A wrapper for the TRAC notify function. So we can use templates
2449	"""
2450	self.logger.debug('function notify()')
2451
2452
2453	class Email2TracNotifyEmail(TicketNotifyEmail):
2454
2455	def __init__(self, env):
2456	TicketNotifyEmail.__init__(self, env)
2457	self.email2trac_notify_reporter = None
2458	self.email2trac_replyto = None
2459
2460	def send(self, torcpts, ccrcpts):
2461	#print 'Yes this works'
2462	dest = self.reporter or 'anonymous'
2463	hdrs = {}
2464	hdrs['Message-ID'] = self.get_message_id(dest, self.modtime)
2465	hdrs['X-Trac-Ticket-ID'] = str(self.ticket.id)
2466	hdrs['X-Trac-Ticket-URL'] = self.data['ticket']['link']
2467	if not self.newticket:
2468	msgid = self.get_message_id(dest)
2469	hdrs['In-Reply-To'] = msgid
2470	hdrs['References'] = msgid
2471
2472
2473	if self.email2trac_notify_reporter:
2474	if not self.email2trac_notify_reporter in torcpts:
2475	torcpts.append(sender_email)
2476
2477	if self.email2trac_replyto:
2478	# use to rewrite reply to
2479	# hdrs does not work, multiple reply addresses
2480	#hdrs['Reply-To'] = 'bas.van.der.vlies@gmail.com'
2481	self.replyto_email = self.email2trac_replyto
2482
2483	NotifyEmail.send(self, torcpts, ccrcpts, hdrs)
2484
2485	if self.parameters.dry_run :
2486	self.logger.info('DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter'])
2487	return
2488
2489	try:
2490
2491
2492	## create false {abs_}href properties, to trick Notify()
2493	#
2494	if self.VERSION in [0.10]:
2495	self.env.abs_href = Href(self.get_config('project', 'url'))
2496	self.env.href = Href(self.get_config('project', 'url'))
2497
2498	tn = Email2TracNotifyEmail(self.env)
2499
2500	## additionally append sender (regardeless of settings in trac.ini)
2501	#
2502	if self.parameters.notify_reporter:
2503
2504	self.logger.debug('\t Notify reporter set')
2505
2506	if not self.email_header_acl('notify_reporter_black_list', self.email_addr, False):
2507	tn.email2trac_notify_reporter = self.email_addr
2508
2509	if self.parameters.notify_replyto_rewrite:
2510
2511	self.logger.debug('\t Notify replyto rewrite set')
2512
2513	action, value = self.parameters.notify_replyto_rewrite.split(':')
2514
2515	if action in ['use_mail_domain']:
2516	self.logger.debug('\t\t use_mail_domain:%s' %value)
2517	tn.email2trac_replyto = '%s@%s' %(self.id, value)
2518
2519	elif action in ['use_trac_smtp_replyto']:
2520	self.logger.debug('\t\t use_trac_smtp_replyto delimiter:%s' %value)
2521	dummy = self.smtp_replyto.split('@')
2522	if len(dummy) > 1:
2523	tn.email2trac_replyto = '%s%s%s@%s' %(dummy[0], value, self.id, dummy[1])
2524	else:
2525	tn.email2trac_replyto = '%s%s%s' %(dummy[0], value, self.id)
2526
2527	if self.parameters.alternate_notify_template:
2528
2529	if self.VERSION >= 0.11:
2530
2531	from trac.web.chrome import Chrome
2532
2533	if self.parameters.alternate_notify_template_update and not new:
2534	tn.template_name = self.parameters.alternate_notify_template_update
2535	else:
2536	tn.template_name = self.parameters.alternate_notify_template
2537
2538	tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
2539
2540	else:
2541
2542	tn.template_name = self.parameters.alternate_notify_template
2543
2544	tn.notify(tkt, new, modtime)
2545
2546	except Exception, e:
2547	self.logger.error('Failure sending notification on creation of ticket #%s: %s' %(self.id, e))
2548
2549	########## END Class Definition ########################################################
2550
2551
2552	########## Parse Config File ###########################################################
2553
2554	def ReadConfig(file, name):
2555	"""
2556	Parse the config file
2557	"""
2558	if not os.path.isfile(file):
2559	print 'File %s does not exist' %file
2560	sys.exit(1)
2561
2562	config = trac_config.Configuration(file)
2563
2564	parentdir = config.get('DEFAULT', 'parentdir')
2565	sections = config.sections()
2566
2567	## use some trac internals to get the defaults
2568	#
2569	tmp = config.parser.defaults()
2570	project = SaraDict()
2571
2572	for option, value in tmp.items():
2573	try:
2574	project[option] = int(value)
2575	except ValueError:
2576	project[option] = value
2577
2578	if name:
2579	if name in sections:
2580	project = SaraDict()
2581	for option, value in config.options(name):
2582	try:
2583	project[option] = int(value)
2584	except ValueError:
2585	project[option] = value
2586
2587	elif not parentdir:
2588	print "Not a valid project name: %s, valid names are: %s" %(name, sections)
2589	print "or set parentdir in the [DEFAULT] section"
2590	sys.exit(1)
2591
2592	## If parentdir then set project dir to parentdir + name
2593	#
2594	if not project.has_key('project'):
2595	if not parentdir:
2596	print "You must set project or parentdir in your configuration file"
2597	sys.exit(1)
2598	elif not name:
2599	print "You must configure a project section in your configuration file"
2600	else:
2601	project['project'] = os.path.join(parentdir, name)
2602
2603	##
2604	# Save the project name
2605	#
2606	project['project_name'] = os.path.basename(project['project'])
2607
2608	return project
2609
2610	########## Setup Logging ###############################################################
2611
2612	def setup_log(parameters, project_name, interactive=None):
2613	"""
2614	Setup logging
2615
2616	Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
2617	would be interpreted by the ConfigParser itself.
2618	"""
2619	logger = logging.getLogger('email2trac %s' %project_name)
2620
2621	if interactive:
2622	parameters.log_type = 'stderr'
2623
2624	if not parameters.log_type:
2625	if sys.platform in ['win32', 'cygwin']:
2626	parameters.log_type = 'eventlog'
2627	else:
2628	parameters.log_type = 'syslog'
2629
2630	if parameters.log_type == 'file':
2631
2632	if not parameters.log_file:
2633	parameters.log_file = 'email2trac.log'
2634
2635	if not os.path.isabs(parameters.log_file):
2636	parameters.log_file = os.path.join(tempfile.gettempdir(), parameters.log_file)
2637
2638	log_handler = logging.FileHandler(parameters.log_file)
2639
2640	elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
2641	## Requires win32 extensions
2642	#
2643	logid = "email2trac"
2644	log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
2645
2646	elif parameters.log_type in ('syslog', 'unix'):
2647	log_handler = logging.handlers.SysLogHandler('/dev/log')
2648
2649	elif parameters.log_type in ('stderr'):
2650	log_handler = logging.StreamHandler(sys.stderr)
2651
2652	else:
2653	log_handler = logging.handlers.BufferingHandler(0)
2654
2655	if parameters.log_format:
2656	parameters.log_format = parameters.log_format.replace('$(', '%(')
2657	else:
2658	parameters.log_format = '%(name)s: %(message)s'
2659	if parameters.log_type in ('file', 'stderr'):
2660	parameters.log_format = '%(asctime)s ' + parameters.log_format
2661
2662	log_formatter = logging.Formatter(parameters.log_format)
2663	log_handler.setFormatter(log_formatter)
2664	logger.addHandler(log_handler)
2665
2666	if (parameters.log_level in ['DEBUG', 'ALL']) or (parameters.debug > 0):
2667	logger.setLevel(logging.DEBUG)
2668	parameters.debug = 1
2669
2670	elif parameters.log_level in ['INFO'] or parameters.verbose:
2671	logger.setLevel(logging.INFO)
2672
2673	elif parameters.log_level in ['WARNING']:
2674	logger.setLevel(logging.WARNING)
2675
2676	elif parameters.log_level in ['ERROR']:
2677	logger.setLevel(logging.ERROR)
2678
2679	elif parameters.log_level in ['CRITICAL']:
2680	logger.setLevel(logging.CRITICAL)
2681
2682	else:
2683	logger.setLevel(logging.INFO)
2684
2685	return logger
2686
2687	########## Own TicketNotifyEmail class ###############################################################
2688
2689	if __name__ == '__main__':
2690	## Default config file
2691	#
2692	agilo = False
2693	bh_product = None
2694	configfile = '@email2trac_conf@'
2695	project = ''
2696	component = ''
2697	ticket_prefix = 'default'
2698	dry_run = None
2699	verbose = None
2700	debug_interactive = None
2701	virtualenv = '@virtualenv@'
2702
2703	SHORT_OPT = 'AB:cdE:hf:np:t:v'
2704	LONG_OPT = ['agilo', 'bloodhound_product=', 'component=', 'debug',
2705	'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=',
2706	'virtualenv=', 'verbose']
2707
2708	try:
2709	opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
2710	except getopt.error,detail:
2711	print __doc__
2712	print detail
2713	sys.exit(1)
2714
2715	project_name = None
2716	for opt,value in opts:
2717	if opt in [ '-h', '--help']:
2718	print __doc__
2719	sys.exit(0)
2720	elif opt in ['-A', '--agilo']:
2721	agilo = True
2722	elif opt in ['-B', '--bloodhound_product']:
2723	bh_product = value
2724	elif opt in ['-c', '--component']:
2725	component = value
2726	elif opt in ['-d', '--debug']:
2727	debug_interactive = 1
2728	elif opt in ['-E', '--virtualenv']:
2729	virtualenv = value
2730	elif opt in ['-f', '--file']:
2731	configfile = value
2732	elif opt in ['-n', '--dry-run']:
2733	dry_run = True
2734	elif opt in ['-p', '--project']:
2735	project_name = value
2736	elif opt in ['-t', '--ticket_prefix']:
2737	ticket_prefix = value
2738	elif opt in ['-v', '--verbose']:
2739	verbose = True
2740
2741	if virtualenv and os.path.exists(virtualenv):
2742	activate_this = os.path.join(virtualenv, 'bin/activate_this.py')
2743	if os.path.exists(activate_this):
2744	execfile(activate_this, dict(__file__=activate_this))
2745
2746	try:
2747	from trac import __version__ as trac_version
2748	from trac import config as trac_config
2749
2750	except ImportError, detail:
2751	print "Can not find a a valid trac installation, solutions could be:"
2752	print "\tset PYTHONPATH"
2753	print "\tuse the --virtualenv <dir> option"
2754	sys.exit(1)
2755
2756	settings = ReadConfig(configfile, project_name)
2757
2758	## The default prefix for ticket values in email2trac.conf
2759	#
2760	settings.ticket_prefix = ticket_prefix
2761	settings.dry_run = dry_run
2762	settings.verbose = verbose
2763
2764	if bh_product:
2765	settings.bh_product = bh_product
2766
2767	if not settings.debug and debug_interactive:
2768	settings.debug = debug_interactive
2769
2770	if not settings.project:
2771	print __doc__
2772	print 'No Trac project is defined in the email2trac config file.'
2773	sys.exit(1)
2774
2775	logger = setup_log(settings, os.path.basename(settings.project), debug_interactive)
2776
2777	if component:
2778	settings['component'] = component
2779
2780	## We are only interested in the major versions
2781	# 0.12.3 --> 0.12
2782	# 1.0.2 --> 1.0
2783	#
2784	l = trac_version.split('.')
2785	version = '.'.join(l[0:2])
2786
2787	logger.debug("Found trac version: %s" %(version))
2788
2789	try:
2790	if version == '0.10':
2791	from trac import attachment
2792	from trac.env import Environment
2793	from trac.ticket import Ticket
2794	from trac.web.href import Href
2795	from trac import util
2796	from trac.ticket.web_ui import TicketModule
2797
2798	#
2799	# return util.text.to_unicode(str)
2800	#
2801	# see http://projects.edgewall.com/trac/changeset/2799
2802	from trac.ticket.notification import TicketNotifyEmail
2803	from trac import config as trac_config
2804	from trac.core import TracError
2805
2806	elif version in ['0.11', '0.12', '0.13', '1.0', '1.1']:
2807	from trac import attachment
2808	from trac import config as trac_config
2809	from trac import util
2810	from trac.core import TracError
2811	from trac.env import Environment
2812	from trac.perm import PermissionSystem
2813	from trac.perm import PermissionCache
2814	from trac.test import Mock, MockPerm
2815	from trac.ticket.api import TicketSystem
2816	from trac.ticket.web_ui import TicketModule
2817	from trac.web.href import Href
2818
2819	try:
2820	import pkg_resources
2821	pkg = pkg_resources.get_distribution('BloodhoundMultiProduct')
2822	bloodhound = pkg.version.split()[:2]
2823	except pkg_resources.DistributionNotFound:
2824	# assume no bloodhound
2825	bloodhound = None
2826
2827	if bloodhound:
2828	from multiproduct.env import Environment, ProductEnvironment
2829	from multiproduct.ticket.web_ui import (ProductTicketModule
2830	as TicketModule)
2831	logger.debug("Found Bloodhound Distribution")
2832
2833	if not settings.bh_product:
2834	print __doc__
2835	print 'No Bloodhound project defined (bh_project) in section:%s email2trac config file.' %(settings.project)
2836	sys.exit(1)
2837
2838	if agilo:
2839
2840	try:
2841
2842	from agilo.ticket.model import AgiloTicket as Ticket
2843
2844	except ImportError, detail:
2845
2846	try:
2847	from agilo.ticket.model import Ticket
2848	except ImportError, detail:
2849	logger.error('Could not find Trac Agilo environemnt')
2850	sys.exit(0)
2851
2852	else:
2853
2854	from trac.ticket import Ticket
2855
2856	#
2857	# return util.text.to_unicode(str)
2858	#
2859	# see http://projects.edgewall.com/trac/changeset/2799
2860	from trac.ticket.notification import TicketNotifyEmail
2861	from trac.notification import NotifyEmail
2862
2863	else:
2864	logger.error('TRAC version %s is not supported' %version)
2865	sys.exit(0)
2866
2867	## Must be set before environment is created
2868	#
2869	if settings.has_key('python_egg_cache'):
2870	python_egg_cache = str(settings['python_egg_cache'])
2871	os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
2872
2873	if settings.debug > 0:
2874	logger.debug('Loading environment %s', settings.project)
2875
2876	try:
2877	env = Environment(settings['project'], create=0)
2878
2879	if bloodhound:
2880	## possibly overkill testing if the multiproduct schema is a
2881	# new enough version
2882	#
2883	from multiproduct.env import MultiProductSystem
2884	mps = MultiProductSystem(env)
2885	if mps.get_version() > 4:
2886	try:
2887	env = ProductEnvironment(env, settings.bh_product, create=0)
2888	except LookupError:
2889	logger.error('%s is not a valid Bloodhound' %settings.bh_product)
2890	sys.exit(0)
2891
2892	except IOError, detail:
2893	logger.error("trac error: %s" %detail)
2894	sys.exit(0)
2895	except TracError, detail:
2896	logger.error("trac error: %s" %detail)
2897	sys.exit(0)
2898
2899	tktparser = TicketEmailParser(env, settings, logger, float(version))
2900	tktparser.parse(sys.stdin)
2901
2902	## Catch all errors and use the logging module
2903	#
2904	except Exception, error:
2905
2906	etype, evalue, etb = sys.exc_info()
2907	for e in traceback.format_exception(etype, evalue, etb):
2908	logger.critical(e)
2909
2910	if m:
2911	tktparser.save_email_for_debug(m, settings.project_name, True)
2912
2913	sys.exit(1)
2914
2915	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format