Context Navigation

source: trunk/email2trac.py.in @ 612

Last change on this file since 612 was 612, checked in by bas, 12 years ago

adjusted strip_siganture_regex to:

^{-- $}

removed this from strip_signature_regex:

^{-----Original Message-----$}

This is an outlook quote and has nothing to do with signatures. People can add this to
email2trac.conf

Property svn:executable set to *
Property svn:keywords set to Id

File size: 85.7 KB

Rev	Line
[22]	1	#!@PYTHON@
	2	#
	3	# This file is part of the email2trac utils
	4	#
[595]	5	# Copyright 2002 SARA
[22]	6	#
[595]	7	# Licensed under the Apache License, Version 2.0 (the "License");
	8	# you may not use this file except in compliance with the License.
	9	# You may obtain a copy of the License at
[22]	10	#
[595]	11	# http://www.apache.org/licenses/LICENSE-2.0
[22]	12	#
[595]	13	# Unless required by applicable law or agreed to in writing, software
	14	# distributed under the License is distributed on an "AS IS" BASIS,
	15	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	16	# See the License for the specific language governing permissions and
	17	# limitations under the License.
	18	#
[80]	19	# For vi/emacs or other use tabstop=4 (vi: set ts=4)
	20	#
[22]	21	"""
[547]	22	email2trac.py -- Email -> TRAC tickets
[22]	23
[547]	24	A MTA filter to create Trac tickets from inbound emails.
[22]	25
[547]	26	first proof of concept from:
	27	Copyright 2005, Daniel Lundin <daniel@edgewall.com>
	28	Copyright 2005, Edgewall Software
[22]	29
[282]	30	Authors:
	31	Bas van der Vlies <basv@sara.nl>
	32	Walter de Jong <walter@sara.nl>
[22]	33
	34	How to use
	35	----------
[282]	36	* See https://subtrac.sara.nl/oss/email2trac/
	37
[22]	38	* Commandline opions:
[205]	39	-h,--help
[507]	40	-d, --debug
[205]	41	-f,--file <configuration file>
[410]	42	-n,--dry-run
[205]	43	-p, --project <project name>
[415]	44	-t, --ticket_prefix <name>
[22]	45
	46	SVN Info:
	47	$Id: email2trac.py.in 612 2012-09-14 07:31:21Z bas $
	48	"""
	49	import os
	50	import sys
	51	import string
	52	import getopt
	53	import time
	54	import email
[136]	55	import email.Iterators
	56	import email.Header
[22]	57	import re
	58	import urllib
	59	import unicodedata
	60	import mimetypes
[96]	61	import traceback
[403]	62	import logging
	63	import logging.handlers
[404]	64	import UserDict
[498]	65	import tempfile
[433]	66
[520]	67	from datetime import timedelta, datetime
[433]	68	from stat import *
[22]	69
[593]	70	try:
	71	from trac import __version__ as trac_version
	72	from trac import config as trac_config
[403]	73
[593]	74	except ImportError, detail:
	75	print "Can not find a a valid trac installation, try setting PYTHONPATH"
	76	sys.exit(1)
[91]	77
[508]	78	## Some global variables
[96]	79	#
	80	m = None
[22]	81
[508]	82	# This is to for the function AlwaysNotifyReporter
	83	sender_email = None
	84
[404]	85	class SaraDict(UserDict.UserDict):
[507]	86	def __init__(self, dictin = None):
	87	UserDict.UserDict.__init__(self)
	88	self.name = None
	89
	90	if dictin:
	91	if dictin.has_key('name'):
	92	self.name = dictin['name']
	93	del dictin['name']
	94	self.data = dictin
	95
	96	def get_value(self, name):
	97	if self.has_key(name):
	98	return self[name]
	99	else:
	100	return None
	101
	102	def __repr__(self):
	103	return repr(self.data)
[182]	104
[507]	105	def __str__(self):
	106	return str(self.data)
	107
	108	def __getattr__(self, name):
	109	"""
	110	override the class attribute get method. Return the value
	111	from the dictionary
	112	"""
	113	if self.data.has_key(name):
	114	return self.data[name]
	115	else:
	116	return None
	117
	118	def __setattr__(self, name, value):
	119	"""
	120	override the class attribute set method only when the UserDict
	121	has set its class attribute
	122	"""
	123	if self.__dict__.has_key('data'):
	124	self.data[name] = value
	125	else:
	126	self.__dict__[name] = value
[404]	127
[507]	128	def __iter__(self):
	129	return iter(self.data.keys())
[404]	130
[22]	131	class TicketEmailParser(object):
[507]	132	env = None
	133	comment = '> '
[359]	134
[507]	135	def __init__(self, env, parameters, logger, version):
	136	self.env = env
[22]	137
[507]	138	# Database connection
	139	#
	140	self.db = None
[22]	141
[507]	142	# Save parameters
	143	#
	144	self.parameters = parameters
	145	self.logger = logger
[206]	146
[507]	147	# Some useful mail constants
	148	#
	149	self.email_name = None
	150	self.email_addr = None
	151	self.email_from = None
	152	self.author = None
	153	self.id = None
[587]	154
	155	## Will be set to True if the user has ben registered
	156	#
	157	self.allow_registered_user = False
[507]	158
	159	self.STRIP_CONTENT_TYPES = list()
[72]	160
[507]	161	## fields properties via body_text
	162	#
	163	self.properties = dict()
[452]	164
[507]	165	self.VERSION = version
[402]	166
[507]	167	self.get_config = self.env.config.get
[404]	168
[507]	169	## init function ##
	170	#
	171	self.setup_parameters()
[402]	172
[507]	173	def setup_parameters(self):
	174	if self.parameters.umask:
	175	os.umask(self.parameters.umask)
[402]	176
[507]	177	if not self.parameters.spam_level:
	178	self.parameters.spam_level = 0
[22]	179
[507]	180	if not self.parameters.spam_header:
	181	self.parameters.spam_header = 'X-Spam-Score'
[207]	182
[507]	183	if not self.parameters.email_quote:
[607]	184	self.parameters.email_quote = '^> .*'
[22]	185
[507]	186	if not self.parameters.ticket_update_by_subject_lookback:
	187	self.parameters.ticket_update_by_subject_lookback = 30
[360]	188
[507]	189	if self.parameters.verbatim_format == None:
	190	self.parameters.verbatim_format = 1
[118]	191
[507]	192	if self.parameters.reflow == None:
	193	self.parameters.reflow = 1
[231]	194
[507]	195	if self.parameters.binhex:
	196	self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
[238]	197
[507]	198	if self.parameters.applesingle:
	199	self.STRIP_CONTENT_TYPES.append('application/applefile')
[238]	200
[507]	201	if self.parameters.appledouble:
	202	self.STRIP_CONTENT_TYPES.append('application/applefile')
[238]	203
[507]	204	if self.parameters.strip_content_types:
	205	items = self.parameters.strip_content_types.split(',')
	206	for item in items:
	207	self.STRIP_CONTENT_TYPES.append(item.strip())
[294]	208
[507]	209	if self.parameters.tmpdir:
	210	self.parameters.tmpdir = os.path.normcase(str(self.parameters['tmpdir']))
	211	else:
	212	self.parameters.tmpdir = os.path.normcase('/tmp')
[257]	213
[507]	214	if self.parameters.email_triggers_workflow == None:
	215	self.parameters.email_triggers_workflow = 1
[191]	216
[507]	217	if not self.parameters.subject_field_separator:
	218	self.parameters.subject_field_separator = '&'
	219	else:
	220	self.parameters.subject_field_separator = self.parameters.subject_field_separator.strip()
[297]	221
[606]	222	if not self.parameters.strip_signature_regex:
[612]	223	self.parameters.strip_signature_regex = '^-- $'
[606]	224
	225
[507]	226	self.trac_smtp_from = self.get_config('notification', 'smtp_from')
	227	self.smtp_default_domain = self.get_config('notification', 'smtp_default_domain')
[305]	228
[484]	229
[507]	230	self.system = None
[359]	231
[341]	232	########## Email Header Functions ###########################################################
[339]	233
[507]	234	def spam(self, message):
	235	"""
	236	# X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
	237	# Note if Spam_level then '*' are included
	238	"""
[580]	239	self.logger.debug('function spam')
	240
[507]	241	spam = False
	242	if message.has_key(self.parameters.spam_header):
	243	spam_l = string.split(message[self.parameters.spam_header])
[578]	244	#self.logger.info('Spam header: %s' %(message[self.parameters.spam_header]))
[22]	245
[507]	246	try:
	247	number = spam_l[0].count('*')
	248	except IndexError, detail:
	249	number = 0
	250
	251	if number >= self.parameters.spam_level:
	252	spam = True
	253
[580]	254	## treat virus mails as spam
[507]	255	#
	256	elif message.has_key('X-Virus-found'):
	257	spam = True
[194]	258
[580]	259	## Astaro firewall spam handling
[507]	260	#
[580]	261	elif message.get('X-Spam-Flag') == "YES" and message.get('X-Spam-Result') == "Spam":
	262	spam = True
	263
	264	## How to handle SPAM messages
	265	#
[507]	266	if self.parameters.drop_spam and spam:
[194]	267
[507]	268	self.logger.info('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' %self.parameters.spam_level)
	269	return 'drop'
[194]	270
[507]	271	elif spam:
[194]	272
[507]	273	return 'Spam'
	274	else:
[22]	275
[507]	276	return False
[191]	277
[572]	278	def email_header_acl(self, keyword, message_field, default):
[507]	279	"""
	280	This function wil check if the email address is allowed or denied
	281	to send mail to the ticket list
	282	"""
	283	self.logger.debug('function email_header_acl: %s' %keyword)
[409]	284
[507]	285	try:
	286	mail_addresses = self.parameters[keyword]
[221]	287
[507]	288	# Check if we have an empty string
	289	#
	290	if not mail_addresses:
	291	return default
[221]	292
[507]	293	except KeyError, detail:
[572]	294	self.logger.debug('\t %s not defined, all messages are allowed.' %(keyword))
[206]	295
[507]	296	return default
[206]	297
[507]	298	mail_addresses = string.split(mail_addresses, ',')
[572]	299	message_addresses = string.split(message_field, ',')
[221]	300
[507]	301	for entry in mail_addresses:
	302	entry = entry.strip()
	303	TO_RE = re.compile(entry, re.VERBOSE\|re.IGNORECASE)
[149]	304
[572]	305	for addr in message_addresses:
	306	addr = addr.strip()
	307
	308	if self.parameters.compare_function_list in [ 'matches', 'match']:
	309	s = '\t%s matches %s' %(addr, entry)
	310	result = TO_RE.match(addr)
	311	else:
	312	s = '\t%s contains %s' %(addr, entry)
	313	result = TO_RE.search(addr)
	314
	315	if result:
	316	self.logger.debug(s)
	317	return True
	318
[507]	319	return False
[208]	320
[507]	321	def email_header_txt(self, m):
	322	"""
	323	Display To and CC addresses in description field
	324	"""
	325	s = ''
[334]	326
[507]	327	if m['To'] and len(m['To']) > 0:
	328	s = "'''To:''' %s\r\n" %(m['To'])
	329	if m['Cc'] and len(m['Cc']) > 0:
	330	s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
[22]	331
[507]	332	return self.email_to_unicode(s)
[22]	333
[138]	334
[507]	335	def get_sender_info(self, message):
	336	"""
	337	Get the default author name and email address from the message
	338	"""
[576]	339	self.logger.debug('function get_sender_info')
[43]	340
[570]	341	to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
	342	self.email_to_addrs = list()
[226]	343
[570]	344	for n,e in to_addrs:
	345	self.email_to_addrs.append(e)
	346
	347	self.email_to_addrs = ','.join(self.email_to_addrs)
	348
	349
[576]	350	self.email_name, self.email_addr = email.Utils.parseaddr(message['from'])
[142]	351
[576]	352	## decode email name can contain charset
	353	#
	354	self.email_name = self.email_to_unicode(self.email_name)
	355
	356	dstr = '\t email name: %s, email address: %s' %(self.email_name, self.email_addr)
	357	self.logger.debug(dstr)
	358
[507]	359	## Trac can not handle author's name that contains spaces
	360	#
[518]	361	if self.email_addr.lower() == self.trac_smtp_from.lower():
[507]	362	if self.email_name:
	363	self.author = self.email_name
	364	else:
	365	self.author = "email2trac"
	366	else:
	367	self.author = self.email_addr
[304]	368
[507]	369	if self.parameters.ignore_trac_user_settings:
	370	return
[194]	371
[507]	372	# Is this a registered user, use email address as search key:
	373	# result:
	374	# u : login name
	375	# n : Name that the user has set in the settings tab
	376	# e : email address that the user has set in the settings tab
	377	#
	378	users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
	379	if (
	380	(e and (e.lower() == self.email_addr.lower())) or
[518]	381	(u.lower() + '@' + self.smtp_default_domain.lower() == self.email_addr.lower())
[507]	382	)
	383	]
[43]	384
[507]	385	if len(users) >= 1:
[587]	386	self.email_from = users[0][2]
[507]	387	self.author = users[0][0]
[45]	388
[590]	389	self.logger.debug('\tget_sender_info: found registered user: %s' %(self.author) )
	390
[587]	391	if self.parameters.white_list_registered_users:
[590]	392	self.logger.debug('\tget_sender_info: added registered user to white list')
[587]	393	self.allow_registered_user = True
	394
	395
[597]	396	def set_cc_fields(self, ticket, message, update=False):
[507]	397	"""
	398	Set all the right fields for a new ticket
	399	"""
[597]	400	self.logger.debug('function set_cc_fields')
[72]	401
[270]	402
[507]	403	# Put all CC-addresses in ticket CC field
	404	#
	405	if self.parameters.reply_all:
[43]	406
[507]	407	email_cc = ''
[299]	408
[507]	409	cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
[299]	410
[507]	411	if not cc_addrs:
	412	return
[43]	413
[507]	414	## Build a list of forbidden CC addresses
	415	#
	416	#to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
	417	#to_list = list()
	418	#for n,e in to_addrs:
	419	# to_list.append(e)
[43]	420
[597]	421	if update:
	422	self.logger.debug("\tupdate cc-fields")
	423
[507]	424	for name,addr in cc_addrs:
	425
	426	## Prevent mail loop
	427	#
	428	if addr == self.trac_smtp_from:
[597]	429	self.logger.debug("\tSkipping %s email address for CC-field, same as smtp_from address in trac.ini " %(addr))
[507]	430	continue
[43]	431
[597]	432	## Alwyas remove reporter email address
	433	#
	434	elif addr == self.email_addr:
	435	self.logger.debug("\tSkipping reporter email address for CC-field")
	436	continue
	437
[507]	438	else:
[597]	439	if email_cc:
	440	email_cc = '%s, %s' %(email_cc, addr)
	441	else:
	442	email_cc = addr
[96]	443
[507]	444	if email_cc:
[597]	445	self.logger.debug('\tCC fields: %s' %email_cc)
[507]	446	ticket['cc'] = self.email_to_unicode(email_cc)
[299]	447
[339]	448
[588]	449	def acl_list_from_file(self, f, keyword):
	450	"""
	451	Read the email address from a file
	452	"""
	453	self.logger.debug('function acl_list_from_file %s : %s' %(f, keyword))
	454
	455	if not os.path.isfile(f):
	456	self.logger.error('%s_file: %s does not exists' %(keyword, f) )
	457	else:
	458	## read whole file and replace '\n' with ''
	459	#
	460	addr_l = open(f, 'r').readlines()
	461	s = ','.join(addr_l).replace('\n','')
	462
	463	try:
	464	self.parameters[keyword] = "%s,%s" %(self.parameters[keyword], s)
	465	except KeyError, detail:
	466	self.parameters[keyword] = s
	467
[339]	468	########## DEBUG functions ###########################################################
	469
[555]	470	def debug_body(self, message_body, temp_file):
	471	"""
	472	"""
	473	self.logger.debug('function debug_body:')
[310]	474
[555]	475	body_file = "%s.body" %(temp_file)
	476	fx = open(body_file, 'wb')
	477
[507]	478	if self.parameters.dry_run:
[561]	479	self.logger.info('DRY-RUN: not saving body to %s' %(body_file))
[507]	480	return
[331]	481
[555]	482	self.logger.debug('writing body to %s' %(body_file))
[507]	483	if not message_body:
	484	message_body = '(None)'
[310]	485
[507]	486	message_body = message_body.encode('utf-8')
[310]	487
[507]	488	fx.write(message_body)
	489	fx.close()
	490	try:
	491	os.chmod(body_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
	492	except OSError:
	493	pass
[310]	494
[555]	495	def debug_attachments(self, message_parts, temp_file):
[507]	496	"""
	497	"""
	498	self.logger.debug('function debug_attachments')
	499
	500	n = 0
	501	for item in message_parts:
[553]	502
	503	## Skip inline text parts
	504	#
[507]	505	if not isinstance(item, tuple):
	506	continue
	507
	508	(original, filename, part) = item
[555]	509	self.logger.debug('\t part%d: Content-Type: %s' % (n, part.get_content_type()) )
[310]	510
[555]	511	dummy_filename, ext = os.path.splitext(filename)
	512
[507]	513	n = n + 1
[555]	514	part_name = 'part%d%s' %(n, ext)
	515	part_file = "%s.%s" %(temp_file, part_name)
	516
	517	s = 'writing %s: filename: %s' %(part_file, filename)
[507]	518	self.print_unicode(s)
[555]	519
	520	## Forbidden chars, just write part files instead of names
[507]	521	#
[555]	522	#filename = filename.replace('\\', '_')
	523	#filename = filename.replace('/', '_')
	524	#filename = filename + '.att_email2trac'
	525	# part_file = os.path.join(self.parameters.tmpdir, filename)
	526	#part_file = util.text.unicode_quote(part_file)
	527	#self.print_unicode(part_file)
[331]	528
[507]	529	if self.parameters.dry_run:
[561]	530	self.logger.info('DRY_RUN: NOT saving attachments')
[507]	531	continue
[331]	532
[507]	533	fx = open(part_file, 'wb')
	534	text = part.get_payload(decode=1)
[331]	535
[507]	536	if not text:
	537	text = '(None)'
[331]	538
[507]	539	fx.write(text)
	540	fx.close()
[331]	541
[507]	542	try:
	543	os.chmod(part_file,S_IRWXU\|S_IRWXG\|S_IRWXO)
	544	except OSError:
	545	pass
[310]	546
[528]	547	def save_email_for_debug(self, message, project_name, create_tempfile=False):
[309]	548
[528]	549	if self.parameters.dry_run:
[555]	550	self.logger.debug('DRY_RUN: NOT saving email message')
[528]	551	return
	552
[507]	553	if create_tempfile:
[555]	554	(fd, tmp_file) = tempfile.mkstemp('.%s.email2trac' %project_name)
[514]	555	fx = os.fdopen(fd, 'wb')
[507]	556	else:
[555]	557	tmp_file = os.path.join(self.parameters.tmpdir, 'msg.%s.email2trac' %project_name)
	558	fx = open(tmp_file, 'wb')
[173]	559
[44]	560
[555]	561	self.logger.debug('saving email to %s' %(tmp_file))
[528]	562	fx.write('%s' % message)
	563	fx.close()
	564
	565	try:
[555]	566	os.chmod(tmp_file, S_IRWXU\|S_IRWXG\|S_IRWXO)
[528]	567	except OSError:
	568	pass
[331]	569
[507]	570	message_parts = self.get_message_parts(message)
	571	message_parts = self.unique_attachment_names(message_parts)
	572	body_text = self.get_body_text(message_parts)
[555]	573	self.debug_body(body_text, tmp_file)
	574	self.debug_attachments(message_parts, tmp_file)
[309]	575
[339]	576	########## Conversion functions ###########################################################
	577
[507]	578	def email_to_unicode(self, message_str):
	579	"""
	580	Email has 7 bit ASCII code, convert it to unicode with the charset
	581	that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
	582	understands it.
	583	"""
	584	self.logger.debug("function email_to_unicode")
[341]	585
[507]	586	results = email.Header.decode_header(message_str)
[341]	587
[507]	588	s = None
	589	for text,format in results:
	590	if format:
	591	try:
	592	temp = unicode(text, format)
	593	except UnicodeError, detail:
	594	# This always works
	595	#
[525]	596	msg = 'ERROR: Could not find charset: %s, please install' %format
	597	self.logger.error(msg)
[507]	598	temp = unicode(text, 'iso-8859-15')
	599	except LookupError, detail:
[514]	600	msg = 'ERROR: Could not find charset: %s, please install' %format
[525]	601	self.logger.error(msg)
[507]	602	#temp = unicode(text, 'iso-8859-15')
	603	temp = message_str
	604
	605	else:
	606	temp = string.strip(text)
	607	temp = unicode(text, 'iso-8859-15')
[341]	608
[507]	609	if s:
	610	s = '%s %s' %(s, temp)
	611	else:
	612	s = '%s' %temp
[341]	613
[507]	614	return s
[341]	615
[507]	616	def str_to_dict(self, s):
	617	"""
	618	Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
	619	"""
	620	self.logger.debug("function str_to_dict")
[164]	621
[507]	622	fields = string.split(s, self.parameters.subject_field_separator)
[262]	623
[507]	624	result = dict()
	625	for field in fields:
	626	try:
	627	index, value = string.split(field, '=')
[169]	628
[507]	629	# We can not change the description of a ticket via the subject
	630	# line. The description is the body of the email
	631	#
	632	if index.lower() in ['description']:
	633	continue
[169]	634
[507]	635	if value:
	636	result[index.lower()] = value
[169]	637
[507]	638	except ValueError:
	639	pass
	640	return result
[167]	641
[507]	642	def print_unicode(self,s):
	643	"""
	644	This function prints unicode strings if possible else it will quote it
	645	"""
	646	try:
	647	self.logger.debug(s)
	648	except UnicodeEncodeError, detail:
	649	self.logger.debug(util.text.unicode_quote(s))
[379]	650
[503]	651
[507]	652	def html_2_txt(self, data):
	653	"""
	654	Various routines to convert html syntax to valid trac wiki syntax
	655	"""
	656	self.logger.debug('function html_2_txt')
[503]	657
[507]	658	## This routine make an safe html that can be include
	659	# in trac, but no further text processing can be done
	660	#
	661	# try:
	662	# from lxml.html.clean import Cleaner
	663	# tags_rm = list()
	664	# tags_rm.append('body')
[503]	665	#
[507]	666	# cleaner = Cleaner(remove_tags=tags_rm )
	667	# parsed_data = cleaner.clean_html(data)
	668	# parsed_data = '\n{{{\n#!html\n' + parsed_data + '\n}}}\n'
[503]	669	#
[507]	670	# return parsed_data
	671	#
	672	# except ImportError::
	673	# pass
[503]	674
[507]	675	parsed_data = None
	676	if self.parameters.html2text_cmd:
[514]	677	(fd, tmp_file) = tempfile.mkstemp('email2trac.html')
	678	f = os.fdopen(fd, 'w')
	679
[507]	680	cmd = '%s %s' %(self.parameters.html2text_cmd, tmp_file)
	681	self.logger.debug('\t html2text conversion %s'%(cmd))
	682
	683	if self.parameters.dry_run:
[561]	684	self.logger.info('DRY_RUN: html2text conversion command: %s\n' %(cmd))
[503]	685
[507]	686	else:
	687	f.write(data)
	688	f.close()
[503]	689
[507]	690	lines = os.popen(cmd).readlines()
	691	parsed_data = ''.join(lines)
[503]	692
[507]	693	os.unlink(tmp_file)
[503]	694
[507]	695	else:
	696	self.logger.debug('\t No html2text conversion tool specified in email2trac.conf')
[503]	697
[507]	698	return parsed_data
[503]	699
[549]	700	def check_filename_length(self, name):
	701	"""
[551]	702	To bypass a bug in Trac
	703	check if the filename lenght is not larger then OS limit.
[549]	704	yes : return truncated filename
	705	no : return unmodified filename
	706	"""
	707	self.logger.debug('function check_filename_length: ')
[550]	708
	709	if not name:
	710	return 'None'
[549]	711
	712	dummy_filename, ext = os.path.splitext(name)
	713
	714	## Trac uses this format
	715	#
	716	quote_format = util.text.unicode_quote(dummy_filename)
	717
	718	## Determine max filename lenght
	719	#
[603]	720	try:
	721	filemax_length = os.pathconf('/', 'PC_NAME_MAX')
	722	except AttributeError, detail:
	723	filemax_lenght = 240
[549]	724
	725	if len(quote_format) <= filemax_length:
	726	return name
	727
	728	else:
	729	## Truncate file to filemax_length and reserve room for extension
	730	# We must break on a boundry
	731	#
	732	length = filemax_length - 6
	733
	734	for i in range(0,10):
	735
	736	truncated = quote_format[ : (length - i)]
	737
	738	try:
	739	unqoute_truncated = util.text.unicode_unquote(truncated)
	740	unqoute_truncated = unqoute_truncated + ext
	741
	742	self.print_unicode('\t ' + unqoute_truncated)
	743
	744	break
	745
	746	except UnicodeDecodeError, detail:
	747	continue
	748
	749
	750	return unqoute_truncated
	751
[339]	752	########## TRAC ticket functions ###########################################################
	753
[546]	754	def mail_workflow(self, tkt):
[545]	755	"""
	756	"""
[546]	757	self.logger.debug('function mail_workflow: ')
[545]	758
	759	req = Mock(authname=self.author, perm=MockPerm(), args={})
	760	ticket_system = TicketSystem(self.env)
	761
	762	try:
	763	workflow = self.parameters['workflow_%s' %tkt['status'].lower()]
	764
	765	except KeyError:
[553]	766	## fallback for compability (Will be deprecated)
	767	# workflow can be none.
[545]	768	#
	769	workflow = None
	770	if tkt['status'] in ['closed']:
	771	workflow = self.parameters.workflow
	772
	773	if workflow:
[556]	774
[557]	775	## process all workflow implementations
	776	#
[556]	777	tkt_module = TicketModule(self.env)
	778	field_changes, problems = tkt_module.get_ticket_changes(req, tkt, workflow)
	779
	780	for field in field_changes.keys():
	781
	782	## We have already processed these fields
	783	#
[558]	784	if not field in ['summary', 'description']:
[556]	785	s = 'workflow : %s, field %s : %s, by %s' \
	786	%(workflow, field, field_changes[field]['new'],field_changes[field]['by'] )
	787	self.logger.debug(s)
	788
	789	tkt[field] = field_changes[field]['new']
	790
[545]	791	return True
	792
	793	else:
	794	return False
	795
[507]	796	def check_permission_participants(self, tkt, action):
	797	"""
	798	Check if the mailer is allowed to update the ticket
	799	"""
[538]	800	self.logger.debug('function check_permission_participants %s')
[398]	801
[518]	802	if tkt['reporter'].lower() in [self.author.lower(), self.email_addr.lower()]:
[507]	803	self.logger.debug('ALLOW, %s is the ticket reporter' %(self.email_addr))
[408]	804
[507]	805	return True
[398]	806
[507]	807	perm = PermissionSystem(self.env)
	808	if perm.check_permission(action, self.author):
	809	self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
[408]	810
[507]	811	return True
	812
	813	# Is the updater in the CC?
	814	try:
	815	cc_list = tkt['cc'].split(',')
	816	for cc in cc_list:
[518]	817	if self.email_addr.lower() in cc.lower.strip():
[507]	818	self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
[398]	819
[507]	820	return True
[398]	821
[507]	822	except KeyError:
	823	pass
[398]	824
[507]	825	return False
[474]	826
[507]	827	def check_permission(self, tkt, action):
	828	"""
	829	check if the reporter has the right permission for the action:
[397]	830	- TICKET_CREATE
	831	- TICKET_MODIFY
[474]	832	- TICKET_APPEND
	833	- TICKET_CHGPROP
[397]	834
[507]	835	There are three models:
	836	- None : no checking at all
	837	- trac : check the permission via trac permission model
	838	- email2trac: ....
	839	"""
[538]	840	self.logger.debug("function check_permission: %s" %(action))
[397]	841
[507]	842	if self.parameters.ticket_permission_system in ['trac']:
[398]	843
[539]	844	perm = PermissionCache(self.env, self.author)
	845	if perm.has_permission(action):
[507]	846	return True
	847	else:
	848	return False
[398]	849
[507]	850	elif self.parameters.ticket_permission_system in ['update_restricted_to_participants']:
	851	return (self.check_permission_participants(tkt, action))
[398]	852
[507]	853	## Default is to allow everybody ticket updates and ticket creation
	854	#
	855	else:
	856	return True
[397]	857
	858
[507]	859	def update_ticket_fields(self, ticket, user_dict, new=None):
	860	"""
	861	This will update the ticket fields. It will check if the
	862	given fields are known and if the right values are specified
	863	It will only update the ticket field value:
	864	- If the field is known
	865	- If the value supplied is valid for the ticket field.
	866	If not then there are two options:
	867	1) Skip the value (new=None)
	868	2) Set default value for field (new=1)
	869	"""
	870	self.logger.debug("function update_ticket_fields")
[169]	871
[507]	872	## Check only permission model on ticket updates
	873	#
	874	if not new:
	875	if self.parameters.ticket_permission_system:
	876	if not self.check_permission(ticket, 'TICKET_CHGPROP'):
	877	self.logger.info('Reporter: %s has no permission to change ticket properties' %self.author)
	878	return False
[474]	879
[507]	880	## Build a system dictionary from the ticket fields
	881	# with field as index and option as value
	882	#
	883	sys_dict = dict()
[530]	884
[507]	885	for field in ticket.fields:
[530]	886
[507]	887	try:
	888	sys_dict[field['name']] = field['options']
[169]	889
[507]	890	except KeyError:
[531]	891	sys_dict[field['name']] = None
[507]	892	pass
[169]	893
[507]	894	## Check user supplied fields an compare them with the
	895	# system one's
	896	#
[549]	897	for field,value in user_dict.items():
	898
	899	s = 'user_field\t %s = %s' %(field,value)
	900	self.print_unicode(s)
[524]	901
[531]	902	if not field in sys_dict.keys():
[524]	903	self.logger.debug('%s is not a valid field for tickets' %(field))
	904	continue
	905
[507]	906	## To prevent mail loop
	907	#
	908	if field == 'cc':
[301]	909
[507]	910	cc_list = user_dict['cc'].split(',')
[301]	911
[507]	912	if self.trac_smtp_from in cc_list:
	913	self.logger.debug('MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
[408]	914
[507]	915	cc_list.remove(self.trac_smtp_from)
[301]	916
[507]	917	value = ','.join(cc_list)
	918
[301]	919
[507]	920	## Check if every value is allowed for this field
	921	#
[531]	922	if sys_dict[field]:
[169]	923
[507]	924	if value in sys_dict[field]:
	925	ticket[field] = value
	926	else:
	927	## Must we set a default if value is not allowed
	928	#
	929	if new:
	930	value = self.get_config('ticket', 'default_%s' %(field) )
[169]	931
[507]	932	else:
[553]	933
[507]	934	ticket[field] = value
[202]	935
[549]	936	s = 'ticket_field\t %s = %s' %(field, ticket[field])
	937	self.print_unicode(s)
[345]	938
[507]	939	def ticket_update(self, m, id, spam):
	940	"""
	941	If the current email is a reply to an existing ticket, this function
	942	will append the contents of this email to that ticket, instead of
	943	creating a new one.
	944	"""
	945	self.logger.debug("function ticket_update")
[202]	946
[507]	947	if not self.parameters.ticket_update:
	948	self.logger.debug("ticket_update disabled")
	949	return False
[456]	950
[507]	951	## Must we update ticket fields
	952	#
	953	update_fields = dict()
	954	try:
	955	id, keywords = string.split(id, '?')
[262]	956
[507]	957	update_fields = self.str_to_dict(keywords)
[165]	958
[507]	959	## Strip '#'
	960	#
	961	self.id = int(id[1:])
[165]	962
[507]	963	except ValueError:
[390]	964
[507]	965	## Strip '#'
	966	#
	967	self.id = int(id[1:])
[164]	968
[597]	969	self.logger.debug("\tticket id: %s" %id)
[71]	970
[507]	971	## When is the change committed
	972	#
	973	if self.VERSION < 0.11:
	974	when = int(time.time())
	975	else:
	976	when = datetime.now(util.datefmt.utc)
[77]	977
[507]	978	try:
	979	tkt = Ticket(self.env, self.id, self.db)
[390]	980
[507]	981	except util.TracError, detail:
[390]	982
[507]	983	## Not a valid ticket
	984	#
[604]	985	self.logger.info("\tCreating a new ticket, ticket id: %s does not exists" %id)
[507]	986	self.id = None
	987	return False
[126]	988
[507]	989	## Check the permission of the reporter
	990	#
	991	if self.parameters.ticket_permission_system:
	992	if not self.check_permission(tkt, 'TICKET_APPEND'):
	993	self.logger.info('Reporter: %s has no permission to add comments or attachments to tickets' %self.author)
	994	return False
[398]	995
[507]	996	## How many changes has this ticket
	997	#
	998	grouped = TicketModule(self.env).grouped_changelog_entries(tkt, self.db)
	999	cnum = sum(1 for e in grouped) + 1
[288]	1000
[486]	1001
[507]	1002	## reopen the ticket if it is was closed
	1003	# We must use the ticket workflow framework
	1004	#
	1005	if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
[257]	1006
[546]	1007	if not self.mail_workflow(tkt):
[538]	1008
[507]	1009	if tkt['status'] in ['closed']:
[526]	1010	tkt['status'] = 'reopened'
	1011	tkt['resolution'] = ''
	1012
[507]	1013	## Old pre 0.11 situation
	1014	#
	1015	elif self.parameters.email_triggers_workflow:
[470]	1016
[597]	1017	self.logger.debug('\temail triggers workflow pre trac 0.11')
[470]	1018
[507]	1019	if tkt['status'] in ['closed']:
	1020	tkt['status'] = 'reopened'
	1021	tkt['resolution'] = ''
[257]	1022
[507]	1023	else:
[597]	1024	self.logger.debug('\temail triggers workflow disabled')
[469]	1025
[507]	1026	## Must we update some ticket fields properties via subject line
	1027	#
	1028	if update_fields:
	1029	self.update_ticket_fields(tkt, update_fields)
[166]	1030
[597]	1031
[507]	1032	message_parts = self.get_message_parts(m)
	1033	message_parts = self.unique_attachment_names(message_parts)
[210]	1034
[597]	1035	## Must we update some ticket fields properties via inline comments
	1036	# in body_text
[507]	1037	#
	1038	if self.properties:
	1039	self.update_ticket_fields(tkt, self.properties)
[309]	1040
[597]	1041	## Must we update the CC ticket field
	1042	#
	1043	self.set_cc_fields(tkt, m, update=True)
	1044
[507]	1045	if self.parameters.email_header:
	1046	message_parts.insert(0, self.email_header_txt(m))
[76]	1047
[507]	1048	body_text = self.get_body_text(message_parts)
[236]	1049
[507]	1050	error_with_attachments = self.attach_attachments(message_parts)
[348]	1051
[507]	1052	if body_text.strip() or update_fields or self.properties:
[543]	1053
[507]	1054	if self.parameters.dry_run:
[561]	1055	s = 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) %s %s' %(self.author, cnum)
	1056	self.logger.info(s)
[543]	1057
[507]	1058	else:
	1059	if error_with_attachments:
	1060	body_text = '%s\\%s' %(error_with_attachments, body_text)
[543]	1061
[597]	1062	self.logger.debug('\ttkt.save_changes(%s, %d)' %(self.author, cnum))
[507]	1063	tkt.save_changes(self.author, body_text, when, None, str(cnum))
	1064
	1065	if not spam:
	1066	self.notify(tkt, False, when)
[72]	1067
[507]	1068	return True
[71]	1069
[507]	1070	def set_ticket_fields(self, ticket):
	1071	"""
	1072	set the ticket fields to value specified
	1073	- /etc/email2trac.conf with <prefix>_<field>
	1074	- trac default values, trac.ini
	1075	"""
	1076	self.logger.debug('function set_ticket_fields')
[410]	1077
[507]	1078	user_dict = dict()
[202]	1079
[507]	1080	for field in ticket.fields:
[202]	1081
[507]	1082	name = field['name']
[202]	1083
[507]	1084	## default trac value
	1085	#
	1086	if not field.get('custom'):
	1087	value = self.get_config('ticket', 'default_%s' %(name) )
[599]	1088
	1089	## skip this field can only be set by email2trac.conf
	1090	#
	1091	if name in ['resolution']:
[507]	1092	value = None
[599]	1093
[507]	1094	else:
	1095	## Else get the default value for reporter
	1096	#
	1097	value = field.get('value')
	1098	options = field.get('options')
[345]	1099
[507]	1100	if value and options and (value not in options):
	1101	value = options[int(value)]
	1102
[549]	1103	s = 'trac[%s] = %s' %(name, value)
	1104	self.print_unicode(s)
[202]	1105
[507]	1106	## email2trac.conf settings
	1107	#
	1108	prefix = self.parameters.ticket_prefix
	1109	try:
	1110	value = self.parameters['%s_%s' %(prefix, name)]
[202]	1111
[549]	1112	s = 'email2trac[%s] = %s ' %(name, value)
	1113	self.print_unicode(s)
	1114
[507]	1115	except KeyError, detail:
	1116	pass
	1117
	1118	if value:
	1119	user_dict[name] = value
[202]	1120
[550]	1121	s = 'used %s = %s' %(name, value)
	1122	self.print_unicode(s)
	1123
[507]	1124	self.update_ticket_fields(ticket, user_dict, new=1)
[202]	1125
[507]	1126	if 'status' not in user_dict.keys():
	1127	ticket['status'] = 'new'
[202]	1128
	1129
[507]	1130	def ticket_update_by_subject(self, subject):
	1131	"""
	1132	This list of Re: prefixes is probably incomplete. Taken from
	1133	wikipedia. Here is how the subject is matched
	1134	- Re: <subject>
	1135	- Re: (<Mail list label>:)+ <subject>
[202]	1136
[507]	1137	So we must have the last column
	1138	"""
	1139	self.logger.debug('function ticket_update_by_subject')
[356]	1140
[533]	1141	found_id = None
[507]	1142	if self.parameters.ticket_update and self.parameters.ticket_update_by_subject:
	1143
	1144	SUBJECT_RE = re.compile(r'^(?:(?:RE\|AW\|VS\|SV\|FW\|FWD):\s)+(.)', re.IGNORECASE)
	1145	result = SUBJECT_RE.search(subject)
[356]	1146
[507]	1147	if result:
	1148	## This is a reply
	1149	#
	1150	orig_subject = result.group(1)
[356]	1151
[507]	1152	self.logger.debug('subject search string: %s' %(orig_subject))
[356]	1153
[507]	1154	cursor = self.db.cursor()
	1155	summaries = [orig_subject, '%%: %s' % orig_subject]
[356]	1156
[520]	1157	## Time resolution is in micoseconds
[507]	1158	#
[520]	1159	search_date = datetime.now(util.datefmt.utc) - timedelta(days=self.parameters.ticket_update_by_subject_lookback)
[356]	1160
[529]	1161	if self.VERSION < 0.12:
	1162	lookback = util.datefmt.to_timestamp(search_date)
	1163	else:
	1164	lookback = util.datefmt.to_utimestamp(search_date)
	1165
[507]	1166	for summary in summaries:
	1167	self.logger.debug('Looking for summary matching: "%s"' % summary)
[408]	1168
[533]	1169	sql = """SELECT id, reporter FROM ticket
[507]	1170	WHERE changetime >= %s AND summary LIKE %s
	1171	ORDER BY changetime DESC"""
[533]	1172
[507]	1173	cursor.execute(sql, [lookback, summary.strip()])
[356]	1174
[507]	1175	for row in cursor:
[408]	1176
[533]	1177	(matched_id, sql_reporter) = row
[408]	1178
[533]	1179	## Save first entry.
	1180	#
	1181	if not found_id:
	1182	found_id = matched_id
	1183
	1184	## If subject and reporter are the same. The we certainly have found the right ticket
	1185	#
	1186	if sql_reporter == self.author:
	1187	self.logger.debug('Found matching reporter: %s with ticket id: %d' %(sql_reporter, matched_id))
	1188	found_id = matched_id
	1189	break
[356]	1190
[533]	1191	if found_id:
	1192	self.logger.debug('Found matching ticket id: %d' % found_id)
	1193	found_id = '#%d' % found_id
	1194	return (found_id, orig_subject)
	1195
[553]	1196	# obsolete !?? 12 Aug 2011
[533]	1197	#subject = orig_subject
[356]	1198
[533]	1199	return (found_id, subject)
[356]	1200
	1201
[507]	1202	def new_ticket(self, msg, subject, spam, set_fields = None):
	1203	"""
	1204	Create a new ticket
	1205	"""
	1206	self.logger.debug('function new_ticket')
[250]	1207
[507]	1208	tkt = Ticket(self.env)
[326]	1209
[597]	1210	## self.author can be email address of an username
	1211	#
	1212	tkt['reporter'] = self.author
[326]	1213
[597]	1214	self.set_cc_fields(tkt, msg)
	1215
[507]	1216	self.set_ticket_fields(tkt)
[202]	1217
[507]	1218	## Check the permission of the reporter
	1219	#
	1220	if self.parameters.ticket_permission_system:
	1221	if not self.check_permission(tkt, 'TICKET_CREATE'):
	1222	self.logger.info('Reporter: %s has no permission to create tickets' %self.author)
	1223	return False
[397]	1224
[507]	1225	## Old style setting for component, will be removed
	1226	#
	1227	if spam:
	1228	tkt['component'] = 'Spam'
[204]	1229
[507]	1230	elif self.parameters.has_key('component'):
	1231	tkt['component'] = self.parameters['component']
[201]	1232
[507]	1233	if not msg['Subject']:
	1234	tkt['summary'] = u'(No subject)'
	1235	else:
	1236	tkt['summary'] = subject
[22]	1237
	1238
[507]	1239	if set_fields:
	1240	rest, keywords = string.split(set_fields, '?')
[262]	1241
[507]	1242	if keywords:
	1243	update_fields = self.str_to_dict(keywords)
	1244	self.update_ticket_fields(tkt, update_fields)
[262]	1245
[296]	1246
[604]	1247	message_parts = self.get_message_parts(msg, True)
[309]	1248
[507]	1249	## Must we update some ticket fields properties via body_text
	1250	#
	1251	if self.properties:
	1252	self.update_ticket_fields(tkt, self.properties)
[309]	1253
[507]	1254	message_parts = self.unique_attachment_names(message_parts)
	1255
	1256	## produce e-mail like header
	1257	#
	1258	head = ''
	1259	if self.parameters.email_header:
	1260	head = self.email_header_txt(msg)
	1261	message_parts.insert(0, head)
	1262
	1263	body_text = self.get_body_text(message_parts)
[45]	1264
[507]	1265	tkt['description'] = body_text
[90]	1266
[507]	1267	## When is the change committed
	1268	#
	1269	if self.VERSION < 0.11:
	1270	when = int(time.time())
	1271	else:
	1272	when = datetime.now(util.datefmt.utc)
[45]	1273
[507]	1274	if self.parameters.dry_run:
[561]	1275	self.logger.info('DRY_RUN: tkt.insert()')
[507]	1276	else:
	1277	self.id = tkt.insert()
	1278
	1279	changed = False
	1280	comment = ''
[77]	1281
[507]	1282	## some routines in trac are dependend on ticket id
	1283	# like alternate notify template
	1284	#
	1285	if self.parameters.alternate_notify_template:
	1286	tkt['id'] = self.id
	1287	changed = True
[273]	1288
[507]	1289	## Rewrite the description if we have mailto enabled
	1290	#
	1291	if self.parameters.mailto_link:
	1292	changed = True
	1293	comment = u'\nadded mailto line\n'
[592]	1294
[581]	1295	#mailto = self.html_mailto_link( m['Subject'])
	1296	mailto = self.html_mailto_link(subject)
[253]	1297
[507]	1298	tkt['description'] = u'%s\r\n%s%s\r\n' \
	1299	%(head, mailto, body_text)
	1300
	1301	## Save the attachments to the ticket
	1302	#
	1303	error_with_attachments = self.attach_attachments(message_parts)
[295]	1304
[507]	1305	if error_with_attachments:
	1306	changed = True
	1307	comment = '%s\n%s\n' %(comment, error_with_attachments)
[45]	1308
[545]	1309	if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
[546]	1310	if self.mail_workflow(tkt):
[545]	1311	changed = True
	1312
[507]	1313	if changed:
	1314	if self.parameters.dry_run:
[561]	1315	s = 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
	1316	self.logger.info(s)
	1317
[507]	1318	else:
	1319	tkt.save_changes(tkt['reporter'], comment)
[90]	1320
[507]	1321	if not spam:
[545]	1322	self.notify(tkt, True)
[45]	1323
[260]	1324
[507]	1325	def attach_attachments(self, message_parts, update=False):
	1326	'''
	1327	save any attachments as files in the ticket's directory
	1328	'''
	1329	self.logger.debug('function attach_attachments()')
[342]	1330
[507]	1331	if self.parameters.dry_run:
[555]	1332	self.logger.debug("DRY_RUN: no attachments attached to tickets")
[507]	1333	return ''
[342]	1334
[507]	1335	count = 0
[342]	1336
[507]	1337	## Get Maxium attachment size
	1338	#
	1339	max_size = int(self.get_config('attachment', 'max_size'))
	1340	status = None
	1341
	1342	for item in message_parts:
	1343	## Skip body parts
	1344	#
	1345	if not isinstance(item, tuple):
	1346	continue
	1347
	1348	(original, filename, part) = item
[477]	1349
[548]	1350	## We have to determine the size so we use this temporary solution.
[507]	1351	#
[548]	1352	path, fd = util.create_unique_file(os.path.join(self.parameters.tmpdir, 'email2trac_tmp.att'))
[507]	1353	text = part.get_payload(decode=1)
	1354	if not text:
	1355	text = '(None)'
	1356	fd.write(text)
	1357	fd.close()
[342]	1358
[507]	1359	## get the file_size
	1360	#
	1361	stats = os.lstat(path)
	1362	file_size = stats[ST_SIZE]
[342]	1363
[507]	1364	## Check if the attachment size is allowed
	1365	#
	1366	if (max_size != -1) and (file_size > max_size):
	1367	status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
	1368	%(status, original, file_size, max_size)
[342]	1369
[507]	1370	os.unlink(path)
	1371	continue
	1372	else:
	1373	count = count + 1
	1374
	1375	## Insert the attachment
	1376	#
	1377	fd = open(path, 'rb')
	1378	if self.system == 'discussion':
[553]	1379	att = attachment.Attachment(self.env, 'discussion', 'topic/%s' % (self.id,))
[585]	1380	elif self.system == 'blog':
	1381	att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,))
[507]	1382	else:
[548]	1383	s = 'Attach %s to ticket %d' %(filename, self.id)
	1384	self.print_unicode(s)
[507]	1385	att = attachment.Attachment(self.env, 'ticket', self.id)
[359]	1386
[507]	1387	## This will break the ticket_update system, the body_text is vaporized
	1388	# ;-(
	1389	#
	1390	if not update:
	1391	att.author = self.author
	1392	att.description = self.email_to_unicode('Added by email2trac')
[342]	1393
[553]	1394	try:
[549]	1395
[507]	1396	self.logger.debug('Insert atachment')
	1397	att.insert(filename, fd, file_size)
[553]	1398
[507]	1399	except OSError, detail:
[553]	1400
[507]	1401	self.logger.info('%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail))
	1402	status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
[342]	1403
[507]	1404	## Remove the created temporary filename
	1405	#
	1406	fd.close()
	1407	os.unlink(path)
[342]	1408
[507]	1409	## return error
	1410	#
	1411	return status
[342]	1412
[359]	1413	########## Fullblog functions #################################################
[339]	1414
[585]	1415	def blog(self, msg, subject, id, params):
[507]	1416	"""
	1417	The blog create/update function
	1418	"""
	1419	## import the modules
	1420	#
	1421	from tracfullblog.core import FullBlogCore
	1422	from tracfullblog.model import BlogPost, BlogComment
[260]	1423
[507]	1424	## instantiate blog core
	1425	#
	1426	blog = FullBlogCore(self.env)
	1427	req = Mock(authname='anonymous', perm=MockPerm(), args={})
[312]	1428
[585]	1429	## parameters from the subject
	1430	#
	1431	params = self.str_to_dict((params or '').lstrip('?'))
[260]	1432
[585]	1433	## preferably get the time from the email date header, or else
	1434	# use current date and time
	1435	date = email.Utils.parsedate_tz(msg.get('date'))
	1436	if date:
	1437	dt = util.datefmt.to_datetime(email.Utils.mktime_tz(date), util.datefmt.utc)
	1438	else:
	1439	self.logger.warn("No valid date header found")
	1440	dt = util.datefmt.to_datetime(None, util.datefmt.utc)
	1441
	1442	## blog entry affected
	1443	#
	1444	self.id = id or util.datefmt.format_datetime(dt, "%Y%m%d%H%M%S", util.datefmt.utc)
	1445
	1446	## check wether a blog post exists
	1447	#
	1448	post = BlogPost(self.env, self.id)
	1449	force_update = self.properties.get('update', params.get('update'))
	1450
	1451	## message parts
	1452	#
	1453	message_parts = self.get_message_parts(msg)
	1454	message_parts = self.unique_attachment_names(message_parts)
	1455
	1456	if post.get_versions() and not force_update:
	1457
	1458	## add comment to blog entry
[507]	1459	#
[585]	1460	comment = BlogComment(self.env, self.id)
	1461	comment.author = self.properties.get('author', params.get('author', self.author))
[507]	1462	comment.comment = self.get_body_text(message_parts)
[585]	1463	comment.time = dt
[312]	1464
[578]	1465	if self.parameters.dry_run:
	1466	self.logger.info('DRY-RUN: not adding comment for blog entry "%s"' % id)
	1467	return
[585]	1468	warnings = blog.create_comment(req, comment)
[260]	1469
[507]	1470	else:
[585]	1471	## create or update blog entry
[507]	1472	#
[585]	1473	post.author = self.properties.get('author', params.get('author', self.author))
	1474	post.categories = self.properties.get('categories', params.get('categories', ''))
[578]	1475	post.title = subject.strip()
[585]	1476	post.publish_time = dt
[507]	1477	post.body = self.get_body_text(message_parts)
	1478
[578]	1479	if self.parameters.dry_run:
	1480	self.logger.info('DRY-RUN: not creating blog entry "%s"' % post.title)
	1481	return
[585]	1482	warnings = blog.create_post(req, post, self.author, u'Created by email2trac', False)
[260]	1483
[585]	1484	## check for problems
	1485	#
	1486	if warnings:
	1487	raise TracError(', '.join('blog:%s:%s' % (w[0], w[1]) for w in warnings))
	1488
	1489	## all seems well, attach attachments
	1490	#
	1491	self.attach_attachments(message_parts)
[260]	1492
[585]	1493
[359]	1494	########## Discussion functions ##############################################
[342]	1495
[507]	1496	def discussion_topic(self, content, subject):
[342]	1497
[507]	1498	## Import modules.
	1499	#
	1500	from tracdiscussion.api import DiscussionApi
	1501	from trac.util.datefmt import to_timestamp, utc
[359]	1502
[507]	1503	self.logger.debug('Creating a new topic in forum:', self.id)
[359]	1504
[507]	1505	## Get dissussion API component.
	1506	#
	1507	api = self.env[DiscussionApi]
	1508	context = self._create_context(content, subject)
[359]	1509
[507]	1510	## Get forum for new topic.
	1511	#
	1512	forum = api.get_forum(context, self.id)
[359]	1513
[507]	1514	if not forum:
	1515	self.logger.error("ERROR: Replied forum doesn't exist")
[359]	1516
[507]	1517	## Prepare topic.
	1518	#
	1519	topic = {'forum' : forum['id'],
	1520	'subject' : context.subject,
	1521	'time': to_timestamp(datetime.now(utc)),
	1522	'author' : self.author,
	1523	'subscribers' : [self.email_addr],
	1524	'body' : self.get_body_text(context.content_parts)}
[359]	1525
[507]	1526	## Add topic to DB and commit it.
	1527	#
	1528	self._add_topic(api, context, topic)
	1529	self.db.commit()
[359]	1530
[507]	1531	def discussion_topic_reply(self, content, subject):
[359]	1532
[507]	1533	## Import modules.
	1534	#
	1535	from tracdiscussion.api import DiscussionApi
	1536	from trac.util.datefmt import to_timestamp, utc
[359]	1537
[507]	1538	self.logger.debug('Replying to discussion topic', self.id)
[359]	1539
[507]	1540	## Get dissussion API component.
	1541	#
	1542	api = self.env[DiscussionApi]
	1543	context = self._create_context(content, subject)
[359]	1544
[507]	1545	## Get replied topic.
	1546	#
	1547	topic = api.get_topic(context, self.id)
[359]	1548
[507]	1549	if not topic:
	1550	self.logger.error("ERROR: Replied topic doesn't exist")
[359]	1551
[507]	1552	## Prepare message.
	1553	#
	1554	message = {'forum' : topic['forum'],
	1555	'topic' : topic['id'],
	1556	'replyto' : -1,
	1557	'time' : to_timestamp(datetime.now(utc)),
	1558	'author' : self.author,
	1559	'body' : self.get_body_text(context.content_parts)}
[359]	1560
[507]	1561	## Add message to DB and commit it.
	1562	#
	1563	self._add_message(api, context, message)
	1564	self.db.commit()
[359]	1565
[507]	1566	def discussion_message_reply(self, content, subject):
[359]	1567
[507]	1568	## Import modules.
	1569	#
	1570	from tracdiscussion.api import DiscussionApi
	1571	from trac.util.datefmt import to_timestamp, utc
[359]	1572
[507]	1573	self.logger.debug('Replying to discussion message', self.id)
[359]	1574
[507]	1575	## Get dissussion API component.
	1576	#
	1577	api = self.env[DiscussionApi]
	1578	context = self._create_context(content, subject)
[359]	1579
[507]	1580	## Get replied message.
	1581	#
	1582	message = api.get_message(context, self.id)
[359]	1583
[507]	1584	if not message:
	1585	self.logger.error("ERROR: Replied message doesn't exist")
[359]	1586
[507]	1587	## Prepare message.
	1588	#
	1589	message = {'forum' : message['forum'],
	1590	'topic' : message['topic'],
	1591	'replyto' : message['id'],
	1592	'time' : to_timestamp(datetime.now(utc)),
	1593	'author' : self.author,
	1594	'body' : self.get_body_text(context.content_parts)}
[359]	1595
[507]	1596	## Add message to DB and commit it.
	1597	#
	1598	self._add_message(api, context, message)
	1599	self.db.commit()
[359]	1600
[507]	1601	def _create_context(self, content, subject):
[359]	1602
[507]	1603	## Import modules.
	1604	#
	1605	from trac.mimeview import Context
	1606	from trac.web.api import Request
	1607	from trac.perm import PermissionCache
[359]	1608
[507]	1609	## TODO: Read server base URL from config.
	1610	# Create request object to mockup context creation.
	1611	#
	1612	environ = {'SERVER_PORT' : 80,
	1613	'SERVER_NAME' : 'test',
	1614	'REQUEST_METHOD' : 'POST',
	1615	'wsgi.url_scheme' : 'http',
	1616	'wsgi.input' : sys.stdin}
	1617	chrome = {'links': {},
	1618	'scripts': [],
	1619	'ctxtnav': [],
	1620	'warnings': [],
	1621	'notices': []}
[359]	1622
[507]	1623	if self.env.base_url_for_redirect:
	1624	environ['trac.base_url'] = self.env.base_url
[359]	1625
[507]	1626	req = Request(environ, None)
	1627	req.chrome = chrome
	1628	req.tz = 'missing'
	1629	req.authname = self.author
	1630	req.perm = PermissionCache(self.env, self.author)
	1631	req.locale = None
[359]	1632
[507]	1633	## Create and return context.
	1634	#
	1635	context = Context.from_request(req)
	1636	context.realm = 'discussion-email2trac'
	1637	context.cursor = self.db.cursor()
	1638	context.content = content
	1639	context.subject = subject
[359]	1640
[507]	1641	## Read content parts from content.
	1642	#
	1643	context.content_parts = self.get_message_parts(content)
	1644	context.content_parts = self.unique_attachment_names(
	1645	context.content_parts)
[359]	1646
[507]	1647	return context
[359]	1648
[507]	1649	def _add_topic(self, api, context, topic):
	1650	context.req.perm.assert_permission('DISCUSSION_APPEND')
[359]	1651
[507]	1652	## Filter topic.
	1653	#
	1654	for discussion_filter in api.discussion_filters:
	1655	accept, topic_or_error = discussion_filter.filter_topic(
	1656	context, topic)
	1657	if accept:
	1658	topic = topic_or_error
	1659	else:
	1660	raise TracError(topic_or_error)
[359]	1661
[507]	1662	## Add a new topic.
	1663	#
	1664	api.add_topic(context, topic)
[359]	1665
[507]	1666	## Get inserted topic with new ID.
	1667	#
	1668	topic = api.get_topic_by_time(context, topic['time'])
[359]	1669
[507]	1670	## Attach attachments.
	1671	#
	1672	self.id = topic['id']
	1673	self.attach_attachments(context.content_parts, True)
[359]	1674
[507]	1675	## Notify change listeners.
	1676	#
	1677	for listener in api.topic_change_listeners:
	1678	listener.topic_created(context, topic)
[359]	1679
[507]	1680	def _add_message(self, api, context, message):
	1681	context.req.perm.assert_permission('DISCUSSION_APPEND')
[359]	1682
[507]	1683	## Filter message.
	1684	#
	1685	for discussion_filter in api.discussion_filters:
	1686	accept, message_or_error = discussion_filter.filter_message(
	1687	context, message)
	1688	if accept:
	1689	message = message_or_error
	1690	else:
	1691	raise TracError(message_or_error)
[359]	1692
[507]	1693	## Add message.
	1694	#
	1695	api.add_message(context, message)
[359]	1696
[507]	1697	## Get inserted message with new ID.
	1698	#
	1699	message = api.get_message_by_time(context, message['time'])
[359]	1700
[507]	1701	## Attach attachments.
	1702	#
[549]	1703
[507]	1704	self.attach_attachments(context.content_parts, True)
[359]	1705
[507]	1706	## Notify change listeners.
	1707	#
	1708	for listener in api.message_change_listeners:
	1709	listener.message_created(context, message)
[359]	1710
	1711	########## MAIN function ######################################################
	1712
[507]	1713	def parse(self, fp):
	1714	"""
	1715	"""
	1716	self.logger.debug('Main function parse')
	1717	global m
[96]	1718
[507]	1719	m = email.message_from_file(fp)
	1720
	1721	if not m:
	1722	self.logger.debug('This is not a valid email message format')
	1723	return
	1724
	1725	## Work around lack of header folding in Python; see http://bugs.python.org/issue4696
	1726	#
	1727	try:
	1728	m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
	1729	except AttributeError, detail:
	1730	pass
[239]	1731
[507]	1732	if self.parameters.debug: # save the entire e-mail message text
[528]	1733	self.save_email_for_debug(m, self.parameters.project_name, True)
[77]	1734
[507]	1735	self.db = self.env.get_db_cnx()
	1736	self.get_sender_info(m)
[152]	1737
[587]	1738
[588]	1739	if self.parameters.white_list_file:
	1740	self.acl_list_from_file(self.parameters.white_list_file, 'white_list')
	1741
[587]	1742	if not ( self.email_header_acl('white_list', self.email_addr, True) or self.allow_registered_user ) :
	1743
[507]	1744	self.logger.info('Message rejected : %s not in white list' %(self.email_addr))
	1745	return False
[77]	1746
[507]	1747	if self.email_header_acl('black_list', self.email_addr, False):
	1748	self.logger.info('Message rejected : %s in black list' %(self.email_addr))
	1749	return False
[221]	1750
[570]	1751	if not self.email_header_acl('recipient_list', self.email_to_addrs, True):
	1752	self.logger.info('Message rejected : %s not in recipient list' %(self.email_to_addrs))
[507]	1753	return False
[226]	1754
[507]	1755	## If spam drop the message
	1756	#
	1757	if self.spam(m) == 'drop':
	1758	return False
[194]	1759
[507]	1760	elif self.spam(m) == 'spam':
	1761	spam_msg = True
	1762	else:
	1763	spam_msg = False
[204]	1764
[507]	1765	if not m['Subject']:
	1766	subject = 'No Subject'
	1767	else:
[564]	1768	subject = self.email_to_unicode(m['Subject'])
[304]	1769
[564]	1770	## Bug in python logging module <2.6
	1771	#
	1772	self.logger.info('subject: %s' %repr(subject))
[359]	1773
[507]	1774	## [hic] #1529: Re: LRZ
	1775	# [hic] #1529?owner=bas,priority=medium: Re: LRZ
	1776	#
	1777	ticket_regex = r'''
	1778	(?P<new_fields>[#][?].*)
	1779	\|(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?.*)?:)
	1780	'''
	1781	## Check if FullBlogPlugin is installed
	1782	#
	1783	blog_enabled = None
	1784	blog_regex = ''
	1785	if self.get_config('components', 'tracfullblog.*') in ['enabled']:
	1786	self.logger.debug('Trac BLOG support enabled')
	1787	blog_enabled = True
[585]	1788	blog_regex = '''\|(?P<blog>blog(?P<blog_params>[?][^:])?:(?P<blog_id>\S))'''
[329]	1789
[77]	1790
[507]	1791	## Check if DiscussionPlugin is installed
	1792	#
	1793	discussion_enabled = None
	1794	discussion_regex = ''
	1795	if self.get_config('components', 'tracdiscussion.api.discussionapi') in ['enabled']:
	1796	self.logger.debug('Trac Discussion support enabled')
	1797	discussion_enabled = True
	1798	discussion_regex = r'''
	1799	\|(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
	1800	\|(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
	1801	\|(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
	1802	'''
[77]	1803
[359]	1804
[507]	1805	regex_str = ticket_regex + blog_regex + discussion_regex
	1806	SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
[359]	1807
[507]	1808	## Find out if this is a ticket, a blog or a discussion
	1809	#
	1810	result = SYSTEM_RE.search(subject)
[390]	1811
[507]	1812	if result:
	1813	## update ticket + fields
	1814	#
	1815	if result.group('reply'):
	1816	self.system = 'ticket'
[260]	1817
[507]	1818	## Skip the last ':' character
	1819	#
	1820	if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
	1821	self.new_ticket(m, subject, spam_msg)
[390]	1822
[507]	1823	## New ticket + fields
	1824	#
	1825	elif result.group('new_fields'):
	1826	self.system = 'ticket'
	1827	self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
[262]	1828
[507]	1829	if blog_enabled:
	1830	if result.group('blog'):
	1831	self.system = 'blog'
[585]	1832	self.blog(m, subject[result.end('blog'):], result.group('blog_id'), result.group('blog_params'))
[359]	1833
[507]	1834	if discussion_enabled:
	1835	## New topic.
	1836	#
	1837	if result.group('forum'):
	1838	self.system = 'discussion'
	1839	self.id = int(result.group('forum_id'))
	1840	self.discussion_topic(m, subject[result.end('forum'):])
[359]	1841
[507]	1842	## Reply to topic.
	1843	#
	1844	elif result.group('topic'):
	1845	self.system = 'discussion'
	1846	self.id = int(result.group('topic_id'))
	1847	self.discussion_topic_reply(m, subject[result.end('topic'):])
[359]	1848
[507]	1849	## Reply to topic message.
	1850	#
	1851	elif result.group('message'):
	1852	self.system = 'discussion'
	1853	self.id = int(result.group('message_id'))
	1854	self.discussion_message_reply(m, subject[result.end('message'):])
[359]	1855
[507]	1856	else:
[553]	1857
[507]	1858	self.system = 'ticket'
	1859	(matched_id, subject) = self.ticket_update_by_subject(subject)
[553]	1860
[507]	1861	if matched_id:
[553]	1862
[507]	1863	if not self.ticket_update(m, matched_id, spam_msg):
	1864	self.new_ticket(m, subject, spam_msg)
[553]	1865
[507]	1866	else:
	1867	## No update by subject, so just create a new ticket
	1868	#
	1869	self.new_ticket(m, subject, spam_msg)
[353]	1870
[356]	1871
[343]	1872	########## BODY TEXT functions ###########################################################
	1873
[507]	1874	def strip_signature(self, text):
	1875	"""
	1876	Strip signature from message, inspired by Mailman software
	1877	"""
[607]	1878	self.logger.debug('function strip_signature: %s' %self.parameters.strip_signature_regex)
[462]	1879
[507]	1880	body = []
[606]	1881
	1882	STRIP_RE = re.compile( self.parameters.strip_signature_regex )
[507]	1883	for line in text.splitlines():
[606]	1884
	1885	match = STRIP_RE.match(line)
	1886	if match:
	1887	self.logger.debug('\t"%s " matched, skiping rest of message' %line)
[507]	1888	break
[606]	1889
[507]	1890	body.append(line)
[136]	1891
[507]	1892	return ('\n'.join(body))
[136]	1893
[507]	1894	def reflow(self, text, delsp = 0):
	1895	"""
	1896	Reflow the message based on the format="flowed" specification (RFC 3676)
	1897	"""
	1898	flowedlines = []
	1899	quotelevel = 0
	1900	prevflowed = 0
[231]	1901
[507]	1902	for line in text.splitlines():
	1903	from re import match
	1904
	1905	## Figure out the quote level and the content of the current line
	1906	#
	1907	m = match('(>)( ?)(.)', line)
	1908	linequotelevel = len(m.group(1))
	1909	line = m.group(3)
[231]	1910
[507]	1911	## Determine whether this line is flowed
	1912	#
	1913	if line and line != '-- ' and line[-1] == ' ':
	1914	flowed = 1
	1915	else:
	1916	flowed = 0
[231]	1917
[507]	1918	if flowed and delsp and line and line[-1] == ' ':
	1919	line = line[:-1]
[231]	1920
[507]	1921	## If the previous line is flowed, append this line to it
	1922	#
	1923	if prevflowed and line != '-- ' and linequotelevel == quotelevel:
	1924	flowedlines[-1] += line
[477]	1925
[507]	1926	## Otherwise, start a new line
	1927	#
	1928	else:
	1929	flowedlines.append('>' * linequotelevel + line)
[231]	1930
[507]	1931	prevflowed = flowed
	1932
[231]	1933
[507]	1934	return '\n'.join(flowedlines)
[231]	1935
[507]	1936	def strip_quotes(self, text):
	1937	"""
	1938	Strip quotes from message by Nicolas Mendoza
	1939	"""
[607]	1940	self.logger.debug('function strip_quotes: %s' %self.parameters.email_quote)
[462]	1941
[507]	1942	body = []
[607]	1943
	1944	STRIP_RE = re.compile( self.parameters.email_quote )
	1945
[507]	1946	for line in text.splitlines():
[607]	1947
[507]	1948	try:
[462]	1949
[607]	1950	match = STRIP_RE.match(line)
	1951	if match:
	1952	print line
[507]	1953	continue
[462]	1954
[507]	1955	except UnicodeDecodeError:
[462]	1956
[507]	1957	tmp_line = self.email_to_unicode(line)
[607]	1958
	1959	match = STRIP_RE.match(tmp_line)
	1960	if match:
	1961	print line
[507]	1962	continue
	1963
	1964	body.append(line)
[151]	1965
[507]	1966	return ('\n'.join(body))
[191]	1967
[507]	1968	def inline_properties(self, text):
	1969	"""
	1970	Parse text if we use inline keywords to set ticket fields
	1971	"""
	1972	self.logger.debug('function inline_properties')
[309]	1973
[507]	1974	properties = dict()
	1975	body = list()
[309]	1976
[507]	1977	INLINE_EXP = re.compile('\s[@]\s(\w+)\s:(.)$')
[309]	1978
[507]	1979	for line in text.splitlines():
	1980	match = INLINE_EXP.match(line)
	1981	if match:
	1982	keyword, value = match.groups()
[509]	1983
	1984	if self.parameters.inline_properties_first_wins:
	1985	if keyword in self.properties.keys():
	1986	continue
	1987
[507]	1988	self.properties[keyword] = value.strip()
[509]	1989	self.logger.debug('\tinline properties: %s : %s' %(keyword,value))
[408]	1990
[507]	1991	else:
	1992	body.append(line)
	1993
	1994	return '\n'.join(body)
[309]	1995
	1996
[507]	1997	def wrap_text(self, text, replace_whitespace = False):
	1998	"""
	1999	Will break a lines longer then given length into several small
	2000	lines of size given length
	2001	"""
	2002	import textwrap
[154]	2003
[507]	2004	LINESEPARATOR = '\n'
	2005	reformat = ''
[151]	2006
[507]	2007	for s in text.split(LINESEPARATOR):
	2008	tmp = textwrap.fill(s, self.parameters.use_textwrap)
	2009	if tmp:
	2010	reformat = '%s\n%s' %(reformat,tmp)
	2011	else:
	2012	reformat = '%s\n' %reformat
[153]	2013
[507]	2014	return reformat
[153]	2015
[507]	2016	# Python2.4 and higher
	2017	#
	2018	#return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
	2019	#
[154]	2020
[343]	2021	########## EMAIL attachements functions ###########################################################
	2022
[507]	2023	def inline_part(self, part):
	2024	"""
	2025	"""
	2026	self.logger.debug('function inline_part()')
[154]	2027
[507]	2028	return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
[340]	2029
[604]	2030	def get_message_parts(self, msg, new_email=False):
[507]	2031	"""
	2032	parses the email message and returns a list of body parts and attachments
	2033	body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
	2034	"""
	2035	self.logger.debug('function get_message_parts()')
[317]	2036
[507]	2037	message_parts = list()
	2038
	2039	ALTERNATIVE_MULTIPART = False
[278]	2040
[507]	2041	for part in msg.walk():
	2042	content_maintype = part.get_content_maintype()
	2043	content_type = part.get_content_type()
[278]	2044
[507]	2045	self.logger.debug('\t Message part: Main-Type: %s' % content_maintype)
	2046	self.logger.debug('\t Message part: Content-Type: %s' % content_type)
[498]	2047
[507]	2048	## Check content type
	2049	#
	2050	if content_type in self.STRIP_CONTENT_TYPES:
	2051	self.logger.debug("\t A %s attachment named '%s' was skipped" %(content_type, part.get_filename()))
	2052	continue
[238]	2053
[507]	2054	## Catch some mulitpart execptions
	2055	#
	2056	if content_type == 'multipart/alternative':
	2057	ALTERNATIVE_MULTIPART = True
	2058	continue
[278]	2059
[507]	2060	## Skip multipart containers
	2061	#
	2062	if content_maintype == 'multipart':
	2063	self.logger.debug("\t Skipping multipart container")
	2064	continue
	2065
[569]	2066	## Check if this is an inline part. It's inline if there is co Cont-Disp header,
	2067	# or if there is one and it says "inline"
[507]	2068	#
	2069	inline = self.inline_part(part)
[236]	2070
[507]	2071	## Drop HTML message
	2072	#
	2073	if ALTERNATIVE_MULTIPART and self.parameters.drop_alternative_html_version:
[553]	2074
[507]	2075	if content_type == 'text/html':
	2076	self.logger.debug('\t Skipping alternative HTML message')
	2077	ALTERNATIVE_MULTIPART = False
	2078	continue
[278]	2079
[569]	2080	filename = part.get_filename()
	2081	s = '\t unicode filename: %s' %(filename)
	2082	self.print_unicode(s)
	2083	self.logger.debug('\t raw filename: %s' %repr(filename))
[498]	2084
[569]	2085	filename = self.check_filename_length(filename)
	2086
[507]	2087	## Save all non plain text message as attachment
	2088	#
	2089	if not content_type in ['text/plain']:
[498]	2090
[569]	2091	message_parts.append( (filename, part) )
[498]	2092
[569]	2093	## We only convert html messages
[507]	2094	#
	2095	if not content_type == 'text/html':
[569]	2096	self.logger.debug('\t Appending %s (%s)' %(repr(filename), content_type))
[507]	2097	continue
[500]	2098
[569]	2099
	2100	## We have an text or html message
	2101	#
[507]	2102	if not inline:
[569]	2103	self.logger.debug('\t Appending %s (%s), not an inline messsage part' %(repr(filename), content_type))
	2104	message_parts.append( (filename, part) )
[507]	2105	continue
	2106
	2107	## Try to decode message part. We have a html or plain text messafe
	2108	#
	2109	body_text = part.get_payload(decode=1)
	2110	if not body_text:
	2111	body_text = part.get_payload(decode=0)
[236]	2112
[507]	2113	## Try to convert html message
	2114	#
	2115	if content_type == 'text/html':
[502]	2116
[507]	2117	body_text = self.html_2_txt(body_text)
	2118	if not body_text:
	2119	continue
[498]	2120
[507]	2121	format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
	2122	delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
[231]	2123
[507]	2124	if self.parameters.reflow and not self.parameters.verbatim_format and format == 'flowed':
	2125	body_text = self.reflow(body_text, delsp == 'yes')
[22]	2126
[604]	2127	if new_email and self.parameters.only_strip_on_update:
	2128	self.logger.debug('Skip signature/quote stripping for new messages')
	2129	else:
	2130	if self.parameters.strip_signature:
	2131	body_text = self.strip_signature(body_text)
[191]	2132
[604]	2133	if self.parameters.strip_quotes:
	2134	body_text = self.strip_quotes(body_text)
	2135
[507]	2136	if self.parameters.inline_properties:
	2137	body_text = self.inline_properties(body_text)
[309]	2138
[507]	2139	if self.parameters.use_textwrap:
	2140	body_text = self.wrap_text(body_text)
[148]	2141
[507]	2142	## Get contents charset (iso-8859-15 if not defined in mail headers)
	2143	#
	2144	charset = part.get_content_charset()
	2145	if not charset:
	2146	charset = 'iso-8859-15'
[102]	2147
[507]	2148	try:
	2149	ubody_text = unicode(body_text, charset)
[100]	2150
[507]	2151	except UnicodeError, detail:
	2152	ubody_text = unicode(body_text, 'iso-8859-15')
[89]	2153
[507]	2154	except LookupError, detail:
	2155	ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
[100]	2156
[507]	2157	if self.parameters.verbatim_format:
	2158	message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
	2159	else:
	2160	message_parts.append('%s' %ubody_text)
[22]	2161
[507]	2162	return message_parts
	2163
	2164	def unique_attachment_names(self, message_parts):
	2165	"""
	2166	Make sure we have unique names attachments:
	2167	- check if it contains illegal characters
	2168	- Rename "None" filenames to "untitled-part"
	2169	"""
	2170	self.logger.debug('function unique_attachment_names()')
	2171	renamed_parts = []
	2172	attachment_names = set()
[296]	2173
[507]	2174	for item in message_parts:
	2175
	2176	## If not an attachment, leave it alone
	2177	#
	2178	if not isinstance(item, tuple):
	2179	renamed_parts.append(item)
	2180	continue
	2181
	2182	(filename, part) = item
[295]	2183
[507]	2184	## If filename = None, use a default one
	2185	#
	2186	if filename in [ 'None']:
	2187	filename = 'untitled-part'
[550]	2188	self.logger.info('\t Rename filename "None" to: %s' %filename)
[22]	2189
[507]	2190	## Guess the extension from the content type, use non strict mode
	2191	# some additional non-standard but commonly used MIME types
	2192	# are also recognized
	2193	#
	2194	ext = mimetypes.guess_extension(part.get_content_type(), False)
	2195	if not ext:
	2196	ext = '.bin'
[22]	2197
[507]	2198	filename = '%s%s' % (filename, ext)
[22]	2199
[507]	2200	## Discard relative paths for windows/unix in attachment names
	2201	#
	2202	filename = filename.replace('\\', '_')
	2203	filename = filename.replace('/', '_')
[347]	2204
[507]	2205	## remove linefeed char
	2206	#
	2207	for forbidden_char in ['\r', '\n']:
	2208	filename = filename.replace(forbidden_char,'')
[465]	2209
[507]	2210	## We try to normalize the filename to utf-8 NFC if we can.
	2211	# Files uploaded from OS X might be in NFD.
	2212	# Check python version and then try it
	2213	#
	2214	#if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
	2215	# try:
	2216	# filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8')
	2217	# except TypeError:
	2218	# pass
[100]	2219
[507]	2220	## Make the filename unique for this ticket
	2221	#
	2222	num = 0
	2223	unique_filename = filename
	2224	dummy_filename, ext = os.path.splitext(filename)
[134]	2225
[507]	2226	while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
	2227	num += 1
	2228	unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
	2229
[550]	2230	s = '\t Attachment with filename %s will be saved as %s' % (filename, unique_filename)
[549]	2231	self.print_unicode(s)
[100]	2232
[507]	2233	attachment_names.add(unique_filename)
[236]	2234
[507]	2235	renamed_parts.append((filename, unique_filename, part))
	2236
	2237	return renamed_parts
	2238
	2239
	2240	def attachment_exists(self, filename):
[250]	2241
[549]	2242	self.logger.debug("function attachment_exists")
[250]	2243
[549]	2244	s = '\t check if attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
	2245	self.print_unicode(s)
	2246
[553]	2247	## Do we have a valid ticket id
[507]	2248	#
	2249	if not self.id:
	2250	return False
[250]	2251
[507]	2252	try:
	2253	if self.system == 'discussion':
[553]	2254
	2255	att = attachment.Attachment(self.env, 'discussion', 'ticket/%s' % (self.id,), filename)
	2256
[585]	2257	elif self.system == 'blog':
	2258
	2259	att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,), filename)
	2260
[507]	2261	else:
[553]	2262
	2263	att = attachment.Attachment(self.env, 'ticket', self.id, filename)
	2264
[507]	2265	return True
[553]	2266
[507]	2267	except attachment.ResourceNotFound:
[553]	2268
[507]	2269	return False
[343]	2270
	2271	########## TRAC Ticket Text ###########################################################
[507]	2272
	2273	def get_body_text(self, message_parts):
	2274	"""
	2275	"""
	2276	self.logger.debug('function get_body_text()')
[498]	2277
[507]	2278	body_text = []
	2279
	2280	for part in message_parts:
	2281
	2282	## Plain text part, append it
	2283	#
	2284	if not isinstance(part, tuple):
	2285	body_text.extend(part.strip().splitlines())
	2286	body_text.append("")
	2287	continue
[490]	2288
[507]	2289	(original, filename, part) = part
	2290	inline = self.inline_part(part)
[491]	2291
[507]	2292	## Skip generation of attachment link if html is converted to text
	2293	#
	2294	if part.get_content_type() == 'text/html' and self.parameters.html2text_cmd and inline:
	2295	s = 'Skipping attachment link for html part: %s' %(filename)
	2296	self.print_unicode(s)
	2297	continue
	2298
	2299	if part.get_content_maintype() == 'image' and inline:
[553]	2300
[507]	2301	if self.system != 'discussion':
	2302	s = 'wiki image link for: %s' %(filename)
	2303	self.print_unicode(s)
	2304	body_text.append('[[Image(%s)]]' % filename)
[553]	2305
[507]	2306	body_text.append("")
[553]	2307
[507]	2308	else:
[553]	2309
[507]	2310	if self.system != 'discussion':
[553]	2311
[507]	2312	s = 'wiki attachment link for: %s' %(filename)
	2313	self.print_unicode(s)
	2314	body_text.append('[attachment:"%s"]' % filename)
[553]	2315
[507]	2316	body_text.append("")
[490]	2317
[507]	2318	## Convert list body_texts to string
	2319	#
	2320	body_text = '\r\n'.join(body_text)
	2321	return body_text
[236]	2322
[507]	2323	def html_mailto_link(self, subject):
	2324	"""
	2325	This function returns a HTML mailto tag with the ticket id and author email address
	2326	"""
	2327	self.logger.debug("function html_mailto_link")
[576]	2328
[507]	2329	if not self.author:
	2330	author = self.email_addr
	2331	else:
	2332	author = self.author
[343]	2333
[507]	2334	if not self.parameters.mailto_cc:
	2335	self.parameters.mailto_cc = ''
[592]	2336
	2337	## Bug in urllib.quote function
	2338	#
	2339	if isinstance(subject, unicode):
	2340	subject = subject.encode('utf-8')
[434]	2341
[507]	2342	## use urllib to escape the chars
	2343	#
[517]	2344	s = '%s?Subject=%s&cc=%s' %(
[507]	2345	urllib.quote(self.email_addr),
	2346	urllib.quote('Re: #%s: %s' %(self.id, subject)),
	2347	urllib.quote(self.parameters.mailto_cc)
	2348	)
[343]	2349
[507]	2350	if self.VERSION in [ 0.10 ]:
	2351	s = '\r\n{{{\r\n#!html\r\n<a\r\n href="mailto:%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
	2352	else:
	2353	s = '[mailto:"%s" Reply to: %s]' %(s, author)
[506]	2354
[507]	2355	self.logger.debug("\tmailto link %s" %s)
	2356	return s
[343]	2357
	2358	########## TRAC notify section ###########################################################
	2359
[507]	2360	def notify(self, tkt, new=True, modtime=0):
	2361	"""
	2362	A wrapper for the TRAC notify function. So we can use templates
	2363	"""
	2364	self.logger.debug('function notify()')
[392]	2365
[509]	2366	if self.parameters.notify_reporter:
	2367	self.logger.debug('\t Notify reporter set')
[508]	2368	global sender_email
	2369	sender_email = self.email_addr
	2370
	2371	self.logger.debug('\t Using Email2TracNotification function AlwaysNotifyReporter')
	2372	import trac.notification as Email2TracNotification
	2373	Email2TracNotification.Notify.notify = AlwaysNotifyReporter
	2374
	2375	if self.parameters.dry_run :
[561]	2376	self.logger.info('DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter'])
[507]	2377	return
	2378	try:
[392]	2379
[507]	2380	#from trac.ticket.notification import TicketNotificationSystem
	2381	#tn_sys = TicketNotificationSystem(self.env)
	2382	#print tn_sys
	2383	#print tn_sys.__dict__
	2384	#sys.exit(0)
[392]	2385
[507]	2386	## create false {abs_}href properties, to trick Notify()
	2387	#
[540]	2388	if self.VERSION in [0.10]:
[507]	2389	self.env.abs_href = Href(self.get_config('project', 'url'))
	2390	self.env.href = Href(self.get_config('project', 'url'))
[22]	2391
[507]	2392	tn = TicketNotifyEmail(self.env)
[213]	2393
[507]	2394	if self.parameters.alternate_notify_template:
[222]	2395
[507]	2396	if self.VERSION >= 0.11:
[222]	2397
[507]	2398	from trac.web.chrome import Chrome
[222]	2399
[507]	2400	if self.parameters.alternate_notify_template_update and not new:
	2401	tn.template_name = self.parameters.alternate_notify_template_update
	2402	else:
	2403	tn.template_name = self.parameters.alternate_notify_template
[222]	2404
[507]	2405	tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
	2406
	2407	else:
[222]	2408
[507]	2409	tn.template_name = self.parameters.alternate_notify_template
[42]	2410
[507]	2411	tn.notify(tkt, new, modtime)
[41]	2412
[507]	2413	except Exception, e:
	2414	self.logger.error('Failure sending notification on creation of ticket #%s: %s' %(self.id, e))
[41]	2415
[508]	2416	########## END Class Definition ########################################################
[22]	2417
[508]	2418	########## Global Notificaition Function ################################################
	2419	def AlwaysNotifyReporter(self, resid):
	2420	"""
	2421	Copy of def notify() to manipulate recipents to always include reporter for the
	2422	notification.
	2423	"""
[569]	2424	#print sender_email, resid
[508]	2425	(torcpts, ccrcpts) = self.get_recipients(resid)
[569]	2426	#print "get_recipients finished"
[74]	2427
[508]	2428	if not tktparser.email_header_acl('notify_reporter_black_list', sender_email, False):
	2429	## additionally append sender (regardeless of settings in trac.ini)
	2430	#
	2431	if not sender_email in torcpts:
	2432	torcpts.append(sender_email)
	2433
	2434	self.begin_send()
	2435	self.send(torcpts, ccrcpts)
	2436	self.finish_send()
	2437
[343]	2438	########## Parse Config File ###########################################################
[22]	2439
	2440	def ReadConfig(file, name):
[507]	2441	"""
	2442	Parse the config file
	2443	"""
	2444	if not os.path.isfile(file):
	2445	print 'File %s does not exist' %file
	2446	sys.exit(1)
[22]	2447
[507]	2448	config = trac_config.Configuration(file)
	2449
	2450	parentdir = config.get('DEFAULT', 'parentdir')
	2451	sections = config.sections()
[22]	2452
[507]	2453	## use some trac internals to get the defaults
	2454	#
	2455	tmp = config.parser.defaults()
	2456	project = SaraDict()
[485]	2457
[507]	2458	for option, value in tmp.items():
	2459	try:
	2460	project[option] = int(value)
	2461	except ValueError:
	2462	project[option] = value
[485]	2463
[507]	2464	if name:
	2465	if name in sections:
	2466	project = SaraDict()
	2467	for option, value in config.options(name):
	2468	try:
	2469	project[option] = int(value)
	2470	except ValueError:
	2471	project[option] = value
[485]	2472
[507]	2473	elif not parentdir:
	2474	print "Not a valid project name: %s, valid names are: %s" %(name, sections)
	2475	print "or set parentdir in the [DEFAULT] section"
	2476	sys.exit(1)
[22]	2477
[507]	2478	## If parentdir then set project dir to parentdir + name
	2479	#
	2480	if not project.has_key('project'):
	2481	if not parentdir:
	2482	print "You must set project or parentdir in your configuration file"
	2483	sys.exit(1)
	2484	elif not name:
	2485	print "You must configure a project section in your configuration file"
	2486	else:
	2487	project['project'] = os.path.join(parentdir, name)
[22]	2488
[528]	2489	##
	2490	# Save the project name
	2491	#
	2492	project['project_name'] = os.path.basename(project['project'])
	2493
[507]	2494	return project
[22]	2495
[410]	2496	########## Setup Logging ###############################################################
[87]	2497
[419]	2498	def setup_log(parameters, project_name, interactive=None):
[507]	2499	"""
[516]	2500	Setup logging
[410]	2501
[507]	2502	Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
[420]	2503	would be interpreted by the ConfigParser itself.
[507]	2504	"""
	2505	logger = logging.getLogger('email2trac %s' %project_name)
[410]	2506
[507]	2507	if interactive:
	2508	parameters.log_type = 'stderr'
[414]	2509
[507]	2510	if not parameters.log_type:
	2511	if sys.platform in ['win32', 'cygwin']:
[521]	2512	parameters.log_type = 'eventlog'
[507]	2513	else:
	2514	parameters.log_type = 'syslog'
[410]	2515
[507]	2516	if parameters.log_type == 'file':
[415]	2517
[507]	2518	if not parameters.log_file:
	2519	parameters.log_file = 'email2trac.log'
[415]	2520
[507]	2521	if not os.path.isabs(parameters.log_file):
	2522	parameters.log_file = os.path.join(tempfile.gettempdir(), parameters.log_file)
[410]	2523
[507]	2524	log_handler = logging.FileHandler(parameters.log_file)
[415]	2525
[507]	2526	elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
	2527	## Requires win32 extensions
	2528	#
[532]	2529	logid = "email2trac"
[507]	2530	log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
[410]	2531
[507]	2532	elif parameters.log_type in ('syslog', 'unix'):
	2533	log_handler = logging.handlers.SysLogHandler('/dev/log')
[410]	2534
[507]	2535	elif parameters.log_type in ('stderr'):
	2536	log_handler = logging.StreamHandler(sys.stderr)
[410]	2537
[507]	2538	else:
	2539	log_handler = logging.handlers.BufferingHandler(0)
[410]	2540
[507]	2541	if parameters.log_format:
	2542	parameters.log_format = parameters.log_format.replace('$(', '%(')
	2543	else:
	2544	parameters.log_format = '%(name)s: %(message)s'
[516]	2545	if parameters.log_type in ('file', 'stderr'):
	2546	parameters.log_format = '%(asctime)s ' + parameters.log_format
[410]	2547
[507]	2548	log_formatter = logging.Formatter(parameters.log_format)
	2549	log_handler.setFormatter(log_formatter)
	2550	logger.addHandler(log_handler)
[410]	2551
[507]	2552	if (parameters.log_level in ['DEBUG', 'ALL']) or (parameters.debug > 0):
	2553	logger.setLevel(logging.DEBUG)
	2554	parameters.debug = 1
[410]	2555
[507]	2556	elif parameters.log_level in ['INFO'] or parameters.verbose:
	2557	logger.setLevel(logging.INFO)
[410]	2558
[507]	2559	elif parameters.log_level in ['WARNING']:
	2560	logger.setLevel(logging.WARNING)
[410]	2561
[507]	2562	elif parameters.log_level in ['ERROR']:
	2563	logger.setLevel(logging.ERROR)
[410]	2564
[507]	2565	elif parameters.log_level in ['CRITICAL']:
	2566	logger.setLevel(logging.CRITICAL)
[410]	2567
[507]	2568	else:
	2569	logger.setLevel(logging.INFO)
[410]	2570
[507]	2571	return logger
[410]	2572
	2573
[22]	2574	if __name__ == '__main__':
[507]	2575	## Default config file
	2576	#
	2577	configfile = '@email2trac_conf@'
	2578	project = ''
	2579	component = ''
	2580	ticket_prefix = 'default'
	2581	dry_run = None
	2582	verbose = None
	2583	debug_interactive = None
[202]	2584
[507]	2585	SHORT_OPT = 'cdhf:np:t:v'
	2586	LONG_OPT = ['component=', 'debug', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=', 'verbose']
[201]	2587
[507]	2588	try:
	2589	opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
	2590	except getopt.error,detail:
	2591	print __doc__
	2592	print detail
	2593	sys.exit(1)
	2594
	2595	project_name = None
	2596	for opt,value in opts:
	2597	if opt in [ '-h', '--help']:
	2598	print __doc__
	2599	sys.exit(0)
	2600	elif opt in ['-c', '--component']:
	2601	component = value
	2602	elif opt in ['-d', '--debug']:
	2603	debug_interactive = 1
	2604	elif opt in ['-f', '--file']:
	2605	configfile = value
	2606	elif opt in ['-n', '--dry-run']:
	2607	dry_run = True
	2608	elif opt in ['-p', '--project']:
	2609	project_name = value
	2610	elif opt in ['-t', '--ticket_prefix']:
	2611	ticket_prefix = value
	2612	elif opt in ['-v', '--verbose']:
	2613	verbose = True
	2614
	2615	settings = ReadConfig(configfile, project_name)
[410]	2616
[507]	2617	## The default prefix for ticket values in email2trac.conf
	2618	#
	2619	settings.ticket_prefix = ticket_prefix
	2620	settings.dry_run = dry_run
	2621	settings.verbose = verbose
[410]	2622
[507]	2623	if not settings.debug and debug_interactive:
	2624	settings.debug = debug_interactive
[412]	2625
[507]	2626	if not settings.project:
	2627	print __doc__
	2628	print 'No Trac project is defined in the email2trac config file.'
	2629	sys.exit(1)
[419]	2630
[507]	2631	logger = setup_log(settings, os.path.basename(settings.project), debug_interactive)
	2632
	2633	if component:
	2634	settings['component'] = component
[202]	2635
[507]	2636	## Determine major trac version used to be in email2trac.conf
	2637	# Quick hack for 0.12
	2638	#
	2639	version = '0.%s' %(trac_version.split('.')[1])
	2640	if version.startswith('0.12'):
	2641	version = '0.12'
[541]	2642	elif version.startswith('0.13'):
	2643	version = '0.13'
[373]	2644
[507]	2645	logger.debug("Found trac version: %s" %(version))
	2646
	2647	try:
	2648	if version == '0.10':
	2649	from trac import attachment
	2650	from trac.env import Environment
	2651	from trac.ticket import Ticket
	2652	from trac.web.href import Href
	2653	from trac import util
	2654	from trac.ticket.web_ui import TicketModule
[486]	2655
[507]	2656	#
	2657	# return util.text.to_unicode(str)
	2658	#
	2659	# see http://projects.edgewall.com/trac/changeset/2799
	2660	from trac.ticket.notification import TicketNotifyEmail
	2661	from trac import config as trac_config
	2662	from trac.core import TracError
[359]	2663
[554]	2664	elif version in ['0.11', '0.12', '0.13']:
[507]	2665	from trac import attachment
	2666	from trac import config as trac_config
	2667	from trac import util
	2668	from trac.core import TracError
[554]	2669	from trac.env import Environment
[507]	2670	from trac.perm import PermissionSystem
[539]	2671	from trac.perm import PermissionCache
[538]	2672	from trac.test import Mock, MockPerm
[507]	2673	from trac.ticket import Ticket
[554]	2674	from trac.ticket.api import TicketSystem
[507]	2675	from trac.ticket.web_ui import TicketModule
[541]	2676	from trac.web.href import Href
[368]	2677
[541]	2678	#
	2679	# return util.text.to_unicode(str)
	2680	#
	2681	# see http://projects.edgewall.com/trac/changeset/2799
	2682	from trac.ticket.notification import TicketNotifyEmail
	2683
[507]	2684	else:
	2685	logger.error('TRAC version %s is not supported' %version)
[519]	2686	sys.exit(0)
[182]	2687
[507]	2688	## Must be set before environment is created
	2689	#
	2690	if settings.has_key('python_egg_cache'):
	2691	python_egg_cache = str(settings['python_egg_cache'])
	2692	os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
[291]	2693
[507]	2694	if settings.debug > 0:
	2695	logger.debug('Loading environment %s', settings.project)
[359]	2696
[507]	2697	try:
	2698	env = Environment(settings['project'], create=0)
	2699	except IOError, detail:
[534]	2700	logger.error("trac error: %s" %detail)
[519]	2701	sys.exit(0)
[333]	2702
[507]	2703	tktparser = TicketEmailParser(env, settings, logger, float(version))
	2704	tktparser.parse(sys.stdin)
[22]	2705
[507]	2706	## Catch all errors and use the logging module
	2707	#
	2708	except Exception, error:
[187]	2709
[507]	2710	etype, evalue, etb = sys.exc_info()
	2711	for e in traceback.format_exception(etype, evalue, etb):
	2712	logger.critical(e)
[187]	2713
[507]	2714	if m:
[528]	2715	tktparser.save_email_for_debug(m, settings.project_name, True)
[97]	2716
[507]	2717	sys.exit(1)
[22]	2718	# EOB

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format