source: trunk/email2trac.py.in @ 636

Last change on this file since 636 was 636, checked in by bas, 11 years ago

ticket cc fields ignored on ticket update, closes #324

  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 90.5 KB
RevLine 
[22]1#!@PYTHON@
2#
3# This file is part of the email2trac utils
4#
[628]5#       Copyright 2002 SURFsara
[22]6#
[595]7#       Licensed under the Apache License, Version 2.0 (the "License");
8#       you may not use this file except in compliance with the License.
9#       You may obtain a copy of the License at
[22]10#
[595]11#       http://www.apache.org/licenses/LICENSE-2.0
[22]12#
[595]13#       Unless required by applicable law or agreed to in writing, software
14#       distributed under the License is distributed on an "AS IS" BASIS,
15#       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16#       See the License for the specific language governing permissions and
17#       limitations under the License.
18#
[80]19# For vi/emacs or other use tabstop=4 (vi: set ts=4)
20#
[22]21"""
[547]22email2trac.py -- Email -> TRAC tickets
[22]23
[547]24A MTA filter to create Trac tickets from inbound emails.
[22]25
[547]26first proof of concept from:
27 Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28 Copyright 2005, Edgewall Software
[22]29
[282]30Authors:
[628]31  Bas van der Vlies <bas.vandervlies@surfsara.nl>
32  Walter de Jong <walter.dejong@surfsara.nl>
[22]33
34How to use
35----------
[628]36 * See https://oss.trac.surfsara.nl/email2trac/
[282]37
[22]38 * Commandline opions:
[205]39                -h,--help
[624]40                -A, --agilo
[507]41                -d, --debug
[625]42                -E, --virtualenv <path>
[205]43                -f,--file  <configuration file>
[410]44                -n,--dry-run
[205]45                -p, --project <project name>
[415]46                -t, --ticket_prefix <name>
[626]47                -v, --verbose
[22]48
49SVN Info:
50        $Id: email2trac.py.in 636 2013-08-07 12:30:49Z bas $
51"""
52import os
53import sys
54import string
55import getopt
56import time
57import email
[136]58import email.Iterators
59import email.Header
[22]60import re
61import urllib
62import unicodedata
63import mimetypes
[96]64import traceback
[403]65import logging
66import logging.handlers
[404]67import UserDict
[498]68import tempfile
[433]69
[520]70from datetime import timedelta, datetime
[433]71from stat import *
[22]72
[508]73## Some global variables
[96]74#
75m = None
[22]76
[404]77class SaraDict(UserDict.UserDict):
[507]78    def __init__(self, dictin = None):
79        UserDict.UserDict.__init__(self)
80        self.name = None
81       
82        if dictin:
83            if dictin.has_key('name'):
84                self.name = dictin['name']
85                del dictin['name']
86            self.data = dictin
87           
88    def get_value(self, name):
89        if self.has_key(name):
90            return self[name]
91        else:
92            return None
93               
94    def __repr__(self):
95        return repr(self.data)
[182]96
[507]97    def __str__(self):
98        return str(self.data)
99           
100    def __getattr__(self, name):
101        """
102        override the class attribute get method. Return the value
103        from the dictionary
104        """
105        if self.data.has_key(name):
106            return self.data[name]
107        else:
108            return None
109           
110    def __setattr__(self, name, value):
111        """
112        override the class attribute set method only when the UserDict
113        has set its class attribute
114        """
115        if self.__dict__.has_key('data'):
116            self.data[name] = value
117        else:
118            self.__dict__[name] = value
[404]119
[507]120    def __iter__(self):
121        return iter(self.data.keys())
[404]122
[633]123
124
[22]125class TicketEmailParser(object):
[507]126    env = None
127    comment = '> '
[359]128
[507]129    def __init__(self, env, parameters, logger, version):
130        self.env = env
[22]131
[507]132        # Database connection
133        #
134        self.db = None
[22]135
[507]136        # Save parameters
137        #
138        self.parameters = parameters
139        self.logger = logger
[206]140
[507]141        # Some useful mail constants
142        #
143        self.email_name = None
144        self.email_addr = None
145        self.email_from = None
146        self.author     = None
147        self.id         = None
[587]148
149        ## Will be set to True if the user has ben registered
150        #
151        self.allow_registered_user = False
[507]152       
153        self.STRIP_CONTENT_TYPES = list()
[72]154
[507]155        ## fields properties via body_text
156        #
157        self.properties = dict()
[452]158
[507]159        self.VERSION = version
[402]160
[507]161        self.get_config = self.env.config.get
[404]162
[507]163        ## init function ##
164        #
165        self.setup_parameters()
[402]166
[507]167    def setup_parameters(self):
168        if self.parameters.umask:
169            os.umask(self.parameters.umask)
[402]170
[507]171        if not self.parameters.spam_level:
172            self.parameters.spam_level = 0
[22]173
[507]174        if not self.parameters.spam_header:
175            self.parameters.spam_header = 'X-Spam-Score'
[207]176
[507]177        if not self.parameters.email_quote:
[607]178            self.parameters.email_quote = '^> .*'
[614]179            self.parameters.skip_line_regex = '^> .*'
[22]180
[507]181        if not self.parameters.ticket_update_by_subject_lookback:
182            self.parameters.ticket_update_by_subject_lookback = 30
[360]183
[507]184        if self.parameters.verbatim_format == None:
185            self.parameters.verbatim_format = 1
[118]186
[507]187        if self.parameters.reflow == None:
188            self.parameters.reflow = 1
[231]189
[507]190        if self.parameters.binhex:
191            self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
[238]192
[507]193        if self.parameters.applesingle:
194            self.STRIP_CONTENT_TYPES.append('application/applefile')
[238]195
[507]196        if self.parameters.appledouble:
197            self.STRIP_CONTENT_TYPES.append('application/applefile')
[238]198
[507]199        if self.parameters.strip_content_types:
200            items = self.parameters.strip_content_types.split(',')
201            for item in items:
202                self.STRIP_CONTENT_TYPES.append(item.strip())
[294]203
[507]204        if self.parameters.tmpdir:
205            self.parameters.tmpdir = os.path.normcase(str(self.parameters['tmpdir']))
206        else:
207            self.parameters.tmpdir = os.path.normcase('/tmp')
[257]208
[507]209        if self.parameters.email_triggers_workflow == None:
210            self.parameters.email_triggers_workflow = 1
[191]211
[507]212        if not self.parameters.subject_field_separator:
213            self.parameters.subject_field_separator = '&'
214        else:
215            self.parameters.subject_field_separator = self.parameters.subject_field_separator.strip()
[297]216
[606]217        if not self.parameters.strip_signature_regex:
[612]218            self.parameters.strip_signature_regex = '^-- $'
[614]219            self.parameters.cut_line_regex = '^-- $'
[606]220
[507]221        self.trac_smtp_from = self.get_config('notification', 'smtp_from')
222        self.smtp_default_domain = self.get_config('notification', 'smtp_default_domain')
[631]223        self.smtp_replyto = self.get_config('notification', 'smtp_replyto')
[636]224        self.trac_smtp_always_cc = self.get_config('notification', 'smtp_always_cc')
225        self.trac_smtp_always_bcc = self.get_config('notification', 'smtp_always_bcc')
[305]226
[484]227
[507]228        self.system = None
[359]229
[341]230########## Email Header Functions ###########################################################
[339]231
[507]232    def spam(self, message):
233        """
234        # X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
235        # Note if Spam_level then '*' are included
236        """
[580]237        self.logger.debug('function spam')
238
[507]239        spam = False
240        if message.has_key(self.parameters.spam_header):
241            spam_l = string.split(message[self.parameters.spam_header])
[578]242            #self.logger.info('Spam header: %s' %(message[self.parameters.spam_header]))
[22]243
[507]244            try:
245                number = spam_l[0].count('*')
246            except IndexError, detail:
247                number = 0
248               
249            if number >= self.parameters.spam_level:
250                spam = True
251               
[580]252        ## treat virus mails as spam
[507]253        #
254        elif message.has_key('X-Virus-found'):         
255            spam = True
[194]256
[580]257        ## Astaro firewall spam handling
[507]258        #
[580]259        elif message.get('X-Spam-Flag') == "YES" and message.get('X-Spam-Result') == "Spam":
260            spam = True
261
262        ## How to handle SPAM messages
263        #
[507]264        if self.parameters.drop_spam and spam:
[194]265
[507]266            self.logger.info('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' %self.parameters.spam_level)
267            return 'drop'   
[194]268
[507]269        elif spam:
[194]270
[507]271            return 'Spam'   
272        else:
[22]273
[507]274            return False
[191]275
[572]276    def email_header_acl(self, keyword, message_field, default):
[507]277        """
278        This function wil check if the email address is allowed or denied
279        to send mail to the ticket list
280        """
281        self.logger.debug('function email_header_acl: %s' %keyword)
[409]282
[507]283        try:
284            mail_addresses = self.parameters[keyword]
[221]285
[507]286            # Check if we have an empty string
287            #
288            if not mail_addresses:
289                return default
[221]290
[507]291        except KeyError, detail:
[572]292            self.logger.debug('\t %s not defined, all messages are allowed.' %(keyword))
[206]293
[507]294            return default
[206]295
[507]296        mail_addresses = string.split(mail_addresses, ',')
[572]297        message_addresses = string.split(message_field, ',')
[221]298
[507]299        for entry in mail_addresses:
300            entry = entry.strip()
301            TO_RE = re.compile(entry, re.VERBOSE|re.IGNORECASE)
[149]302
[572]303            for addr in message_addresses:
304                addr = addr.strip()
305
306                if self.parameters.compare_function_list in [ 'matches', 'match']:
307                    s = '\t%s matches %s' %(addr, entry)
308                    result =  TO_RE.match(addr)
309                else:
310                    s = '\t%s contains %s' %(addr, entry)
311                    result =  TO_RE.search(addr)
312
313                if result:
314                    self.logger.debug(s)
315                    return True
316
[507]317        return False
[208]318
[507]319    def email_header_txt(self, m):
320        """
321        Display To and CC addresses in description field
322        """
323        s = ''
[334]324
[507]325        if m['To'] and len(m['To']) > 0:
326            s = "'''To:''' %s\r\n" %(m['To'])
327        if m['Cc'] and len(m['Cc']) > 0:
328            s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
[22]329
[507]330        return  self.email_to_unicode(s)
[22]331
[138]332
[507]333    def get_sender_info(self, message):
334        """
335        Get the default author name and email address from the message
336        """
[576]337        self.logger.debug('function get_sender_info')
[43]338
[570]339        to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
340        self.email_to_addrs = list()
[226]341
[570]342        for n,e in to_addrs:
343            self.email_to_addrs.append(e)
344
345        self.email_to_addrs = ','.join(self.email_to_addrs)
346
347
[576]348        self.email_name, self.email_addr  = email.Utils.parseaddr(message['from'])
[142]349
[576]350        ## decode email name can contain charset
351        #
352        self.email_name = self.email_to_unicode(self.email_name)
353
354        dstr = '\t email name: %s, email address: %s' %(self.email_name, self.email_addr) 
355        self.logger.debug(dstr)
356
[507]357        ## Trac can not handle author's name that contains spaces
358        #
[518]359        if self.email_addr.lower() == self.trac_smtp_from.lower():
[507]360            if self.email_name:
361                self.author = self.email_name
362            else:
363                self.author = "email2trac"
364        else:
365            self.author = self.email_addr
[304]366
[507]367        if self.parameters.ignore_trac_user_settings:
368            return
[194]369
[507]370        # Is this a registered user, use email address as search key:
371        # result:
372        #   u : login name
373        #   n : Name that the user has set in the settings tab
374        #   e : email address that the user has set in the settings tab
375        #
376        users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
377            if (
378                (e and (e.lower() == self.email_addr.lower())) or
[518]379                (u.lower() + '@' + self.smtp_default_domain.lower() == self.email_addr.lower())
[507]380            )
381            ]
[43]382
[507]383        if len(users) >= 1:
[587]384            self.email_from = users[0][2]
[507]385            self.author = users[0][0]
[45]386
[590]387            self.logger.debug('\tget_sender_info: found registered user: %s' %(self.author) )
388
[587]389            if self.parameters.white_list_registered_users:
[590]390                self.logger.debug('\tget_sender_info: added registered user to white list')
[587]391                self.allow_registered_user = True
392                   
393
[597]394    def set_cc_fields(self, ticket, message, update=False):
[507]395        """
396        Set all the right fields for a new ticket
397        """
[597]398        self.logger.debug('function set_cc_fields')
[72]399
[270]400
[507]401        # Put all CC-addresses in ticket CC field
402        #
403        if self.parameters.reply_all:
[43]404
[636]405            ticket_cc = ''
[299]406
[636]407            msg_cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
[299]408
[636]409            if not msg_cc_addrs:
[507]410                return
[43]411
[597]412            if update:
[636]413                self.logger.debug("\tupdate ticket cc-field")
414                ticket_cc = ticket['cc']
415                ticket_cc_list = ticket_cc.split(',')
[597]416
[636]417            for name,addr in msg_cc_addrs:
[507]418       
419                ## Prevent mail loop
420                #
421                if addr == self.trac_smtp_from:
[597]422                    self.logger.debug("\tSkipping %s email address for CC-field, same as smtp_from address in trac.ini " %(addr))
[507]423                    continue
[43]424
[597]425                ## Alwyas remove reporter email address
426                #
427                elif addr == self.email_addr:
428                    self.logger.debug("\tSkipping reporter email address for CC-field")
429                    continue
430
[636]431                ## Always remove the always_cc address
432                #
433                elif addr == self.trac_smtp_always_cc:
434                    self.logger.debug("\tSkipping smtp_always_cc email address for CC-field")
435                    continue
436
437                ## Always remove the always_bcc address
438                #
439                elif addr == self.trac_smtp_always_bcc:
440                    self.logger.debug("\tSkipping smtp_always_bcc email address for CC-field")
441                    continue
442
[507]443                else:
[636]444                    ## On update, prevent adding duplicates
445                    #
446                    if update:
447                      if addr in ticket_cc_list:
448                          continue
449                   
450                    if ticket_cc:
451                        ticket_cc = '%s, %s' %(ticket_cc, addr)
[597]452                    else:
[636]453                        ticket_cc = addr
[96]454
[636]455            if ticket_cc:
456                self.logger.debug('\tCC fields: %s' %ticket_cc)
457                ticket['cc'] = self.email_to_unicode(ticket_cc)
[299]458
[339]459
[588]460    def acl_list_from_file(self, f, keyword):
461        """
462        Read the email address from a file
463        """
464        self.logger.debug('function acl_list_from_file %s : %s' %(f, keyword))
465       
466        if  not os.path.isfile(f):
467            self.logger.error('%s_file: %s does not exists' %(keyword, f) )
468        else:
469            ## read whole file and replace '\n' with ''
470            #
471            addr_l = open(f, 'r').readlines()
472            s = ','.join(addr_l).replace('\n','')
473
474            try:
475                self.parameters[keyword] = "%s,%s" %(self.parameters[keyword], s)
476            except KeyError, detail:
477                self.parameters[keyword] = s
478
[339]479########## DEBUG functions  ###########################################################
480
[555]481    def debug_body(self, message_body, temp_file):
482        """
483        """
484        self.logger.debug('function debug_body:')
[310]485
[555]486        body_file = "%s.body" %(temp_file)
487        fx = open(body_file, 'wb')
488
[507]489        if self.parameters.dry_run:
[561]490            self.logger.info('DRY-RUN: not saving body to %s' %(body_file))
[507]491            return
[331]492
[555]493        self.logger.debug('writing body to %s' %(body_file))
[507]494        if not message_body:
495                message_body = '(None)'
[310]496
[507]497        message_body = message_body.encode('utf-8')
[310]498
[507]499        fx.write(message_body)
500        fx.close()
501        try:
502            os.chmod(body_file,S_IRWXU|S_IRWXG|S_IRWXO)
503        except OSError:
504            pass
[310]505
[555]506    def debug_attachments(self, message_parts, temp_file):
[507]507        """
508        """
509        self.logger.debug('function debug_attachments')
510       
511        n = 0
512        for item in message_parts:
[553]513
514            ## Skip inline text parts
515            #
[507]516            if not isinstance(item, tuple):
517                continue
518               
519            (original, filename, part) = item
[555]520            self.logger.debug('\t part%d: Content-Type: %s' % (n, part.get_content_type()) )
[310]521
[555]522            dummy_filename, ext = os.path.splitext(filename)
523
[507]524            n = n + 1
[555]525            part_name = 'part%d%s' %(n, ext)
526            part_file = "%s.%s" %(temp_file, part_name)
527
528            s = 'writing %s: filename: %s' %(part_file, filename)
[507]529            self.print_unicode(s)
[555]530
531            ## Forbidden chars, just write part files instead of names
[507]532            #
[555]533            #filename = filename.replace('\\', '_')
534            #filename = filename.replace('/', '_')
535            #filename = filename + '.att_email2trac'
536            # part_file = os.path.join(self.parameters.tmpdir, filename)
537            #part_file = util.text.unicode_quote(part_file)
538            #self.print_unicode(part_file)
[331]539
[507]540            if self.parameters.dry_run:
[561]541                self.logger.info('DRY_RUN: NOT saving attachments')
[507]542                continue
[331]543
[507]544            fx = open(part_file, 'wb')
545            text = part.get_payload(decode=1)
[331]546
[507]547            if not text:
548                text = '(None)'
[331]549
[507]550            fx.write(text)
551            fx.close()
[331]552
[507]553            try:
554                os.chmod(part_file,S_IRWXU|S_IRWXG|S_IRWXO)
555            except OSError:
556                pass
[310]557
[528]558    def save_email_for_debug(self, message, project_name, create_tempfile=False):
[309]559
[528]560        if self.parameters.dry_run:
[555]561            self.logger.debug('DRY_RUN: NOT saving email message')
[528]562            return
563
[507]564        if create_tempfile:
[555]565            (fd, tmp_file) = tempfile.mkstemp('.%s.email2trac' %project_name)
[514]566            fx = os.fdopen(fd, 'wb')
[507]567        else:
[555]568            tmp_file = os.path.join(self.parameters.tmpdir, 'msg.%s.email2trac' %project_name)
569            fx = open(tmp_file, 'wb')
[173]570
[44]571
[555]572        self.logger.debug('saving email to %s' %(tmp_file))
[528]573        fx.write('%s' % message)
574        fx.close()
575       
576        try:
[555]577            os.chmod(tmp_file, S_IRWXU|S_IRWXG|S_IRWXO)
[528]578        except OSError:
579            pass
[331]580
[507]581        message_parts = self.get_message_parts(message)
582        message_parts = self.unique_attachment_names(message_parts)
583        body_text = self.get_body_text(message_parts)
[555]584        self.debug_body(body_text, tmp_file)
585        self.debug_attachments(message_parts, tmp_file)
[309]586
[339]587########## Conversion functions  ###########################################################
588
[507]589    def email_to_unicode(self, message_str):
590        """
591        Email has 7 bit ASCII code, convert it to unicode with the charset
592        that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
593        understands it.
594        """
595        self.logger.debug("function email_to_unicode")
[341]596
[507]597        results =  email.Header.decode_header(message_str)
[341]598
[507]599        s = None
600        for text,format in results:
601            if format:
602                try:
603                    temp = unicode(text, format)
604                except UnicodeError, detail:
605                    # This always works
606                    #
[525]607                    msg = 'ERROR: Could not find charset: %s, please install' %format
608                    self.logger.error(msg)
[507]609                    temp = unicode(text, 'iso-8859-15')
610                except LookupError, detail:
[514]611                    msg = 'ERROR: Could not find charset: %s, please install' %format
[525]612                    self.logger.error(msg)
[507]613                    #temp = unicode(text, 'iso-8859-15')
614                    temp = message_str
615                       
616            else:
617                temp = string.strip(text)
618                temp = unicode(text, 'iso-8859-15')
[341]619
[507]620            if s:
621                s = '%s %s' %(s, temp)
622            else:
623                s = '%s' %temp
[341]624
[507]625        return s
[341]626
[507]627    def str_to_dict(self, s):
628        """
629        Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
630        """
631        self.logger.debug("function str_to_dict")
[164]632
[507]633        fields = string.split(s, self.parameters.subject_field_separator)
[262]634
[507]635        result = dict()
636        for field in fields:
637            try:
638                index, value = string.split(field, '=')
[169]639
[507]640                # We can not change the description of a ticket via the subject
641                # line. The description is the body of the email
642                #
643                if index.lower() in ['description']:
644                    continue
[169]645
[507]646                if value:
647                    result[index.lower()] = value
[169]648
[507]649            except ValueError:
650                pass
651        return result
[167]652
[507]653    def print_unicode(self,s):
654        """
655        This function prints unicode strings if possible else it will quote it
656        """
657        try:
658            self.logger.debug(s)
659        except UnicodeEncodeError, detail:
660            self.logger.debug(util.text.unicode_quote(s))
[379]661
[503]662
[507]663    def html_2_txt(self, data):
664        """
665        Various routines to convert html syntax to valid trac wiki syntax
666        """
667        self.logger.debug('function html_2_txt')
[503]668
[507]669        ## This routine make an safe html that can be include
670        #  in trac, but no further text processing can be done
671        #
672#       try:
673#           from lxml.html.clean import Cleaner
674#           tags_rm = list()
675#           tags_rm.append('body')
[503]676#
[507]677#           cleaner = Cleaner(remove_tags=tags_rm )
678#           parsed_data = cleaner.clean_html(data)
679#           parsed_data = '\n{{{\n#!html\n' + parsed_data + '\n}}}\n'
[503]680#
[507]681#           return parsed_data
682#           
683#       except ImportError::
684#           pass
[503]685
[507]686        parsed_data = None
687        if self.parameters.html2text_cmd:
[514]688            (fd, tmp_file) = tempfile.mkstemp('email2trac.html')
689            f = os.fdopen(fd, 'w')
690
[507]691            cmd = '%s %s' %(self.parameters.html2text_cmd, tmp_file)
692            self.logger.debug('\t html2text conversion %s'%(cmd))
693   
694            if self.parameters.dry_run:
[561]695                self.logger.info('DRY_RUN: html2text conversion command: %s\n' %(cmd))
[503]696
[507]697            else:
698                f.write(data)
699                f.close()
[503]700
[507]701                lines = os.popen(cmd).readlines()
702                parsed_data =  ''.join(lines)
[503]703
[507]704                os.unlink(tmp_file)
[503]705
[507]706        else:
707            self.logger.debug('\t No html2text conversion tool specified in email2trac.conf')
[503]708
[507]709        return parsed_data
[503]710
[549]711    def check_filename_length(self, name):
712        """
[551]713        To bypass a bug in Trac
714        check if the filename lenght is not larger then OS limit.
[549]715          yes : return truncated filename
716          no  : return unmodified filename
717        """
718        self.logger.debug('function check_filename_length: ')
[550]719
720        if not name:
721            return 'None'
[549]722       
723        dummy_filename, ext = os.path.splitext(name)
724
725        ## Trac uses this format
726        #
[620]727        try:
728            quote_format = util.text.unicode_quote(dummy_filename)
[549]729
[620]730        except UnicodeDecodeError, detail:
731            ## last resort convert to unicode
732            #
733            dummy_filename = util.text.to_unicode(dummy_filename)
734            quote_format = util.text.unicode_quote(dummy_filename)
735
[549]736        ## Determine max filename lenght
737        #
[603]738        try:
739            filemax_length = os.pathconf('/', 'PC_NAME_MAX')
740        except AttributeError, detail:
741            filemax_lenght = 240
[549]742
743        if len(quote_format) <= filemax_length:
744            return name
745
746        else:
747            ## Truncate file to filemax_length and reserve room for extension
748            #  We must break on a boundry
749            #
750            length = filemax_length - 6
751
752            for i in range(0,10):
753
754                truncated = quote_format[ : (length - i)]
755
756                try:
757                    unqoute_truncated = util.text.unicode_unquote(truncated)
758                    unqoute_truncated = unqoute_truncated + ext
759
760                    self.print_unicode('\t ' + unqoute_truncated)
761
762                    break
763
764                except UnicodeDecodeError, detail:
765                    continue
766
767
768            return unqoute_truncated
769
[339]770########## TRAC ticket functions  ###########################################################
771
[546]772    def mail_workflow(self, tkt):
[545]773        """
774        """
[546]775        self.logger.debug('function mail_workflow: ')
[545]776       
777        req = Mock(authname=self.author, perm=MockPerm(), args={})
778        ticket_system = TicketSystem(self.env)
779       
780        try:
781            workflow = self.parameters['workflow_%s' %tkt['status'].lower()]
782
783        except KeyError:
[553]784            ## fallback for compability (Will be deprecated)
785            #  workflow can be none.
[545]786            #
787            workflow = None
788            if tkt['status'] in ['closed']:
789                workflow = self.parameters.workflow
790
791        if workflow:
[556]792
[557]793            ## process all workflow implementations
794            #
[556]795            tkt_module = TicketModule(self.env)
796            field_changes, problems = tkt_module.get_ticket_changes(req, tkt, workflow)
797
798            for field in field_changes.keys():
799
800                ## We have already processed these fields
801                #
[558]802                if not field in ['summary', 'description']:
[556]803                    s = 'workflow : %s, field %s : %s, by %s' \
804                       %(workflow, field, field_changes[field]['new'],field_changes[field]['by'] )
805                    self.logger.debug(s)
806
807                    tkt[field] = field_changes[field]['new']
808
[545]809            return True
810
811        else:
812            return False
813
[507]814    def check_permission_participants(self, tkt, action):
815        """
816        Check if the mailer is allowed to update the ticket
817        """
[538]818        self.logger.debug('function check_permission_participants %s')
[398]819
[518]820        if tkt['reporter'].lower() in [self.author.lower(), self.email_addr.lower()]:
[507]821            self.logger.debug('ALLOW, %s is the ticket reporter' %(self.email_addr))
[408]822
[507]823            return True
[398]824
[507]825        perm = PermissionSystem(self.env)
826        if perm.check_permission(action, self.author):
827            self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
[408]828
[507]829            return True
830       
831        # Is the updater in the CC?
832        try:
833            cc_list = tkt['cc'].split(',')
834            for cc in cc_list:
[518]835                if self.email_addr.lower() in cc.lower.strip():
[507]836                    self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
[398]837
[507]838                    return True
[398]839
[507]840        except KeyError:
841            pass
[398]842
[507]843        return False
[474]844
[507]845    def check_permission(self, tkt, action):
846        """
847        check if the reporter has the right permission for the action:
[397]848          - TICKET_CREATE
849          - TICKET_MODIFY
[474]850          - TICKET_APPEND
851          - TICKET_CHGPROP
[397]852
[507]853        There are three models:
854            - None      : no checking at all
855            - trac      : check the permission via trac permission model
856            - email2trac: ....
857        """
[538]858        self.logger.debug("function check_permission: %s" %(action))
[397]859
[507]860        if self.parameters.ticket_permission_system in ['trac']:
[398]861
[539]862            perm = PermissionCache(self.env, self.author)
863            if perm.has_permission(action):
[507]864                return True
865            else:
866                return False
[398]867
[507]868        elif self.parameters.ticket_permission_system in ['update_restricted_to_participants']:
869            return (self.check_permission_participants(tkt, action))   
[398]870
[507]871        ## Default is to allow everybody ticket updates and ticket creation
872        #
873        else:
874                return True
[397]875
876
[507]877    def update_ticket_fields(self, ticket, user_dict, new=None):
878        """
879        This will update the ticket fields. It will check if the
880        given fields are known and if the right values are specified
881        It will only update the ticket field value:
882            - If the field is known
883            - If the value supplied is valid for the ticket field.
884              If not then there are two options:
885               1) Skip the value (new=None)
886               2) Set default value for field (new=1)
887        """
888        self.logger.debug("function update_ticket_fields")
[169]889
[507]890        ## Check only permission model on ticket updates
891        #
892        if not new:
893            if self.parameters.ticket_permission_system:
894                if not self.check_permission(ticket, 'TICKET_CHGPROP'):
895                    self.logger.info('Reporter: %s has no permission to change ticket properties' %self.author)
896                    return False
[474]897
[507]898        ## Build a system dictionary from the ticket fields
899        #  with field as index and option as value
900        #
901        sys_dict = dict()
[530]902
[507]903        for field in ticket.fields:
[530]904
[507]905            try:
906                sys_dict[field['name']] = field['options']
[169]907
[507]908            except KeyError:
[531]909                sys_dict[field['name']] = None
[507]910                pass
[169]911
[507]912        ## Check user supplied fields an compare them with the
913        #  system one's
914        #
[549]915        for field,value in user_dict.items():
916       
917            s = 'user_field\t %s = %s' %(field,value)
918            self.print_unicode(s)
[524]919
[531]920            if not field in sys_dict.keys(): 
[524]921                self.logger.debug('%s is not a valid field for tickets' %(field))
922                continue
923
[507]924            ## To prevent mail loop
925            #
926            if field == 'cc':
[301]927
[507]928                cc_list = user_dict['cc'].split(',')
[301]929
[507]930                if self.trac_smtp_from in cc_list:
931                    self.logger.debug('MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
[408]932
[507]933                    cc_list.remove(self.trac_smtp_from)
[301]934
[507]935                value = ','.join(cc_list)
936               
[301]937
[507]938            ## Check if every value is allowed for this field
939            #
[531]940            if sys_dict[field]:
[169]941
[507]942                if value in sys_dict[field]:
943                    ticket[field] = value
944                else:
945                    ## Must we set a default if value is not allowed
946                    #
947                    if new:
948                        value = self.get_config('ticket', 'default_%s' %(field) )
[169]949
[507]950            else:
[553]951
[507]952                ticket[field] = value
[202]953
[549]954            s = 'ticket_field\t %s = %s' %(field,  ticket[field])
955            self.print_unicode(s)
[345]956
[507]957    def ticket_update(self, m, id, spam):
958        """
959        If the current email is a reply to an existing ticket, this function
960        will append the contents of this email to that ticket, instead of
961        creating a new one.
962        """
963        self.logger.debug("function ticket_update")
[202]964
[507]965        if not self.parameters.ticket_update:
966            self.logger.debug("ticket_update disabled")
967            return False
[456]968
[507]969        ## Must we update ticket fields
970        #
971        update_fields = dict()
972        try:
973            id, keywords = string.split(id, '?')
[262]974
[507]975            update_fields = self.str_to_dict(keywords)
[165]976
[507]977            ## Strip '#'
978            #
979            self.id = int(id[1:])
[165]980
[507]981        except ValueError:
[390]982
[507]983            ## Strip '#'
984            #
985            self.id = int(id[1:])
[164]986
[597]987        self.logger.debug("\tticket id: %s" %id)
[71]988
[507]989        ## When is the change committed
990        #
991        if self.VERSION < 0.11:
992            when = int(time.time())
993        else:
994            when = datetime.now(util.datefmt.utc)
[77]995
[507]996        try:
997            tkt = Ticket(self.env, self.id, self.db)
[390]998
[507]999        except util.TracError, detail:
[390]1000
[507]1001            ## Not a valid ticket
1002            #
[604]1003            self.logger.info("\tCreating a new ticket, ticket id: %s does not exists" %id)
[507]1004            self.id = None
1005            return False
[126]1006
[507]1007        ## Check the permission of the reporter
1008        #
1009        if self.parameters.ticket_permission_system:
1010            if not self.check_permission(tkt, 'TICKET_APPEND'):
1011                self.logger.info('Reporter: %s has no permission to add comments or attachments to tickets' %self.author)
1012                return False
[398]1013
[507]1014        ## How many changes has this ticket
1015        #
1016        grouped = TicketModule(self.env).grouped_changelog_entries(tkt, self.db)
1017        cnum = sum(1 for e in grouped) + 1
[288]1018
[486]1019
[507]1020        ## reopen the ticket if it is was closed
1021        #  We must use the ticket workflow framework
1022        #
1023        if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
[257]1024
[546]1025            if not self.mail_workflow(tkt):
[538]1026
[507]1027                if tkt['status'] in ['closed']:
[526]1028                    tkt['status'] = 'reopened'
1029                    tkt['resolution'] = ''
1030
[507]1031        ## Old pre 0.11 situation
1032        #
1033        elif self.parameters.email_triggers_workflow:
[470]1034
[597]1035            self.logger.debug('\temail triggers workflow pre trac 0.11')
[470]1036
[507]1037            if tkt['status'] in ['closed']:
1038                tkt['status'] = 'reopened'
1039                tkt['resolution'] = ''
[257]1040
[507]1041        else:
[597]1042            self.logger.debug('\temail triggers workflow disabled')
[469]1043
[507]1044        ## Must we update some ticket fields properties via subject line
1045        #
1046        if update_fields:
1047            self.update_ticket_fields(tkt, update_fields)
[166]1048
[597]1049
[507]1050        message_parts = self.get_message_parts(m)
1051        message_parts = self.unique_attachment_names(message_parts)
[210]1052
[597]1053        ## Must we update some ticket fields properties via inline comments
1054        # in body_text
[507]1055        #
1056        if self.properties:
1057                self.update_ticket_fields(tkt, self.properties)
[309]1058
[597]1059        ## Must we update the CC ticket field
1060        #
1061        self.set_cc_fields(tkt, m, update=True)
1062
[507]1063        if self.parameters.email_header:
1064            message_parts.insert(0, self.email_header_txt(m))
[76]1065
[507]1066        body_text = self.get_body_text(message_parts)
[236]1067
[507]1068        error_with_attachments = self.attach_attachments(message_parts)
[348]1069
[507]1070        if body_text.strip() or update_fields or self.properties:
[543]1071
[507]1072            if self.parameters.dry_run:
[561]1073                s = 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) %s %s' %(self.author, cnum)
1074                self.logger.info(s)
[543]1075
[507]1076            else:
1077                if error_with_attachments:
1078                    body_text = '%s\\%s' %(error_with_attachments, body_text)
[543]1079
[597]1080                self.logger.debug('\ttkt.save_changes(%s, %d)' %(self.author, cnum))
[507]1081                tkt.save_changes(self.author, body_text, when, None, str(cnum))
1082           
1083        if not spam:
1084            self.notify(tkt, False, when)
[72]1085
[507]1086        return True
[71]1087
[507]1088    def set_ticket_fields(self, ticket):
1089        """
1090        set the ticket fields to value specified
1091            - /etc/email2trac.conf with <prefix>_<field>
1092            - trac default values, trac.ini
1093        """
1094        self.logger.debug('function set_ticket_fields')
[410]1095
[507]1096        user_dict = dict()
[202]1097
[507]1098        for field in ticket.fields:
[202]1099
[507]1100            name = field['name']
[202]1101
[507]1102            ## default trac value
1103            #
1104            if not field.get('custom'):
1105                value = self.get_config('ticket', 'default_%s' %(name) )
[599]1106
1107                ## skip this field can only be set by email2trac.conf
1108                #
1109                if name in ['resolution']:
[507]1110                    value = None
[599]1111
[507]1112            else:
1113                ##  Else get the default value for reporter
1114                #
1115                value = field.get('value')
1116                options = field.get('options')
[345]1117
[507]1118                if value and options and (value not in options):
1119                     value = options[int(value)]
1120   
[549]1121            s = 'trac[%s] = %s' %(name, value)
1122            self.print_unicode(s)
[202]1123
[507]1124            ## email2trac.conf settings
1125            #
1126            prefix = self.parameters.ticket_prefix
1127            try:
1128                value = self.parameters['%s_%s' %(prefix, name)]
[202]1129
[549]1130                s = 'email2trac[%s] = %s ' %(name, value)
1131                self.print_unicode(s)
1132
[507]1133            except KeyError, detail:
1134                pass
1135       
1136            if value:
1137                user_dict[name] = value
[202]1138
[550]1139                s = 'used %s = %s' %(name, value)
1140                self.print_unicode(s)
1141
[507]1142        self.update_ticket_fields(ticket, user_dict, new=1)
[202]1143
[507]1144        if 'status' not in user_dict.keys():
1145            ticket['status'] = 'new'
[202]1146
1147
[507]1148    def ticket_update_by_subject(self, subject):
1149        """
1150        This list of Re: prefixes is probably incomplete. Taken from
1151        wikipedia. Here is how the subject is matched
1152          - Re: <subject>
1153          - Re: (<Mail list label>:)+ <subject>
[202]1154
[507]1155        So we must have the last column
1156        """
1157        self.logger.debug('function ticket_update_by_subject')
[356]1158
[533]1159        found_id = None
[507]1160        if self.parameters.ticket_update and self.parameters.ticket_update_by_subject:
1161               
1162            SUBJECT_RE = re.compile(r'^(?:(?:RE|AW|VS|SV|FW|FWD):\s*)+(.*)', re.IGNORECASE)
1163            result = SUBJECT_RE.search(subject)
[356]1164
[507]1165            if result:
1166                ## This is a reply
1167                #
1168                orig_subject = result.group(1)
[356]1169
[507]1170                self.logger.debug('subject search string: %s' %(orig_subject))
[356]1171
[507]1172                cursor = self.db.cursor()
1173                summaries = [orig_subject, '%%: %s' % orig_subject]
[356]1174
[520]1175                ## Time resolution is in micoseconds
[507]1176                #
[520]1177                search_date = datetime.now(util.datefmt.utc) - timedelta(days=self.parameters.ticket_update_by_subject_lookback)
[356]1178
[529]1179                if self.VERSION < 0.12:
1180                    lookback = util.datefmt.to_timestamp(search_date)
1181                else:
1182                    lookback = util.datefmt.to_utimestamp(search_date)
1183
[507]1184                for summary in summaries:
1185                    self.logger.debug('Looking for summary matching: "%s"' % summary)
[408]1186
[533]1187                    sql = """SELECT id, reporter FROM ticket
[507]1188                            WHERE changetime >= %s AND summary LIKE %s
1189                            ORDER BY changetime DESC"""
[533]1190
[507]1191                    cursor.execute(sql, [lookback, summary.strip()])
[356]1192
[507]1193                    for row in cursor:
[408]1194
[533]1195                        (matched_id, sql_reporter) = row
[408]1196
[533]1197                        ## Save first entry.
1198                        #
1199                        if not found_id:
1200                            found_id = matched_id
1201                           
1202                        ## If subject and reporter are the same. The we certainly have found the right ticket
1203                        #
1204                        if sql_reporter == self.author:
1205                            self.logger.debug('Found matching reporter: %s with ticket id: %d' %(sql_reporter, matched_id))
1206                            found_id = matched_id
1207                            break
[356]1208
[533]1209                    if found_id:
1210                        self.logger.debug('Found matching ticket id: %d' % found_id)
1211                        found_id = '#%d' % found_id
1212                        return (found_id, orig_subject)
1213                   
[553]1214                    # obsolete !?? 12 Aug 2011
[533]1215                    #subject = orig_subject
[356]1216
[533]1217        return (found_id, subject)
[356]1218
1219
[507]1220    def new_ticket(self, msg, subject, spam, set_fields = None):
1221        """
1222        Create a new ticket
1223        """
1224        self.logger.debug('function new_ticket')
[250]1225
[507]1226        tkt = Ticket(self.env)
[326]1227
[597]1228        ## self.author can be email address of an username
1229        #
1230        tkt['reporter'] = self.author
[326]1231
[597]1232        self.set_cc_fields(tkt, msg)
1233
[507]1234        self.set_ticket_fields(tkt)
[202]1235
[507]1236        ## Check the permission of the reporter
1237        #
1238        if self.parameters.ticket_permission_system:
1239            if not self.check_permission(tkt, 'TICKET_CREATE'):
1240                self.logger.info('Reporter: %s has no permission to create tickets' %self.author)
1241                return False
[397]1242
[507]1243        ## Old style setting for component, will be removed
1244        #
1245        if spam:
1246            tkt['component'] = 'Spam'
[204]1247
[507]1248        elif self.parameters.has_key('component'):
1249            tkt['component'] = self.parameters['component']
[201]1250
[507]1251        if not msg['Subject']:
1252            tkt['summary'] = u'(No subject)'
1253        else:
1254            tkt['summary'] = subject
[22]1255
1256
[507]1257        if set_fields:
1258            rest, keywords = string.split(set_fields, '?')
[262]1259
[507]1260            if keywords:
1261                update_fields = self.str_to_dict(keywords)
1262                self.update_ticket_fields(tkt, update_fields)
[262]1263
[296]1264
[604]1265        message_parts = self.get_message_parts(msg, True)
[309]1266
[507]1267        ## Must we update some ticket fields properties via body_text
1268        #
1269        if self.properties:
1270                self.update_ticket_fields(tkt, self.properties)
[309]1271
[507]1272        message_parts = self.unique_attachment_names(message_parts)
1273       
1274        ## produce e-mail like header
1275        #
1276        head = ''
1277        if self.parameters.email_header:
1278            head = self.email_header_txt(msg)
1279            message_parts.insert(0, head)
1280           
1281        body_text = self.get_body_text(message_parts)
[45]1282
[507]1283        tkt['description'] = body_text
[90]1284
[507]1285        ## When is the change committed
1286        #
1287        if self.VERSION < 0.11:
1288            when = int(time.time())
1289        else:
1290            when = datetime.now(util.datefmt.utc)
[45]1291
[507]1292        if self.parameters.dry_run:
[561]1293            self.logger.info('DRY_RUN: tkt.insert()')
[507]1294        else:
1295            self.id = tkt.insert()
1296   
1297        changed = False
1298        comment = ''
[77]1299
[507]1300        ## some routines in trac are dependend on ticket id
1301        #  like alternate notify template
1302        #
1303        if self.parameters.alternate_notify_template:
1304            tkt['id'] = self.id
1305            changed = True
[273]1306
[507]1307        ## Rewrite the description if we have mailto enabled
1308        #
1309        if self.parameters.mailto_link:
1310            changed = True
1311            comment = u'\nadded mailto line\n'
[592]1312
[581]1313            #mailto = self.html_mailto_link( m['Subject'])
1314            mailto = self.html_mailto_link(subject)
[253]1315
[507]1316            tkt['description'] = u'%s\r\n%s%s\r\n' \
1317                %(head, mailto, body_text)
1318   
1319        ## Save the attachments to the ticket   
1320        #
1321        error_with_attachments =  self.attach_attachments(message_parts)
[295]1322
[507]1323        if error_with_attachments:
1324            changed = True
1325            comment = '%s\n%s\n' %(comment, error_with_attachments)
[45]1326
[545]1327        if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
[546]1328            if self.mail_workflow(tkt):
[545]1329                changed = True
1330
[507]1331        if changed:
1332            if self.parameters.dry_run:
[561]1333                s = 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
1334                self.logger.info(s)
1335
[507]1336            else:
1337                tkt.save_changes(tkt['reporter'], comment)
[90]1338
[507]1339        if not spam:
[545]1340            self.notify(tkt, True)
[45]1341
[260]1342
[507]1343    def attach_attachments(self, message_parts, update=False):
1344        '''
1345        save any attachments as files in the ticket's directory
1346        '''
1347        self.logger.debug('function attach_attachments()')
[342]1348
[507]1349        if self.parameters.dry_run:
[555]1350            self.logger.debug("DRY_RUN: no attachments attached to tickets")
[507]1351            return ''
[342]1352
[507]1353        count = 0
[342]1354
[507]1355        ## Get Maxium attachment size
1356        #
1357        max_size = int(self.get_config('attachment', 'max_size'))
1358        status   = None
1359       
1360        for item in message_parts:
1361            ## Skip body parts
1362            #
1363            if not isinstance(item, tuple):
1364                continue
1365               
1366            (original, filename, part) = item
[477]1367
[548]1368            ## We have to determine the size so we use this temporary solution.
[507]1369            #
[548]1370            path, fd =  util.create_unique_file(os.path.join(self.parameters.tmpdir, 'email2trac_tmp.att'))
[507]1371            text = part.get_payload(decode=1)
1372            if not text:
1373                text = '(None)'
1374            fd.write(text)
1375            fd.close()
[342]1376
[507]1377            ## get the file_size
1378            #
1379            stats = os.lstat(path)
1380            file_size = stats[ST_SIZE]
[342]1381
[507]1382            ## Check if the attachment size is allowed
1383            #
1384            if (max_size != -1) and (file_size > max_size):
1385                status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1386                    %(status, original, file_size, max_size)
[342]1387
[507]1388                os.unlink(path)
1389                continue
1390            else:
1391                count = count + 1
1392                   
1393            ## Insert the attachment
1394            #
1395            fd = open(path, 'rb')
1396            if self.system == 'discussion':
[553]1397                att = attachment.Attachment(self.env, 'discussion', 'topic/%s' % (self.id,))
[585]1398            elif self.system == 'blog':
1399                att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,))
[507]1400            else:
[548]1401                s = 'Attach %s to ticket %d' %(filename, self.id)
1402                self.print_unicode(s)
[507]1403                att = attachment.Attachment(self.env, 'ticket', self.id)
[359]1404 
[507]1405            ## This will break the ticket_update system, the body_text is vaporized
1406            #  ;-(
1407            #
1408            if not update:
1409                att.author = self.author
1410                att.description = self.email_to_unicode('Added by email2trac')
[342]1411
[553]1412            try:
[549]1413
[507]1414                self.logger.debug('Insert atachment')
1415                att.insert(filename, fd, file_size)
[553]1416
[507]1417            except OSError, detail:
[553]1418
[507]1419                self.logger.info('%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail))
1420                status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
[342]1421
[507]1422            ## Remove the created temporary filename
1423            #
1424            fd.close()
1425            os.unlink(path)
[342]1426
[507]1427        ## return error
1428        #
1429        return status
[342]1430
[359]1431########## Fullblog functions  #################################################
[339]1432
[585]1433    def blog(self, msg, subject, id, params):
[507]1434        """
1435        The blog create/update function
1436        """
1437        ## import the modules
1438        #
1439        from tracfullblog.core import FullBlogCore
1440        from tracfullblog.model import BlogPost, BlogComment
[260]1441
[507]1442        ## instantiate blog core
1443        #
1444        blog = FullBlogCore(self.env)
1445        req = Mock(authname='anonymous', perm=MockPerm(), args={})
[312]1446
[585]1447        ## parameters from the subject
1448        #
1449        params = self.str_to_dict((params or '').lstrip('?'))
[260]1450
[585]1451        ## preferably get the time from the email date header, or else
1452        #  use current date and time
1453        date = email.Utils.parsedate_tz(msg.get('date'))
1454        if date:
1455            dt = util.datefmt.to_datetime(email.Utils.mktime_tz(date), util.datefmt.utc)
1456        else:
1457            self.logger.warn("No valid date header found")
1458            dt = util.datefmt.to_datetime(None, util.datefmt.utc)
1459
1460        ## blog entry affected
1461        #
1462        self.id = id or util.datefmt.format_datetime(dt, "%Y%m%d%H%M%S", util.datefmt.utc)
1463
1464        ## check wether a blog post exists
1465        #
1466        post = BlogPost(self.env, self.id)
1467        force_update = self.properties.get('update', params.get('update'))
1468
1469        ## message parts
1470        #
1471        message_parts = self.get_message_parts(msg)
1472        message_parts = self.unique_attachment_names(message_parts)
1473
1474        if post.get_versions() and not force_update:
1475
1476            ## add comment to blog entry
[507]1477            #
[585]1478            comment = BlogComment(self.env, self.id)
1479            comment.author = self.properties.get('author', params.get('author', self.author))
[507]1480            comment.comment = self.get_body_text(message_parts)
[585]1481            comment.time = dt
[312]1482
[578]1483            if self.parameters.dry_run:
1484                self.logger.info('DRY-RUN: not adding comment for blog entry "%s"' % id)
1485                return
[585]1486            warnings = blog.create_comment(req, comment)
[260]1487
[507]1488        else:
[585]1489            ## create or update blog entry
[507]1490            #
[585]1491            post.author = self.properties.get('author', params.get('author', self.author))
1492            post.categories = self.properties.get('categories', params.get('categories', ''))
[578]1493            post.title = subject.strip()
[585]1494            post.publish_time = dt
[507]1495            post.body = self.get_body_text(message_parts)
1496           
[578]1497            if self.parameters.dry_run:
1498                self.logger.info('DRY-RUN: not creating blog entry "%s"' % post.title)
1499                return
[585]1500            warnings = blog.create_post(req, post, self.author, u'Created by email2trac', False)
[260]1501
[585]1502        ## check for problems
1503        #
1504        if warnings:
1505            raise TracError(', '.join('blog:%s:%s' % (w[0], w[1]) for w in warnings))
1506       
1507        ## all seems well, attach attachments
1508        #
1509        self.attach_attachments(message_parts)
[260]1510
[585]1511
[359]1512########## Discussion functions  ##############################################
[342]1513
[507]1514    def discussion_topic(self, content, subject):
[342]1515
[507]1516        ## Import modules.
1517        #
1518        from tracdiscussion.api import DiscussionApi
1519        from trac.util.datefmt import to_timestamp, utc
[359]1520
[507]1521        self.logger.debug('Creating a new topic in forum:', self.id)
[359]1522
[507]1523        ## Get dissussion API component.
1524        #
1525        api = self.env[DiscussionApi]
1526        context = self._create_context(content, subject)
[359]1527
[507]1528        ## Get forum for new topic.
1529        #
1530        forum = api.get_forum(context, self.id)
[359]1531
[507]1532        if not forum:
1533            self.logger.error("ERROR: Replied forum doesn't exist")
[359]1534
[507]1535        ## Prepare topic.
1536        #
1537        topic = {'forum' : forum['id'],
1538                 'subject' : context.subject,
1539                 'time': to_timestamp(datetime.now(utc)),
1540                 'author' : self.author,
1541                 'subscribers' : [self.email_addr],
1542                 'body' : self.get_body_text(context.content_parts)}
[359]1543
[507]1544        ## Add topic to DB and commit it.
1545        #
1546        self._add_topic(api, context, topic)
1547        self.db.commit()
[359]1548
[507]1549    def discussion_topic_reply(self, content, subject):
[359]1550
[507]1551        ## Import modules.
1552        #
1553        from tracdiscussion.api import DiscussionApi
1554        from trac.util.datefmt import to_timestamp, utc
[359]1555
[507]1556        self.logger.debug('Replying to discussion topic', self.id)
[359]1557
[507]1558        ## Get dissussion API component.
1559        #
1560        api = self.env[DiscussionApi]
1561        context = self._create_context(content, subject)
[359]1562
[507]1563        ## Get replied topic.
1564        #
1565        topic = api.get_topic(context, self.id)
[359]1566
[507]1567        if not topic:
1568            self.logger.error("ERROR: Replied topic doesn't exist")
[359]1569
[507]1570        ## Prepare message.
1571        #
1572        message = {'forum' : topic['forum'],
1573                   'topic' : topic['id'],
1574                   'replyto' : -1,
1575                   'time' : to_timestamp(datetime.now(utc)),
1576                   'author' : self.author,
1577                   'body' : self.get_body_text(context.content_parts)}
[359]1578
[507]1579        ## Add message to DB and commit it.
1580        #
1581        self._add_message(api, context, message)
1582        self.db.commit()
[359]1583
[507]1584    def discussion_message_reply(self, content, subject):
[359]1585
[507]1586        ## Import modules.
1587        #
1588        from tracdiscussion.api import DiscussionApi
1589        from trac.util.datefmt import to_timestamp, utc
[359]1590
[507]1591        self.logger.debug('Replying to discussion message', self.id)
[359]1592
[507]1593        ## Get dissussion API component.
1594        #
1595        api = self.env[DiscussionApi]
1596        context = self._create_context(content, subject)
[359]1597
[507]1598        ## Get replied message.
1599        #
1600        message = api.get_message(context, self.id)
[359]1601
[507]1602        if not message:
1603            self.logger.error("ERROR: Replied message doesn't exist")
[359]1604
[507]1605        ## Prepare message.
1606        #
1607        message = {'forum' : message['forum'],
1608                   'topic' : message['topic'],
1609                   'replyto' : message['id'],
1610                   'time' : to_timestamp(datetime.now(utc)),
1611                   'author' : self.author,
1612                   'body' : self.get_body_text(context.content_parts)}
[359]1613
[507]1614        ## Add message to DB and commit it.
1615        #
1616        self._add_message(api, context, message)
1617        self.db.commit()
[359]1618
[507]1619    def _create_context(self, content, subject):
[359]1620
[507]1621        ## Import modules.
1622        #
1623        from trac.mimeview import Context
1624        from trac.web.api import Request
1625        from trac.perm import PermissionCache
[359]1626
[507]1627        ## TODO: Read server base URL from config.
1628        #  Create request object to mockup context creation.
1629        #
1630        environ = {'SERVER_PORT' : 80,
1631                   'SERVER_NAME' : 'test',
1632                   'REQUEST_METHOD' : 'POST',
1633                   'wsgi.url_scheme' : 'http',
1634                   'wsgi.input' : sys.stdin}
1635        chrome =  {'links': {},
1636                   'scripts': [],
1637                   'ctxtnav': [],
1638                   'warnings': [],
1639                   'notices': []}
[359]1640
[507]1641        if self.env.base_url_for_redirect:
1642            environ['trac.base_url'] = self.env.base_url
[359]1643
[507]1644        req = Request(environ, None)
1645        req.chrome = chrome
1646        req.tz = 'missing'
1647        req.authname = self.author
1648        req.perm = PermissionCache(self.env, self.author)
1649        req.locale = None
[359]1650
[507]1651        ## Create and return context.
1652        #
1653        context = Context.from_request(req)
1654        context.realm = 'discussion-email2trac'
1655        context.cursor = self.db.cursor()
1656        context.content = content
1657        context.subject = subject
[359]1658
[507]1659        ## Read content parts from content.
1660        #
1661        context.content_parts = self.get_message_parts(content)
1662        context.content_parts = self.unique_attachment_names(
1663          context.content_parts)
[359]1664
[507]1665        return context
[359]1666
[507]1667    def _add_topic(self, api, context, topic):
1668        context.req.perm.assert_permission('DISCUSSION_APPEND')
[359]1669
[507]1670        ## Filter topic.
1671        #
1672        for discussion_filter in api.discussion_filters:
1673            accept, topic_or_error = discussion_filter.filter_topic(
1674              context, topic)
1675            if accept:
1676                topic = topic_or_error
1677            else:
1678                raise TracError(topic_or_error)
[359]1679
[507]1680        ## Add a new topic.
1681        #
1682        api.add_topic(context, topic)
[359]1683
[507]1684        ## Get inserted topic with new ID.
1685        #
1686        topic = api.get_topic_by_time(context, topic['time'])
[359]1687
[507]1688        ## Attach attachments.
1689        #
1690        self.id = topic['id']
1691        self.attach_attachments(context.content_parts, True)
[359]1692
[507]1693        ## Notify change listeners.
1694        #
1695        for listener in api.topic_change_listeners:
1696            listener.topic_created(context, topic)
[359]1697
[507]1698    def _add_message(self, api, context, message):
1699        context.req.perm.assert_permission('DISCUSSION_APPEND')
[359]1700
[507]1701        ## Filter message.
1702        #
1703        for discussion_filter in api.discussion_filters:
1704            accept, message_or_error = discussion_filter.filter_message(
1705              context, message)
1706            if accept:
1707                message = message_or_error
1708            else:
1709                raise TracError(message_or_error)
[359]1710
[507]1711        ## Add message.
1712        #
1713        api.add_message(context, message)
[359]1714
[507]1715        ## Get inserted message with new ID.
1716        #
1717        message = api.get_message_by_time(context, message['time'])
[359]1718
[507]1719        ## Attach attachments.
1720        #
[549]1721
[507]1722        self.attach_attachments(context.content_parts, True)
[359]1723
[507]1724        ## Notify change listeners.
1725        #
1726        for listener in api.message_change_listeners:
1727            listener.message_created(context, message)
[359]1728
1729########## MAIN function  ######################################################
1730
[507]1731    def parse(self, fp):
1732        """
1733        """
1734        self.logger.debug('Main function parse')
1735        global m
[96]1736
[507]1737        m = email.message_from_file(fp)
1738       
1739        if not m:
1740            self.logger.debug('This is not a valid email message format')
1741            return
1742           
1743        ## Work around lack of header folding in Python; see http://bugs.python.org/issue4696
1744        #
1745        try:
1746            m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
1747        except AttributeError, detail:
1748            pass
[239]1749
[507]1750        if self.parameters.debug:     # save the entire e-mail message text
[528]1751            self.save_email_for_debug(m, self.parameters.project_name, True)
[77]1752
[618]1753        if self.VERSION <= 0.11:
1754            self.db = self.env.get_db_cnx()
1755        else:
1756            self.db = self.env.get_read_db()
1757
[507]1758        self.get_sender_info(m)
[152]1759
[587]1760
[588]1761        if self.parameters.white_list_file:
1762            self.acl_list_from_file(self.parameters.white_list_file, 'white_list')
1763
[587]1764        if not ( self.email_header_acl('white_list', self.email_addr, True) or self.allow_registered_user ) :
1765
[507]1766            self.logger.info('Message rejected : %s not in white list' %(self.email_addr))
1767            return False
[77]1768
[507]1769        if self.email_header_acl('black_list', self.email_addr, False):
1770            self.logger.info('Message rejected : %s in black list' %(self.email_addr))
1771            return False
[221]1772
[570]1773        if not self.email_header_acl('recipient_list', self.email_to_addrs, True):
1774            self.logger.info('Message rejected : %s not in recipient list' %(self.email_to_addrs))
[507]1775            return False
[226]1776
[507]1777        ## If spam drop the message
1778        #
1779        if self.spam(m) == 'drop':
1780            return False
[194]1781
[507]1782        elif self.spam(m) == 'spam':
1783            spam_msg = True
1784        else:
1785            spam_msg = False
[204]1786
[507]1787        if not m['Subject']:
1788            subject  = 'No Subject'
1789        else:
[564]1790            subject  = self.email_to_unicode(m['Subject'])
[304]1791
[564]1792        ## Bug in python logging module <2.6
1793        #
1794        self.logger.info('subject: %s' %repr(subject))
[359]1795
[631]1796        ## First try the new method and then fall back to old method
[621]1797        #
[631]1798        if not self.parse_delivered_to_field(m, subject, spam_msg):
1799            self.parse_subject_field(m, subject, spam_msg)
[621]1800
[631]1801    def parse_delivered_to_field(self, m, subject, spam_msg):
1802        """
1803        """
1804        self.logger.debug('function parse_delivered_to_field')
[621]1805
[631]1806        ## Ticket id is in Delivered-To Field. Trac notify must send this
1807        #  eg: is+390@surfsara.nl
1808        try:
1809
1810            #print self.parameters.project_name
1811            self.logger.debug('Delivered To: %s' %m['Delivered-To'])
1812
1813            id = m['Delivered-To']
1814            id = id.split(self.parameters.recipient_delimiter)[1]
1815            id = id.split('@')[0]
1816
1817            self.logger.debug('\t Found ticket id: %s' %id)
1818
1819            if not self.ticket_update(m, id, spam_msg):
1820                return False
1821
1822        except KeyError, detail:
1823            pass
1824        except IndexError, detail:
1825            pass
1826
1827        return False
1828
1829    def parse_subject_field(self, m, subject, spam_msg):
1830        """
1831        """
1832        self.logger.debug('function parse_subject_header')
1833
[507]1834        ## [hic] #1529: Re: LRZ
1835        #  [hic] #1529?owner=bas,priority=medium: Re: LRZ
1836        #
1837        ticket_regex = r'''
1838            (?P<new_fields>[#][?].*)
1839            |(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?.*)?:)
1840            '''
[631]1841
[507]1842        ## Check if  FullBlogPlugin is installed
1843        #
1844        blog_enabled = None
1845        blog_regex = ''
1846        if self.get_config('components', 'tracfullblog.*') in ['enabled']:
1847            self.logger.debug('Trac BLOG support enabled')
1848            blog_enabled = True
[585]1849            blog_regex = '''|(?P<blog>blog(?P<blog_params>[?][^:]*)?:(?P<blog_id>\S*))'''
[329]1850
[77]1851
[507]1852        ## Check if DiscussionPlugin is installed
1853        #
1854        discussion_enabled = None
1855        discussion_regex = ''
1856        if self.get_config('components', 'tracdiscussion.api.discussionapi') in ['enabled']:
1857            self.logger.debug('Trac Discussion support enabled')
1858            discussion_enabled = True
1859            discussion_regex = r'''
1860            |(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
1861            |(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
1862            |(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
1863            '''
[77]1864
[359]1865
[507]1866        regex_str = ticket_regex + blog_regex + discussion_regex
1867        SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
[359]1868
[507]1869        ## Find out if this is a ticket, a blog or a discussion
1870        #
1871        result =  SYSTEM_RE.search(subject)
[390]1872
[507]1873        if result:
1874            ## update ticket + fields
1875            #
1876            if result.group('reply'):
1877                self.system = 'ticket'
[260]1878
[507]1879                ## Skip the last ':' character
1880                #
1881                if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
1882                    self.new_ticket(m, subject, spam_msg)
[390]1883
[507]1884            ## New ticket + fields
1885            #
1886            elif result.group('new_fields'):
1887                self.system = 'ticket'
1888                self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
[262]1889
[507]1890            if blog_enabled:
1891                if result.group('blog'):
1892                    self.system = 'blog'
[585]1893                    self.blog(m, subject[result.end('blog'):], result.group('blog_id'), result.group('blog_params'))
[359]1894
[507]1895            if discussion_enabled:
1896                ## New topic.
1897                #
1898                if result.group('forum'):
1899                    self.system = 'discussion'
1900                    self.id = int(result.group('forum_id'))
1901                    self.discussion_topic(m, subject[result.end('forum'):])
[359]1902
[507]1903                ## Reply to topic.
1904                #
1905                elif result.group('topic'):
1906                    self.system = 'discussion'
1907                    self.id = int(result.group('topic_id'))
1908                    self.discussion_topic_reply(m, subject[result.end('topic'):])
[359]1909
[507]1910                ## Reply to topic message.
1911                #
1912                elif result.group('message'):
1913                    self.system = 'discussion'
1914                    self.id = int(result.group('message_id'))
1915                    self.discussion_message_reply(m, subject[result.end('message'):])
[359]1916
[507]1917        else:
[553]1918
[507]1919            self.system = 'ticket'
1920            (matched_id, subject) = self.ticket_update_by_subject(subject)
[553]1921
[507]1922            if matched_id:
[553]1923
[507]1924                if not self.ticket_update(m, matched_id, spam_msg):
1925                    self.new_ticket(m, subject, spam_msg)
[553]1926
[507]1927            else:
1928                ## No update by subject, so just create a new ticket
1929                #
1930                self.new_ticket(m, subject, spam_msg)
[353]1931
[356]1932
[343]1933########## BODY TEXT functions  ###########################################################
1934
[507]1935    def strip_signature(self, text):
1936        """
1937        Strip signature from message, inspired by Mailman software
1938        """
[607]1939        self.logger.debug('function strip_signature: %s' %self.parameters.strip_signature_regex)
[462]1940
[507]1941        body = []
[606]1942
1943        STRIP_RE = re.compile( self.parameters.strip_signature_regex )
[507]1944        for line in text.splitlines():
[606]1945
1946            match = STRIP_RE.match(line)
1947            if match:
1948                self.logger.debug('\t"%s "  matched, skiping rest of message' %line)
[507]1949                break
[606]1950
[507]1951            body.append(line)
[136]1952
[507]1953        return ('\n'.join(body))
[136]1954
[507]1955    def reflow(self, text, delsp = 0):
1956        """
1957        Reflow the message based on the format="flowed" specification (RFC 3676)
1958        """
1959        flowedlines = []
1960        quotelevel = 0
1961        prevflowed = 0
[231]1962
[507]1963        for line in text.splitlines():
1964            from re import match
1965           
1966            ## Figure out the quote level and the content of the current line
1967            #
1968            m = match('(>*)( ?)(.*)', line)
1969            linequotelevel = len(m.group(1))
1970            line = m.group(3)
[231]1971
[507]1972            ## Determine whether this line is flowed
1973            #
1974            if line and line != '-- ' and line[-1] == ' ':
1975                flowed = 1
1976            else:
1977                flowed = 0
[231]1978
[507]1979            if flowed and delsp and line and line[-1] == ' ':
1980                line = line[:-1]
[231]1981
[507]1982            ## If the previous line is flowed, append this line to it
1983            #
1984            if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1985                flowedlines[-1] += line
[477]1986
[507]1987            ## Otherwise, start a new line
1988            #
1989            else:
1990                flowedlines.append('>' * linequotelevel + line)
[231]1991
[507]1992            prevflowed = flowed
1993           
[231]1994
[507]1995        return '\n'.join(flowedlines)
[231]1996
[507]1997    def strip_quotes(self, text):
1998        """
1999        Strip quotes from message by Nicolas Mendoza
2000        """
[607]2001        self.logger.debug('function strip_quotes: %s' %self.parameters.email_quote)
[462]2002
[507]2003        body = []
[607]2004
2005        STRIP_RE = re.compile( self.parameters.email_quote )
2006
[507]2007        for line in text.splitlines():
[607]2008
[507]2009            try:
[462]2010
[607]2011                match = STRIP_RE.match(line)
2012                if match:
2013                    print line
[507]2014                    continue
[462]2015
[507]2016            except UnicodeDecodeError:
[462]2017
[507]2018                tmp_line = self.email_to_unicode(line)
[607]2019
2020                match = STRIP_RE.match(tmp_line)
2021                if match:
2022                    print line
[507]2023                    continue
2024               
2025            body.append(line)
[151]2026
[507]2027        return ('\n'.join(body))
[191]2028
[507]2029    def inline_properties(self, text):
2030        """
2031        Parse text if we use inline keywords to set ticket fields
2032        """
2033        self.logger.debug('function inline_properties')
[309]2034
[507]2035        properties = dict()
2036        body = list()
[309]2037
[507]2038        INLINE_EXP = re.compile('\s*[@]\s*(\w+)\s*:(.*)$')
[309]2039
[507]2040        for line in text.splitlines():
2041            match = INLINE_EXP.match(line)
2042            if match:
2043                keyword, value = match.groups()
[509]2044
2045                if self.parameters.inline_properties_first_wins:
2046                    if keyword in self.properties.keys():
2047                        continue
2048
[507]2049                self.properties[keyword] = value.strip()
[509]2050                self.logger.debug('\tinline properties: %s : %s' %(keyword,value))
[408]2051
[507]2052            else:
2053                body.append(line)
2054               
2055        return '\n'.join(body)
[309]2056
2057
[507]2058    def wrap_text(self, text, replace_whitespace = False):
2059        """
2060        Will break a lines longer then given length into several small
2061        lines of size given length
2062        """
2063        import textwrap
[154]2064
[507]2065        LINESEPARATOR = '\n'
2066        reformat = ''
[151]2067
[507]2068        for s in text.split(LINESEPARATOR):
2069            tmp = textwrap.fill(s, self.parameters.use_textwrap)
2070            if tmp:
2071                reformat = '%s\n%s' %(reformat,tmp)
2072            else:
2073                reformat = '%s\n' %reformat
[153]2074
[507]2075        return reformat
[153]2076
[507]2077        # Python2.4 and higher
2078        #
2079        #return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
2080        #
[154]2081
[343]2082########## EMAIL attachements functions ###########################################################
2083
[507]2084    def inline_part(self, part):
2085        """
2086        """
2087        self.logger.debug('function inline_part()')
[154]2088
[507]2089        return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
[340]2090
[604]2091    def get_message_parts(self, msg, new_email=False):
[507]2092        """
2093        parses the email message and returns a list of body parts and attachments
2094        body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
2095        """
2096        self.logger.debug('function get_message_parts()')
[317]2097
[507]2098        message_parts = list()
2099   
2100        ALTERNATIVE_MULTIPART = False
[278]2101
[507]2102        for part in msg.walk():
2103            content_maintype = part.get_content_maintype()
2104            content_type =  part.get_content_type()
[278]2105
[507]2106            self.logger.debug('\t Message part: Main-Type: %s' % content_maintype)
2107            self.logger.debug('\t Message part: Content-Type: %s' % content_type)
[498]2108
[507]2109            ## Check content type
2110            #
2111            if content_type in self.STRIP_CONTENT_TYPES:
2112                self.logger.debug("\t A %s attachment named '%s' was skipped" %(content_type, part.get_filename()))
2113                continue
[238]2114
[507]2115            ## Catch some mulitpart execptions
2116            #
2117            if content_type == 'multipart/alternative':
2118                ALTERNATIVE_MULTIPART = True
2119                continue
[278]2120
[507]2121            ## Skip multipart containers
2122            #
2123            if content_maintype == 'multipart':
2124                self.logger.debug("\t Skipping multipart container")
2125                continue
2126           
[569]2127            ## Check if this is an inline part. It's inline if there is co Cont-Disp header,
2128            #  or if there is one and it says "inline"
[507]2129            #
2130            inline = self.inline_part(part)
[236]2131
[507]2132            ## Drop HTML message
2133            #
2134            if ALTERNATIVE_MULTIPART and self.parameters.drop_alternative_html_version:
[553]2135
[507]2136                if content_type == 'text/html':
2137                    self.logger.debug('\t Skipping alternative HTML message')
2138                    ALTERNATIVE_MULTIPART = False
2139                    continue
[278]2140
[569]2141            filename = part.get_filename()
2142            s = '\t unicode filename: %s' %(filename)
2143            self.print_unicode(s)
2144            self.logger.debug('\t raw filename: %s' %repr(filename))
[498]2145
[569]2146            filename = self.check_filename_length(filename)
2147
[507]2148            ## Save all non plain text message as attachment
2149            #
2150            if not content_type in ['text/plain']:
[498]2151
[569]2152                message_parts.append( (filename, part) )
[498]2153
[569]2154                ## We only convert html messages
[507]2155                #
2156                if not content_type == 'text/html':
[569]2157                    self.logger.debug('\t Appending %s (%s)' %(repr(filename), content_type))
[507]2158                    continue
[500]2159
[569]2160
2161            ## We have an text or html message
2162            #
[507]2163            if not inline:
[569]2164                    self.logger.debug('\t Appending %s (%s), not an inline messsage part' %(repr(filename), content_type))
2165                    message_parts.append( (filename, part) )
[507]2166                    continue
2167               
2168            ## Try to decode message part. We have a html or plain text messafe
2169            #
2170            body_text = part.get_payload(decode=1)
2171            if not body_text:           
2172                body_text = part.get_payload(decode=0)
[236]2173
[507]2174            ## Try to convert html message
2175            #
2176            if content_type == 'text/html':
[502]2177
[507]2178                body_text = self.html_2_txt(body_text)
2179                if not body_text:
2180                    continue
[498]2181
[507]2182            format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
2183            delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
[231]2184
[507]2185            if self.parameters.reflow and not self.parameters.verbatim_format and format == 'flowed':
2186                body_text = self.reflow(body_text, delsp == 'yes')
[22]2187
[604]2188            if new_email and self.parameters.only_strip_on_update:
2189                self.logger.debug('Skip signature/quote stripping for new messages')
2190            else:
2191                if self.parameters.strip_signature:
2192                    body_text = self.strip_signature(body_text)
[191]2193
[604]2194                if self.parameters.strip_quotes:
2195                    body_text = self.strip_quotes(body_text)
2196
[507]2197            if self.parameters.inline_properties:
2198                body_text = self.inline_properties(body_text)
[309]2199
[507]2200            if self.parameters.use_textwrap:
2201                body_text = self.wrap_text(body_text)
[148]2202
[507]2203            ## Get contents charset (iso-8859-15 if not defined in mail headers)
2204            #
2205            charset = part.get_content_charset()
2206            if not charset:
2207                charset = 'iso-8859-15'
[102]2208
[507]2209            try:
2210                ubody_text = unicode(body_text, charset)
[100]2211
[507]2212            except UnicodeError, detail:
2213                ubody_text = unicode(body_text, 'iso-8859-15')
[89]2214
[507]2215            except LookupError, detail:
2216                ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
[100]2217
[507]2218            if self.parameters.verbatim_format:
2219                message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
2220            else:
2221                message_parts.append('%s' %ubody_text)
[22]2222
[507]2223        return message_parts
2224       
2225    def unique_attachment_names(self, message_parts):
2226        """
2227        Make sure we have unique names attachments:
2228          - check if it contains illegal characters
2229          - Rename "None" filenames to "untitled-part"
2230        """
2231        self.logger.debug('function unique_attachment_names()')
2232        renamed_parts = []
2233        attachment_names = set()
[296]2234
[507]2235        for item in message_parts:
2236           
2237            ## If not an attachment, leave it alone
2238            #
2239            if not isinstance(item, tuple):
2240                renamed_parts.append(item)
2241                continue
2242               
2243            (filename, part) = item
[295]2244
[507]2245            ## If filename = None, use a default one
2246            #
2247            if filename in [ 'None']:
2248                filename = 'untitled-part'
[550]2249                self.logger.info('\t Rename filename "None" to: %s' %filename)
[22]2250
[507]2251                ## Guess the extension from the content type, use non strict mode
2252                #  some additional non-standard but commonly used MIME types
2253                #  are also recognized
2254                #
2255                ext = mimetypes.guess_extension(part.get_content_type(), False)
2256                if not ext:
2257                    ext = '.bin'
[22]2258
[507]2259                filename = '%s%s' % (filename, ext)
[22]2260
[507]2261            ## Discard relative paths for windows/unix in attachment names
2262            #
2263            filename = filename.replace('\\', '_')
2264            filename = filename.replace('/', '_')
[347]2265
[507]2266            ## remove linefeed char
2267            #
2268            for forbidden_char in ['\r', '\n']:
2269                filename = filename.replace(forbidden_char,'')
[465]2270
[507]2271            ## We try to normalize the filename to utf-8 NFC if we can.
2272            #  Files uploaded from OS X might be in NFD.
2273            #  Check python version and then try it
2274            #
2275            #if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
2276            #   try:
2277            #       filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8') 
2278            #   except TypeError:
2279            #       pass
[100]2280
[507]2281            ## Make the filename unique for this ticket
2282            #
2283            num = 0
2284            unique_filename = filename
2285            dummy_filename, ext = os.path.splitext(filename)
[134]2286
[507]2287            while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
2288                num += 1
2289                unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
2290               
[550]2291            s = '\t Attachment with filename %s will be saved as %s' % (filename, unique_filename)
[549]2292            self.print_unicode(s)
[100]2293
[507]2294            attachment_names.add(unique_filename)
[236]2295
[507]2296            renamed_parts.append((filename, unique_filename, part))
2297   
2298        return renamed_parts
2299           
2300           
2301    def attachment_exists(self, filename):
[250]2302
[549]2303        self.logger.debug("function attachment_exists")
[250]2304
[549]2305        s = '\t check if attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
2306        self.print_unicode(s)
2307
[553]2308        ## Do we have a valid ticket id
[507]2309        #
2310        if not self.id:
2311            return False
[250]2312
[507]2313        try:
2314            if self.system == 'discussion':
[553]2315
2316                att = attachment.Attachment(self.env, 'discussion', 'ticket/%s' % (self.id,), filename)
2317
[585]2318            elif self.system == 'blog':
2319
2320                att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,), filename)
2321
[507]2322            else:
[553]2323
2324                att = attachment.Attachment(self.env, 'ticket', self.id, filename)
2325
[507]2326            return True
[553]2327
[507]2328        except attachment.ResourceNotFound:
[553]2329
[507]2330            return False
[343]2331
2332########## TRAC Ticket Text ###########################################################
[507]2333           
2334    def get_body_text(self, message_parts):
2335        """
2336        """
2337        self.logger.debug('function get_body_text()')
[498]2338
[507]2339        body_text = []
2340       
2341        for part in message_parts:
2342       
2343            ## Plain text part, append it
2344            #
2345            if not isinstance(part, tuple):
2346                body_text.extend(part.strip().splitlines())
2347                body_text.append("")
2348                continue
[490]2349
[507]2350            (original, filename, part) = part
2351            inline = self.inline_part(part)
[491]2352
[507]2353            ## Skip generation of attachment link if html is converted to text
2354            #
2355            if part.get_content_type() == 'text/html' and self.parameters.html2text_cmd and inline:
2356                s = 'Skipping attachment link for html part: %s' %(filename)
2357                self.print_unicode(s)
2358                continue
2359           
2360            if part.get_content_maintype() == 'image' and inline:
[553]2361
[507]2362                if self.system != 'discussion':
2363                    s = 'wiki image link for: %s' %(filename)
2364                    self.print_unicode(s)
2365                    body_text.append('[[Image(%s)]]' % filename)
[553]2366
[507]2367                body_text.append("")
[553]2368
[507]2369            else:
[553]2370
[507]2371                if self.system != 'discussion':
[553]2372
[507]2373                    s = 'wiki attachment link for: %s' %(filename)
2374                    self.print_unicode(s)
2375                    body_text.append('[attachment:"%s"]' % filename)
[553]2376
[507]2377                body_text.append("")
[490]2378
[507]2379        ## Convert list body_texts to string
2380        #
2381        body_text = '\r\n'.join(body_text)
2382        return body_text
[236]2383
[507]2384    def html_mailto_link(self, subject):
2385        """
2386        This function returns a HTML mailto tag with the ticket id and author email address
2387        """
2388        self.logger.debug("function html_mailto_link")
[576]2389
[507]2390        if not self.author:
2391            author = self.email_addr
2392        else:   
2393            author = self.author
[343]2394
[507]2395        if not self.parameters.mailto_cc:
2396            self.parameters.mailto_cc = ''
[592]2397           
2398        ## Bug in urllib.quote function
2399        #
2400        if isinstance(subject, unicode):
2401            subject = subject.encode('utf-8')
[434]2402
[507]2403        ## use urllib to escape the chars
2404        #
[517]2405        s = '%s?Subject=%s&cc=%s' %(
[507]2406               urllib.quote(self.email_addr),
2407               urllib.quote('Re: #%s: %s' %(self.id, subject)),
2408               urllib.quote(self.parameters.mailto_cc)
2409               )
[343]2410
[507]2411        if self.VERSION in [ 0.10 ]:
2412            s = '\r\n{{{\r\n#!html\r\n<a\r\n href="mailto:%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
2413        else:
2414            s = '[mailto:"%s" Reply to: %s]' %(s, author)
[506]2415
[507]2416        self.logger.debug("\tmailto link %s" %s)
2417        return s
[343]2418
2419########## TRAC notify section ###########################################################
2420
[507]2421    def notify(self, tkt, new=True, modtime=0):
2422        """
2423        A wrapper for the TRAC notify function. So we can use templates
2424        """
2425        self.logger.debug('function notify()')
[392]2426
[508]2427
[633]2428        class Email2TracNotifyEmail(TicketNotifyEmail):
2429
2430            def __init__(self, env):
2431                TicketNotifyEmail.__init__(self, env)
2432                self.email2trac_notify_reporter = None
2433                self.email2trac_replyto = None
2434
2435            def send(self, torcpts, ccrcpts):
2436                #print 'Yes this works'
2437                dest = self.reporter or 'anonymous'
2438                hdrs = {}
2439                hdrs['Message-ID'] = self.get_message_id(dest, self.modtime)
2440                hdrs['X-Trac-Ticket-ID'] = str(self.ticket.id)
2441                hdrs['X-Trac-Ticket-URL'] = self.data['ticket']['link']
2442                if not self.newticket:
2443                    msgid = self.get_message_id(dest)
2444                    hdrs['In-Reply-To'] = msgid
2445                    hdrs['References'] = msgid
2446
2447
2448                if self.email2trac_notify_reporter:
2449                    if not self.email2trac_notify_reporter in torcpts:
2450                        torcpts.append(sender_email)
2451
2452                if self.email2trac_replyto:
2453                    # use to rewrite reply to
2454                    # hdrs does not work, multiple reply addresses
2455                    #hdrs['Reply-To'] = 'bas.van.der.vlies@gmail.com'
2456                    self.replyto_email = self.email2trac_replyto
2457       
2458                NotifyEmail.send(self, torcpts, ccrcpts, hdrs)
2459
[508]2460        if self.parameters.dry_run  :
[561]2461                self.logger.info('DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter'])
[507]2462                return
2463        try:
[392]2464
[507]2465            ## create false {abs_}href properties, to trick Notify()
2466            #
[540]2467            if self.VERSION in [0.10]:
[507]2468                self.env.abs_href = Href(self.get_config('project', 'url'))
2469                self.env.href = Href(self.get_config('project', 'url'))
[22]2470
[633]2471            tn = Email2TracNotifyEmail(self.env)
[213]2472
[633]2473            ## additionally append sender (regardeless of settings in trac.ini)
2474            #
2475            if self.parameters.notify_reporter:
2476
2477                self.logger.debug('\t Notify reporter set')
2478
2479                if not self.email_header_acl('notify_reporter_black_list', self.email_addr, False):
2480                    tn.email2trac_notify_reporter = self.email_addr
2481
2482            if self.parameters.notify_replyto_rewrite:
2483
2484                self.logger.debug('\t Notify replyto rewrite set')
2485
2486                action, value = self.parameters.notify_replyto_rewrite.split(':')
2487
2488                if action in ['use_mail_domain']:
[634]2489                    self.logger.debug('\t\t use_mail_domain:%s' %value)
[633]2490                    tn.email2trac_replyto = '%s@%s' %(self.id, value)
2491
2492                elif action in ['use_trac_smtp_replyto']:
[634]2493                    self.logger.debug('\t\t use_trac_smtp_replyto delimiter:%s' %value)
[633]2494                    dummy = self.smtp_replyto.split('@')
2495                    if len(dummy) > 1:
2496                        tn.email2trac_replyto = '%s%s%s@%s' %(dummy[0], value, self.id, dummy[1])
2497                    else:
2498                        tn.email2trac_replyto = '%s%s%s' %(dummy[0], value, self.id)
2499
[507]2500            if self.parameters.alternate_notify_template:
[222]2501
[507]2502                if self.VERSION >= 0.11:
[222]2503
[507]2504                    from trac.web.chrome import Chrome
[222]2505
[507]2506                    if  self.parameters.alternate_notify_template_update and not new:
2507                        tn.template_name = self.parameters.alternate_notify_template_update
2508                    else:
2509                        tn.template_name = self.parameters.alternate_notify_template
[222]2510
[507]2511                    tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
2512                       
2513                else:
[222]2514
[507]2515                    tn.template_name = self.parameters.alternate_notify_template
[42]2516
[507]2517            tn.notify(tkt, new, modtime)
[41]2518
[507]2519        except Exception, e:
2520            self.logger.error('Failure sending notification on creation of ticket #%s: %s' %(self.id, e))
[41]2521
[508]2522########## END Class Definition  ########################################################
[22]2523
[74]2524
[343]2525########## Parse Config File  ###########################################################
[22]2526
2527def ReadConfig(file, name):
[507]2528    """
2529    Parse the config file
2530    """
2531    if not os.path.isfile(file):
2532        print 'File %s does not exist' %file
2533        sys.exit(1)
[22]2534
[507]2535    config = trac_config.Configuration(file)
2536   
2537    parentdir = config.get('DEFAULT', 'parentdir')
2538    sections = config.sections()
[22]2539
[507]2540    ## use some trac internals to get the defaults
2541    #
2542    tmp = config.parser.defaults()
2543    project =  SaraDict()
[485]2544
[507]2545    for option, value in tmp.items():
2546        try:
2547            project[option] = int(value)
2548        except ValueError:
2549            project[option] = value
[485]2550
[507]2551    if name:
2552        if name in sections:
2553            project =  SaraDict()
2554            for option, value in  config.options(name):
2555                try:
2556                    project[option] = int(value)
2557                except ValueError:
2558                    project[option] = value
[485]2559
[507]2560        elif not parentdir:
2561            print "Not a valid project name: %s, valid names are: %s" %(name, sections)
2562            print "or set parentdir in the [DEFAULT] section"
2563            sys.exit(1)
[22]2564
[507]2565    ## If parentdir then set project dir to parentdir + name
2566    #
2567    if not project.has_key('project'):
2568        if not parentdir:
2569            print "You must set project or parentdir in your configuration file"
2570            sys.exit(1)
2571        elif not name:
2572            print "You must configure a  project section in your configuration file"
2573        else:
2574            project['project'] = os.path.join(parentdir, name)
[22]2575
[528]2576    ##
2577    # Save the project name
2578    #
2579    project['project_name'] = os.path.basename(project['project'])
2580
[507]2581    return project
[22]2582
[410]2583########## Setup Logging ###############################################################
[87]2584
[419]2585def setup_log(parameters, project_name, interactive=None):
[507]2586    """
[516]2587    Setup logging
[410]2588
[507]2589    Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
[420]2590    would be interpreted by the ConfigParser itself.
[507]2591    """
2592    logger = logging.getLogger('email2trac %s' %project_name)
[410]2593
[507]2594    if interactive:
2595        parameters.log_type = 'stderr'
[414]2596
[507]2597    if not parameters.log_type:
2598        if sys.platform in ['win32', 'cygwin']:
[521]2599            parameters.log_type = 'eventlog'
[507]2600        else:
2601            parameters.log_type = 'syslog'
[410]2602
[507]2603    if parameters.log_type == 'file':
[415]2604
[507]2605        if not parameters.log_file:
2606            parameters.log_file = 'email2trac.log'
[415]2607
[507]2608        if not os.path.isabs(parameters.log_file):
2609            parameters.log_file = os.path.join(tempfile.gettempdir(), parameters.log_file)
[410]2610
[507]2611        log_handler = logging.FileHandler(parameters.log_file)
[415]2612
[507]2613    elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
2614        ## Requires win32 extensions
2615        #
[532]2616        logid = "email2trac"
[507]2617        log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
[410]2618
[507]2619    elif parameters.log_type in ('syslog', 'unix'):
2620        log_handler = logging.handlers.SysLogHandler('/dev/log')
[410]2621
[507]2622    elif parameters.log_type in ('stderr'):
2623        log_handler = logging.StreamHandler(sys.stderr)
[410]2624
[507]2625    else:
2626        log_handler = logging.handlers.BufferingHandler(0)
[410]2627
[507]2628    if parameters.log_format:
2629        parameters.log_format = parameters.log_format.replace('$(', '%(')
2630    else:
2631        parameters.log_format = '%(name)s: %(message)s'
[516]2632        if parameters.log_type in ('file', 'stderr'):
2633            parameters.log_format = '%(asctime)s ' + parameters.log_format
[410]2634
[507]2635    log_formatter = logging.Formatter(parameters.log_format)
2636    log_handler.setFormatter(log_formatter)
2637    logger.addHandler(log_handler)
[410]2638
[507]2639    if (parameters.log_level in ['DEBUG', 'ALL']) or (parameters.debug > 0):
2640        logger.setLevel(logging.DEBUG)
2641        parameters.debug = 1
[410]2642
[507]2643    elif parameters.log_level in ['INFO'] or parameters.verbose:
2644        logger.setLevel(logging.INFO)
[410]2645
[507]2646    elif parameters.log_level in ['WARNING']:
2647        logger.setLevel(logging.WARNING)
[410]2648
[507]2649    elif parameters.log_level in ['ERROR']:
2650        logger.setLevel(logging.ERROR)
[410]2651
[507]2652    elif parameters.log_level in ['CRITICAL']:
2653        logger.setLevel(logging.CRITICAL)
[410]2654
[507]2655    else:
2656        logger.setLevel(logging.INFO)
[410]2657
[507]2658    return logger
[410]2659
[633]2660########## Own TicketNotifyEmail class ###############################################################
[410]2661
[22]2662if __name__ == '__main__':
[507]2663    ## Default config file
2664    #
[622]2665    agilo = False
[507]2666    configfile = '@email2trac_conf@'
2667    project = ''
2668    component = ''
2669    ticket_prefix = 'default'
2670    dry_run = None
2671    verbose = None
2672    debug_interactive = None
[621]2673    virtualenv = '@virtualenv@'
[202]2674
[630]2675    SHORT_OPT = 'AcdE:hf:np:t:v'
[622]2676    LONG_OPT  =  ['agilo', 'component=', 'debug', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=', 'virtualenv=', 'verbose']
[201]2677
[507]2678    try:
2679        opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
2680    except getopt.error,detail:
2681        print __doc__
2682        print detail
2683        sys.exit(1)
2684   
2685    project_name = None
2686    for opt,value in opts:
2687        if opt in [ '-h', '--help']:
2688            print __doc__
2689            sys.exit(0)
[622]2690        elif opt in ['-A', '--agilo']:
2691            agilo = True
[507]2692        elif opt in ['-c', '--component']:
2693            component = value
2694        elif opt in ['-d', '--debug']:
2695            debug_interactive = 1
[625]2696        elif opt in ['-E', '--virtualenv']:
[621]2697            virtualenv = value
[507]2698        elif opt in ['-f', '--file']:
2699            configfile = value
2700        elif opt in ['-n', '--dry-run']:
2701            dry_run = True
2702        elif opt in ['-p', '--project']:
2703            project_name = value
2704        elif opt in ['-t', '--ticket_prefix']:
2705            ticket_prefix = value
2706        elif opt in ['-v', '--verbose']:
2707            verbose = True
[621]2708
2709    if virtualenv and os.path.exists(virtualenv):
2710        activate_this = os.path.join(virtualenv, 'bin/activate_this.py')
2711        if os.path.exists(activate_this):
2712            execfile(activate_this, dict(__file__=activate_this))
2713
2714    try:
2715        from trac import __version__ as trac_version
2716        from trac import config as trac_config
2717
2718    except ImportError, detail:
2719        print "Can not find a a valid trac installation, solutions could be:"
2720        print "\tset PYTHONPATH"
2721        print "\tuse the --virtualenv <dir> option"
2722        sys.exit(1)
[507]2723   
2724    settings = ReadConfig(configfile, project_name)
[410]2725
[507]2726    ## The default prefix for ticket values in email2trac.conf
2727    #
2728    settings.ticket_prefix = ticket_prefix
2729    settings.dry_run = dry_run
2730    settings.verbose = verbose
[410]2731
[507]2732    if not settings.debug and debug_interactive:
2733        settings.debug = debug_interactive
[412]2734
[507]2735    if not settings.project:
2736        print __doc__
2737        print 'No Trac project is defined in the email2trac config file.'
2738        sys.exit(1)
[419]2739
[507]2740    logger = setup_log(settings, os.path.basename(settings.project), debug_interactive)
2741   
2742    if component:
2743        settings['component'] = component
[202]2744
[615]2745    ## We are only interested in the major versions
2746    # 0.12.3 --> 0.12
2747    # 1.0.2  --> 1.0
[507]2748    #
[615]2749    l = trac_version.split('.')
2750    version = '.'.join(l[0:2])
[373]2751
[507]2752    logger.debug("Found trac version: %s" %(version))
2753   
2754    try:
2755        if version == '0.10':
2756            from trac import attachment
2757            from trac.env import Environment
2758            from trac.ticket import Ticket
2759            from trac.web.href import Href
2760            from trac import util
2761            from trac.ticket.web_ui import TicketModule
[486]2762
[507]2763            #
2764            # return  util.text.to_unicode(str)
2765            #
2766            # see http://projects.edgewall.com/trac/changeset/2799
2767            from trac.ticket.notification import TicketNotifyEmail
2768            from trac import config as trac_config
2769            from trac.core import TracError
[359]2770
[633]2771        elif version in ['0.11', '0.12', '0.13', '1.0', '1.1']:
[507]2772            from trac import attachment
2773            from trac import config as trac_config
2774            from trac import util
2775            from trac.core import TracError
[554]2776            from trac.env import Environment
[507]2777            from trac.perm import PermissionSystem
[539]2778            from trac.perm import PermissionCache
[538]2779            from trac.test import Mock, MockPerm
[554]2780            from trac.ticket.api import TicketSystem
[507]2781            from trac.ticket.web_ui import TicketModule
[541]2782            from trac.web.href import Href
[368]2783
[622]2784            if agilo:
2785
2786                try:
2787                    #from agilo.utils.config import AgiloConfig
2788                    #if AgiloConfig(self.env).is_agilo_enabled:
2789                    from agilo.ticket.model import Ticket
2790                except ImportError, detail:
2791                    logger.error('Could not find Trac  Agilo environemnt')
2792                    sys.exit(0)
2793
2794            else:
2795
2796                from trac.ticket import Ticket
2797
[541]2798            #
2799            # return  util.text.to_unicode(str)
2800            #
2801            # see http://projects.edgewall.com/trac/changeset/2799
2802            from trac.ticket.notification import TicketNotifyEmail
[633]2803            from trac.notification import NotifyEmail
[541]2804
[507]2805        else:
2806            logger.error('TRAC version %s is not supported' %version)
[519]2807            sys.exit(0)
[182]2808
[507]2809        ## Must be set before environment is created
2810        #
2811        if settings.has_key('python_egg_cache'):
2812            python_egg_cache = str(settings['python_egg_cache'])
2813            os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
[291]2814
[507]2815        if settings.debug > 0:
2816            logger.debug('Loading environment %s', settings.project)
[359]2817
[507]2818        try:
2819            env = Environment(settings['project'], create=0)
2820        except IOError, detail:
[534]2821            logger.error("trac error: %s" %detail)
[519]2822            sys.exit(0)
[621]2823        except TracError, detail:
2824            logger.error("trac error: %s" %detail)
2825            sys.exit(0)
[333]2826
[507]2827        tktparser = TicketEmailParser(env, settings, logger, float(version))
2828        tktparser.parse(sys.stdin)
[22]2829
[507]2830    ## Catch all errors and use the logging module
2831    #
2832    except Exception, error:
[187]2833
[507]2834        etype, evalue, etb = sys.exc_info()
2835        for e in traceback.format_exception(etype, evalue, etb):
2836            logger.critical(e)
[187]2837
[507]2838        if m:
[528]2839            tktparser.save_email_for_debug(m, settings.project_name, True)
[97]2840
[507]2841        sys.exit(1)
[633]2842
[22]2843# EOB
Note: See TracBrowser for help on using the repository browser.