source: trunk/email2trac.py.in

Last change on this file was 672, checked in by bas, 8 years ago

made email2trac compatible with trac version 1.1 and adjusted msg.txt to new surfsara.nl address

  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 92.9 KB
Line 
1#!@PYTHON@
2#
3# This file is part of the email2trac utils
4#
5#       Copyright 2002 SURFsara
6#
7#       Licensed under the Apache License, Version 2.0 (the "License");
8#       you may not use this file except in compliance with the License.
9#       You may obtain a copy of the License at
10#
11#       http://www.apache.org/licenses/LICENSE-2.0
12#
13#       Unless required by applicable law or agreed to in writing, software
14#       distributed under the License is distributed on an "AS IS" BASIS,
15#       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16#       See the License for the specific language governing permissions and
17#       limitations under the License.
18#
19# For vi/emacs or other use tabstop=4 (vi: set ts=4)
20#
21"""
22email2trac.py -- Email -> TRAC tickets
23
24A MTA filter to create Trac tickets from inbound emails.
25
26first proof of concept from:
27 Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28 Copyright 2005, Edgewall Software
29
30Authors:
31  Bas van der Vlies <bas.vandervlies@surfsara.nl>
32  Walter de Jong <walter.dejong@surfsara.nl>
33
34How to use
35----------
36 * See https://oss.trac.surfsara.nl/email2trac/
37
38 * Commandline opions:
39                -h,--help
40                -A, --agilo
41                -B, --bh_product <name>
42                -d, --debug
43                -E, --virtualenv <path>
44                -f,--file  <configuration file>
45                -n,--dry-run
46                -p, --project <project name>
47                -t, --ticket_prefix <name>
48                -v, --verbose
49
50SVN Info:
51        $Id: email2trac.py.in 672 2015-10-19 10:42:25Z bas $
52"""
53import os
54import sys
55import string
56import getopt
57import time
58import email
59import email.Iterators
60import email.Header
61import re
62import urllib
63import unicodedata
64import mimetypes
65import traceback
66import logging
67import logging.handlers
68import UserDict
69import tempfile
70
71from datetime import timedelta, datetime
72from stat import *
73
74## Some global variables
75#
76m = None
77
78class SaraDict(UserDict.UserDict):
79    def __init__(self, dictin = None):
80        UserDict.UserDict.__init__(self)
81        self.name = None
82       
83        if dictin:
84            if dictin.has_key('name'):
85                self.name = dictin['name']
86                del dictin['name']
87            self.data = dictin
88           
89    def get_value(self, name):
90        if self.has_key(name):
91            return self[name]
92        else:
93            return None
94               
95    def __repr__(self):
96        return repr(self.data)
97
98    def __str__(self):
99        return str(self.data)
100           
101    def __getattr__(self, name):
102        """
103        override the class attribute get method. Return the value
104        from the dictionary
105        """
106        if self.data.has_key(name):
107            return self.data[name]
108        else:
109            return None
110           
111    def __setattr__(self, name, value):
112        """
113        override the class attribute set method only when the UserDict
114        has set its class attribute
115        """
116        if self.__dict__.has_key('data'):
117            self.data[name] = value
118        else:
119            self.__dict__[name] = value
120
121    def __iter__(self):
122        return iter(self.data.keys())
123
124
125
126class TicketEmailParser(object):
127    env = None
128    comment = '> '
129
130    def __init__(self, env, parameters, logger, version):
131        self.env = env
132
133        # Database connection
134        #
135        self.db = None
136
137        # Save parameters
138        #
139        self.parameters = parameters
140        self.logger = logger
141
142        # Some useful mail constants
143        #
144        self.email_name = None
145        self.email_addr = None
146        self.email_from = None
147        self.author     = None
148        self.id         = None
149
150        ## Will be set to True if the user has ben registered
151        #
152        self.allow_registered_user = False
153       
154        self.STRIP_CONTENT_TYPES = list()
155
156        ## fields properties via body_text
157        #
158        self.properties = dict()
159
160        self.VERSION = version
161
162        self.get_config = self.env.config.get
163
164        ## init function ##
165        #
166        self.setup_parameters()
167
168    def setup_parameters(self):
169        if self.parameters.umask:
170            os.umask(self.parameters.umask)
171
172        if not self.parameters.spam_level:
173            self.parameters.spam_level = 0
174
175        if not self.parameters.spam_header:
176            self.parameters.spam_header = 'X-Spam-Score'
177
178        if not self.parameters.email_quote:
179            self.parameters.email_quote = '^> .*'
180            self.parameters.skip_line_regex = '^> .*'
181
182        if not self.parameters.ticket_update_by_subject_lookback:
183            self.parameters.ticket_update_by_subject_lookback = 30
184
185        if self.parameters.verbatim_format == None:
186            self.parameters.verbatim_format = 1
187
188        if self.parameters.reflow == None:
189            self.parameters.reflow = 1
190
191        if self.parameters.binhex:
192            self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
193
194        if self.parameters.applesingle:
195            self.STRIP_CONTENT_TYPES.append('application/applefile')
196
197        if self.parameters.appledouble:
198            self.STRIP_CONTENT_TYPES.append('application/applefile')
199
200        if self.parameters.strip_content_types:
201            items = self.parameters.strip_content_types.split(',')
202            for item in items:
203                self.STRIP_CONTENT_TYPES.append(item.strip())
204
205        if self.parameters.tmpdir:
206            self.parameters.tmpdir = os.path.normcase(str(self.parameters['tmpdir']))
207        else:
208            self.parameters.tmpdir = os.path.normcase('/tmp')
209
210        if self.parameters.email_triggers_workflow == None:
211            self.parameters.email_triggers_workflow = 1
212
213        if not self.parameters.subject_field_separator:
214            self.parameters.subject_field_separator = '&'
215        else:
216            self.parameters.subject_field_separator = self.parameters.subject_field_separator.strip()
217
218        if not self.parameters.strip_signature_regex:
219            self.parameters.strip_signature_regex = '^-- $'
220            self.parameters.cut_line_regex = '^-- $'
221
222        self.trac_smtp_from = self.get_config('notification', 'smtp_from')
223        self.smtp_default_domain = self.get_config('notification', 'smtp_default_domain')
224        self.smtp_replyto = self.get_config('notification', 'smtp_replyto')
225        self.trac_smtp_always_cc = self.get_config('notification', 'smtp_always_cc')
226        self.trac_smtp_always_bcc = self.get_config('notification', 'smtp_always_bcc')
227
228        self.system = None
229
230########## Email Header Functions ###########################################################
231
232    def spam(self, message):
233        """
234        # X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
235        # Note if Spam_level then '*' are included
236        """
237        self.logger.debug('function spam')
238
239        spam = False
240        if message.has_key(self.parameters.spam_header):
241            spam_l = string.split(message[self.parameters.spam_header])
242            #self.logger.info('Spam header: %s' %(message[self.parameters.spam_header]))
243
244            try:
245                number = spam_l[0].count('*')
246            except IndexError, detail:
247                number = 0
248               
249            if number >= self.parameters.spam_level:
250                spam = True
251               
252        ## treat virus mails as spam
253        #
254        elif message.has_key('X-Virus-found'):         
255            spam = True
256
257        ## Astaro firewall spam handling
258        #
259        elif message.get('X-Spam-Flag') == "YES" and message.get('X-Spam-Result') == "Spam":
260            spam = True
261
262        ## How to handle SPAM messages
263        #
264        if self.parameters.drop_spam and spam:
265
266            self.logger.info('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' %self.parameters.spam_level)
267            return 'drop'   
268
269        elif spam:
270
271            return 'Spam'   
272        else:
273
274            return False
275
276    def email_header_acl(self, keyword, message_field, default):
277        """
278        This function wil check if the email address is allowed or denied
279        to send mail to the ticket list
280        """
281        self.logger.debug('function email_header_acl: %s' %keyword)
282
283        try:
284            mail_addresses = self.parameters[keyword]
285
286            # Check if we have an empty string
287            #
288            if not mail_addresses:
289                return default
290
291        except KeyError, detail:
292            self.logger.debug('\t %s not defined, all messages are allowed.' %(keyword))
293
294            return default
295
296        mail_addresses = string.split(mail_addresses, ',')
297        message_addresses = string.split(message_field, ',')
298
299        for entry in mail_addresses:
300            entry = entry.strip()
301            TO_RE = re.compile(entry, re.VERBOSE|re.IGNORECASE)
302
303            for addr in message_addresses:
304                addr = addr.strip()
305
306                if self.parameters.compare_function_list in [ 'matches', 'match']:
307                    s = '\t%s matches %s' %(addr, entry)
308                    result =  TO_RE.match(addr)
309                else:
310                    s = '\t%s contains %s' %(addr, entry)
311                    result =  TO_RE.search(addr)
312
313                if result:
314                    self.logger.debug(s)
315                    return True
316
317        return False
318
319    def email_header_txt(self, m):
320        """
321        Display To and CC addresses in description field
322        """
323        s = ''
324
325        if m['To'] and len(m['To']) > 0:
326            s = "'''To:''' %s\r\n" %(m['To'])
327        if m['Cc'] and len(m['Cc']) > 0:
328            s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
329
330        return  self.email_to_unicode(s)
331
332
333    def get_sender_info(self, message):
334        """
335        Get the default author name and email address from the message
336        """
337        self.logger.debug('function get_sender_info')
338
339        to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
340        self.email_to_addrs = list()
341
342        for n,e in to_addrs:
343            self.email_to_addrs.append(e)
344
345        self.email_to_addrs = ','.join(self.email_to_addrs)
346
347
348        self.email_name, self.email_addr  = email.Utils.parseaddr(message['from'])
349
350        ## decode email name can contain charset
351        #
352        self.email_name = self.email_to_unicode(self.email_name)
353
354        dstr = '\t email name: %s, email address: %s' %(self.email_name, self.email_addr) 
355        self.logger.debug(dstr)
356
357        ## Trac can not handle author's name that contains spaces
358        #
359        if self.email_addr.lower() == self.trac_smtp_from.lower():
360            if self.email_name:
361                self.author = self.email_name
362            else:
363                self.author = "email2trac"
364        else:
365            self.author = self.email_addr
366
367        if self.parameters.ignore_trac_user_settings:
368            return
369
370        # Is this a registered user, use email address as search key:
371        # result:
372        #   u : login name
373        #   n : Name that the user has set in the settings tab
374        #   e : email address that the user has set in the settings tab
375        #
376        #users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
377        users = [ (u,n,e) for (u, n, e) in self.env.get_known_users()
378            if (
379                (e and (e.lower() == self.email_addr.lower())) or
380                (u and (u.lower() + '@' + self.smtp_default_domain.lower() == self.email_addr.lower()))
381            )
382            ]
383
384        if len(users) >= 1:
385            self.email_from = users[0][2]
386            self.author = users[0][0]
387
388            self.logger.debug('\tget_sender_info: found registered user: %s' %(self.author) )
389
390            if self.parameters.white_list_registered_users:
391                self.logger.debug('\tget_sender_info: added registered user to white list')
392                self.allow_registered_user = True
393                   
394
395    def set_cc_fields(self, ticket, message, update=False):
396        """
397        Set all the right fields for a new ticket
398        """
399        self.logger.debug('function set_cc_fields')
400
401
402        # Put all CC-addresses in ticket CC field
403        #
404        if self.parameters.reply_all:
405
406            ticket_cc = ''
407
408            msg_cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
409
410            if not msg_cc_addrs:
411                return
412
413            if update:
414                self.logger.debug("\tupdate ticket cc-field")
415                ticket_cc = ticket['cc']
416                ticket_cc_list = ticket_cc.split(',')
417
418            for name,addr in msg_cc_addrs:
419       
420                ## Prevent mail loop
421                #
422                if addr == self.trac_smtp_from:
423                    self.logger.debug("\tSkipping %s email address for CC-field, same as smtp_from address in trac.ini " %(addr))
424                    continue
425
426                ## Alwyas remove reporter email address
427                #
428                elif addr == self.email_addr:
429                    self.logger.debug("\tSkipping reporter email address for CC-field")
430                    continue
431
432                ## Always remove the always_cc address
433                #
434                elif addr == self.trac_smtp_always_cc:
435                    self.logger.debug("\tSkipping smtp_always_cc email address for CC-field")
436                    continue
437
438                ## Always remove the always_bcc address
439                #
440                elif addr == self.trac_smtp_always_bcc:
441                    self.logger.debug("\tSkipping smtp_always_bcc email address for CC-field")
442                    continue
443
444                ## Skip also addr in cc_black_list (email2trac.conf)
445                #
446                if self.email_header_acl('cc_black_list', addr, False):
447                    self.logger.debug("\tSkipping address for CC-field due to cc_black_list")
448                    continue
449
450                else:
451                    ## On update, prevent adding duplicates
452                    #
453                    if update:
454                      if addr in ticket_cc_list:
455                          continue
456                   
457                    if ticket_cc:
458                        ticket_cc = '%s, %s' %(ticket_cc, addr)
459                    else:
460                        ticket_cc = addr
461
462            if ticket_cc:
463                self.logger.debug('\tCC fields: %s' %ticket_cc)
464                ticket['cc'] = self.email_to_unicode(ticket_cc)
465
466
467    def acl_list_from_file(self, f, keyword):
468        """
469        Read the email address from a file
470        """
471        self.logger.debug('function acl_list_from_file %s : %s' %(f, keyword))
472       
473        if  not os.path.isfile(f):
474            self.logger.error('%s_file: %s does not exists' %(keyword, f) )
475        else:
476            ## read whole file and replace '\n' with ''
477            #
478            addr_l = open(f, 'r').readlines()
479            s = ','.join(addr_l).replace('\n','')
480
481            try:
482                self.parameters[keyword] = "%s,%s" %(self.parameters[keyword], s)
483            except KeyError, detail:
484                self.parameters[keyword] = s
485
486########## DEBUG functions  ###########################################################
487
488    def debug_body(self, message_body, temp_file):
489        """
490        """
491        self.logger.debug('function debug_body:')
492
493        body_file = "%s.body" %(temp_file)
494        fx = open(body_file, 'wb')
495
496        if self.parameters.dry_run:
497            self.logger.info('DRY-RUN: not saving body to %s' %(body_file))
498            return
499
500        self.logger.debug('writing body to %s' %(body_file))
501        if not message_body:
502                message_body = '(None)'
503
504        message_body = message_body.encode('utf-8')
505
506        fx.write(message_body)
507        fx.close()
508        try:
509            os.chmod(body_file,S_IRWXU|S_IRWXG|S_IRWXO)
510        except OSError:
511            pass
512
513    def debug_attachments(self, message_parts, temp_file):
514        """
515        """
516        self.logger.debug('function debug_attachments')
517       
518        n = 0
519        for item in message_parts:
520
521            ## Skip inline text parts
522            #
523            if not isinstance(item, tuple):
524                continue
525               
526            (original, filename, part) = item
527            self.logger.debug('\t part%d: Content-Type: %s' % (n, part.get_content_type()) )
528
529            dummy_filename, ext = os.path.splitext(filename)
530
531            n = n + 1
532            part_name = 'part%d%s' %(n, ext)
533            part_file = "%s.%s" %(temp_file, part_name)
534
535            s = 'writing %s: filename: %s' %(part_file, filename)
536            self.print_unicode(s)
537
538            ## Forbidden chars, just write part files instead of names
539            #
540            #filename = filename.replace('\\', '_')
541            #filename = filename.replace('/', '_')
542            #filename = filename + '.att_email2trac'
543            # part_file = os.path.join(self.parameters.tmpdir, filename)
544            #part_file = util.text.unicode_quote(part_file)
545            #self.print_unicode(part_file)
546
547            if self.parameters.dry_run:
548                self.logger.info('DRY_RUN: NOT saving attachments')
549                continue
550
551            fx = open(part_file, 'wb')
552            text = part.get_payload(decode=1)
553
554            if not text:
555                text = '(None)'
556
557            fx.write(text)
558            fx.close()
559
560            try:
561                os.chmod(part_file,S_IRWXU|S_IRWXG|S_IRWXO)
562            except OSError:
563                pass
564
565    def save_email_for_debug(self, message, project_name, save_only_raw_message=False):
566
567        if self.parameters.dry_run:
568            self.logger.debug('DRY_RUN: NOT saving email message')
569            return
570
571        (fd, tmp_file) = tempfile.mkstemp('.email2trac', project_name, self.parameters.tmpdir)
572        fx = os.fdopen(fd, 'wb')
573
574        self.logger.debug('saving email to %s' %(tmp_file))
575        fx.write('%s' % message)
576        fx.close()
577       
578        try:
579            os.chmod(tmp_file, S_IRWXU|S_IRWXG|S_IRWXO)
580        except OSError:
581            pass
582
583        if not save_only_raw_message:
584            message_parts = self.get_message_parts(message)
585            message_parts = self.unique_attachment_names(message_parts)
586            body_text = self.get_body_text(message_parts)
587            self.debug_body(body_text, tmp_file)
588            self.debug_attachments(message_parts, tmp_file)
589
590########## Conversion functions  ###########################################################
591
592    def email_to_unicode(self, message_str):
593        """
594        Email has 7 bit ASCII code, convert it to unicode with the charset
595        that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
596        understands it.
597        """
598        self.logger.debug("function email_to_unicode")
599
600        self.logger.debug("\t repr:%s type:%s" %(repr(message_str), type(message_str)))
601
602        ## Skip unicode strings, there are already converted
603        #
604        if type(message_str) is unicode:
605            return message_str
606
607        results =  email.Header.decode_header(message_str)
608
609        s = None
610        for text,format in results:
611            if format:
612                try:
613                    temp = unicode(text, format)
614                except UnicodeError, detail:
615                    # This always works
616                    #
617                    msg = 'ERROR: Could not find charset: %s, please install' %format
618                    self.logger.error(msg)
619                    temp = unicode(text, 'iso-8859-15')
620                except LookupError, detail:
621                    msg = 'ERROR: Could not find charset: %s, please install' %format
622                    self.logger.error(msg)
623                    #temp = unicode(text, 'iso-8859-15')
624                    temp = message_str
625                       
626            else:
627                temp = string.strip(text)
628                temp = unicode(text, 'iso-8859-15')
629
630            if s:
631                s = '%s %s' %(s, temp)
632            else:
633                s = '%s' %temp
634
635        return s
636
637    def str_to_dict(self, s):
638        """
639        Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
640        """
641        self.logger.debug("function str_to_dict")
642
643        fields = string.split(s, self.parameters.subject_field_separator)
644
645        result = dict()
646        for field in fields:
647            try:
648                index, value = string.split(field, '=')
649
650                # We can not change the description of a ticket via the subject
651                # line. The description is the body of the email
652                #
653                if index.lower() in ['description']:
654                    continue
655
656                if value:
657                    result[index.lower()] = value
658
659            except ValueError:
660                pass
661        return result
662
663    def print_unicode(self,s):
664        """
665        This function prints unicode strings if possible else it will quote it
666        """
667        try:
668            self.logger.debug(s)
669        except UnicodeEncodeError, detail:
670            self.logger.debug(util.text.unicode_quote(s))
671
672
673    def html_2_txt(self, data):
674        """
675        Various routines to convert html syntax to valid trac wiki syntax
676        """
677        self.logger.debug('function html_2_txt')
678
679        ## This routine make an safe html that can be include
680        #  in trac, but no further text processing can be done
681        #
682#       try:
683#           from lxml.html.clean import Cleaner
684#           tags_rm = list()
685#           tags_rm.append('body')
686#
687#           cleaner = Cleaner(remove_tags=tags_rm )
688#           parsed_data = cleaner.clean_html(data)
689#           parsed_data = '\n{{{\n#!html\n' + parsed_data + '\n}}}\n'
690#
691#           return parsed_data
692#           
693#       except ImportError::
694#           pass
695
696        parsed_data = None
697        if self.parameters.html2text_cmd:
698            (fd, tmp_file) = tempfile.mkstemp('email2trac.html')
699            f = os.fdopen(fd, 'w')
700
701            cmd = '%s %s' %(self.parameters.html2text_cmd, tmp_file)
702            self.logger.debug('\t html2text conversion %s'%(cmd))
703   
704            if self.parameters.dry_run:
705                self.logger.info('DRY_RUN: html2text conversion command: %s\n' %(cmd))
706
707            else:
708                f.write(data)
709                f.close()
710
711                lines = os.popen(cmd).readlines()
712                parsed_data =  ''.join(lines)
713
714                os.unlink(tmp_file)
715
716        else:
717            self.logger.debug('\t No html2text conversion tool specified in email2trac.conf')
718
719        return parsed_data
720
721    def check_filename_length(self, name):
722        """
723        To bypass a bug in Trac
724        check if the filename length is not larger then OS limit.
725          yes : return truncated filename
726          no  : return unmodified filename
727        """
728        self.logger.debug('function check_filename_length: ')
729
730        if not name:
731            return 'None'
732       
733        dummy_filename, ext = os.path.splitext(name)
734
735        ## Trac uses this format
736        #
737        try:
738            quote_format = util.text.unicode_quote(dummy_filename)
739
740        except UnicodeDecodeError, detail:
741            ## last resort convert to unicode
742            #
743            dummy_filename = util.text.to_unicode(dummy_filename)
744            quote_format = util.text.unicode_quote(dummy_filename)
745
746        ## Determine max filename length
747        #
748        try:
749            filemax_length = os.pathconf('/', 'PC_NAME_MAX')
750        except AttributeError, detail:
751            filemax_length = 240
752
753        if len(quote_format) <= filemax_length:
754            return name
755
756        else:
757            ## Truncate file to filemax_length and reserve room for extension
758            #  We must break on a boundry
759            #
760            length = filemax_length - 6
761
762            for i in range(0,10):
763
764                truncated = quote_format[ : (length - i)]
765
766                try:
767                    unqoute_truncated = util.text.unicode_unquote(truncated)
768                    unqoute_truncated = unqoute_truncated + ext
769
770                    self.print_unicode('\t ' + unqoute_truncated)
771
772                    break
773
774                except UnicodeDecodeError, detail:
775                    continue
776
777
778            return unqoute_truncated
779
780########## TRAC ticket functions  ###########################################################
781
782    def mail_workflow(self, tkt):
783        """
784        """
785        self.logger.debug('function mail_workflow: ')
786       
787        req = Mock(authname=self.author, perm=MockPerm(), args={})
788        ticket_system = TicketSystem(self.env)
789       
790        try:
791            workflow = self.parameters['workflow_%s' %tkt['status'].lower()]
792
793        except KeyError:
794            ## fallback for compability (Will be deprecated)
795            #  workflow can be none.
796            #
797            workflow = None
798            if tkt['status'] in ['closed']:
799                workflow = self.parameters.workflow
800
801        if workflow:
802
803            ## process all workflow implementations
804            #
805            tkt_module = TicketModule(self.env)
806            field_changes, problems = tkt_module.get_ticket_changes(req, tkt, workflow)
807
808            for field in field_changes.keys():
809
810                ## We have already processed these fields
811                #
812                if not field in ['summary', 'description']:
813                    s = 'workflow : %s, field %s : %s, by %s' \
814                       %(workflow, field, field_changes[field]['new'],field_changes[field]['by'] )
815                    self.logger.debug(s)
816
817                    tkt[field] = field_changes[field]['new']
818
819            return True
820
821        else:
822            return False
823
824    def check_permission_participants(self, tkt, action):
825        """
826        Check if the mailer is allowed to update the ticket
827        """
828        self.logger.debug('function check_permission_participants %s')
829
830        if tkt['reporter'].lower() in [self.author.lower(), self.email_addr.lower()]:
831            self.logger.debug('ALLOW, %s is the ticket reporter' %(self.email_addr))
832
833            return True
834
835        perm = PermissionSystem(self.env)
836        if perm.check_permission(action, self.author):
837            self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
838
839            return True
840       
841        # Is the updater in the CC?
842        try:
843            cc_list = tkt['cc'].split(',')
844            for cc in cc_list:
845                if self.email_addr.lower() in cc.lower().strip():
846                    self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
847
848                    return True
849
850        except KeyError:
851            pass
852
853        return False
854
855    def check_permission(self, tkt, action):
856        """
857        check if the reporter has the right permission for the action:
858          - TICKET_CREATE
859          - TICKET_MODIFY
860          - TICKET_APPEND
861          - TICKET_CHGPROP
862
863        There are three models:
864            - None      : no checking at all
865            - trac      : check the permission via trac permission model
866            - email2trac: ....
867        """
868        self.logger.debug("function check_permission: %s" %(action))
869
870        if self.parameters.ticket_permission_system in ['trac']:
871
872            perm = PermissionCache(self.env, self.author)
873            if perm.has_permission(action):
874                return True
875            else:
876                return False
877
878        elif self.parameters.ticket_permission_system in ['update_restricted_to_participants']:
879            return (self.check_permission_participants(tkt, action))   
880
881        ## Default is to allow everybody ticket updates and ticket creation
882        #
883        else:
884                return True
885
886
887    def update_ticket_fields(self, ticket, user_dict, new=None):
888        """
889        This will update the ticket fields. It will check if the
890        given fields are known and if the right values are specified
891        It will only update the ticket field value:
892            - If the field is known
893            - If the value supplied is valid for the ticket field.
894              If not then there are two options:
895               1) Skip the value (new=None)
896               2) Set default value for field (new=1)
897        """
898        self.logger.debug("function update_ticket_fields")
899
900        if self.parameters.bh_product:
901            if 'product' in user_dict:
902                if user_dict['product'] != ticket.env.product.prefix:
903                    self.logging.warning("bloodhound products cannot be changed "
904                                         "- ignoring")
905                user_dict.pop('product')
906
907        ## Check only permission model on ticket updates
908        #
909        if not new:
910            if self.parameters.ticket_permission_system:
911                if not self.check_permission(ticket, 'TICKET_CHGPROP'):
912                    self.logger.info('Reporter: %s has no permission to change ticket properties' %self.author)
913                    return False
914
915        ## Build a system dictionary from the ticket fields
916        #  with field as index and option as value
917        #
918        sys_dict = dict()
919
920        for field in ticket.fields:
921
922            try:
923                sys_dict[field['name']] = field['options']
924
925            except KeyError:
926                sys_dict[field['name']] = None
927                pass
928
929        ## Check user supplied fields an compare them with the
930        #  system one's
931        #
932        for field,value in user_dict.items():
933       
934            s = 'user_field\t %s = %s' %(field,value)
935            self.print_unicode(s)
936
937            if not field in sys_dict.keys(): 
938                self.logger.debug('%s is not a valid field for tickets' %(field))
939                continue
940
941            ## To prevent mail loop
942            #
943            if field == 'cc':
944
945                cc_list = user_dict['cc'].split(',')
946
947                if self.trac_smtp_from in cc_list:
948                    self.logger.debug('MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
949
950                    cc_list.remove(self.trac_smtp_from)
951
952                value = ','.join(cc_list)
953               
954
955            ## Check if every value is allowed for this field
956            #
957            if sys_dict[field]:
958
959                if value in sys_dict[field]:
960                    ticket[field] = value
961                else:
962                    ## Must we set a default if value is not allowed
963                    #
964                    if new:
965                        value = self.get_config('ticket', 'default_%s' %(field) )
966
967            else:
968
969                ticket[field] = value
970
971            s = 'ticket_field\t %s = %s' %(field,  ticket[field])
972            self.print_unicode(s)
973
974    def ticket_update(self, m, id, spam):
975        """
976        If the current email is a reply to an existing ticket, this function
977        will append the contents of this email to that ticket, instead of
978        creating a new one.
979        """
980        self.logger.debug("function ticket_update")
981
982        if not self.parameters.ticket_update:
983            self.logger.debug("ticket_update disabled")
984            return False
985
986        ## Must we update ticket fields
987        #
988        update_fields = dict()
989        try:
990            id, keywords = string.split(id, '?')
991
992            update_fields = self.str_to_dict(keywords)
993
994            ## Strip '#'
995            #
996            self.id = int(id[1:])
997
998        except ValueError:
999
1000            ## Strip '#'
1001            #
1002            self.id = int(id[1:])
1003
1004        self.logger.debug("\tticket id: %s" %id)
1005
1006        ## When is the change committed
1007        #
1008        when = datetime.now(util.datefmt.utc)
1009
1010        try:
1011            tkt = Ticket(self.env, self.id, self.db)
1012
1013        except util.TracError, detail:
1014
1015            ## Not a valid ticket
1016            #
1017            self.logger.info("\tCreating a new ticket, ticket id: %s does not exists" %id)
1018            self.id = None
1019            return False
1020
1021        ## Check the permission of the reporter
1022        #
1023        if self.parameters.ticket_permission_system:
1024            if not self.check_permission(tkt, 'TICKET_APPEND'):
1025                self.logger.info('Reporter: %s has no permission to add comments or attachments to tickets' %self.author)
1026                return False
1027
1028        ## How many changes has this ticket
1029        #
1030        #grouped = TicketModule(self.env).grouped_changelog_entries(tkt, self.db)
1031        grouped = TicketModule(self.env).grouped_changelog_entries(tkt)
1032        cnum = sum(1 for e in grouped) + 1
1033
1034
1035        ## reopen the ticket if it is was closed
1036        #  We must use the ticket workflow framework
1037        #
1038        if self.parameters.email_triggers_workflow:
1039            if not self.mail_workflow(tkt):
1040                if tkt['status'] in ['closed']:
1041                    tkt['status'] = 'reopened'
1042                    tkt['resolution'] = ''
1043        else:
1044            self.logger.debug('\temail triggers workflow disabled')
1045
1046        ## Must we update some ticket fields properties via subject line
1047        #
1048        if update_fields:
1049            self.update_ticket_fields(tkt, update_fields)
1050
1051
1052        message_parts = self.get_message_parts(m)
1053        message_parts = self.unique_attachment_names(message_parts)
1054
1055        ## Must we update some ticket fields properties via inline comments
1056        # in body_text
1057        #
1058        if self.properties:
1059                self.update_ticket_fields(tkt, self.properties)
1060
1061        ## Must we update the CC ticket field
1062        #
1063        self.set_cc_fields(tkt, m, update=True)
1064
1065        if self.parameters.email_header:
1066            message_parts.insert(0, self.email_header_txt(m))
1067
1068        body_text = self.get_body_text(message_parts)
1069
1070        error_with_attachments = self.attach_attachments(message_parts)
1071
1072        if body_text.strip() or update_fields or self.properties:
1073
1074            if self.parameters.dry_run:
1075                s = 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) %s %s' %(self.author, cnum)
1076                self.logger.info(s)
1077
1078            else:
1079                if error_with_attachments:
1080                    body_text = '%s\\%s' %(error_with_attachments, body_text)
1081
1082                self.logger.debug('\ttkt.save_changes(%s, %d)' %(self.author, cnum))
1083                tkt.save_changes(self.author, body_text, when, None, str(cnum))
1084           
1085        if not spam:
1086            self.notify(tkt, False, when)
1087
1088        return True
1089
1090    def set_ticket_fields(self, ticket):
1091        """
1092        set the ticket fields to value specified
1093            - /etc/email2trac.conf with <prefix>_<field>
1094            - trac default values, trac.ini
1095        """
1096        self.logger.debug('function set_ticket_fields')
1097
1098        user_dict = dict()
1099
1100        for field in ticket.fields:
1101
1102            name = field['name']
1103
1104            ## default trac value
1105            #
1106            CUSTOM_FIELD = False
1107            if not field.get('custom'):
1108                value = self.get_config('ticket', 'default_%s' %(name) )
1109
1110                ## skip this field can only be set by email2trac.conf
1111                #
1112                if name in ['resolution']:
1113                    value = None
1114
1115            else:
1116                ##  Else get the default value for custom fields
1117                #
1118                CUSTOM_FIELD = True
1119                value = field.get('value')
1120                options = field.get('options')
1121
1122                if value and options and (value not in options):
1123                     value = options[int(value)]
1124   
1125            s = 'trac[%s] = %s' %(name, value)
1126            self.print_unicode(s)
1127
1128            ## email2trac.conf settings
1129            #
1130            prefix = self.parameters.ticket_prefix
1131            try:
1132                value = self.parameters['%s_%s' %(prefix, name)]
1133
1134                s = 'email2trac[%s] = %s ' %(name, value)
1135                self.print_unicode(s)
1136
1137            except KeyError, detail:
1138                pass
1139       
1140            if value:
1141                user_dict[name] = value
1142
1143                s = 'used %s = %s' %(name, value)
1144                self.print_unicode(s)
1145
1146            else:
1147                ## custom fields need some initialisation
1148                #
1149                if CUSTOM_FIELD:
1150                    user_dict[name] = ''
1151                   
1152        self.update_ticket_fields(ticket, user_dict, new=1)
1153
1154        if 'status' not in user_dict.keys():
1155            ticket['status'] = 'new'
1156
1157    def ticket_update_by_subject(self, subject):
1158        """
1159        This list of Re: prefixes is probably incomplete. Taken from
1160        wikipedia. Here is how the subject is matched
1161          - Re: <subject>
1162          - Re: (<Mail list label>:)+ <subject>
1163
1164        So we must have the last column
1165        """
1166        self.logger.debug('function ticket_update_by_subject')
1167
1168        found_id = None
1169        if self.parameters.ticket_update and self.parameters.ticket_update_by_subject:
1170               
1171            SUBJECT_RE = re.compile(r'^(?:(?:RE|AW|VS|SV|FW|FWD):\s*)+(.*)', re.IGNORECASE)
1172            result = SUBJECT_RE.search(subject)
1173
1174            if result:
1175                ## This is a reply
1176                #
1177                orig_subject = result.group(1)
1178
1179                self.logger.debug('subject search string: %s' %(orig_subject))
1180
1181                cursor = self.db.cursor()
1182                summaries = [orig_subject, '%%: %s' % orig_subject]
1183
1184                ## Time resolution is in micoseconds
1185                #
1186                search_date = datetime.now(util.datefmt.utc) - timedelta(days=self.parameters.ticket_update_by_subject_lookback)
1187
1188                if self.VERSION < 0.12:
1189                    lookback = util.datefmt.to_timestamp(search_date)
1190                else:
1191                    lookback = util.datefmt.to_utimestamp(search_date)
1192
1193                for summary in summaries:
1194                    self.logger.debug('Looking for summary matching: "%s"' % summary)
1195
1196                    sql = """SELECT id, reporter FROM ticket
1197                            WHERE changetime >= %s AND summary LIKE %s
1198                            ORDER BY changetime DESC"""
1199
1200                    cursor.execute(sql, [lookback, summary.strip()])
1201
1202                    for row in cursor:
1203
1204                        (matched_id, sql_reporter) = row
1205
1206                        ## Save first entry.
1207                        #
1208                        if not found_id:
1209                            found_id = matched_id
1210                           
1211                        ## If subject and reporter are the same. The we certainly have found the right ticket
1212                        #
1213                        if sql_reporter == self.author:
1214                            self.logger.debug('Found matching reporter: %s with ticket id: %d' %(sql_reporter, matched_id))
1215                            found_id = matched_id
1216                            break
1217
1218                    if found_id:
1219                        self.logger.debug('Found matching ticket id: %d' % found_id)
1220                        found_id = '#%d' % found_id
1221                        return (found_id, orig_subject)
1222                   
1223        return (found_id, subject)
1224
1225
1226    def new_ticket(self, msg, subject, spam, set_fields = None):
1227        """
1228        Create a new ticket
1229        """
1230        self.logger.debug('function new_ticket')
1231
1232        tkt = Ticket(self.env)
1233
1234        ## self.author can be email address of an username
1235        #
1236        tkt['reporter'] = self.author
1237
1238        self.set_cc_fields(tkt, msg)
1239
1240        self.set_ticket_fields(tkt)
1241
1242        ## Check the permission of the reporter
1243        #
1244        if self.parameters.ticket_permission_system:
1245            if not self.check_permission(tkt, 'TICKET_CREATE'):
1246                self.logger.info('Reporter: %s has no permission to create tickets' %self.author)
1247                return False
1248
1249        ## Old style setting for component, will be removed
1250        #
1251        if spam:
1252            tkt['component'] = 'Spam'
1253
1254        elif self.parameters.has_key('component'):
1255            tkt['component'] = self.parameters['component']
1256
1257        if not msg['Subject']:
1258            tkt['summary'] = u'(No subject)'
1259        else:
1260            tkt['summary'] = subject
1261
1262
1263        if set_fields:
1264            rest, keywords = string.split(set_fields, '?')
1265
1266            if keywords:
1267                update_fields = self.str_to_dict(keywords)
1268                self.update_ticket_fields(tkt, update_fields)
1269
1270
1271        message_parts = self.get_message_parts(msg, True)
1272
1273        ## Must we update some ticket fields properties via body_text
1274        #
1275        if self.properties:
1276                self.update_ticket_fields(tkt, self.properties)
1277
1278        message_parts = self.unique_attachment_names(message_parts)
1279       
1280        ## produce e-mail like header
1281        #
1282        head = ''
1283        if self.parameters.email_header:
1284            head = self.email_header_txt(msg)
1285            message_parts.insert(0, head)
1286           
1287        body_text = self.get_body_text(message_parts)
1288
1289        tkt['description'] = body_text
1290
1291        ## When is the change committed
1292        #
1293        when = datetime.now(util.datefmt.utc)
1294
1295        if self.parameters.dry_run:
1296            self.logger.info('DRY_RUN: tkt.insert()')
1297        else:
1298            self.id = tkt.insert()
1299   
1300        changed = False
1301        comment = ''
1302
1303        ## some routines in trac are dependend on ticket id
1304        #  like alternate notify template
1305        #
1306        if self.parameters.alternate_notify_template:
1307            tkt['id'] = self.id
1308            changed = True
1309
1310        ## Rewrite the description if we have mailto enabled
1311        #
1312        if self.parameters.mailto_link:
1313            changed = True
1314            comment = u'\nadded mailto line\n'
1315
1316            #mailto = self.html_mailto_link( m['Subject'])
1317            mailto = self.html_mailto_link(subject)
1318
1319            tkt['description'] = u'%s\r\n%s%s\r\n' \
1320                %(head, mailto, body_text)
1321   
1322        ## Save the attachments to the ticket   
1323        #
1324        error_with_attachments =  self.attach_attachments(message_parts)
1325
1326        if error_with_attachments:
1327            changed = True
1328            comment = '%s\n%s\n' %(comment, error_with_attachments)
1329
1330        if self.parameters.email_triggers_workflow:
1331            if self.mail_workflow(tkt):
1332                changed = True
1333
1334        if changed:
1335            if self.parameters.dry_run:
1336                s = 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
1337                self.logger.info(s)
1338
1339            else:
1340                tkt.save_changes(tkt['reporter'], comment)
1341
1342        if not spam:
1343            self.notify(tkt, True)
1344
1345
1346    def attach_attachments(self, message_parts, update=False):
1347        '''
1348        save any attachments as files in the ticket's directory
1349        '''
1350        self.logger.debug('function attach_attachments()')
1351
1352        if self.parameters.dry_run:
1353            self.logger.debug("DRY_RUN: no attachments attached to tickets")
1354            return ''
1355
1356        count = 0
1357
1358        ## Get Maxium attachment size
1359        #
1360        max_size = int(self.get_config('attachment', 'max_size'))
1361        status   = None
1362       
1363        for item in message_parts:
1364            ## Skip body parts
1365            #
1366            if not isinstance(item, tuple):
1367                continue
1368               
1369            (original, filename, part) = item
1370
1371            ## We have to determine the size so we use this temporary solution.
1372            #
1373            path, fd =  util.create_unique_file(os.path.join(self.parameters.tmpdir, 'email2trac_tmp.att'))
1374            text = part.get_payload(decode=1)
1375            if not text:
1376                text = '(None)'
1377            fd.write(text)
1378            fd.close()
1379
1380            ## get the file_size
1381            #
1382            stats = os.lstat(path)
1383            file_size = stats[ST_SIZE]
1384
1385            ## Check if the attachment size is allowed
1386            #
1387            if (max_size != -1) and (file_size > max_size):
1388                status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1389                    %(status, original, file_size, max_size)
1390
1391                os.unlink(path)
1392                continue
1393            else:
1394                count = count + 1
1395                   
1396            ## Insert the attachment
1397            #
1398            fd = open(path, 'rb')
1399            if self.system == 'discussion':
1400                att = attachment.Attachment(self.env, 'discussion', 'topic/%s' % (self.id,))
1401            elif self.system == 'blog':
1402                att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,))
1403            else:
1404                s = 'Attach %s to ticket %d' %(filename, self.id)
1405                self.print_unicode(s)
1406                att = attachment.Attachment(self.env, 'ticket', self.id)
1407 
1408            ## This will break the ticket_update system, the body_text is vaporized
1409            #  ;-(
1410            #
1411            if not update:
1412                att.author = self.author
1413                att.description = self.email_to_unicode('Added by email2trac')
1414
1415            try:
1416
1417                self.logger.debug('Insert atachment')
1418                att.insert(filename, fd, file_size)
1419
1420            except OSError, detail:
1421
1422                self.logger.info('%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail))
1423                status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
1424
1425            ## Remove the created temporary filename
1426            #
1427            fd.close()
1428            os.unlink(path)
1429
1430        ## return error
1431        #
1432        return status
1433
1434########## Fullblog functions  #################################################
1435
1436    def blog(self, msg, subject, id, params):
1437        """
1438        The blog create/update function
1439        """
1440        ## import the modules
1441        #
1442        from tracfullblog.core import FullBlogCore
1443        from tracfullblog.model import BlogPost, BlogComment
1444
1445        ## instantiate blog core
1446        #
1447        blog = FullBlogCore(self.env)
1448        req = Mock(authname='anonymous', perm=MockPerm(), args={})
1449
1450        ## parameters from the subject
1451        #
1452        params = self.str_to_dict((params or '').lstrip('?'))
1453
1454        ## preferably get the time from the email date header, or else
1455        #  use current date and time
1456        date = email.Utils.parsedate_tz(msg.get('date'))
1457        if date:
1458            dt = util.datefmt.to_datetime(email.Utils.mktime_tz(date), util.datefmt.utc)
1459        else:
1460            self.logger.warn("No valid date header found")
1461            dt = util.datefmt.to_datetime(None, util.datefmt.utc)
1462
1463        ## blog entry affected
1464        #
1465        self.id = id or util.datefmt.format_datetime(dt, "%Y%m%d%H%M%S", util.datefmt.utc)
1466
1467        ## check wether a blog post exists
1468        #
1469        post = BlogPost(self.env, self.id)
1470        force_update = self.properties.get('update', params.get('update'))
1471
1472        ## message parts
1473        #
1474        message_parts = self.get_message_parts(msg)
1475        message_parts = self.unique_attachment_names(message_parts)
1476
1477        if post.get_versions() and not force_update:
1478
1479            ## add comment to blog entry
1480            #
1481            comment = BlogComment(self.env, self.id)
1482            comment.author = self.properties.get('author', params.get('author', self.author))
1483            comment.comment = self.get_body_text(message_parts)
1484            comment.time = dt
1485
1486            if self.parameters.dry_run:
1487                self.logger.info('DRY-RUN: not adding comment for blog entry "%s"' % id)
1488                return
1489            warnings = blog.create_comment(req, comment)
1490
1491        else:
1492            ## create or update blog entry
1493            #
1494            post.author = self.properties.get('author', params.get('author', self.author))
1495            post.categories = self.properties.get('categories', params.get('categories', ''))
1496            post.title = subject.strip()
1497            post.publish_time = dt
1498            post.body = self.get_body_text(message_parts)
1499           
1500            if self.parameters.dry_run:
1501                self.logger.info('DRY-RUN: not creating blog entry "%s"' % post.title)
1502                return
1503            warnings = blog.create_post(req, post, self.author, u'Created by email2trac', False)
1504
1505        ## check for problems
1506        #
1507        if warnings:
1508            raise TracError(', '.join('blog:%s:%s' % (w[0], w[1]) for w in warnings))
1509       
1510        ## all seems well, attach attachments
1511        #
1512        self.attach_attachments(message_parts)
1513
1514
1515########## Discussion functions  ##############################################
1516
1517    def discussion_topic(self, content, subject):
1518
1519        ## Import modules.
1520        #
1521        from tracdiscussion.api import DiscussionApi
1522        from trac.util.datefmt import to_timestamp, utc
1523
1524        self.logger.debug('Creating a new topic in forum:', self.id)
1525
1526        ## Get dissussion API component.
1527        #
1528        api = self.env[DiscussionApi]
1529        args = {'forum' : self.id}
1530        context = self._create_context(api, args, content, subject)
1531
1532        ## Get forum for new topic.
1533        #
1534        forum = context.forum
1535
1536        if not forum:
1537            self.logger.error("ERROR: Replied forum doesn't exist")
1538
1539        ## Prepare topic.
1540        #
1541        topic = {'forum' : forum['id'],
1542                 'subject' : context.subject,
1543                 'time': to_timestamp(datetime.now(utc)),
1544                 'author' : self.author,
1545                 'subscribers' : [self.email_addr],
1546                 'body' : self.get_body_text(context.content_parts)}
1547
1548        ## Add topic to DB and commit it.
1549        #
1550        self._add_topic(api, context, topic)
1551
1552    def discussion_topic_reply(self, content, subject):
1553
1554        ## Import modules.
1555        #
1556        from tracdiscussion.api import DiscussionApi
1557        from trac.util.datefmt import to_timestamp, utc
1558
1559        self.logger.debug('Replying to discussion topic', self.id)
1560
1561        ## Get dissussion API component.
1562        #
1563        api = self.env[DiscussionApi]
1564        args = {'topic' : self.id}
1565        context = self._create_context(api, args, content, subject)
1566
1567        ## Get replied topic.
1568        #
1569        topic = context.topic
1570
1571        if not topic:
1572            self.logger.error("ERROR: Replied topic doesn't exist")
1573
1574        ## Prepare message.
1575        #
1576        message = {'forum' : topic['forum'],
1577                   'topic' : topic['id'],
1578                   'replyto' : -1,
1579                   'time' : to_timestamp(datetime.now(utc)),
1580                   'author' : self.author,
1581                   'body' : self.get_body_text(context.content_parts)}
1582
1583        ## Add message to DB and commit it.
1584        #
1585        self._add_message(api, context, message)
1586
1587    def discussion_message_reply(self, content, subject):
1588
1589        ## Import modules.
1590        #
1591        from tracdiscussion.api import DiscussionApi
1592        from trac.util.datefmt import to_timestamp, utc
1593
1594        self.logger.debug('Replying to discussion message', self.id)
1595
1596        ## Get dissussion API component.
1597        #
1598        api = self.env[DiscussionApi]
1599        args = {'message' : self.id}
1600        context = self._create_context(api, args, content, subject)
1601
1602        ## Get replied message.
1603        #
1604        message = context.message
1605
1606        if not message:
1607            self.logger.error("ERROR: Replied message doesn't exist")
1608
1609        ## Prepare message.
1610        #
1611        message = {'forum' : message['forum'],
1612                   'topic' : message['topic'],
1613                   'replyto' : message['id'],
1614                   'time' : to_timestamp(datetime.now(utc)),
1615                   'author' : self.author,
1616                   'body' : self.get_body_text(context.content_parts)}
1617
1618        ## Add message to DB and commit it.
1619        #
1620        self._add_message(api, context, message)
1621
1622    def _create_context(self, api, args, content, subject):
1623
1624        ## Import modules.
1625        #
1626        from trac.mimeview import Context
1627        from trac.web.api import Request
1628        from trac.web.session import Session
1629        from trac.perm import PermissionCache
1630
1631        ## TODO: Read server base URL from config.
1632        #  Create request object to mockup context creation.
1633        #
1634        environ = {'SERVER_PORT' : 80,
1635                   'SERVER_NAME' : 'test',
1636                   'REQUEST_METHOD' : 'POST',
1637                   'wsgi.url_scheme' : 'http',
1638                   'wsgi.input' : sys.stdin}
1639        chrome =  {'links': {},
1640                   'scripts': [],
1641                   'ctxtnav': [],
1642                   'warnings': [],
1643                   'notices': []}
1644
1645        if self.env.base_url_for_redirect:
1646            environ['trac.base_url'] = self.env.base_url
1647
1648        req = Request(environ, None)
1649        req.chrome = chrome
1650        req.tz = 'missing'
1651        req.authname = self.author
1652        req.perm = PermissionCache(self.env, self.author)
1653        req.locale = None
1654        req.args = args
1655        req.session = Session(env, req)
1656
1657        ## Create and return context.
1658        #
1659        context = Context.from_request(req)
1660        context.realm = 'discussion-email2trac'
1661        context.db = self.env.get_db_cnx()
1662        context.content = content
1663        context.subject = subject
1664
1665        ## Read content parts from content.
1666        #
1667        context.content_parts = self.get_message_parts(content)
1668        context.content_parts = self.unique_attachment_names(
1669          context.content_parts)
1670
1671        api._prepare_context(context)
1672
1673        return context
1674
1675    def _add_topic(self, api, context, topic):
1676        context.req.perm.assert_permission('DISCUSSION_APPEND')
1677
1678        ## Filter topic.
1679        #
1680        for discussion_filter in api.discussion_filters:
1681            accept, topic_or_error = discussion_filter.filter_topic(
1682              context, topic)
1683            if accept:
1684                topic = topic_or_error
1685            else:
1686                raise TracError(topic_or_error)
1687
1688        ## Add a new topic.
1689        #
1690        new_topic_id = api.add_topic(context, topic)
1691
1692        ## Get inserted topic with new ID.
1693        #
1694        topic = api.get_topic(context, new_topic_id)
1695
1696        ## Attach attachments.
1697        #
1698        self.id = topic['id']
1699        self.attach_attachments(context.content_parts, True)
1700
1701        ## Notify change listeners.
1702        #
1703        for listener in api.topic_change_listeners:
1704            listener.topic_created(context, topic)
1705
1706    def _add_message(self, api, context, message):
1707        context.req.perm.assert_permission('DISCUSSION_APPEND')
1708
1709        ## Filter message.
1710        #
1711        for discussion_filter in api.discussion_filters:
1712            accept, message_or_error = discussion_filter.filter_message(
1713              context, message)
1714            if accept:
1715                message = message_or_error
1716            else:
1717                raise TracError(message_or_error)
1718
1719        ## Add message.
1720        #
1721        new_msg_id = api.add_message(context, message)
1722
1723        ## Get inserted message with new ID.
1724        #
1725        message = api.get_message(context, new_msg_id)
1726
1727        ## Attach attachments.
1728        #
1729
1730        self.attach_attachments(context.content_parts, True)
1731
1732        ## Notify change listeners.
1733        #
1734        for listener in api.message_change_listeners:
1735            listener.message_created(context, message)
1736
1737########## MAIN function  ######################################################
1738
1739    def parse(self, fp):
1740        """
1741        """
1742        self.logger.debug('Main function parse')
1743        global m
1744
1745        m = email.message_from_file(fp)
1746       
1747        if not m:
1748            self.logger.debug('This is not a valid email message format')
1749            return
1750           
1751        ## Work around lack of header folding in Python; see http://bugs.python.org/issue4696
1752        #
1753        try:
1754            m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
1755        except AttributeError, detail:
1756            pass
1757
1758        if self.parameters.debug: # save email + try to decode message part
1759            self.save_email_for_debug(m, self.parameters.project_name)
1760        elif self.parameters.save_raw_message:     # save only the raw e-mail message text
1761            self.save_email_for_debug(m, self.parameters.project_name, True)
1762
1763        self.db = self.env.get_read_db()
1764
1765        self.get_sender_info(m)
1766
1767
1768        if self.parameters.white_list_file:
1769            self.acl_list_from_file(self.parameters.white_list_file, 'white_list')
1770
1771        if not ( self.email_header_acl('white_list', self.email_addr, True) or self.allow_registered_user ) :
1772
1773            self.logger.info('Message rejected : %s not in white list' %(self.email_addr))
1774            return False
1775
1776        if self.email_header_acl('black_list', self.email_addr, False):
1777            self.logger.info('Message rejected : %s in black list' %(self.email_addr))
1778            return False
1779
1780        if not self.email_header_acl('recipient_list', self.email_to_addrs, True):
1781            self.logger.info('Message rejected : %s not in recipient list' %(self.email_to_addrs))
1782            return False
1783
1784        ## If spam drop the message
1785        #
1786        if self.spam(m) == 'drop':
1787            return False
1788
1789        elif self.spam(m) == 'spam':
1790            spam_msg = True
1791        else:
1792            spam_msg = False
1793
1794        if not m['Subject']:
1795            subject  = 'No Subject'
1796        else:
1797            subject  = self.email_to_unicode(m['Subject'])
1798
1799        ## Bug in python logging module <2.6
1800        #
1801        self.logger.info('subject: %s' %repr(subject))
1802
1803        ## First try the new method and then fall back to old method
1804        #
1805        if not self.parse_delivered_to_field(m, subject, spam_msg):
1806            self.parse_subject_field(m, subject, spam_msg)
1807
1808    def parse_delivered_to_field(self, m, subject, spam_msg):
1809        """
1810        See if we have replied to an existing ticket
1811        """
1812        self.logger.debug('function parse_delivered_to_field')
1813
1814        ## Ticket id is in Delivered-To Field. Trac notify must send this
1815        #  eg: is+390@surfsara.nl
1816       
1817        ## Only run if this parameter is se
1818        #
1819        if not self.parameters.recipient_delimiter:
1820            return False
1821
1822        try:
1823
1824            #print self.parameters.project_name
1825            self.logger.debug('Delivered To: %s' %m['Delivered-To'])
1826
1827            id = m['Delivered-To']
1828            id = id.split(self.parameters.recipient_delimiter)[1]
1829            id = id.split('@')[0]
1830
1831            self.logger.debug('\t Found ticket id: %s' %id)
1832
1833            ## The ticket_update expects a # in front of the ticket id
1834            #
1835            id = "#%s" %(id)
1836
1837            ## true if ticket update
1838            #
1839            return self.ticket_update(m, id, spam_msg)
1840
1841        except KeyError, detail:
1842            pass
1843        except IndexError, detail:
1844            pass
1845
1846        return False
1847
1848
1849    def parse_subject_field(self, m, subject, spam_msg):
1850        """
1851        """
1852        self.logger.debug('function parse_subject_header')
1853
1854        ## [hic] #1529: Re: LRZ
1855        #  [hic] #1529?owner=bas,priority=medium: Re: LRZ
1856        #
1857        ticket_regex = r'''
1858            (?P<new_fields>[#][?].*)
1859            |(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?[^:]*)?:)
1860            '''
1861
1862        ## Check if  FullBlogPlugin is installed
1863        #
1864        blog_enabled = None
1865        blog_regex = ''
1866        if self.get_config('components', 'tracfullblog.*') in ['enabled']:
1867            self.logger.debug('Trac BLOG support enabled')
1868            blog_enabled = True
1869            blog_regex = '''|(?P<blog>blog(?P<blog_params>[?][^:]*)?:(?P<blog_id>\S*))'''
1870
1871
1872        ## Check if DiscussionPlugin is installed
1873        #
1874        discussion_enabled = None
1875        discussion_regex = ''
1876        if self.get_config('components', 'tracdiscussion.api.discussionapi') in ['enabled']:
1877            self.logger.debug('Trac Discussion support enabled')
1878            discussion_enabled = True
1879            discussion_regex = r'''
1880            |(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
1881            |(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
1882            |(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
1883            '''
1884
1885
1886        regex_str = ticket_regex + blog_regex + discussion_regex
1887        SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
1888
1889        ## Find out if this is a ticket, a blog or a discussion
1890        #
1891        result =  SYSTEM_RE.search(subject)
1892
1893        if result:
1894            ## update ticket + fields
1895            #
1896            if result.group('reply'):
1897                self.system = 'ticket'
1898
1899                ## Skip the last ':' character
1900                #
1901                if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
1902                    self.new_ticket(m, subject, spam_msg)
1903
1904            ## New ticket + fields
1905            #
1906            elif result.group('new_fields'):
1907                self.system = 'ticket'
1908                self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
1909
1910            if blog_enabled:
1911                if result.group('blog'):
1912                    self.system = 'blog'
1913                    self.blog(m, subject[result.end('blog'):], result.group('blog_id'), result.group('blog_params'))
1914
1915            if discussion_enabled:
1916                ## New topic.
1917                #
1918                if result.group('forum'):
1919                    self.system = 'discussion'
1920                    self.id = int(result.group('forum_id'))
1921                    self.discussion_topic(m, subject[result.end('forum'):])
1922
1923                ## Reply to topic.
1924                #
1925                elif result.group('topic'):
1926                    self.system = 'discussion'
1927                    self.id = int(result.group('topic_id'))
1928                    self.discussion_topic_reply(m, subject[result.end('topic'):])
1929
1930                ## Reply to topic message.
1931                #
1932                elif result.group('message'):
1933                    self.system = 'discussion'
1934                    self.id = int(result.group('message_id'))
1935                    self.discussion_message_reply(m, subject[result.end('message'):])
1936
1937        else:
1938
1939            self.system = 'ticket'
1940            (matched_id, subject) = self.ticket_update_by_subject(subject)
1941
1942            if matched_id:
1943
1944                if not self.ticket_update(m, matched_id, spam_msg):
1945                    self.new_ticket(m, subject, spam_msg)
1946
1947            else:
1948                ## No update by subject, so just create a new ticket
1949                #
1950                self.new_ticket(m, subject, spam_msg)
1951
1952
1953########## BODY TEXT functions  ###########################################################
1954
1955    def strip_signature(self, text):
1956        """
1957        Strip signature from message, inspired by Mailman software
1958        """
1959        self.logger.debug('function strip_signature: %s' %self.parameters.strip_signature_regex)
1960
1961        body = []
1962
1963        STRIP_RE = re.compile( self.parameters.strip_signature_regex )
1964        for line in text.splitlines():
1965
1966            match = STRIP_RE.match(line)
1967            if match:
1968                self.logger.debug('\t"%s "  matched, skiping rest of message' %line)
1969                break
1970
1971            body.append(line)
1972
1973        return ('\n'.join(body))
1974
1975    def reflow(self, text, delsp = 0):
1976        """
1977        Reflow the message based on the format="flowed" specification (RFC 3676)
1978        """
1979        flowedlines = []
1980        quotelevel = 0
1981        prevflowed = 0
1982
1983        for line in text.splitlines():
1984            from re import match
1985           
1986            ## Figure out the quote level and the content of the current line
1987            #
1988            m = match('(>*)( ?)(.*)', line)
1989            linequotelevel = len(m.group(1))
1990            line = m.group(3)
1991
1992            ## Determine whether this line is flowed
1993            #
1994            if line and line != '-- ' and line[-1] == ' ':
1995                flowed = 1
1996            else:
1997                flowed = 0
1998
1999            if flowed and delsp and line and line[-1] == ' ':
2000                line = line[:-1]
2001
2002            ## If the previous line is flowed, append this line to it
2003            #
2004            if prevflowed and line != '-- ' and linequotelevel == quotelevel:
2005                flowedlines[-1] += line
2006
2007            ## Otherwise, start a new line
2008            #
2009            else:
2010                flowedlines.append('>' * linequotelevel + line)
2011
2012            prevflowed = flowed
2013           
2014
2015        return '\n'.join(flowedlines)
2016
2017    def strip_quotes(self, text):
2018        """
2019        Strip quotes from message by Nicolas Mendoza
2020        """
2021        self.logger.debug('function strip_quotes: %s' %self.parameters.email_quote)
2022
2023        body = []
2024
2025        STRIP_RE = re.compile( self.parameters.email_quote )
2026
2027        for line in text.splitlines():
2028
2029            try:
2030
2031                match = STRIP_RE.match(line)
2032                if match:
2033                    self.logger.debug('\t"%s "  matched, skipping rest of message' %line)
2034                    continue
2035
2036            except UnicodeDecodeError:
2037
2038                tmp_line = self.email_to_unicode(line)
2039
2040                match = STRIP_RE.match(tmp_line)
2041                if match:
2042                    self.logger.debug('\t"%s "  matched, skipping rest of message' %line)
2043                    continue
2044               
2045            body.append(line)
2046
2047        return ('\n'.join(body))
2048
2049    def inline_properties(self, text):
2050        """
2051        Parse text if we use inline keywords to set ticket fields
2052        """
2053        self.logger.debug('function inline_properties')
2054
2055        properties = dict()
2056        body = list()
2057
2058        INLINE_EXP = re.compile('\s*[@]\s*(\w+)\s*:(.*)$')
2059
2060        for line in text.splitlines():
2061            match = INLINE_EXP.match(line)
2062            if match:
2063                keyword, value = match.groups()
2064
2065                if self.parameters.inline_properties_first_wins:
2066                    if keyword in self.properties.keys():
2067                        continue
2068
2069                self.properties[keyword] = value.strip()
2070                self.logger.debug('\tinline properties: %s : %s' %(keyword,value))
2071
2072            else:
2073                body.append(line)
2074               
2075        return '\n'.join(body)
2076
2077
2078    def wrap_text(self, text, replace_whitespace = False):
2079        """
2080        Will break a lines longer then given length into several small
2081        lines of size given length
2082        """
2083        import textwrap
2084
2085        LINESEPARATOR = '\n'
2086        reformat = ''
2087
2088        for s in text.split(LINESEPARATOR):
2089            tmp = textwrap.fill(s, self.parameters.use_textwrap)
2090            if tmp:
2091                reformat = '%s\n%s' %(reformat,tmp)
2092            else:
2093                reformat = '%s\n' %reformat
2094
2095        return reformat
2096
2097        # Python2.4 and higher
2098        #
2099        #return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
2100        #
2101
2102########## EMAIL attachements functions ###########################################################
2103
2104    def inline_part(self, part):
2105        """
2106        """
2107        self.logger.debug('function inline_part()')
2108
2109        return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
2110
2111    def get_message_parts(self, msg, new_email=False):
2112        """
2113        parses the email message and returns a list of body parts and attachments
2114        body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
2115        """
2116        self.logger.debug('function get_message_parts()')
2117
2118        message_parts = list()
2119   
2120        ALTERNATIVE_MULTIPART = False
2121
2122        for part in msg.walk():
2123            content_maintype = part.get_content_maintype()
2124            content_type =  part.get_content_type()
2125
2126            self.logger.debug('\t Message part: Main-Type: %s' % content_maintype)
2127            self.logger.debug('\t Message part: Content-Type: %s' % content_type)
2128
2129            ## Check content type
2130            #
2131            if content_type in self.STRIP_CONTENT_TYPES:
2132                self.logger.debug("\t A %s attachment named '%s' was skipped" %(content_type, part.get_filename()))
2133                continue
2134
2135            ## Catch some mulitpart execptions
2136            #
2137            if content_type == 'multipart/alternative':
2138                ALTERNATIVE_MULTIPART = True
2139                continue
2140
2141            ## Skip multipart containers
2142            #
2143            if content_maintype == 'multipart':
2144                self.logger.debug("\t Skipping multipart container")
2145                continue
2146           
2147            ## Check if this is an inline part. It's inline if there is co Cont-Disp header,
2148            #  or if there is one and it says "inline"
2149            #
2150            inline = self.inline_part(part)
2151
2152            ## Drop HTML message
2153            #
2154            if ALTERNATIVE_MULTIPART and self.parameters.drop_alternative_html_version:
2155
2156                if content_type == 'text/html':
2157                    self.logger.debug('\t Skipping alternative HTML message')
2158                    ALTERNATIVE_MULTIPART = False
2159                    continue
2160
2161
2162            #if self.VERSION < 1.0:
2163            #    filename = part.get_filename()
2164
2165            ## convert 7 bit filename to 8 bit unicode
2166            #
2167            raw_filename = part.get_filename()
2168            filename = self.email_to_unicode(raw_filename);
2169
2170            s = '\t unicode filename: %s' %(filename)
2171            self.print_unicode(s)
2172            self.logger.debug('\t raw filename: %s' %repr(raw_filename))
2173
2174            if self.VERSION < 1.0:
2175                filename = self.check_filename_length(filename)
2176
2177            ## Save all non plain text message as attachment
2178            #
2179            if not content_type in ['text/plain']:
2180
2181                message_parts.append( (filename, part) )
2182
2183                ## We only convert html messages
2184                #
2185                if not content_type == 'text/html':
2186                    self.logger.debug('\t Appending %s (%s)' %(repr(filename), content_type))
2187                    continue
2188
2189
2190            ## We have an text or html message
2191            #
2192            if not inline:
2193                    self.logger.debug('\t Appending %s (%s), not an inline messsage part' %(repr(filename), content_type))
2194                    message_parts.append( (filename, part) )
2195                    continue
2196               
2197            ## Try to decode message part. We have a html or plain text messafe
2198            #
2199            body_text = part.get_payload(decode=1)
2200            if not body_text:           
2201                body_text = part.get_payload(decode=0)
2202
2203            ## Try to convert html message
2204            #
2205            if content_type == 'text/html':
2206
2207                body_text = self.html_2_txt(body_text)
2208                if not body_text:
2209                    continue
2210
2211            format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
2212            delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
2213
2214            if self.parameters.reflow and not self.parameters.verbatim_format and format == 'flowed':
2215                body_text = self.reflow(body_text, delsp == 'yes')
2216
2217            if new_email and self.parameters.only_strip_on_update:
2218                self.logger.debug('Skip signature/quote stripping for new messages')
2219            else:
2220                if self.parameters.strip_signature:
2221                    body_text = self.strip_signature(body_text)
2222
2223                if self.parameters.strip_quotes:
2224                    body_text = self.strip_quotes(body_text)
2225
2226            if self.parameters.inline_properties:
2227                body_text = self.inline_properties(body_text)
2228
2229            if self.parameters.use_textwrap:
2230                body_text = self.wrap_text(body_text)
2231
2232            ## Get contents charset (iso-8859-15 if not defined in mail headers)
2233            #
2234            charset = part.get_content_charset()
2235            if not charset:
2236                charset = 'iso-8859-15'
2237
2238            try:
2239                ubody_text = unicode(body_text, charset)
2240
2241            except UnicodeError, detail:
2242                ubody_text = unicode(body_text, 'iso-8859-15')
2243
2244            except LookupError, detail:
2245                ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
2246
2247            if self.parameters.verbatim_format:
2248                message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
2249            else:
2250                message_parts.append('%s' %ubody_text)
2251
2252        return message_parts
2253       
2254    def unique_attachment_names(self, message_parts):
2255        """
2256        Make sure we have unique names attachments:
2257          - check if it contains illegal characters
2258          - Rename "None" filenames to "untitled-part"
2259        """
2260        self.logger.debug('function unique_attachment_names()')
2261        renamed_parts = []
2262        attachment_names = set()
2263
2264        for item in message_parts:
2265           
2266            ## If not an attachment, leave it alone
2267            #
2268            if not isinstance(item, tuple):
2269                renamed_parts.append(item)
2270                continue
2271               
2272            (filename, part) = item
2273
2274            ## If filename = None, use a default one
2275            #
2276            if filename in [ 'None']:
2277                filename = 'untitled-part'
2278                self.logger.info('\t Rename filename "None" to: %s' %filename)
2279
2280                ## Guess the extension from the content type, use non strict mode
2281                #  some additional non-standard but commonly used MIME types
2282                #  are also recognized
2283                #
2284                ext = mimetypes.guess_extension(part.get_content_type(), False)
2285                if not ext:
2286                    ext = '.bin'
2287
2288                filename = '%s%s' % (filename, ext)
2289
2290            ## Discard relative paths for windows/unix in attachment names
2291            #
2292            filename = filename.replace('\\', '_')
2293            filename = filename.replace('/', '_')
2294
2295            ## remove linefeed char
2296            #
2297            for forbidden_char in ['\r', '\n']:
2298                filename = filename.replace(forbidden_char,'')
2299
2300            ## We try to normalize the filename to utf-8 NFC if we can.
2301            #  Files uploaded from OS X might be in NFD.
2302            #  Check python version and then try it
2303            #
2304            #if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
2305            #   try:
2306            #       filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8') 
2307            #   except TypeError:
2308            #       pass
2309
2310            ## Make the filename unique for this ticket
2311            #
2312            num = 0
2313            unique_filename = filename
2314            dummy_filename, ext = os.path.splitext(filename)
2315
2316            while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
2317                num += 1
2318                unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
2319               
2320            s = '\t Attachment with filename %s will be saved as %s' % (filename, unique_filename)
2321            self.print_unicode(s)
2322
2323            attachment_names.add(unique_filename)
2324
2325            renamed_parts.append((filename, unique_filename, part))
2326   
2327        return renamed_parts
2328           
2329           
2330    def attachment_exists(self, filename):
2331
2332        self.logger.debug("function attachment_exists")
2333
2334        s = '\t check if attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
2335        self.print_unicode(s)
2336
2337        ## Do we have a valid ticket id
2338        #
2339        if not self.id:
2340            return False
2341
2342        try:
2343            if self.system == 'discussion':
2344
2345                att = attachment.Attachment(self.env, 'discussion', 'ticket/%s' % (self.id,), filename)
2346
2347            elif self.system == 'blog':
2348
2349                att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,), filename)
2350
2351            else:
2352
2353                att = attachment.Attachment(self.env, 'ticket', self.id, filename)
2354
2355            return True
2356
2357        except attachment.ResourceNotFound:
2358
2359            return False
2360
2361########## TRAC Ticket Text ###########################################################
2362           
2363    def get_body_text(self, message_parts):
2364        """
2365        """
2366        self.logger.debug('function get_body_text()')
2367
2368        body_text = []
2369       
2370        for part in message_parts:
2371       
2372            ## Plain text part, append it
2373            #
2374            if not isinstance(part, tuple):
2375                body_text.extend(part.strip().splitlines())
2376                body_text.append("")
2377                continue
2378
2379            (original, filename, part) = part
2380            inline = self.inline_part(part)
2381
2382            ## Skip generation of attachment link if html is converted to text
2383            #
2384            if part.get_content_type() == 'text/html' and self.parameters.html2text_cmd and inline:
2385                s = 'Skipping attachment link for html part: %s' %(filename)
2386                self.print_unicode(s)
2387                continue
2388           
2389            if part.get_content_maintype() == 'image' and inline:
2390
2391                if self.system != 'discussion':
2392                    s = 'wiki image link for: %s' %(filename)
2393                    self.print_unicode(s)
2394                    body_text.append('[[Image(%s)]]' % filename)
2395
2396                body_text.append("")
2397
2398            else:
2399
2400                if self.system != 'discussion':
2401
2402                    s = 'wiki attachment link for: %s' %(filename)
2403                    self.print_unicode(s)
2404                    body_text.append('[attachment:"%s"]' % filename)
2405
2406                body_text.append("")
2407
2408        ## Convert list body_texts to string
2409        #
2410        body_text = '\r\n'.join(body_text)
2411        return body_text
2412
2413    def html_mailto_link(self, subject):
2414        """
2415        This function returns a HTML mailto tag with the ticket id and author email address
2416        """
2417        self.logger.debug("function html_mailto_link")
2418
2419        if not self.author:
2420            author = self.email_addr
2421        else:   
2422            author = self.author
2423
2424        if not self.parameters.mailto_cc:
2425            self.parameters.mailto_cc = ''
2426           
2427        ## Bug in urllib.quote function
2428        #
2429        if isinstance(subject, unicode):
2430            subject = subject.encode('utf-8')
2431
2432        ## use urllib to escape the chars
2433        #
2434        s = '%s?Subject=%s&cc=%s' %(
2435               urllib.quote(self.email_addr),
2436               urllib.quote('Re: #%s: %s' %(self.id, subject)),
2437               urllib.quote(self.parameters.mailto_cc)
2438               )
2439
2440        s = '[mailto:"%s" Reply to: %s]' %(s, author)
2441
2442        self.logger.debug("\tmailto link %s" %s)
2443        return s
2444
2445########## TRAC notify section ###########################################################
2446
2447    def notify(self, tkt, new=True, modtime=0):
2448        """
2449        A wrapper for the TRAC notify function. So we can use templates
2450        """
2451        self.logger.debug('function notify()')
2452
2453
2454        class Email2TracNotifyEmail(TicketNotifyEmail):
2455
2456            def __init__(self, env):
2457                TicketNotifyEmail.__init__(self, env)
2458                self.email2trac_notify_reporter = None
2459                self.email2trac_replyto = None
2460
2461            def send(self, torcpts, ccrcpts):
2462                #print 'Yes this works'
2463                dest = self.reporter or 'anonymous'
2464                hdrs = {}
2465                hdrs['Message-ID'] = self.get_message_id(dest, self.modtime)
2466                hdrs['X-Trac-Ticket-ID'] = str(self.ticket.id)
2467                hdrs['X-Trac-Ticket-URL'] = self.data['ticket']['link']
2468                if not self.newticket:
2469                    msgid = self.get_message_id(dest)
2470                    hdrs['In-Reply-To'] = msgid
2471                    hdrs['References'] = msgid
2472
2473
2474                if self.email2trac_notify_reporter:
2475                    if not self.email2trac_notify_reporter in torcpts:
2476                        torcpts.append(self.email2trac_notify_reporter)
2477
2478                if self.email2trac_replyto:
2479                    # use to rewrite reply to
2480                    # hdrs does not work, multiple reply addresses
2481                    #hdrs['Reply-To'] = 'bas.van.der.vlies@gmail.com'
2482                    self.replyto_email = self.email2trac_replyto
2483       
2484                NotifyEmail.send(self, torcpts, ccrcpts, hdrs)
2485
2486        if self.parameters.dry_run  :
2487                self.logger.info('DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter'])
2488                return
2489
2490        try:
2491
2492            tn = Email2TracNotifyEmail(self.env)
2493
2494            ## additionally append sender (regardeless of settings in trac.ini)
2495            #
2496            if self.parameters.notify_reporter:
2497
2498                self.logger.debug('\t Notify reporter set')
2499
2500                if not self.email_header_acl('notify_reporter_black_list', self.email_addr, False):
2501                    tn.email2trac_notify_reporter = self.email_addr
2502
2503            if self.parameters.notify_replyto_rewrite:
2504
2505                self.logger.debug('\t Notify replyto rewrite set to:%s' %self.parameters.notify_replyto_rewrite)
2506
2507                if self.parameters.notify_replyto_rewrite in ['use_mail_domain']:
2508                    self.logger.debug('\t\t use_mail_domain:%s' %self.smtp_default_domain)
2509                    tn.email2trac_replyto = '%s@%s' %(self.id, self.smtp_default_domain )
2510
2511                elif self.parameters.notify_replyto_rewrite in ['use_trac_smtp_replyto']:
2512                    self.logger.debug('\t\t use_trac_smtp_replyto delimiter:%s' %self.parameters.recipient_delimiter)
2513
2514                    ## handle addres with @ and without
2515                    #
2516                    dummy = self.smtp_replyto.split('@')
2517                    if len(dummy) > 1:
2518                        tn.email2trac_replyto = '%s%s%s@%s' %(dummy[0], self.parameters.recipient_delimiter, self.id, dummy[1])
2519                    else:
2520                        tn.email2trac_replyto = '%s%s%s' %(dummy[0], self.parameters.recipient_delimiter, self.id)
2521
2522            if self.parameters.alternate_notify_template:
2523
2524                if self.VERSION >= 0.12:
2525
2526                    from trac.web.chrome import Chrome
2527
2528                    if  self.parameters.alternate_notify_template_update and not new:
2529                        tn.template_name = self.parameters.alternate_notify_template_update
2530                    else:
2531                        tn.template_name = self.parameters.alternate_notify_template
2532
2533                    tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
2534                       
2535                else:
2536
2537                    tn.template_name = self.parameters.alternate_notify_template
2538
2539            tn.notify(tkt, new, modtime)
2540
2541        except Exception, e:
2542            self.logger.error('Failure sending notification on creation of ticket #%s: %s' %(self.id, e))
2543
2544########## END Class Definition  ########################################################
2545
2546
2547########## Parse Config File  ###########################################################
2548
2549def ReadConfig(file, name):
2550    """
2551    Parse the config file
2552    """
2553    if not os.path.isfile(file):
2554        print 'File %s does not exist' %file
2555        sys.exit(1)
2556
2557    config = trac_config.Configuration(file)
2558   
2559    parentdir = config.get('DEFAULT', 'parentdir')
2560    sections = config.sections()
2561
2562    ## use some trac internals to get the defaults
2563    #
2564    tmp = config.parser.defaults()
2565    project =  SaraDict()
2566
2567    for option, value in tmp.items():
2568        try:
2569            project[option] = int(value)
2570        except ValueError:
2571            project[option] = value
2572
2573    if name:
2574        if name in sections:
2575            project =  SaraDict()
2576            for option, value in  config.options(name):
2577                try:
2578                    project[option] = int(value)
2579                except ValueError:
2580                    project[option] = value
2581
2582        elif not parentdir:
2583            print "Not a valid project name: %s, valid names are: %s" %(name, sections)
2584            print "or set parentdir in the [DEFAULT] section"
2585            sys.exit(1)
2586
2587    ## If parentdir then set project dir to parentdir + name
2588    #
2589    if not project.has_key('project'):
2590        if not parentdir:
2591            print "You must set project or parentdir in your configuration file"
2592            sys.exit(1)
2593        elif not name:
2594            print "You must configure a  project section in your configuration file"
2595        else:
2596            project['project'] = os.path.join(parentdir, name)
2597
2598    ##
2599    # Save the project name
2600    #
2601    project['project_name'] = os.path.basename(project['project'])
2602
2603    return project
2604
2605########## Setup Logging ###############################################################
2606
2607def setup_log(parameters, project_name, interactive=None):
2608    """
2609    Setup logging
2610
2611    Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
2612    would be interpreted by the ConfigParser itself.
2613    """
2614    logger = logging.getLogger('email2trac %s' %project_name)
2615
2616    if interactive:
2617        parameters.log_type = 'stderr'
2618
2619    if not parameters.log_type:
2620        if sys.platform in ['win32', 'cygwin']:
2621            parameters.log_type = 'eventlog'
2622        else:
2623            parameters.log_type = 'syslog'
2624
2625    if parameters.log_type == 'file':
2626
2627        if not parameters.log_file:
2628            parameters.log_file = 'email2trac.log'
2629
2630        if not os.path.isabs(parameters.log_file):
2631            parameters.log_file = os.path.join(tempfile.gettempdir(), parameters.log_file)
2632
2633        log_handler = logging.FileHandler(parameters.log_file)
2634
2635    elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
2636        ## Requires win32 extensions
2637        #
2638        logid = "email2trac"
2639        log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
2640
2641    elif parameters.log_type in ('syslog', 'unix'):
2642        log_handler = logging.handlers.SysLogHandler('/dev/log')
2643
2644    elif parameters.log_type in ('stderr'):
2645        log_handler = logging.StreamHandler(sys.stderr)
2646
2647    else:
2648        log_handler = logging.handlers.BufferingHandler(0)
2649
2650    if parameters.log_format:
2651        parameters.log_format = parameters.log_format.replace('$(', '%(')
2652    else:
2653        parameters.log_format = '%(name)s: %(message)s'
2654        if parameters.log_type in ('file', 'stderr'):
2655            parameters.log_format = '%(asctime)s ' + parameters.log_format
2656
2657    log_formatter = logging.Formatter(parameters.log_format)
2658    log_handler.setFormatter(log_formatter)
2659    logger.addHandler(log_handler)
2660
2661    if (parameters.log_level in ['DEBUG', 'ALL']) or (parameters.debug > 0):
2662        logger.setLevel(logging.DEBUG)
2663        parameters.debug = 1
2664
2665    elif parameters.log_level in ['INFO'] or parameters.verbose:
2666        logger.setLevel(logging.INFO)
2667
2668    elif parameters.log_level in ['WARNING']:
2669        logger.setLevel(logging.WARNING)
2670
2671    elif parameters.log_level in ['ERROR']:
2672        logger.setLevel(logging.ERROR)
2673
2674    elif parameters.log_level in ['CRITICAL']:
2675        logger.setLevel(logging.CRITICAL)
2676
2677    else:
2678        logger.setLevel(logging.INFO)
2679
2680    return logger
2681
2682########## Own TicketNotifyEmail class ###############################################################
2683
2684if __name__ == '__main__':
2685    ## Default config file
2686    #
2687    agilo = False
2688    bh_product = None
2689    configfile = '@email2trac_conf@'
2690    project = ''
2691    component = ''
2692    ticket_prefix = 'default'
2693    dry_run = None
2694    verbose = None
2695    debug_interactive = None
2696    virtualenv = '@virtualenv@'
2697
2698    SHORT_OPT = 'AB:cdE:hf:np:t:v'
2699    LONG_OPT  =  ['agilo', 'bh_product=', 'component=', 'debug',
2700                  'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=',
2701                  'virtualenv=', 'verbose']
2702
2703    try:
2704        opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
2705    except getopt.error,detail:
2706        print __doc__
2707        print detail
2708        sys.exit(1)
2709   
2710    project_name = None
2711    for opt,value in opts:
2712        if opt in [ '-h', '--help']:
2713            print __doc__
2714            sys.exit(0)
2715        elif opt in ['-A', '--agilo']:
2716            agilo = True
2717        elif opt in ['-B', '--bh_product']:
2718            bh_product = value
2719        elif opt in ['-c', '--component']:
2720            component = value
2721        elif opt in ['-d', '--debug']:
2722            debug_interactive = 1
2723        elif opt in ['-E', '--virtualenv']:
2724            virtualenv = value
2725        elif opt in ['-f', '--file']:
2726            configfile = value
2727        elif opt in ['-n', '--dry-run']:
2728            dry_run = True
2729        elif opt in ['-p', '--project']:
2730            project_name = value
2731        elif opt in ['-t', '--ticket_prefix']:
2732            ticket_prefix = value
2733        elif opt in ['-v', '--verbose']:
2734            verbose = True
2735
2736    if virtualenv and os.path.exists(virtualenv):
2737        activate_this = os.path.join(virtualenv, 'bin/activate_this.py')
2738        if os.path.exists(activate_this):
2739            execfile(activate_this, dict(__file__=activate_this))
2740
2741    try:
2742        from trac import __version__ as trac_version
2743        from trac import config as trac_config
2744
2745    except ImportError, detail:
2746        print detail
2747        print "Can not find a a valid trac installation, solutions could be:"
2748        print "\tset PYTHONPATH"
2749        print "\tuse the --virtualenv <dir> option"
2750        sys.exit(1)
2751   
2752    settings = ReadConfig(configfile, project_name)
2753
2754    ## The default prefix for ticket values in email2trac.conf
2755    #
2756    settings.ticket_prefix = ticket_prefix
2757    settings.dry_run = dry_run
2758    settings.verbose = verbose
2759
2760    if bh_product:
2761        settings.bh_product = bh_product
2762
2763    if not settings.debug and debug_interactive:
2764        settings.debug = debug_interactive
2765
2766    if not settings.project:
2767        print __doc__
2768        print 'No Trac project is defined in the email2trac config file.'
2769        sys.exit(1)
2770
2771    logger = setup_log(settings, os.path.basename(settings.project), debug_interactive)
2772   
2773    if component:
2774        settings['component'] = component
2775
2776    ## We are only interested in the major versions
2777    # 0.12.3 --> 0.12
2778    # 1.0.2  --> 1.0
2779    #
2780    l = trac_version.split('.')
2781    version = '.'.join(l[0:2])
2782
2783    logger.debug("Found trac version: %s" %(version))
2784   
2785    try:
2786
2787        if version in ['0.12', '0.13', '1.0', '1.1']:
2788            from trac import attachment
2789            from trac import config as trac_config
2790            from trac import util
2791            from trac.core import TracError
2792            from trac.env import Environment
2793            from trac.perm import PermissionSystem
2794            from trac.perm import PermissionCache
2795            from trac.test import Mock, MockPerm
2796            from trac.ticket.api import TicketSystem
2797            from trac.ticket.web_ui import TicketModule
2798            from trac.web.href import Href
2799
2800            try:
2801                import pkg_resources
2802                pkg = pkg_resources.get_distribution('BloodhoundMultiProduct')
2803                bloodhound = pkg.version.split()[:2]
2804            except pkg_resources.DistributionNotFound:
2805                # assume no bloodhound
2806                bloodhound = None
2807
2808            if bloodhound:
2809                from multiproduct.env import Environment, ProductEnvironment
2810                from multiproduct.ticket.web_ui import (ProductTicketModule
2811                                                        as TicketModule)
2812                logger.debug("Found Bloodhound Distribution")
2813
2814                if not settings.bh_product:
2815                    print __doc__
2816                    print 'No Bloodhound project defined (bh_project) in section:%s email2trac config file.' %(settings.project)
2817                    sys.exit(1)
2818
2819            if agilo:
2820
2821                try:
2822
2823                    from agilo.ticket.model import AgiloTicket as Ticket
2824
2825                except ImportError, detail:
2826
2827                    try:
2828                        from agilo.ticket.model import Ticket
2829                    except ImportError, detail:
2830                        logger.error('Could not find Trac  Agilo environemnt')
2831                        sys.exit(0)
2832
2833            else:
2834
2835                from trac.ticket import Ticket
2836
2837            #
2838            # return  util.text.to_unicode(str)
2839            #
2840            # see http://projects.edgewall.com/trac/changeset/2799
2841            from trac.ticket.notification import TicketNotifyEmail
2842            from trac.notification import NotifyEmail
2843
2844        else:
2845            logger.error('TRAC version %s is not supported' %version)
2846            sys.exit(0)
2847
2848        ## Must be set before environment is created
2849        #
2850        if settings.has_key('python_egg_cache'):
2851            python_egg_cache = str(settings['python_egg_cache'])
2852            os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
2853
2854        if settings.debug > 0:
2855            logger.debug('Loading environment %s', settings.project)
2856
2857        try:
2858            env = Environment(settings['project'], create=0)
2859
2860            if bloodhound:
2861                ## possibly overkill testing if the multiproduct schema is a
2862                # new enough version
2863                #
2864                from multiproduct.env import MultiProductSystem
2865                mps = MultiProductSystem(env)
2866                if mps.get_version() > 4:
2867                    try:
2868                        env = ProductEnvironment(env, settings.bh_product, create=0)
2869                    except LookupError:
2870                        logger.error('%s is not a valid Bloodhound' %settings.bh_product)
2871                        sys.exit(0)
2872               
2873        except IOError, detail:
2874            logger.error("trac error: %s" %detail)
2875            sys.exit(0)
2876        except TracError, detail:
2877            logger.error("trac error: %s" %detail)
2878            sys.exit(0)
2879
2880        tktparser = TicketEmailParser(env, settings, logger, float(version))
2881        tktparser.parse(sys.stdin)
2882
2883    ## Catch all errors and use the logging module
2884    #
2885    except Exception, error:
2886
2887        etype, evalue, etb = sys.exc_info()
2888        for e in traceback.format_exception(etype, evalue, etb):
2889            logger.critical(e)
2890
2891        if m:
2892            tktparser.save_email_for_debug(m, settings.project_name)
2893
2894        sys.exit(1)
2895
2896# EOB
Note: See TracBrowser for help on using the repository browser.